def main(args):
assert args.output.endswith('.pkl'), "Output file must be a pkl file"
# Clear up temp folder if it exists
if os.path.exists(args.tmp_folder) and not args.tpn_only:
shutil.rmtree(args.tmp_folder)
frame_dir = os.path.join(args.tmp_folder, 'frames')
bbox_dir = os.path.join(args.tmp_folder, 'bboxes')
keypoint_file = os.path.join(args.tmp_folder, 'keypoints.json')
if not args.tpn_only:
# split to frames:
ensuredir(os.path.join(args.tmp_folder, 'frames'))
out = os.system("ffmpeg -i %s -qscale:v 2 %s/frames/img_%%06d.jpg" %
(args.vid_path, args.tmp_folder))
if out != 0:
print("could not split to frames, error code: " + str(out))
sys.exit(1)
# Mask-RCNN
maskrcnn_bboxes.predict_imgs(frame_dir, bbox_dir)
# hrnet
hrnet_predict.predict(
'../hrnet/experiments/coco/hrnet/w32_256x192_adam_lr1e-3.yaml',
frame_dir, bbox_dir, keypoint_file)
# Run TPN
print("Running TPN...")
poses = run_tpn(args.model, frame_dir, keypoint_file, args.pose_refine,
args.focal_length, args.cx, args.cy)
save(args.output, poses)
def main(img_folder, metadata, poses_path, depth_folder, out_path, visualize):
config, model = load_model('unnormalized')
test_set = ImageFolderDataset(img_folder, metadata, poses_path, depth_folder)
transforms = load_transforms('unnormalized', config, test_set) + [lambda x: x['pose2d']]
test_set.transform = Compose(transforms)
test_loader = DataLoader(test_set)
pred = torch_predict(model, test_loader)
mean3d = transforms[1].normalizer.mean
std3d = transforms[1].normalizer.std
pred = combine_pose_and_trans(pred, std3d, mean3d, MuPoTSJoints(), 'hip')
result = {}
for image in test_set.images:
inds = test_set.img_names == image
result[image] = pred[inds]
save(out_path, result)
if visualize:
image = test_set.images[0]
image_path = os.path.join(img_folder, image)
show_result(image_path, result[image])
def predict(in_folder, out_folder):
model = load_model()
print("Running model...")
for file in os.listdir(in_folder):
img = cv2.imread(os.path.join(in_folder, file))
with c2_utils.NamedCudaScope(GPU_ID):
cls_boxes, _, _ = infer_engine.im_detect_all(model, img, None)
save(os.path.join(out_folder, "%s.pkl" % file), cls_boxes[1])
def run_experiment(test_dict_2d, test_dict_3d, train_dict_2d, train_dict_3d,
train_dict_3d_absolute, test_dict_actionwise, cam_dict,
normalisation_params, out_path, p):
"""
Runs the experiment specified by the params object `p`. It saves additional data in a
timestamped folder under LOG_PATH, like network structure, trained model, training loss/errors, etc.
"""
model = siamese_network(p)
print "Parameters:"
print p
ensuredir(out_path)
save(os.path.join(out_path, 'params.txt'), str(p))
save(os.path.join(out_path, 'normalisation.pkl'), normalisation_params)
train_generator = PosNegBatchGenerator(
train_dict_2d,
train_dict_3d,
p,
shuffle=True,
cams=cam_dict,
train_dict_3d_absolute=train_dict_3d_absolute)
valid_generator = SplitBatchGenerator(test_dict_2d, test_dict_3d, p)
callbacks = []
if p.weight_decay:
callbacks.append(LearningRateScheduler(exp_decay(p), verbose=1))
callbacks.append(
LogAllMillimeterError(normalisation_params['std_3d'],
test_dict_actionwise,
siamese=len(model.inputs) >= 2,
csv=os.path.join(out_path, 'metrics.log')))
print "Output folder is", out_path
result = model.fit_generator(train_generator,
steps_per_epoch=p.TRAIN_SIZE / p.batch_size,
epochs=p.num_epochs,
callbacks=callbacks,
validation_data=valid_generator,
validation_steps=p.VALID_SIZE / p.batch_size,
verbose=2)
model.save(os.path.join(out_path, 'model.h5'))
return result
def predict_dataset(model, dataset, out_folder, batch_size=16):
loader = DataLoader(dataset,
batch_size,
collate_fn=lambda x: x,
num_workers=3)
with torch.no_grad():
for batch in loader:
predictions = model(batch)
for i in range(len(batch)):
boxes = predictions[i]['instances'].pred_boxes.tensor.cpu(
).numpy()
scores = predictions[i]['instances'].scores.cpu().numpy(
)[:, np.newaxis]
output = np.concatenate([boxes, scores], axis=1)
assert output.shape[1] == 5
save(os.path.join(out_folder, "%s.pkl" % batch[i]['name']),
output)
def run_experiment(output_path, _config, exp: Experiment):
exp.log_parameters(flatten_params(_config))
save(os.path.join(output_path, "config.json"), _config)
ensuredir(output_path)
if _config["train_data"] == "mpii_train":
print("Training data is mpii-train")
train_data = Mpi3dTrainDataset(
_config["pose2d_type"],
_config["pose3d_scaling"],
_config["cap_25fps"],
_config["stride"],
)
elif _config["train_data"] == "mpii+muco":
print("Training data is mpii-train and muco_temp concatenated")
mpi_data = Mpi3dTrainDataset(
_config["pose2d_type"],
_config["pose3d_scaling"],
_config["cap_25fps"],
_config["stride"],
)
muco_data = PersonStackedMucoTempDataset(_config["pose2d_type"],
_config["pose3d_scaling"])
train_data = ConcatPoseDataset(mpi_data, muco_data)
elif _config["train_data"].startswith("muco_temp"):
train_data = PersonStackedMucoTempDataset(_config["pose2d_type"],
_config["pose3d_scaling"])
test_data = Mpi3dTestDataset(_config["pose2d_type"],
_config["pose3d_scaling"],
eval_frames_only=True)
if _config["simple_aug"]:
train_data.augment(False)
assert _config["orient_norm"] == "gauss"
normalizer_orient = MeanNormalizeOrient(train_data)
# Load the preprocessing steps
train_data.transform = None
transforms_train = [
decode_trfrm(_config["preprocess_2d"], globals())(train_data,
cache=False),
decode_trfrm(_config["preprocess_3d"], globals())(train_data,
cache=False),
normalizer_orient,
]
normalizer2d = transforms_train[0].normalizer
normalizer3d = transforms_train[1].normalizer
transforms_test = [
decode_trfrm(_config["preprocess_2d"], globals())(test_data,
normalizer2d),
decode_trfrm(_config["preprocess_3d"], globals())(test_data,
normalizer3d),
normalizer_orient,
]
transforms_train.append(RemoveIndex())
transforms_test.append(RemoveIndex())
train_data.transform = SaveableCompose(transforms_train)
test_data.transform = SaveableCompose(transforms_test)
# save normalisation params
save(output_path + "/preprocess_params.pkl",
train_data.transform.state_dict())
len_train = len(train_data)
len_test = len(test_data)
print("Length of training data:", len_train)
print("Length of test data:", len_test)
exp.log_parameter("train data length", len_train)
exp.log_parameter("test data length", len_test)
bos = train_data[[0]]["orientation"].shape
out_shape = (bos[1] * bos[2] if _config["model"]["loss"] == "orient" else
MuPoTSJoints.NUM_JOINTS * 3)
model = TemporalModelOptimized1f(
train_data[[0]]["pose2d"].shape[-1],
out_shape,
_config["model"]["filter_widths"],
dropout=_config["model"]["dropout"],
channels=_config["model"]["channels"],
layernorm=_config["model"]["layernorm"],
)
test_model = TemporalModel(
train_data[[0]]["pose2d"].shape[-1],
out_shape,
_config["model"]["filter_widths"],
dropout=_config["model"]["dropout"],
channels=_config["model"]["channels"],
layernorm=_config["model"]["layernorm"],
)
model.cuda()
test_model.cuda()
save(output_path + "/model_summary.txt", str(model))
pad = (model.receptive_field() - 1) // 2
train_loader = ChunkedGenerator(
train_data,
_config["batch_size"],
pad,
_config["train_time_flip"],
shuffle=_config["shuffle"],
ordered_batch=_config["ordered_batch"],
)
tester = ModelCopyTemporalEvaluator(test_model,
test_data,
_config["model"]["loss"],
_config["test_time_flip"],
post_process3d=get_postprocessor(
_config, test_data, normalizer3d),
prefix="test",
orient_norm=_config["orient_norm"],
normalizer_orient=normalizer_orient)
torch_train(
exp,
train_loader,
model,
lambda m, b: calc_loss(m, b, _config, None, None, None,
normalizer_orient),
# lambda m, b: calc_loss(
# m,
# b,
# _config,
# torch.tensor(normalizer2d.mean[2::3]).cuda(),
# torch.tensor(normalizer2d.std[2::3]).cuda(),
# torch.tensor(normalizer3d.std).cuda(),
# ),
_config,
callbacks=[tester],
)
model_path = os.path.join(output_path, "model_params.pkl")
torch.save(model.state_dict(), model_path)
exp.log_model("model", model_path)
save(
output_path + "/test_results.pkl",
{
"index": test_data.index,
"pred": preds_from_logger(test_data, tester),
"pose3d": test_data.poses3d,
},
)
def run_experiment(output_path, _config, exp: Experiment):
config, m = eval.load_model(_config["weights"])
# config.update(_config)
config["model"].update(_config["model"])
_config["model"] = config["model"]
# tmp = _config["model"]["loss"]
# _config["model"]["loss"] = "v * mse + e_smooth_small"
exp.log_parameters(train.flatten_params(_config))
# _config["model"]["loss"] = tmp
save(os.path.join(output_path, "config.json"), _config)
ensuredir(output_path)
if _config["train_data"] == "mpii_train":
print("Training data is mpii-train")
train_data = Mpi3dTrainDataset(
_config["pose2d_type"],
_config["pose3d_scaling"],
_config["cap_25fps"],
_config["stride"],
)
elif _config["train_data"] == "mpii+muco":
print("Training data is mpii-train and muco_temp concatenated")
mpi_data = Mpi3dTrainDataset(
_config["pose2d_type"],
_config["pose3d_scaling"],
_config["cap_25fps"],
_config["stride"],
)
muco_data = PersonStackedMucoTempDataset(_config["pose2d_type"],
_config["pose3d_scaling"])
train_data = ConcatPoseDataset(mpi_data, muco_data)
elif _config["train_data"].startswith("muco_temp"):
train_data = PersonStackedMucoTempDataset(_config["pose2d_type"],
_config["pose3d_scaling"])
test_data = Mpi3dTestDataset(_config["pose2d_type"],
_config["pose3d_scaling"],
eval_frames_only=True)
if _config["simple_aug"]:
train_data.augment(False)
# Load the preprocessing steps
params_path = os.path.join(LOG_PATH, _config["weights"],
"preprocess_params.pkl")
transform = SaveableCompose.from_file(params_path, test_data, globals())
train_data.transform = None
transforms_train = [
decode_trfrm(_config["preprocess_2d"], globals())(train_data,
cache=False),
decode_trfrm(_config["preprocess_3d"], globals())(train_data,
cache=False),
]
normalizer2d = transforms_train[0].normalizer
normalizer3d = transforms_train[1].normalizer
transforms_test = [
decode_trfrm(_config["preprocess_2d"], globals())(test_data,
normalizer2d),
decode_trfrm(_config["preprocess_3d"], globals())(test_data,
normalizer3d),
]
transforms_train.append(RemoveIndex())
transforms_test.append(RemoveIndex())
train_data.transform = SaveableCompose(transforms_train)
test_data.transform = SaveableCompose(transforms_test)
# train_data.transform = SaveableCompose.from_file(params_path, train_data, globals())
# test_data.transform = SaveableCompose.from_file(params_path, test_data, globals())
# save normalisation params
save(output_path + "/preprocess_params.pkl",
train_data.transform.state_dict())
len_train = len(train_data)
len_test = len(test_data)
print("Length of training data:", len_train)
print("Length of test data:", len_test)
exp.log_parameter("train data length", len_train)
exp.log_parameter("test data length", len_test)
model = TemporalModelOptimized1f(
train_data[[0]]["pose2d"].shape[-1],
MuPoTSJoints.NUM_JOINTS,
config["model"]["filter_widths"],
dropout=config["model"]["dropout"],
channels=config["model"]["channels"],
layernorm=config["model"]["layernorm"],
)
model.load_state_dict(m.state_dict())
test_model = TemporalModel(
train_data[[0]]["pose2d"].shape[-1],
MuPoTSJoints.NUM_JOINTS,
config["model"]["filter_widths"],
dropout=config["model"]["dropout"],
channels=config["model"]["channels"],
layernorm=config["model"]["layernorm"],
)
model.cuda()
test_model.cuda()
save(output_path + "/model_summary.txt", str(model))
# normalizer2d = train_data.transform.transforms[0].normalizer
# normalizer3d = train_data.transform.transforms[1].normalizer
pad = (model.receptive_field() - 1) // 2
train_loader = ChunkedGenerator(
train_data,
_config["batch_size"],
pad,
_config["train_time_flip"],
shuffle=_config["shuffle"],
ordered_batch=_config["ordered_batch"],
)
tester = ModelCopyTemporalEvaluator(
test_model,
test_data,
config["model"]["loss"],
_config["test_time_flip"],
post_process3d=get_postprocessor(_config, test_data, normalizer3d),
prefix="test",
)
torch_train(
exp,
train_loader,
model,
lambda m, b: train.calc_loss(
m,
b,
_config,
torch.tensor(normalizer2d.mean[2::3]).cuda(),
torch.tensor(normalizer2d.std[2::3]).cuda(),
torch.tensor(normalizer3d.std).cuda(),
),
_config,
callbacks=[tester],
)
model_path = os.path.join(output_path, "model_params.pkl")
torch.save(model.state_dict(), model_path)
exp.log_model("model", model_path)
def run_experiment(output_path, _config):
save(os.path.join(output_path, 'config.json'), _config)
ensuredir(output_path)
if _config['train_data'] == 'mpii_train':
print("Training data is mpii-train")
train_data = Mpi3dTrainDataset(_config['pose2d_type'],
_config['pose3d_scaling'],
_config['cap_25fps'], _config['stride'])
elif _config['train_data'] == 'mpii+muco':
print("Training data is mpii-train and muco_temp concatenated")
mpi_data = Mpi3dTrainDataset(_config['pose2d_type'],
_config['pose3d_scaling'],
_config['cap_25fps'], _config['stride'])
muco_data = PersonStackedMucoTempDataset(_config['pose2d_type'],
_config['pose3d_scaling'])
train_data = ConcatPoseDataset(mpi_data, muco_data)
elif _config['train_data'].startswith('muco_temp'):
train_data = PersonStackedMucoTempDataset(_config['pose2d_type'],
_config['pose3d_scaling'])
test_data = Mpi3dTestDataset(_config['pose2d_type'],
_config['pose3d_scaling'],
eval_frames_only=True)
if _config['simple_aug']:
train_data.augment(False)
# Load the preprocessing steps
train_data.transform = None
transforms_train = [
decode_trfrm(_config['preprocess_2d'], globals())(train_data,
cache=False),
decode_trfrm(_config['preprocess_3d'], globals())(train_data,
cache=False)
]
normalizer2d = transforms_train[0].normalizer
normalizer3d = transforms_train[1].normalizer
transforms_test = [
decode_trfrm(_config['preprocess_2d'], globals())(test_data,
normalizer2d),
decode_trfrm(_config['preprocess_3d'], globals())(test_data,
normalizer3d)
]
transforms_train.append(RemoveIndex())
transforms_test.append(RemoveIndex())
train_data.transform = SaveableCompose(transforms_train)
test_data.transform = SaveableCompose(transforms_test)
# save normalisation params
save(output_path + '/preprocess_params.pkl',
train_data.transform.state_dict())
print("Length of training data:", len(train_data))
print("Length of test data:", len(test_data))
model = TemporalModelOptimized1f(train_data[[0]]['pose2d'].shape[-1],
MuPoTSJoints.NUM_JOINTS,
_config['model']['filter_widths'],
dropout=_config['model']['dropout'],
channels=_config['model']['channels'],
layernorm=_config['model']['layernorm'])
test_model = TemporalModel(train_data[[0]]['pose2d'].shape[-1],
MuPoTSJoints.NUM_JOINTS,
_config['model']['filter_widths'],
dropout=_config['model']['dropout'],
channels=_config['model']['channels'],
layernorm=_config['model']['layernorm'])
model.cuda()
test_model.cuda()
save(output_path + '/model_summary.txt', str(model))
pad = (model.receptive_field() - 1) // 2
train_loader = ChunkedGenerator(train_data,
_config['batch_size'],
pad,
_config['train_time_flip'],
shuffle=True)
tester = ModelCopyTemporalEvaluator(test_model,
test_data,
_config['model']['loss'],
_config['test_time_flip'],
post_process3d=get_postprocessor(
_config, test_data, normalizer3d),
prefix='test')
torch_train(train_loader,
model,
lambda m, b: calc_loss(m, b, _config),
_config,
callbacks=[tester])
torch.save(model.state_dict(), os.path.join(output_path,
'model_params.pkl'))
save(
output_path + '/test_results.pkl', {
'index': test_data.index,
'pred': preds_from_logger(test_data, tester),
'pose3d': test_data.poses3d
})