def predict_fused_rn_seq(fusion_weights_path,
dataset_name,
dataset_fold,
config_filepaths,
freeze_g_theta=False,
fuse_at_fc1=False,
flat_seqs=False,
batch_size=32,
verbose=2,
gpus=1,
return_acc=False,
use_data_gen=True):
if verbose > 0:
print("***** Predicting parameters *****")
print("\t fusion_weights_path:", fusion_weights_path)
print("\t Dataset:", dataset_name)
print("\t Dataset fold:", dataset_fold)
print("\t Fusion info")
print("\t > config_filepaths:", config_filepaths)
print("\t > freeze_g_theta:", freeze_g_theta)
print("\t > fuse_at_fc1:", fuse_at_fc1)
print("\t Predicting options")
print("\t > Batch Size:", batch_size)
print("\t > flat_seqs:", flat_seqs)
print("\t > Use Data Generator:", use_data_gen)
####
if dataset_name == 'UT':
dataset = UT
elif dataset_name == 'SBU':
dataset = SBU
elif dataset_name == 'NTU':
dataset = NTU
elif dataset_name == 'YMJA':
dataset = YMJA
data_kwargs, _, _ = read_config(config_filepaths[0])
data_kwargs['sample_method'] = 'all'
data_kwargs['seq_step'] = data_kwargs.get('seq_step',
data_kwargs['timesteps'] // 2)
if verbose > 0:
print("Reading data...")
if use_data_gen:
if verbose > 0:
print("> Using DataGenerator")
val_generator = DataGenerator(dataset_name,
dataset_fold,
'validation',
batch_size=batch_size,
reshuffle=False,
shuffle_indiv_order=False,
**data_kwargs)
X_val, Y_val = val_generator[0]
num_joints = len(X_val) // 2
object_shape = (len(X_val[0][0]), )
if verbose > 0:
print("> Reading Y_val...")
Y_val_flat = []
for batch_idx in range(len(val_generator)):
_, y_val = val_generator[batch_idx]
Y_val_flat += y_val.tolist()
videos_address = []
prvs_video_idx = val_generator.seqs_mapping[0][0]
pointer, num_seqs = 0, 0
for video_idx, seq_idx in val_generator.seqs_mapping:
if prvs_video_idx == video_idx:
num_seqs += 1
else:
videos_address.append(slice(pointer, pointer + num_seqs))
pointer += num_seqs
num_seqs = 1
prvs_video_idx = video_idx
videos_address.append(slice(pointer, pointer + num_seqs))
Y_val = []
for video_address in videos_address:
Y_val.append(Y_val_flat[video_address][0])
else:
if verbose > 0:
print("> Reading all data at once")
X_val, Y_val = dataset.get_val(dataset_fold, **data_kwargs)
if flat_seqs: # Accuracy in this case will be per sequence and not per video
num_joints = len(X_val) // 2
object_shape = (len(X_val[0][0]), )
else: # Accuracy in this case will be per video, after averaging the seqs
num_joints = len(X_val[0][0]) // 2
object_shape = (len(X_val[0][0][0]), )
## Flatten X_val at axis = 1 (num_seqs), and swap axis (1,0,2)
## Num_videos replaced by -> SUM num_seqs
## Keep "address" of each input, so unflatten from Y_pred can take place
reshaped_X_val = [
] # reshaped X_val dropping axis which represents the video seqs
videos_address = [] # video address in reshaped_X_val
pointer = 0
for video_seqs in X_val:
num_seqs = len(video_seqs)
videos_address.append(slice(pointer, pointer + num_seqs))
pointer += num_seqs
reshaped_X_val += video_seqs
X_val = np.array(reshaped_X_val).transpose((1, 0, 2)).tolist()
output_size = len(Y_val[0])
if verbose > 0:
print("Creating model...")
models_kwargs = []
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath)
timesteps = data_kwargs['timesteps']
add_joint_idx = data_kwargs['add_joint_idx']
add_body_part = data_kwargs['add_body_part']
overhead = add_joint_idx + add_body_part # True/False = 1/0
num_dim = (object_shape[0] - overhead) // timesteps
model_kwargs['num_dim'] = num_dim
model_kwargs['overhead'] = overhead
models_kwargs.append(model_kwargs)
train_kwargs['drop_rate'] = 0
weights_filepaths = [[] for _ in config_filepaths]
model = fuse_rn(num_joints,
object_shape,
output_size,
train_kwargs,
models_kwargs,
weights_filepaths,
freeze_g_theta=freeze_g_theta,
fuse_at_fc1=fuse_at_fc1)
if verbose > 0:
print("Loading weights...")
model.load_weights(fusion_weights_path)
if verbose > 0:
print("Starting predicting...")
if use_data_gen:
reshaped_Y_pred = model.predict_generator(val_generator,
max_queue_size=10,
workers=5,
use_multiprocessing=True,
verbose=verbose)
else:
reshaped_Y_pred = model.predict(X_val,
batch_size=batch_size,
verbose=verbose)
use_gauss_weight = True
if not flat_seqs: # Undo and avg reshaped_Y_pred (SUM num_seqs, ...) -> (Num_videos, ...)
Y_pred = []
for video_address in videos_address:
if use_gauss_weight:
avg_scores = np.average(
reshaped_Y_pred[video_address],
axis=0,
weights=gauss(len(reshaped_Y_pred[video_address])))
else:
avg_scores = np.average(reshaped_Y_pred[video_address], axis=0)
Y_pred.append(avg_scores.tolist())
acc_tensor = categorical_accuracy(Y_val, Y_pred)
acc = K.eval(acc_tensor).mean()
if verbose > 0:
print("Validation acc: {:.2%}".format(acc))
# Convert back from to_categorical
Y_pred = np.argmax(Y_pred, axis=1, out=None).tolist()
Y_val = np.argmax(Y_val, axis=1, out=None).tolist()
if return_acc:
return acc
else:
return Y_pred, Y_val
def train_fused_rn(output_path,
dataset_name,
dataset_fold,
config_filepaths,
weights_filepaths,
batch_size=32,
epochs=100,
checkpoint_period=5,
learning_rate=1e-4,
drop_rate=0.1,
freeze_g_theta=False,
fuse_at_fc1=False,
initial_epoch=0,
initial_weights=None,
use_data_gen=True,
subsample_ratio=None,
gpus=1,
verbose=2):
data_kwargs, _, _ = read_config(config_filepaths[0])
if verbose > 0:
print("***** Training parameters *****")
print("\t Output path:", output_path)
print("\t Dataset:", dataset_name)
print("\t Dataset fold:", dataset_fold)
print("\t Fusion info")
print("\t > config_filepaths:", config_filepaths)
print("\t > weights_filepaths:", weights_filepaths)
print("\t > freeze_g_theta:", freeze_g_theta)
print("\t > fuse_at_fc1:", fuse_at_fc1)
print("\t Training options")
print("\t > Batch Size:", batch_size)
print("\t > Epochs:", epochs)
print("\t > Checkpoint Period:", checkpoint_period)
print("\t > Learning Rate:", learning_rate)
print("\t > Dropout rate:", drop_rate)
print("\t > Training Subsample Ratio:", subsample_ratio)
print("\t GPUs:", gpus)
print("\t Skeleton info")
for key, value in data_kwargs.items():
print("\t > {}: {}".format(key, value))
if dataset_name == 'UT':
dataset = UT
elif dataset_name == 'SBU':
dataset = SBU
if verbose > 0:
print("Reading data...")
if use_data_gen:
train_generator = DataGenerator(dataset_name,
dataset_fold,
'train',
batch_size=batch_size,
reshuffle=True,
shuffle_indiv_order=True,
**data_kwargs)
val_generator = DataGenerator(dataset_name,
dataset_fold,
'validation',
batch_size=batch_size,
reshuffle=False,
shuffle_indiv_order=False,
**data_kwargs)
X_train, Y_train = train_generator[0]
X_val, Y_val = val_generator[0]
train_data = train_generator
val_data = val_generator
else:
X_train, Y_train = dataset.get_train(dataset_fold, **data_kwargs)
X_val, Y_val = dataset.get_val(dataset_fold, **data_kwargs)
train_data = [X_train, Y_train]
val_data = [X_val, Y_val]
num_joints = len(X_train) // 2
object_shape = (len(X_train[0][0]), )
output_size = len(Y_train[0])
models_kwargs = []
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath)
timesteps = data_kwargs['timesteps']
add_joint_idx = data_kwargs['add_joint_idx']
add_body_part = data_kwargs['add_body_part']
overhead = add_joint_idx + add_body_part # True/False = 1/0
num_dim = (object_shape[0] - overhead) // timesteps
model_kwargs['num_dim'] = num_dim
model_kwargs['overhead'] = overhead
models_kwargs.append(model_kwargs)
train_kwargs['drop_rate'] = drop_rate
if verbose > 0:
print("Creating model...")
model = fuse_rn(num_joints,
object_shape,
output_size,
train_kwargs,
models_kwargs,
weights_filepaths,
freeze_g_theta=freeze_g_theta,
fuse_at_fc1=fuse_at_fc1)
if initial_weights is not None:
model.load_weights(initial_weights)
fit_history = train_model(model=model,
verbose=verbose,
learning_rate=learning_rate,
output_path=output_path,
checkpoint_period=checkpoint_period,
batch_size=batch_size,
epochs=epochs,
use_data_gen=use_data_gen,
train_data=train_data,
val_data=val_data,
subsample_ratio=subsample_ratio)
return fit_history
def predict_fused_rn(fusion_weights_path,
dataset_name,
dataset_fold,
config_filepaths,
freeze_g_theta=False,
fuse_at_fc1=False,
batch_size=32,
verbose=2,
gpus=1):
if verbose > 0:
print("***** Predicting parameters *****")
print("\t fusion_weights_path:", fusion_weights_path)
print("\t Dataset:", dataset_name)
print("\t Dataset fold:", dataset_fold)
print("\t Fusion info")
print("\t > config_filepaths:", config_filepaths)
print("\t > freeze_g_theta:", freeze_g_theta)
print("\t > fuse_at_fc1:", fuse_at_fc1)
print("\t Predicting options")
print("\t > Batch Size:", batch_size)
data_kwargs, _, _ = read_config(config_filepaths[0])
val_generator = DataGenerator(dataset_name,
dataset_fold,
'validation',
batch_size=batch_size,
reshuffle=False,
shuffle_indiv_order=False,
**data_kwargs)
X_val, Y_val = val_generator[0]
num_joints = len(X_val) // 2
object_shape = (len(X_val[0][0]), )
output_size = len(Y_val[0])
if verbose > 0:
print("Reading Y_val...")
Y_val = []
for batch_idx in range(len(val_generator)):
_, y_val = val_generator[batch_idx]
Y_val += y_val.tolist()
models_kwargs = []
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath)
timesteps = data_kwargs['timesteps']
add_joint_idx = data_kwargs['add_joint_idx']
add_body_part = data_kwargs['add_body_part']
overhead = add_joint_idx + add_body_part # True/False = 1/0
num_dim = (object_shape[0] - overhead) // timesteps
model_kwargs['num_dim'] = num_dim
model_kwargs['overhead'] = overhead
models_kwargs.append(model_kwargs)
train_kwargs['drop_rate'] = 0
weights_filepaths = [[] for _ in config_filepaths]
if verbose > 0:
print("Creating model...")
model = fuse_rn(num_joints,
object_shape,
output_size,
train_kwargs,
models_kwargs,
weights_filepaths,
freeze_g_theta=freeze_g_theta,
fuse_at_fc1=fuse_at_fc1)
if verbose > 0:
print("Loading weights...")
model.load_weights(fusion_weights_path)
if verbose > 0:
print("Starting predicting...")
Y_pred = model.predict_generator(val_generator,
max_queue_size=10,
workers=5,
use_multiprocessing=True,
verbose=verbose)
acc_tensor = categorical_accuracy(Y_val, Y_pred)
acc = K.eval(acc_tensor).mean()
if verbose > 0:
print("Validation acc: {:.2%}".format(acc))
# Convert back from to_categorical
Y_pred = np.argmax(Y_pred, axis=1, out=None).tolist()
Y_val = np.argmax(Y_val, axis=1, out=None).tolist()
return Y_pred, Y_val
def train_fused_rn(output_path,
dataset_name,
dataset_fold,
config_filepaths,
weights_filepaths,
batch_size=32,
epochs=100,
checkpoint_period=5,
learning_rate=1e-4,
drop_rate=0.1,
freeze_g_theta=False,
fuse_at_fc1=False,
new_arch=False,
avg_at_end=False,
initial_epoch=0,
initial_weights=None,
use_data_gen=True,
subsample_ratio=None,
gpus=1,
verbose=2):
data_kwargs, _, _ = read_config(config_filepaths[0])
if new_arch:
data_kwargs['arch'] = 'joint_temp_fused'
if verbose > 0:
print("***** Training parameters *****")
print("\t Output path:", output_path)
print("\t Dataset:", dataset_name)
print("\t Dataset fold:", dataset_fold)
print("\t Fusion info")
print("\t > config_filepaths:", config_filepaths)
print("\t > weights_filepaths:", weights_filepaths)
print("\t > freeze_g_theta:", freeze_g_theta)
print("\t > fuse_at_fc1:", fuse_at_fc1)
print("\t > New architecture:", new_arch)
print("\t Training options")
print("\t > Batch Size:", batch_size)
print("\t > Epochs:", epochs)
print("\t > Checkpoint Period:", checkpoint_period)
print("\t > Learning Rate:", learning_rate)
print("\t > Dropout rate:", drop_rate)
print("\t > Training Subsample Ratio:", subsample_ratio)
print("\t GPUs:", gpus)
print("\t Skeleton info")
for key, value in data_kwargs.items():
print("\t > {}: {}".format(key, value))
if dataset_name == 'UT':
dataset = UT
elif dataset_name == 'SBU':
dataset = SBU
elif dataset_name == 'YMJA':
dataset = YMJA
if verbose > 0:
print("Reading data...")
if use_data_gen:
train_generator = DataGenerator(dataset_name,
dataset_fold,
'train',
batch_size=batch_size,
reshuffle=True,
shuffle_indiv_order=True,
**data_kwargs)
val_generator = DataGenerator(dataset_name,
dataset_fold,
'validation',
batch_size=batch_size,
reshuffle=False,
shuffle_indiv_order=False,
**data_kwargs)
X_train, Y_train = train_generator.getSampleData(0, new_arch)
X_val, Y_val = val_generator.getSampleData(0, new_arch)
train_data = train_generator
val_data = val_generator
else:
X_train, Y_train = dataset.get_train(dataset_fold, **data_kwargs)
X_val, Y_val = dataset.get_val(dataset_fold, **data_kwargs)
train_data = [X_train, Y_train]
val_data = [X_val, Y_val]
models_kwargs = []
output_size = len(Y_train[0])
if new_arch:
check_configs = []
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(
config_filepath)
check_configs.append(model_kwargs)
# Ensure that temporal stream and joint stream both included. Reorder if necessary so that joint stream first.
if (len(check_configs) != 2):
print("Error: Expecting Joint Stream and Temporal Stream")
exit(0)
# Should reorder both weights and config.
if (check_configs[0]['rel_type'] == 'temp_stream'):
config_filepaths.reverse()
weights_filepaths.reverse()
check_configs = []
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(
config_filepath)
check_configs.append(model_kwargs)
# Ensure that both joint and temporal stream exist
if (check_configs[0]['rel_type'] != 'joint_stream'
or check_configs[1]['rel_type'] != 'temp_stream'):
print("Error: Expecting Joint Stream and Temporal Stream")
exit(0)
# Ensure X_train has two components
if (len(X_train) != 2):
print(
"Error: Expecting X_train to consist of both joint and temporal components"
)
exit(0)
for config_filepath, X_train_comp in zip(config_filepaths, X_train):
num_joints = len(X_train_comp)
object_shape = (len(X_train_comp[0][0]), )
output_size = len(Y_train[0])
data_kwargs, model_kwargs, train_kwargs = read_config(
config_filepath)
timesteps = data_kwargs['timesteps']
add_joint_idx = data_kwargs['add_joint_idx']
add_body_part = data_kwargs['add_body_part']
overhead = add_joint_idx + add_body_part # True/False = 1/0
num_dim = (object_shape[0] - overhead) // timesteps
model_kwargs['num_dim'] = num_dim
model_kwargs['overhead'] = overhead
model_kwargs['num_objs'] = num_joints
model_kwargs['object_shape'] = object_shape
models_kwargs.append(model_kwargs)
else:
num_joints = len(X_train) // 2
object_shape = (len(X_train[0][0]), )
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(
config_filepath)
timesteps = data_kwargs['timesteps']
add_joint_idx = data_kwargs['add_joint_idx']
add_body_part = data_kwargs['add_body_part']
overhead = add_joint_idx + add_body_part # True/False = 1/0
num_dim = (object_shape[0] - overhead) // timesteps
model_kwargs['num_dim'] = num_dim
model_kwargs['overhead'] = overhead
model_kwargs['num_objs'] = num_joints
model_kwargs['object_shape'] = object_shape
models_kwargs.append(model_kwargs)
for mod_kwargs in models_kwargs:
mod_kwargs[
'return_attention'] = False # Don't return attention for fused mdels
train_kwargs['drop_rate'] = drop_rate
if verbose > 0:
print("Creating model...")
model = fuse_rn(output_size,
new_arch,
train_kwargs,
models_kwargs,
weights_filepaths,
freeze_g_theta=freeze_g_theta,
fuse_at_fc1=fuse_at_fc1,
avg_at_end=avg_at_end)
if initial_weights is not None:
model.load_weights(initial_weights)
data_len = None
fit_history = train_model(model=model,
verbose=verbose,
learning_rate=learning_rate,
output_path=output_path,
checkpoint_period=checkpoint_period,
batch_size=batch_size,
epochs=epochs,
use_data_gen=use_data_gen,
train_data=train_data,
val_data=val_data,
subsample_ratio=subsample_ratio,
data_len=data_len)
return fit_history
"--normalize",
action="store_true",
help="Normalize the embedding before averaging and output.")
parser.add_argument("--config", type=str, help="The configuration file.")
parser.add_argument("model_dir", type=str, help="The model directory.")
parser.add_argument("rspecifier",
type=str,
help="Kaldi feature rspecifier (or ark file).")
parser.add_argument("wspecifier",
type=str,
help="Kaldi output wspecifier (or ark file).")
if __name__ == "__main__":
args = parser.parse_args()
params = read_config(args.config)
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
nnet_dir = os.path.join(args.model_dir, "nnet")
with open(os.path.join(nnet_dir, "num_speakers"), 'r') as f:
num_speakers = f.readline().strip()
trainer = Trainer(params=params,
model_dir=args.model_dir,
num_speakers=int(num_speakers))
load_model_dir = os.path.join(args.model_dir, "checkpoint")
trainer.load(model=trainer.network,
model_name=os.path.join(load_model_dir, "net.pth"))
trainer.network.eval()
if args.rspecifier.rsplit(".", 1)[1] == "scp":
def run_protocol(experiment_name,
config_filepath,
dataset_name,
num_reruns=2,
folds=None,
verbose=0,
batch_size=64,
seqs_eval=False,
fusion_mode=None,
temp_rn=False):
if fusion_mode is None:
data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath)
else:
fusion_kwargs, train_kwargs = read_config(config_filepath, fusion=True)
weights_base_paths = [
'/'.join(weights_filepath.split('/')[:-3])
for weights_filepath in fusion_kwargs['weights_filepaths']
]
fusion_kwargs.pop('weights_filepaths', None)
criteria = fusion_kwargs.pop(
'criteria', 'val_loss' if fusion_mode == 'middle' else 'val_acc')
data_kwargs, model_kwargs, _ = read_config(
fusion_kwargs['config_filepaths'][0])
if dataset_name == 'UT-1':
dataset_folds = UT.get_folds(setid=1)
dataset_codename = 'UT'
elif dataset_name == 'UT-2':
dataset_folds = UT.get_folds(setid=2)
dataset_codename = 'UT'
elif dataset_name == 'SBU':
dataset_folds = SBU.get_folds()
dataset_codename = 'SBU'
elif dataset_name == 'YMJA':
dataset_folds = YMJA.get_folds()
dataset_codename = 'YMJA'
elif dataset_name == 'NTU':
dataset_folds = NTU.get_folds()
dataset_codename = 'NTU'
elif dataset_name == 'NTU-V2':
dataset_folds = NTU_V2.get_folds()
dataset_codename = 'NTU-V2'
if folds is not None:
if dataset_codename.startswith('NTU'):
dataset_folds = folds
else:
dataset_folds = list(map(int, folds))
base_path = 'models/{}/{}'.format(dataset_name, experiment_name)
print("Running for dataset:", dataset_name)
print("Running for folds:", dataset_folds)
print("Experiment name:", experiment_name)
print("Experiment folder:", base_path)
print("Number of reruns per dataset fold:", num_reruns)
if os.path.exists(base_path):
print("Resuming run protocol...")
else:
os.makedirs(base_path)
if config_filepath != os.path.join(base_path, 'parameters.cfg'):
copyfile(config_filepath, os.path.join(base_path, 'parameters.cfg'))
print("***** Experiment parameters *****")
print("Configuration file:", config_filepath)
print("\t Skeleton info")
for key, value in data_kwargs.items():
print("\t > {}: {}".format(key, value))
if fusion_mode is None:
print("\t Model info")
for key, value in model_kwargs.items():
print("\t > {}: {}".format(key, value))
else:
print("\t Fusion info")
print("\t > fusion mode:", fusion_mode)
for key, value in fusion_kwargs.items():
print("\t > {}: {}".format(key, value))
print("\t > Criteria for weights:", criteria)
print("\t > Using Temporal RN:", temp_rn)
print("\t Training options")
for key, value in train_kwargs.items():
print("\t > {}: {}".format(key, value))
print("\t > Batch Size:", batch_size)
print("\t Evaluation options")
print("\t > Pool average from videos sequences:", seqs_eval)
if seqs_eval:
fold_results_seqs = []
data_kwargs_seqs = data_kwargs.copy()
data_kwargs_seqs.setdefault('seq_step', data_kwargs['timesteps'] // 2)
data_kwargs_seqs['flat_seqs'] = False
print("\t > Step between sequences:", data_kwargs_seqs['seq_step'])
fold_results = []
for dataset_fold in dataset_folds:
print("Running for fold:", dataset_fold)
fold_path = base_path + '/fold_{}/'.format(dataset_fold)
if fusion_mode == 'middle':
fold_weights_filepaths = [
find_best_weights(weights_base_path +
'/fold_{}/'.format(dataset_fold),
criteria=criteria)
for weights_base_path in weights_base_paths
]
print('> fold weights filepaths:', fold_weights_filepaths)
if seqs_eval: reruns_val_acc_seqs = []
best_epochs = []
for rerun_idx in range(num_reruns):
output_path = fold_path + '/rerun_{}'.format(rerun_idx)
if os.path.exists(output_path + '/fit_history.csv'):
hist_df = pd.read_csv(output_path + '/fit_history.csv')
else:
if fusion_mode is None:
if not temp_rn:
fit_history = train_rn(output_path,
dataset_codename,
dataset_fold=dataset_fold,
**train_kwargs,
data_kwargs=data_kwargs,
model_kwargs=model_kwargs,
batch_size=batch_size,
gpus=1,
verbose=verbose)
else:
fit_history = train_temp_rn(output_path,
dataset_codename,
dataset_fold=dataset_fold,
**train_kwargs,
data_kwargs=data_kwargs,
model_kwargs=model_kwargs,
batch_size=batch_size,
gpus=1,
verbose=verbose)
elif fusion_mode == 'middle':
if not temp_rn:
fit_history = train_fused_rn(
output_path,
dataset_codename,
dataset_fold=dataset_fold,
weights_filepaths=fold_weights_filepaths,
**fusion_kwargs,
**train_kwargs,
batch_size=batch_size,
gpus=1,
verbose=verbose)
else:
fit_history = train_fused_temp_rn(
output_path,
dataset_codename,
dataset_fold=dataset_fold,
weights_filepaths=fold_weights_filepaths,
**fusion_kwargs,
**train_kwargs,
batch_size=batch_size,
gpus=1,
verbose=verbose)
hist_df = parse_fit_history(fit_history)
hist_df.to_csv(output_path + '/fit_history.csv', index=False)
sorted_hist_df = hist_df.sort_values(['val_acc', 'val_loss'],
ascending=[False, True])
best_epoch = sorted_hist_df.iloc[0]
print(
"> {}: ACC: {:.2%} Loss: {:.4f} - MAX val_ACC: {:.2%} val_Loss: {:.4f}"
.format(rerun_idx, best_epoch.acc, best_epoch.loss,
best_epoch.val_acc, best_epoch.val_loss))
best_epochs.append(best_epoch)
if seqs_eval:
rerun_path = output_path
pooled_val_acc_filepath = rerun_path + '/pooled_val_acc.csv'
if os.path.exists(pooled_val_acc_filepath):
val_acc_df = pd.read_csv(pooled_val_acc_filepath)
else:
weights_path = rerun_path + '/relnet_weights-val_acc.hdf5'
if fusion_mode is None:
val_acc = predict_rn_seq(
weights_path,
dataset_codename,
dataset_fold=dataset_fold,
batch_size=batch_size,
verbose=verbose,
data_kwargs_seqs=data_kwargs_seqs,
model_kwargs=model_kwargs,
return_acc=True)
elif fusion_mode == 'middle':
val_acc = predict_fused_rn_seq(
weights_path,
dataset_codename,
dataset_fold=dataset_fold,
batch_size=batch_size,
verbose=verbose,
**fusion_kwargs,
return_acc=True)
else:
print("Seqs eval not implemented for fusion_mode:",
fusion_mode)
val_acc = -1
val_acc_df = pd.DataFrame.from_dict({'val_acc': [val_acc]})
val_acc_df.to_csv(pooled_val_acc_filepath, index=False)
val_acc_series = val_acc_df.iloc[0]
print("> Pooled Seqs val_ACC: {:.2%}".format(
val_acc_series.val_acc))
reruns_val_acc_seqs.append(val_acc_series)
summary_fold = pd.concat(best_epochs,
axis=1).transpose().reset_index(drop=True)
summary_fold.index.name = 'rerun'
summary_fold.to_csv(fold_path + '/summary.csv')
sorted_summary_fold = summary_fold.sort_values(['val_acc', 'val_loss'],
ascending=[False, True])
best_fold = sorted_summary_fold.iloc[0]
fold_results.append(best_fold)
if seqs_eval:
summary_fold_seqs = pd.concat(
reruns_val_acc_seqs, axis=1).transpose().reset_index(drop=True)
summary_fold_seqs.index.name = 'rerun'
summary_fold_seqs.to_csv(fold_path + '/summary-pooled_val_acc.csv')
best_fold_seqs = summary_fold_seqs.loc[
summary_fold_seqs.val_acc.idxmax()]
fold_results_seqs.append(best_fold_seqs)
summary_df = pd.concat(fold_results,
axis=1).transpose().reset_index(drop=True)
summary_df.index.name = 'fold'
summary_df.to_csv(base_path + '/summary.csv')
mean = summary_df.mean().rename('mean')
std = summary_df.std().rename('std')
stats = pd.concat([mean, std], axis=1).T
summary_df = pd.concat([summary_df, stats])
print("+++++ Summarized results:")
print("\nSummary for Dataset, Folds, Experiment:")
print("> {} | {} | {}".format(dataset_codename, dataset_folds,
experiment_name))
pretty_print_stats(summary_df)
print("Val ACC Mean: {:.2%} Std: {:.4f}".format(mean.val_acc, std.val_acc))
if seqs_eval:
summary_seqs = pd.concat(fold_results_seqs,
axis=1).transpose().reset_index(drop=True)
summary_seqs.index.name = 'fold'
summary_seqs.to_csv(base_path + '/summary-pooled_val_acc.csv')
print("+++++ Seqs eval results:")
mean = summary_seqs.mean().rename('mean')
std = summary_seqs.std().rename('std')
stats = pd.concat([mean, std], axis=1).T
summary_seqs = pd.concat([summary_seqs, stats])
print(summary_seqs.to_string(formatters=["{:.2%}".format]))
def predict_fused_temp_rn(fusion_weights_path,
dataset_name,
dataset_fold,
config_filepaths,
freeze_g_theta=False,
fuse_at_fc1=False,
batch_size=32,
verbose=2,
gpus=1):
if verbose > 0:
print("***** Predicting parameters *****")
print("\t fusion_weights_path:", fusion_weights_path)
print("\t Dataset:", dataset_name)
print("\t Dataset fold:", dataset_fold)
print("\t Fusion info")
print("\t > config_filepaths:", config_filepaths)
print("\t > freeze_g_theta:", freeze_g_theta)
print("\t > fuse_at_fc1:", fuse_at_fc1)
print("\t Predicting options")
print("\t > Batch Size:", batch_size)
####
data_kwargs, _, _ = read_config(config_filepaths[0])
if verbose > 0:
print("Initializing Data Generator...")
val_generator = DataGeneratorSeq(dataset_name,
dataset_fold,
'validation',
batch_size=batch_size,
reshuffle=False,
shuffle_indiv_order=False,
pad_sequences=True,
**data_kwargs)
X_val, Y_val = val_generator[0]
_, seq_len, num_joints, *object_shape = np.array(X_val).shape
num_joints = num_joints // 2
object_shape = tuple(object_shape)
output_size = len(Y_val[0])
models_kwargs = []
for config_filepath in config_filepaths:
data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath)
timesteps = data_kwargs['timesteps']
add_joint_idx = data_kwargs['add_joint_idx']
add_body_part = data_kwargs['add_body_part']
overhead = add_joint_idx + add_body_part # True/False = 1/0
num_dim = (object_shape[0] - overhead) // timesteps
model_kwargs['num_dim'] = num_dim
model_kwargs['overhead'] = overhead
models_kwargs.append(model_kwargs)
train_kwargs['drop_rate'] = 0
weights_filepaths = [[] for _ in config_filepaths]
if verbose > 0:
print("Creating model...")
model = get_fusion_model(num_joints,
object_shape,
output_size,
seq_len,
train_kwargs,
models_kwargs,
weights_filepaths,
freeze_g_theta=freeze_g_theta,
fuse_at_fc1=fuse_at_fc1)
if verbose > 0:
print("Loading weights...")
model.load_weights(fusion_weights_path)
if verbose > 0:
print("Starting predicting...")
Y_pred = []
Y_val = []
if verbose > 0:
progbar = progressbar.ProgressBar(max_value=len(val_generator))
for batch_idx in range(len(val_generator)):
if verbose > 0: progbar.update(batch_idx)
batch_x, batch_y = val_generator[batch_idx]
Y_pred += list(model.predict_on_batch(batch_x))
Y_val += batch_y.tolist()
if verbose > 0: progbar.finish()
y_true = np.array(Y_val)
y_pred = np.array(Y_pred)
n_hits = np.sum(y_true.argmax(axis=-1) == y_pred.argmax(axis=-1))
acc = n_hits / y_true.shape[0]
if verbose > 0:
print("Validation acc: {:.2%}".format(acc))
# Convert back from to_categorical
Y_pred = np.argmax(Y_pred, axis=1).tolist()
Y_val = np.argmax(Y_val, axis=1).tolist()
return Y_pred, Y_val