def train_fused_rn(output_path, dataset_name, dataset_fold, config_filepaths, weights_filepaths, batch_size=32, epochs=100, checkpoint_period=5, learning_rate=1e-4, drop_rate=0.1, freeze_g_theta=False, fuse_at_fc1=False, initial_epoch=0, initial_weights=None, use_data_gen=True, subsample_ratio=None, gpus=1, verbose=2): data_kwargs, _, _ = read_config(config_filepaths[0]) if verbose > 0: print("***** Training parameters *****") print("\t Output path:", output_path) print("\t Dataset:", dataset_name) print("\t Dataset fold:", dataset_fold) print("\t Fusion info") print("\t > config_filepaths:", config_filepaths) print("\t > weights_filepaths:", weights_filepaths) print("\t > freeze_g_theta:", freeze_g_theta) print("\t > fuse_at_fc1:", fuse_at_fc1) print("\t Training options") print("\t > Batch Size:", batch_size) print("\t > Epochs:", epochs) print("\t > Checkpoint Period:", checkpoint_period) print("\t > Learning Rate:", learning_rate) print("\t > Dropout rate:", drop_rate) print("\t > Training Subsample Ratio:", subsample_ratio) print("\t GPUs:", gpus) print("\t Skeleton info") for key, value in data_kwargs.items(): print("\t > {}: {}".format(key, value)) if dataset_name == 'UT': dataset = UT elif dataset_name == 'SBU': dataset = SBU if verbose > 0: print("Reading data...") if use_data_gen: train_generator = DataGenerator(dataset_name, dataset_fold, 'train', batch_size=batch_size, reshuffle=True, shuffle_indiv_order=True, **data_kwargs) val_generator = DataGenerator(dataset_name, dataset_fold, 'validation', batch_size=batch_size, reshuffle=False, shuffle_indiv_order=False, **data_kwargs) X_train, Y_train = train_generator[0] X_val, Y_val = val_generator[0] train_data = train_generator val_data = val_generator else: X_train, Y_train = dataset.get_train(dataset_fold, **data_kwargs) X_val, Y_val = dataset.get_val(dataset_fold, **data_kwargs) train_data = [X_train, Y_train] val_data = [X_val, Y_val] num_joints = len(X_train) // 2 object_shape = (len(X_train[0][0]), ) output_size = len(Y_train[0]) models_kwargs = [] for config_filepath in config_filepaths: data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath) timesteps = data_kwargs['timesteps'] add_joint_idx = data_kwargs['add_joint_idx'] add_body_part = data_kwargs['add_body_part'] overhead = add_joint_idx + add_body_part # True/False = 1/0 num_dim = (object_shape[0] - overhead) // timesteps model_kwargs['num_dim'] = num_dim model_kwargs['overhead'] = overhead models_kwargs.append(model_kwargs) train_kwargs['drop_rate'] = drop_rate if verbose > 0: print("Creating model...") model = fuse_rn(num_joints, object_shape, output_size, train_kwargs, models_kwargs, weights_filepaths, freeze_g_theta=freeze_g_theta, fuse_at_fc1=fuse_at_fc1) if initial_weights is not None: model.load_weights(initial_weights) fit_history = train_model(model=model, verbose=verbose, learning_rate=learning_rate, output_path=output_path, checkpoint_period=checkpoint_period, batch_size=batch_size, epochs=epochs, use_data_gen=use_data_gen, train_data=train_data, val_data=val_data, subsample_ratio=subsample_ratio) return fit_history
def predict_fused_rn(fusion_weights_path, dataset_name, dataset_fold, config_filepaths, freeze_g_theta=False, fuse_at_fc1=False, batch_size=32, verbose=2, gpus=1): if verbose > 0: print("***** Predicting parameters *****") print("\t fusion_weights_path:", fusion_weights_path) print("\t Dataset:", dataset_name) print("\t Dataset fold:", dataset_fold) print("\t Fusion info") print("\t > config_filepaths:", config_filepaths) print("\t > freeze_g_theta:", freeze_g_theta) print("\t > fuse_at_fc1:", fuse_at_fc1) print("\t Predicting options") print("\t > Batch Size:", batch_size) data_kwargs, _, _ = read_config(config_filepaths[0]) val_generator = DataGenerator(dataset_name, dataset_fold, 'validation', batch_size=batch_size, reshuffle=False, shuffle_indiv_order=False, **data_kwargs) X_val, Y_val = val_generator[0] num_joints = len(X_val) // 2 object_shape = (len(X_val[0][0]), ) output_size = len(Y_val[0]) if verbose > 0: print("Reading Y_val...") Y_val = [] for batch_idx in range(len(val_generator)): _, y_val = val_generator[batch_idx] Y_val += y_val.tolist() models_kwargs = [] for config_filepath in config_filepaths: data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath) timesteps = data_kwargs['timesteps'] add_joint_idx = data_kwargs['add_joint_idx'] add_body_part = data_kwargs['add_body_part'] overhead = add_joint_idx + add_body_part # True/False = 1/0 num_dim = (object_shape[0] - overhead) // timesteps model_kwargs['num_dim'] = num_dim model_kwargs['overhead'] = overhead models_kwargs.append(model_kwargs) train_kwargs['drop_rate'] = 0 weights_filepaths = [[] for _ in config_filepaths] if verbose > 0: print("Creating model...") model = fuse_rn(num_joints, object_shape, output_size, train_kwargs, models_kwargs, weights_filepaths, freeze_g_theta=freeze_g_theta, fuse_at_fc1=fuse_at_fc1) if verbose > 0: print("Loading weights...") model.load_weights(fusion_weights_path) if verbose > 0: print("Starting predicting...") Y_pred = model.predict_generator(val_generator, max_queue_size=10, workers=5, use_multiprocessing=True, verbose=verbose) acc_tensor = categorical_accuracy(Y_val, Y_pred) acc = K.eval(acc_tensor).mean() if verbose > 0: print("Validation acc: {:.2%}".format(acc)) # Convert back from to_categorical Y_pred = np.argmax(Y_pred, axis=1, out=None).tolist() Y_val = np.argmax(Y_val, axis=1, out=None).tolist() return Y_pred, Y_val
def predict_fused_rn_seq(fusion_weights_path, dataset_name, dataset_fold, config_filepaths, freeze_g_theta=False, fuse_at_fc1=False, flat_seqs=False, batch_size=32, verbose=2, gpus=1, return_acc=False, use_data_gen=True): if verbose > 0: print("***** Predicting parameters *****") print("\t fusion_weights_path:", fusion_weights_path) print("\t Dataset:", dataset_name) print("\t Dataset fold:", dataset_fold) print("\t Fusion info") print("\t > config_filepaths:", config_filepaths) print("\t > freeze_g_theta:", freeze_g_theta) print("\t > fuse_at_fc1:", fuse_at_fc1) print("\t Predicting options") print("\t > Batch Size:", batch_size) print("\t > flat_seqs:", flat_seqs) print("\t > Use Data Generator:", use_data_gen) #### if dataset_name == 'UT': dataset = UT elif dataset_name == 'SBU': dataset = SBU elif dataset_name == 'NTU': dataset = NTU elif dataset_name == 'YMJA': dataset = YMJA data_kwargs, _, _ = read_config(config_filepaths[0]) data_kwargs['sample_method'] = 'all' data_kwargs['seq_step'] = data_kwargs.get('seq_step', data_kwargs['timesteps'] // 2) if verbose > 0: print("Reading data...") if use_data_gen: if verbose > 0: print("> Using DataGenerator") val_generator = DataGenerator(dataset_name, dataset_fold, 'validation', batch_size=batch_size, reshuffle=False, shuffle_indiv_order=False, **data_kwargs) X_val, Y_val = val_generator[0] num_joints = len(X_val) // 2 object_shape = (len(X_val[0][0]), ) if verbose > 0: print("> Reading Y_val...") Y_val_flat = [] for batch_idx in range(len(val_generator)): _, y_val = val_generator[batch_idx] Y_val_flat += y_val.tolist() videos_address = [] prvs_video_idx = val_generator.seqs_mapping[0][0] pointer, num_seqs = 0, 0 for video_idx, seq_idx in val_generator.seqs_mapping: if prvs_video_idx == video_idx: num_seqs += 1 else: videos_address.append(slice(pointer, pointer + num_seqs)) pointer += num_seqs num_seqs = 1 prvs_video_idx = video_idx videos_address.append(slice(pointer, pointer + num_seqs)) Y_val = [] for video_address in videos_address: Y_val.append(Y_val_flat[video_address][0]) else: if verbose > 0: print("> Reading all data at once") X_val, Y_val = dataset.get_val(dataset_fold, **data_kwargs) if flat_seqs: # Accuracy in this case will be per sequence and not per video num_joints = len(X_val) // 2 object_shape = (len(X_val[0][0]), ) else: # Accuracy in this case will be per video, after averaging the seqs num_joints = len(X_val[0][0]) // 2 object_shape = (len(X_val[0][0][0]), ) ## Flatten X_val at axis = 1 (num_seqs), and swap axis (1,0,2) ## Num_videos replaced by -> SUM num_seqs ## Keep "address" of each input, so unflatten from Y_pred can take place reshaped_X_val = [ ] # reshaped X_val dropping axis which represents the video seqs videos_address = [] # video address in reshaped_X_val pointer = 0 for video_seqs in X_val: num_seqs = len(video_seqs) videos_address.append(slice(pointer, pointer + num_seqs)) pointer += num_seqs reshaped_X_val += video_seqs X_val = np.array(reshaped_X_val).transpose((1, 0, 2)).tolist() output_size = len(Y_val[0]) if verbose > 0: print("Creating model...") models_kwargs = [] for config_filepath in config_filepaths: data_kwargs, model_kwargs, train_kwargs = read_config(config_filepath) timesteps = data_kwargs['timesteps'] add_joint_idx = data_kwargs['add_joint_idx'] add_body_part = data_kwargs['add_body_part'] overhead = add_joint_idx + add_body_part # True/False = 1/0 num_dim = (object_shape[0] - overhead) // timesteps model_kwargs['num_dim'] = num_dim model_kwargs['overhead'] = overhead models_kwargs.append(model_kwargs) train_kwargs['drop_rate'] = 0 weights_filepaths = [[] for _ in config_filepaths] model = fuse_rn(num_joints, object_shape, output_size, train_kwargs, models_kwargs, weights_filepaths, freeze_g_theta=freeze_g_theta, fuse_at_fc1=fuse_at_fc1) if verbose > 0: print("Loading weights...") model.load_weights(fusion_weights_path) if verbose > 0: print("Starting predicting...") if use_data_gen: reshaped_Y_pred = model.predict_generator(val_generator, max_queue_size=10, workers=5, use_multiprocessing=True, verbose=verbose) else: reshaped_Y_pred = model.predict(X_val, batch_size=batch_size, verbose=verbose) use_gauss_weight = True if not flat_seqs: # Undo and avg reshaped_Y_pred (SUM num_seqs, ...) -> (Num_videos, ...) Y_pred = [] for video_address in videos_address: if use_gauss_weight: avg_scores = np.average( reshaped_Y_pred[video_address], axis=0, weights=gauss(len(reshaped_Y_pred[video_address]))) else: avg_scores = np.average(reshaped_Y_pred[video_address], axis=0) Y_pred.append(avg_scores.tolist()) acc_tensor = categorical_accuracy(Y_val, Y_pred) acc = K.eval(acc_tensor).mean() if verbose > 0: print("Validation acc: {:.2%}".format(acc)) # Convert back from to_categorical Y_pred = np.argmax(Y_pred, axis=1, out=None).tolist() Y_val = np.argmax(Y_val, axis=1, out=None).tolist() if return_acc: return acc else: return Y_pred, Y_val
def train_fused_rn(output_path, dataset_name, dataset_fold, config_filepaths, weights_filepaths, batch_size=32, epochs=100, checkpoint_period=5, learning_rate=1e-4, drop_rate=0.1, freeze_g_theta=False, fuse_at_fc1=False, new_arch=False, avg_at_end=False, initial_epoch=0, initial_weights=None, use_data_gen=True, subsample_ratio=None, gpus=1, verbose=2): data_kwargs, _, _ = read_config(config_filepaths[0]) if new_arch: data_kwargs['arch'] = 'joint_temp_fused' if verbose > 0: print("***** Training parameters *****") print("\t Output path:", output_path) print("\t Dataset:", dataset_name) print("\t Dataset fold:", dataset_fold) print("\t Fusion info") print("\t > config_filepaths:", config_filepaths) print("\t > weights_filepaths:", weights_filepaths) print("\t > freeze_g_theta:", freeze_g_theta) print("\t > fuse_at_fc1:", fuse_at_fc1) print("\t > New architecture:", new_arch) print("\t Training options") print("\t > Batch Size:", batch_size) print("\t > Epochs:", epochs) print("\t > Checkpoint Period:", checkpoint_period) print("\t > Learning Rate:", learning_rate) print("\t > Dropout rate:", drop_rate) print("\t > Training Subsample Ratio:", subsample_ratio) print("\t GPUs:", gpus) print("\t Skeleton info") for key, value in data_kwargs.items(): print("\t > {}: {}".format(key, value)) if dataset_name == 'UT': dataset = UT elif dataset_name == 'SBU': dataset = SBU elif dataset_name == 'YMJA': dataset = YMJA if verbose > 0: print("Reading data...") if use_data_gen: train_generator = DataGenerator(dataset_name, dataset_fold, 'train', batch_size=batch_size, reshuffle=True, shuffle_indiv_order=True, **data_kwargs) val_generator = DataGenerator(dataset_name, dataset_fold, 'validation', batch_size=batch_size, reshuffle=False, shuffle_indiv_order=False, **data_kwargs) X_train, Y_train = train_generator.getSampleData(0, new_arch) X_val, Y_val = val_generator.getSampleData(0, new_arch) train_data = train_generator val_data = val_generator else: X_train, Y_train = dataset.get_train(dataset_fold, **data_kwargs) X_val, Y_val = dataset.get_val(dataset_fold, **data_kwargs) train_data = [X_train, Y_train] val_data = [X_val, Y_val] models_kwargs = [] output_size = len(Y_train[0]) if new_arch: check_configs = [] for config_filepath in config_filepaths: data_kwargs, model_kwargs, train_kwargs = read_config( config_filepath) check_configs.append(model_kwargs) # Ensure that temporal stream and joint stream both included. Reorder if necessary so that joint stream first. if (len(check_configs) != 2): print("Error: Expecting Joint Stream and Temporal Stream") exit(0) # Should reorder both weights and config. if (check_configs[0]['rel_type'] == 'temp_stream'): config_filepaths.reverse() weights_filepaths.reverse() check_configs = [] for config_filepath in config_filepaths: data_kwargs, model_kwargs, train_kwargs = read_config( config_filepath) check_configs.append(model_kwargs) # Ensure that both joint and temporal stream exist if (check_configs[0]['rel_type'] != 'joint_stream' or check_configs[1]['rel_type'] != 'temp_stream'): print("Error: Expecting Joint Stream and Temporal Stream") exit(0) # Ensure X_train has two components if (len(X_train) != 2): print( "Error: Expecting X_train to consist of both joint and temporal components" ) exit(0) for config_filepath, X_train_comp in zip(config_filepaths, X_train): num_joints = len(X_train_comp) object_shape = (len(X_train_comp[0][0]), ) output_size = len(Y_train[0]) data_kwargs, model_kwargs, train_kwargs = read_config( config_filepath) timesteps = data_kwargs['timesteps'] add_joint_idx = data_kwargs['add_joint_idx'] add_body_part = data_kwargs['add_body_part'] overhead = add_joint_idx + add_body_part # True/False = 1/0 num_dim = (object_shape[0] - overhead) // timesteps model_kwargs['num_dim'] = num_dim model_kwargs['overhead'] = overhead model_kwargs['num_objs'] = num_joints model_kwargs['object_shape'] = object_shape models_kwargs.append(model_kwargs) else: num_joints = len(X_train) // 2 object_shape = (len(X_train[0][0]), ) for config_filepath in config_filepaths: data_kwargs, model_kwargs, train_kwargs = read_config( config_filepath) timesteps = data_kwargs['timesteps'] add_joint_idx = data_kwargs['add_joint_idx'] add_body_part = data_kwargs['add_body_part'] overhead = add_joint_idx + add_body_part # True/False = 1/0 num_dim = (object_shape[0] - overhead) // timesteps model_kwargs['num_dim'] = num_dim model_kwargs['overhead'] = overhead model_kwargs['num_objs'] = num_joints model_kwargs['object_shape'] = object_shape models_kwargs.append(model_kwargs) for mod_kwargs in models_kwargs: mod_kwargs[ 'return_attention'] = False # Don't return attention for fused mdels train_kwargs['drop_rate'] = drop_rate if verbose > 0: print("Creating model...") model = fuse_rn(output_size, new_arch, train_kwargs, models_kwargs, weights_filepaths, freeze_g_theta=freeze_g_theta, fuse_at_fc1=fuse_at_fc1, avg_at_end=avg_at_end) if initial_weights is not None: model.load_weights(initial_weights) data_len = None fit_history = train_model(model=model, verbose=verbose, learning_rate=learning_rate, output_path=output_path, checkpoint_period=checkpoint_period, batch_size=batch_size, epochs=epochs, use_data_gen=use_data_gen, train_data=train_data, val_data=val_data, subsample_ratio=subsample_ratio, data_len=data_len) return fit_history