def isoFusionTestImageGenerator(filepath_1, filepath_2, batch_size, seq_len, num_classes): # filepath_1:RGB, filepath_2:Flow X_train_1, y_train_1 = data.load_iso_video_list(filepath_1) X_train_2, y_train_2 = data.load_iso_video_list(filepath_2) X_tridx_1 = np.asarray(np.arange(0, len(y_train_1)), dtype=np.int32) X_tridx_2 = np.asarray(np.arange(0, len(y_train_2)), dtype=np.int32) y_train_1 = np.asarray(y_train_1, dtype=np.int32) y_train_2 = np.asarray(y_train_2, dtype=np.int32) # RGB while 1: for X_indices, y_label_t_1 in minibatches(X_tridx_1, y_train_1, batch_size, shuffle=True): # minibatches(X_tridx_2, y_train_2, batch_size, shuffle=True)): # Read data for each batch image_path_1 = [] image_fcnt_1 = [] image_path_2 = [] image_fcnt_2 = [] image_olen = [] image_start = [] is_training = [] for data_a in range(batch_size): X_index_a = X_indices[data_a] key_str = '%06d' % X_index_a image_path_1.append(X_train_1[key_str]['videopath']) image_fcnt_1.append(X_train_1[key_str]['framecnt']) image_path_2.append(X_train_2[key_str]['videopath']) image_fcnt_2.append(X_train_2[key_str]['framecnt']) image_olen.append(seq_len) image_start.append(1) is_training.append(False) # Testing image_info_1 = zip(image_path_1, image_fcnt_1, image_olen, image_start, is_training) image_info_2 = zip(image_path_2, image_fcnt_2, image_olen, image_start, is_training) X_data_t_1 = threading_data([_ for _ in image_info_1], data.prepare_iso_rgb_data) X_data_t_2 = threading_data([_ for _ in image_info_2], data.prepare_iso_flow_data) y_hot_label_t_1 = keras.utils.to_categorical( y_label_t_1, num_classes=num_classes) # import keras.backend as K # X_data_t = keras.layers.Concatenate(axis=-1)([X_data_t_1, X_data_t_2]) # print(X_data_t_1.shape) # print(X_data_t_2.shape) # X_data_t = tf.concat([X_data_t_1, X_data_t_2], axis=-1) # print('*************') yield ([X_data_t_1, X_data_t_2], y_hot_label_t_1)
def jesterTestImageGenerator(filepath, batch_size, seq_len, num_classes, modality): X_test, y_test = data.load_iso_video_list(filepath) X_teidx = np.asarray(np.arange(0, len(y_test)), dtype=np.int32) y_test = np.asarray(y_test, dtype=np.int32) while 1: for X_indices, y_label_t in minibatches(X_teidx, y_test, batch_size, shuffle=False): # Read data for each batch image_path = [] image_fcnt = [] image_olen = [] is_training = [] for data_a in range(batch_size): X_index_a = X_indices[data_a] key_str = '%06d' % X_index_a image_path.append(X_test[key_str]['videopath']) image_fcnt.append(X_test[key_str]['framecnt']) image_olen.append(seq_len) is_training.append(False) # Testing image_info = zip(image_path, image_fcnt, image_olen, is_training) if modality == 0: #RGB X_data_t = threading_data([_ for _ in image_info], data.prepare_jester_rgb_data) if modality == 2: #Flow X_data_t = threading_data([_ for _ in image_info], data.prepare_jester_flow_data) y_hot_label_t = keras.utils.to_categorical(y_label_t, num_classes=num_classes) yield (X_data_t, y_hot_label_t)
def isoTrainImageGenerator(filepath, batch_size, depth, num_classes, modality): X_train,y_train = data.load_iso_video_list(filepath) X_tridx = np.asarray(np.arange(0, len(y_train)), dtype=np.int32) y_train = np.asarray(y_train, dtype=np.int32) while 1: for X_indices, y_label_t in minibatches(X_tridx, y_train, batch_size, shuffle=True): # Read data for each batch image_path = [] image_fcnt = [] image_olen = [] image_start = [] is_training = [] for data_a in range(batch_size): X_index_a = X_indices[data_a] key_str = '%06d' % X_index_a image_path.append(X_train[key_str]['videopath']) image_fcnt.append(X_train[key_str]['framecnt']) image_olen.append(depth) image_start.append(1) is_training.append(True) # Training image_info = zip(image_path,image_fcnt,image_olen,image_start,is_training) if modality==0: #RGB X_data_t = threading_data([_ for _ in image_info], data.prepare_iso_rgb_data) elif modality==1: #Depth X_data_t = threading_data([_ for _ in image_info], data.prepare_iso_depth_data) elif modality==2: #Flow X_data_t = threading_data([_ for _ in image_info], data.prepare_iso_flow_data) y_hot_label_t = keras.utils.to_categorical(y_label_t, num_classes=num_classes) yield (X_data_t, y_hot_label_t)
kernel_regularizer=l2(weight_decay), name='Classes')(flatten) outputs = keras.layers.Activation('softmax', name='Output')(classes) model = keras.models.Model(inputs=inputs, outputs=outputs) optimizer = keras.optimizers.SGD(lr=0.001, decay=0, momentum=0.9, nesterov=False) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) if cfg_type == ATTENTIONX: pretrained_model = '%s/isogr_rgb_attenxclstm_weights.h5' % model_prefix elif cfg_type == ATTENTIONI: pretrained_model = '%s/isogr_rgb_atteniclstm_weights.h5' % model_prefix elif cfg_type == ATTENTIONO: pretrained_model = '%s/isogr_rgb_attenoclstm_weights.h5' % model_prefix print 'Loading pretrained model from %s' % pretrained_model model.load_weights(pretrained_model, by_name=False) for i in range(len(model.trainable_weights)): print model.trainable_weights[i] _, test_labels = data.load_iso_video_list(testing_datalist) test_steps = len(test_labels) / batch_size print model.evaluate_generator( isoTestImageGenerator(testing_datalist, batch_size, seq_len, num_classes, cfg_modality), steps=test_steps, )
init_epoch = 0 seq_len = 32 batch_size = 4 num_classes = 249 dataset_name = 'isogr_Fusion' RGB_training_datalist = './dataset_splits/IsoGD/train_rgb_list.txt' RGB_testing_datalist = './dataset_splits/IsoGD/valid_rgb_list.txt' Flow_training_datalist = './dataset_splits/IsoGD/train_flow_list.txt' Flow_testing_datalist = './dataset_splits/IsoGD/valid_flow_list.txt' weight_decay = 0.00005 model_prefix = './models/Fusion/' weights_file = '%s/%s_weights.{epoch:02d}-{val_loss:.2f}.h5' % (model_prefix, dataset_name) _, train_labels = data.load_iso_video_list(RGB_training_datalist) train_steps = len(train_labels) / batch_size _, test_labels = data.load_iso_video_list(RGB_testing_datalist) test_steps = len(test_labels) / batch_size print 'nb_epoch: %d - seq_len: %d - batch_size: %d - weight_decay: %.6f' % ( nb_epoch, seq_len, batch_size, weight_decay) def lr_polynomial_decay(global_step): learning_rate = 0.001 end_learning_rate = 0.000001 decay_steps = train_steps * nb_epoch power = 0.9 p = float(global_step) / float(decay_steps) lr = (learning_rate - end_learning_rate) * \ np.power(1-p, power)+end_learning_rate
def jesterTrainImageGenerator(filepath, batch_size, seq_len, num_classes, modality): X_train, y_train = data.load_iso_video_list(filepath) X_tridx = np.asarray(np.arange(0, len(y_train)), dtype=np.int32) y_train = np.asarray(y_train, dtype=np.int32) while 1: for X_indices, y_label_t in minibatches(X_tridx, y_train, batch_size, shuffle=True): # Read data for each batch image_path = [] image_fcnt = [] image_olen = [] image_start = [] is_training = [] for data_a in range(batch_size): X_index_a = X_indices[data_a] key_str = '%06d' % X_index_a if USING_PYRAMID_INPUT == True: framecnt = X_train[key_str]['framecnt'] if framecnt <= seq_len: for pid in range(3): image_path.append(X_train[key_str]['videopath']) image_fcnt.append(X_train[key_str]['framecnt']) image_olen.append(seq_len) image_start.append(1) is_training.append(True) # Training elif framecnt < seq_len * 3: image_path.append(X_train[key_str]['videopath']) image_fcnt.append(seq_len) image_olen.append(seq_len) image_start.append(1) is_training.append(True) # Training image_path.append(X_train[key_str]['videopath']) image_fcnt.append(seq_len) image_olen.append(seq_len) image_start.append(framecnt / 2 - seq_len / 2 + 1) is_training.append(True) # Training image_path.append(X_train[key_str]['videopath']) image_fcnt.append(seq_len) image_olen.append(seq_len) image_start.append(framecnt - seq_len + 1) is_training.append(True) # Training else: for pid in range(3): image_path.append(X_train[key_str]['videopath']) image_fcnt.append(framecnt / 3) image_olen.append(seq_len) image_start.append(framecnt * pid / 3 + 1) is_training.append(True) # Training image_path.append(X_train[key_str]['videopath']) image_fcnt.append(X_train[key_str]['framecnt']) image_olen.append(seq_len) image_start.append(1) is_training.append(True) # Training image_info = zip(image_path, image_fcnt, image_olen, image_start, is_training) if modality == 0: #RGB X_data_t = threading_data([_ for _ in image_info], data.prepare_jester_rgb_data) if modality == 2: #Flow X_data_t = threading_data([_ for _ in image_info], data.prepare_jester_flow_data) y_hot_label_t = keras.utils.to_categorical(y_label_t, num_classes=num_classes) yield (X_data_t, y_hot_label_t)