trainable=False)(errors) # calculate weighted error by layer errors_by_time = Flatten()(errors_by_time) # will be (batch_size, nt) final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) # weight errors by time #Create Model model = Model(inputs=inputs, outputs=final_errors) # Loading Pretrained weights #model.load_weights('/home/daniel/AnacondaProjects/prednet-master_2/model_data_keras2/prednet_kitti_weights-extrapfinetuned.hdf5') # configuring Model model.compile(loss='mean_absolute_error', optimizer='adam') train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True) val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val) lr_schedule = lambda epoch: 0.001 if epoch < 75 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs callbacks = [LearningRateScheduler(lr_schedule)] if save_model: if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) callbacks.append( ModelCheckpoint(filepath=weights_file, monitor='val_loss',
data_format = layer_config[ 'data_format'] if 'data_format' in layer_config else layer_config[ 'dim_ordering'] prednet = PredNet(weights=orig_model.layers[1].get_weights(), **layer_config) input_shape = list(orig_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt inputs = Input(input_shape) predictions = prednet(inputs) model = Model(inputs=inputs, outputs=predictions) model.compile(loss=extrap_loss, optimizer='adam') train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True, output_mode='prediction') val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val, output_mode='prediction') lr_schedule = lambda epoch: 0.0001 if epoch < 75 else 0.00001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs callbacks = [LearningRateScheduler(lr_schedule)] if save_model: if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) callbacks.append( ModelCheckpoint(filepath=weights_file,
layer_config = train_model.layers[1].get_config() layer_config['output_mode'] = 'prediction' data_format = layer_config[ 'data_format'] if 'data_format' in layer_config else layer_config[ 'dim_ordering'] test_pred_rgcLSTM = Pred_rgcLSTM(weights=train_model.layers[1].get_weights(), **layer_config) input_shape = list(train_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt inputs = Input(shape=tuple(input_shape)) predictions = test_pred_rgcLSTM(inputs) test_model = Model(inputs=inputs, outputs=predictions) test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', data_format=data_format) X_test = test_generator.create_all() X_hat = test_model.predict(X_test, batch_size) if data_format == 'channels_first': X_test = np.transpose(X_test, (0, 1, 3, 4, 2)) X_hat = np.transpose(X_hat, (0, 1, 3, 4, 2)) # Compare MSE of Predrgc_LSTM predictions vs. using last frame. Write results to prediction_scores.txt mse_model = np.mean( (X_test[:, 1:] - X_hat[:, 1:])**2) # look at all timesteps except the first mae_model = np.mean(np.abs(X_test[:, 1:] - X_hat[:, 1:])) mse_prev = np.mean((X_test[:, :-1] - X_test[:, 1:])**2) ssim = evaluu.compare_ssim(X_test[:, 1:],
# Create testing model (to output predictions) layer_config = train_model.layers[1].get_config() layer_config['output_mode'] = 'prediction' dim_ordering = layer_config['dim_ordering'] test_prednet = PredNet(weights=train_model.layers[1].get_weights(), **layer_config) input_shape = list(train_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt inputs = Input(shape=tuple(input_shape)) predictions = test_prednet(inputs) test_model = Model(input=inputs, output=predictions) test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', dim_ordering=dim_ordering) X_test = test_generator.create_all() X_hat = test_model.predict(X_test, batch_size) if dim_ordering == 'th': X_test = np.transpose(X_test, (0, 1, 3, 4, 2)) X_hat = np.transpose(X_hat, (0, 1, 3, 4, 2)) # Compare MSE of PredNet predictions vs. using last frame. Write results to prediction_scores.txt mse_model = np.mean( (X_test[:, 1:] - X_hat[:, 1:])**2) # look at all timesteps except the first mse_prev = np.mean((X_test[:, :-1] - X_test[:, 1:])**2) if not os.path.exists(RESULTS_SAVE_DIR): os.mkdir(RESULTS_SAVE_DIR) f = open(RESULTS_SAVE_DIR + 'prediction_scores.txt', 'w')
def main(): client = storage.Client() bucket = client.bucket('meniscus_cloud_data_1000') val_recordings = [('clouds', 'validation')] test_recordings = [('clouds', 'testing')] categories = 'clouds' desired_im_sz = (1000, 1000) #### Trainiing data ####### training_dataset = [] for blob in bucket.list_blobs(prefix='kitti_data/clouds/training'): training_dataset.append(blob.name) training_images = [] for i in len(training_dataset): name = 'kitti_data/clouds/validation/' + training_dataset[i] for blob in bucket.list_blobs(prefix='kitti_data/clouds/training/' + training_dataset[i]): #training_images += name + '/' + blob.name training_images += blob.id X = np.zeros((len(training_images), ) + desired_im_sz + (3, ), np.uint8) for i, filename in enumerate(training_images): with open(filename, "wb") as file_obj: im_file = blob.download_to_file(file_obj) im = scipy.misc.imread(im_file) X[i] = im train_sources = training_images train_file = X #### Validation data ####### validation_dataset = [] for blob in bucket.list_blobs(prefix='kitti_data/clouds/validation'): validation_dataset.append(blob.name) validation_images = [] for i in len(validation_dataset): name = 'kitti_data/clouds/validation/' + validation_dataset[i] for blob in bucket.list_blobs(prefix=name): #validation_images_blob += name + '/' + blob.name validation_images += blob.id X = np.zeros((len(validation_images), ) + desired_im_sz + (3, ), np.uint8) for i, im_file in enumerate(validation_images): with open(filename, "wb") as file_obj: im_file = blob.download_to_file(file_obj) im = scipy.misc.imread(im_file) X[i] = im val_sources = validation_images val_file = X save_model = True # if weights will be saved weights_file = WEIGHTS_DIR + 'prednet_kitti_weights.hdf5' # where weights will be saved json_file = WEIGHTS_DIR + 'prednet_kitti_model.json' saved_models = WEIGHTS_DIR + 'weights.{epoch:02d}-{val_loss:.2f}.hdf5' #Training parameters nb_epoch = 50 samples_per_epoch = 28 batch_size = 8 N_seq_val = 4 # number of sequences to use for validation # number of timesteps used for sequences in training nt = 4 lr = 0.001 #learning rate up_lr = 0.0001 #learinig rate is updated to this new value up_lr_ep = 40 #point at which learinig rate should be updated # Model parameters #n_channels, im_height, im_width = (3, 128, 160) n_channels, im_height, im_width = (3, 1000, 1000) input_shape = ( n_channels, im_height, im_width) if K.image_data_format() == 'channels_first' else ( im_height, im_width, n_channels) stack_sizes = (n_channels, 48, 96, 192) #Lstm stack_sizes R_stack_sizes = stack_sizes #convolutional filter size A_filt_sizes = (3, 3, 3) #prdiction convloutinal filter size Ahat_filt_sizes = (3, 3, 3, 3) # recurrent convolution filter size R_filt_sizes = (3, 3, 3, 3) # weighting for each layer in final loss; "L_0" model: [1, 0, 0, 0], "L_all": [1, 0.1, 0.1, 0.1] layer_loss_weights = np.array([1., 0., 0., 0.]) layer_loss_weights = np.expand_dims(layer_loss_weights, 1) # equally weight all timesteps except the first time_loss_weights = 1. / (nt - 1) * np.ones((nt, 1)) time_loss_weights[0] = 0 #-----------------------------------------Arcitecture---------------------------------------------------# prednet = PredNet(stack_sizes, R_stack_sizes, A_filt_sizes, Ahat_filt_sizes, R_filt_sizes, output_mode='error', return_sequences=True) #-----------------------------------------Layers--------------------------------------------------------# #initializing a tensor for input # inputs = Input(shape=(nt, ) + input_shape) # # errors will be (batch_size, nt, nb_layers) # errors = prednet(inputs) # # calculate weighted error by layer # errors_by_time = TimeDistributed( Dense(1, trainable=False), weights=[layer_loss_weights, np.zeros(1)], # trainable=False)(errors) # # will be (batch_size, nt) # errors_by_time = Flatten()(errors_by_time) # # weight errors by time # # dense() creates a densely connected network # # final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) # #-------------------------------------------------------------------------------------------------------# #----------------------------------Create model---------------------------------------------------------# model = Model(inputs=inputs, outputs=final_errors) #model = load_model(weights_file,custom_objects={'prednet': PredNet}) # model.compile(loss='mean_absolute_error', optimizer='adam') #model.load_weights(weights_file) #-------------------------------------------------------------------------------------------------------# #-------------------------------------Data Preprocessing------------------------------------------------# train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True) # val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val) # #-------------------------------------------------------------------------------------------------------# #-------------------------------------Callback functions for training--------------------------------------# # start with lesrning rate of 0.001 and then drop to 0.0001 after 75 epochs # lr_schedule = lambda epoch: lr if epoch < up_lr_ep else up_lr # callbacks = [LearningRateScheduler(lr_schedule)] # # #save model best model check points # # if save_model: # # if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) # # callbacks.append(ModelCheckpoint(filepath=weights_file, monitor='val_loss', save_best_only=True)) # callbacks.append( ModelCheckpoint(filepath=weights_file, monitor='val_loss', save_best_only=True)) #Tensorboard for visualization # tb = TensorBoard(log_dir=GRAPH_DIR, batch_size=batch_size, histogram_freq=2, write_graph=True, write_images=True) tb.set_model(model) callbacks.append(tb) checkPoints = ModelCheckpoint(saved_models, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1) # callbacks.append(checkPoints) #earlyStops = EarlyStopping(monitor='val_loss', min_delta=0, # patience=0, verbose=0, mode='auto') #callbacks.append(earlyStops) #--------------------------------------------------------------------------------------------------------------# #-------------------------------------Training-----------------------------------------------------------------# history = model.fit_generator( train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, # validation_data=val_generator, validation_steps=N_seq_val / batch_size) # # #save model # # if save_model: # # json_string = model.to_json() # # with open(json_file, "w") as f: # # f.write(json_string) json_string = model.to_json() with file_io.FileIO(json_string, mode='r') as input_f: with file_io.FileIO(json_file, mode='w+') as output_f: output_f.write(input_f.read())
print("%\n%\n%\n%\n%\n%\n%\n%") print("\n======== Confirming PredNet class ========\n") print("prednet_static:", PredNet_static) print("\nprednet_dynamic:", PredNet_dynamic) print("%\n%\n%\n%\n%\n%\n%\n%") # Replicate the model on G GPUs parallel_model = multi_gpu_model(model, gpus=G) parallel_model.compile(loss=weighted_loss, optimizer="adam", metrics=[full_loss, dynamic_loss]) train_generator = SequenceGenerator(train_path, train_source_path, num_tsteps, batch_size=batch_size, shuffle=True, output_mode="prediction") val_generator = SequenceGenerator(val_path, val_source_path, num_tsteps, batch_size=batch_size, N_seq=num_seq_val, output_mode="prediction") print("Shapes: ", train_generator.X.shape, val_generator.X.shape) print("train generator", np.amax(train_generator.X), np.amin(train_generator.X)) lr_schedule = lambda epoch: 0.00001 if epoch < 75 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs
f.close() train_model = model_from_json(json_string, custom_objects = {'PredNet': PredNet}) train_model.load_weights(weights_file) # Create testing model (to output predictions) layer_config = train_model.layers[1].get_config() layer_config['output_mode'] = 'prediction' dim_ordering = layer_config['dim_ordering'] test_prednet = PredNet(weights=train_model.layers[1].get_weights(), **layer_config) input_shape = list(train_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt inputs = Input(shape=tuple(input_shape)) predictions = test_prednet(inputs) test_model = Model(input=inputs, output=predictions) test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', dim_ordering=dim_ordering) X_test = test_generator.create_all() X_hat = test_model.predict(X_test, batch_size) if dim_ordering == 'th': X_test = np.transpose(X_test, (0, 1, 3, 4, 2)) X_hat = np.transpose(X_hat, (0, 1, 3, 4, 2)) # Compare MSE of PredNet predictions vs. using last frame. Write results to prediction_scores.txt mse_model = np.mean( (X_test[:, 1:] - X_hat[:, 1:])**2 ) # look at all timesteps except the first mse_prev = np.mean( (X_test[:, :-1] - X_test[:, 1:])**2 ) if not os.path.exists(RESULTS_SAVE_DIR): os.mkdir(RESULTS_SAVE_DIR) f = open(RESULTS_SAVE_DIR + 'prediction_scores.txt', 'w') f.write("Model MSE: %f\n" % mse_model) f.write("Previous Frame MSE: %f" % mse_prev) f.close()
layer_config = train_model.layers[1].get_config() layer_config['output_mode'] = 'prediction' data_format = layer_config[ 'data_format'] if 'data_format' in layer_config else layer_config[ 'dim_ordering'] test_prednet = PredNet(weights=train_model.layers[1].get_weights(), **layer_config) input_shape = list(train_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt_1stp inputs = Input(shape=tuple(input_shape)) predictions = test_prednet(inputs) test_model = Model(inputs=inputs, outputs=predictions) test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', data_format=data_format, batch_size=batch_size) def predict_multistep(nt, nt_1stp, X_test, test_model): ''' Multi-step prediction using PredNet model ''' X_hat = np.zeros(X_test.shape, dtype=np.float32) X_tmp = X_test.copy() ntpred = nt - nt_1stp for n in range(ntpred): n1 = n n2 = nt_1stp + n
batch_size = 1 nt = 24 nt_1stp = 12 #time dimension for one-step prediction # scale factor for converting [0-1] range data to [0-201.0] mm/h scale_factor = 201.0 test_file = os.path.join(DATA_DIR, 'jma_2hr_128_train_2015-2016_data.hkl') test_sources = os.path.join(DATA_DIR, 'jma_2hr_128_train_2015-2016_sources.hkl') # Prep data test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', data_format='channels_last', batch_size=batch_size) # First load all the data, then select by index X_test = test_generator.create_all() X_test = X_test def print_range_by_value(X_test, value, fwrite): f = open(RESULTS_SAVE_DIR + fwrite, 'w') for i in range(X_test.shape[0]): print('index:', i, ', :max value', np.max(X_test[i, :, :, :, :])) Xtmp = np.max(X_test[i, :, :, :, :], axis=(1, 2, 3))
# 对每一个误差进行权重赋值,这里取第一时间 final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) # weight errors by time # 构建预测模型,输入为时间轴,HxWxC,输出为final_errors,使用优化器adam,loss为MSE model = Model(inputs=inputs, outputs=final_errors) model.compile(loss='mean_absolute_error', optimizer='adam') # 继续原先的训练 if os.path.exists(weights_file): model.load_weights(weights_file) # 训练数据生成器和校准数据生成器 train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True, sequence_start_mode='unique') val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val, sequence_start_mode='unique') # 学习率lr在75周期后从0.001下降到0.0001 lr_schedule = lambda epoch: 0.001 if epoch < 75 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs callbacks = [LearningRateScheduler(lr_schedule)] # 如果存储模型则增加该模块 if save_model: if not os.path.exists(WEIGHTS_DIR):
def main(verbose=False): save_model = True # if weights will be saved weights_file = os.path.join(WEIGHTS_DIR, 'prednet_kitti_weights.hdf5') # where weights will be saved json_file = os.path.join(WEIGHTS_DIR, 'prednet_kitti_model.json') # Data files train_file = os.path.join(DATA_DIR, 'X_train.hkl') train_sources = os.path.join(DATA_DIR, 'sources_train.hkl') val_file = os.path.join(DATA_DIR, 'X_val.hkl') val_sources = os.path.join(DATA_DIR, 'sources_val.hkl') # Training parameters nb_epoch = 75 batch_size = 4 samples_per_epoch = 250 N_seq_val = 100 # number of sequences to use for validation # Model parameters n_channels, im_height, im_width = (3, 128, 160) input_shape = (n_channels, im_height, im_width) if K.image_data_format() == 'channels_first' else (im_height, im_width, n_channels) stack_sizes = (n_channels, 48, 96, 192) R_stack_sizes = stack_sizes A_filt_sizes = (3, 3, 3) Ahat_filt_sizes = (3, 3, 3, 3) R_filt_sizes = (3, 3, 3, 3) layer_loss_weights = np.array([1., 0., 0., 0.]) # weighting for each layer in final loss; "L_0" model: [1, 0, 0, 0], "L_all": [1, 0.1, 0.1, 0.1] layer_loss_weights = np.expand_dims(layer_loss_weights, 1) nt = 10 # number of timesteps used for sequences in training time_loss_weights = 1./ (nt - 1) * np.ones((nt,1)) # equally weight all timesteps except the first time_loss_weights[0] = 0 prednet = PredNet(stack_sizes, R_stack_sizes, A_filt_sizes, Ahat_filt_sizes, R_filt_sizes, output_mode='error', return_sequences=True) inputs = Input(shape=(nt,) + input_shape) # The output will have shape (batch_size, nt, nb_layers). The outputs correspond # to the errors at each time step and layer. errors = prednet(inputs) # This merely computes a weighted sum of the errors layer by layer throughout time. # The output has shape (batch_size, nt, 1). errors_by_time = TimeDistributed( Dense(1, trainable=False), weights=[layer_loss_weights, np.zeros(1)], trainable=False)(errors) # Will have shape (batch_size, nt) errors_by_time = Flatten()(errors_by_time) # The output of this final layer is the weighted sum over time of the weighted # sums of the errors layer-by-layer, which is the final L_train function from # the original paper. final_errors = Dense( 1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) model = Model(inputs=inputs, outputs=final_errors) model.compile(loss='mean_absolute_error', optimizer='adam') if verbose: model.summary() # dense_2 = model.layers[-1] # from pprint import pprint # pprint(dir(dense_2)) # print("_initial_weights: ", dense_2._initial_weights) # print("_losses: ", dense_2._losses) # print("_per_input_losses: ", dense_2._per_input_losses) # print("_per_input_updates: ", dense_2._per_input_updates) # print("_trainable_weights: ", dense_2._trainable_weights) # print("_updates: ", dense_2._updates) # print("bias:", dense_2.bias) # print("count_params: ", dense_2.count_params()) # print("config: ", dense_2.get_config()) # print("weights: ", dense_2.get_weights()) # print("losses: ", dense_2.losses) # print("kernel: ", dense_2.kernel.name) # print("trainable_weights: ", dense_2.trainable_weights) # print("This is going to be interesting...") # model._make_train_function() # training_function = model.train_function # print("Training Function: ", training_function) # pprint(dir(training_function)) # print("_callable_fn: ", training_function._callable_fn) # print("_feed_arrays: ", training_function._feed_arrays) # print("_feed_symbols: ", training_function._feed_symbols) # print("feed_dict: ", training_function.feed_dict) # print("fetches: ", training_function.fetches) # print("inputs: ", training_function.inputs) # print("outputs: ", training_function.outputs) # print("name: ", training_function.name) # print("session_kwargs: ", training_function.session_kwargs) # print("updates_op: ", training_function.updates_op) train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True) val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val) # start with lr of 0.001 and then drop to 0.0001 after 75 epochs lr_schedule = lambda epoch: 0.001 if epoch < 75 else 0.0001 callbacks = [LearningRateScheduler(lr_schedule)] if save_model: if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) callbacks.append(ModelCheckpoint(filepath=weights_file, monitor='val_loss', save_best_only=True)) history = model.fit_generator(train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, validation_data=val_generator, validation_steps=N_seq_val / batch_size) if save_model: json_string = model.to_json() with open(json_file, "w") as f: f.write(json_string)
def execute_test(): print "Preparing to execute the test..." # Load trained model f = open(json_file, 'r') json_string = f.read() f.close() train_model = model_from_json(json_string, custom_objects={'PredNet': PredNet}) train_model.load_weights(weights_file) # Create testing model (to output predictions) layer_config = train_model.layers[1].get_config() layer_config['output_mode'] = 'prediction' #'prediction' layer_config['extrap_start_time'] = extrap data_format = layer_config[ 'data_format'] if 'data_format' in layer_config else layer_config[ 'dim_ordering'] test_prednet = PredNet(weights=train_model.layers[1].get_weights(), **layer_config) input_shape = list(train_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt inputs = Input(shape=tuple(input_shape)) predictions = test_prednet(inputs) test_model = Model(inputs=inputs, outputs=predictions) test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', data_format=data_format) # orig: unique X_test = test_generator.create_all() X_hat = test_model.predict(X_test, batch_size) if data_format == 'channels_first': X_test = np.transpose(X_test, (0, 1, 3, 4, 2)) X_hat = np.transpose(X_hat, (0, 1, 3, 4, 2)) # Compare MSE of PredNet predictions vs. using last frame. Write results to prediction_scores.txt mse_model = np.mean( (X_test[:, 1:] - X_hat[:, 1:])**2) # look at all timesteps except the first mse_prev = np.mean((X_test[:, :-1] - X_test[:, 1:])**2) if not os.path.exists(RESULTS_DIR): os.mkdir(RESULTS_DIR) f = open(os.path.join(RESULTS_DIR, 'prediction_scores.txt'), 'w') f.write("Model MSE: %f\n" % mse_model) f.write("Previous Frame MSE: %f" % mse_prev) f.close() # Plot some predictions aspect_ratio = float(X_hat.shape[2]) / X_hat.shape[3] plt.figure(figsize=(nt, 2 * aspect_ratio)) gs = gridspec.GridSpec(2, nt) gs.update(wspace=0., hspace=0.) plot_save_dir = os.path.join(RESULTS_DIR, 'prediction_plots/') if not os.path.exists(plot_save_dir): os.mkdir(plot_save_dir) # Output the sequence of all the predicted images for test in range(numtests): testdir = os.path.join("single/", testdir_name) testdir = os.path.join(plot_save_dir, testdir) if not os.path.exists(testdir): os.makedirs(testdir) print "///////// NT: " + str(nt) for t in range(nt): imsave(testdir + "/pred-%02d.jpg" % (t, ), X_hat[test, t]) imsave(testdir + "/orig-%02d.jpg" % (t, ), X_test[test, t]) print "Test data saved in " + testdir
model.summary() print("%\n%\n%\n%\n%\n%\n%\n%") print("\n======== Confirming PredNet class ========\n") print("prednet_static:", PredNet_static) print("\nprednet_dynamic:", PredNet_dynamic) print("%\n%\n%\n%\n%\n%\n%\n%") # Replicate the model on G GPUs parallel_model = multi_gpu_model(model, gpus=G) parallel_model.compile(loss=weighted_loss, optimizer="adam", metrics=[err_loss_static, err_loss_dynamic]) train_generator = SequenceGenerator(train_path, train_source_path, num_tsteps, batch_size=batch_size, shuffle=True) val_generator = SequenceGenerator(val_path, val_source_path, num_tsteps, batch_size=batch_size, N_seq=num_seq_val) print("Shapes: ", train_generator.X.shape, val_generator.X.shape) print("train generator", np.amax(train_generator.X), np.amin(train_generator.X)) lr_schedule = lambda epoch: 0.0001 if epoch < 75 else 0.0001 lr_callback = LearningRateScheduler(lr_schedule)
inputs = Input(shape=(nt, ) + input_shape) errors = prednet(inputs) # errors will be (batch_size, nt, nb_layers) errors_by_time = TimeDistributed( Dense(1, trainable=False), weights=[layer_loss_weights, np.zeros(1)], trainable=False)(errors) # calculate weighted error by layer errors_by_time = Flatten()(errors_by_time) # will be (batch_size, nt) final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) # weight errors by time model = Model(inputs=inputs, outputs=final_errors) model.compile(loss='mean_absolute_error', optimizer='adam') train_generator = SequenceGenerator(train_file, nt, batch_size=batch_size, shuffle=True) val_generator = SequenceGenerator(val_file, nt, batch_size=batch_size, N_seq=N_seq_val) lr_schedule = lambda epoch: 0.001 if epoch < 75 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs callbacks = [LearningRateScheduler(lr_schedule)] history = model.fit_generator(train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, validation_data=val_generator, validation_steps=N_seq_val / batch_size)
def run_training(subdir, hyperparams): ''' Run the training of Prednet with the given dataset and choice of hyperparameters, log the training metrics and save the trained model for running inference. ''' save_model = True # if weights will be saved if not os.path.exists(os.path.join(WEIGHTS_DIR, subdir)): os.mkdir(os.path.join(WEIGHTS_DIR, subdir)) weights_file = os.path.join( WEIGHTS_DIR, subdir, 'prednet_ucsd_weights.hdf5') # where weights will be saved json_file = os.path.join(WEIGHTS_DIR, subdir, 'prednet_ucsd_model.json') # Data files train_file = os.path.join(DATA_DIR, subdir, 'X_Train.hkl') train_sources = os.path.join(DATA_DIR, subdir, 'sources_Train.hkl') val_file = os.path.join(DATA_DIR, subdir, 'X_Val.hkl') val_sources = os.path.join(DATA_DIR, subdir, 'sources_Val.hkl') if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) if not os.path.exists(os.path.join(LOG_DIR, subdir)): os.mkdir(os.path.join(LOG_DIR, subdir)) now = datetime.now folder_now = now().strftime("%Y_%m_%d-%H%M") if not os.path.exists(os.path.join(LOG_DIR, subdir, folder_now)): os.mkdir(os.path.join(LOG_DIR, subdir, folder_now)) training_log = os.path.join(LOG_DIR, subdir, folder_now, 'log.csv') model_weights = os.path.join(LOG_DIR, subdir, folder_now, 'weights.h5') hyperparam = os.path.join(LOG_DIR, subdir, folder_now, 'hyperparam.json') # Training parameters nb_epoch = hyperparams['nb_epoch'] batch_size = hyperparams['batch_size'] samples_per_epoch = hyperparams['samples_per_epoch'] N_seq_val = hyperparams[ 'N_seq_val'] # number of sequences to use for validation old_learning_rate = hyperparams['old_learning_rate'] new_learning_rate = hyperparams['new_learning_rate'] epoch_learning_rate_number = hyperparams['epoch_learning_rate_number'] # Model parameters n_channels, im_height, im_width = (1, 128, 160) input_shape = ( n_channels, im_height, im_width) if K.image_data_format() == 'channels_first' else ( im_height, im_width, n_channels) sz1 = hyperparams['sz1'] sz2 = hyperparams['sz2'] sz3 = hyperparams['sz3'] stack_sizes = (n_channels, sz1, sz2, sz3) R_stack_sizes = stack_sizes fz = hyperparams['fz'] A_filt_sizes = (fz, fz, 1) Ahat_filt_sizes = (fz, fz, fz, 1) R_filt_sizes = (fz, fz, fz, 1) layer_loss_weights = np.array( [1., 0., 0., 0.] ) # weighting for each layer in final loss; "L_0" model: [1, 0, 0, 0], "L_all": [1, 0.1, 0.1, 0.1] layer_loss_weights = np.expand_dims(layer_loss_weights, 1) nt = 10 #5 # number of timesteps used for sequences in training time_loss_weights = 1. / (nt - 1) * np.ones( (nt, 1)) # equally weight all timesteps except the first time_loss_weights[0] = 0 hyperparam_dict = { 'epoch': nb_epoch, 'batch_size': batch_size, 'samples_per_epoch': samples_per_epoch, 'N_seq_val': N_seq_val, 'stack_sz': sz1, 'stack_sz2': sz2, 'stack_sz3': sz3, 'A_filt_sz': fz, 'old_learning_rate': old_learning_rate, 'new_learning_rate': new_learning_rate, 'epoch_learning_rate': epoch_learning_rate_number } with open(hyperparam, 'w') as f: json.dump(hyperparam_dict, f) prednet = PredNet(stack_sizes, R_stack_sizes, A_filt_sizes, Ahat_filt_sizes, R_filt_sizes, output_mode='error', return_sequences=True) inputs = Input(shape=(nt, ) + input_shape) errors = prednet(inputs) # errors will be (batch_size, nt, nb_layers) errors = Dropout(0.2)(errors) errors_by_time = TimeDistributed( Dense(1, trainable=False), weights=[layer_loss_weights, np.zeros(1)], trainable=False)(errors) # calculate weighted error by layer errors_by_time = Flatten()(errors_by_time) # will be (batch_size, nt) final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)( errors_by_time) # weight errors by time model = Model(inputs=inputs, outputs=final_errors) model.compile(loss='mean_absolute_error', optimizer='adam') train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True) val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val) lr_schedule = lambda epoch: old_learning_rate if epoch < epoch_learning_rate_number else new_learning_rate # start with lr of 0.001 and then drop to 0.0001 after 75 epochs callbacks = [LearningRateScheduler(lr_schedule)] if save_model: if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) callbacks.append( ModelCheckpoint(filepath=weights_file, monitor='val_loss', save_best_only=True)) callbacks.append(CSVLogger(training_log)) history = model.fit_generator(train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, validation_data=val_generator, validation_steps=N_seq_val / batch_size) model.save(model_weights) if save_model: json_string = model.to_json() with open(json_file, "w") as f: f.write(json_string)
def run_evaluation(subdir_model, subdir_test, n_plot=20, batch_size=10, nt=10): ''' This function runs the evalution of the trained deep learning network over the selected test dataset, calculates the various metrics such as MSE, SD and PSNR, and generates and saves the results. ''' weights_file = os.path.join(WEIGHTS_DIR, subdir_model, 'prednet_ucsd_weights.hdf5') json_file = os.path.join(WEIGHTS_DIR, subdir_model, 'prednet_ucsd_model.json') test_file = os.path.join(DATA_DIR, subdir_test, 'X_Test.hkl') test_sources = os.path.join(DATA_DIR, subdir_test, 'sources_Test.hkl') # Load trained model f = open(json_file, 'r') json_string = f.read() f.close() train_model = model_from_json(json_string, custom_objects={'PredNet': PredNet}) train_model.load_weights(weights_file) # Create testing model (to output predictions) layer_config = train_model.layers[1].get_config() layer_config['output_mode'] = 'prediction' data_format = layer_config[ 'data_format'] if 'data_format' in layer_config else layer_config[ 'dim_ordering'] test_prednet = PredNet(weights=train_model.layers[1].get_weights(), **layer_config) input_shape = list(train_model.layers[0].batch_input_shape[1:]) input_shape[0] = nt inputs = Input(shape=tuple(input_shape)) predictions = test_prednet(inputs) test_model = Model(inputs=inputs, outputs=predictions) test_generator = SequenceGenerator(test_file, test_sources, nt, sequence_start_mode='unique', data_format=data_format) X_test = test_generator.create_all() X_hat = test_model.predict(X_test, batch_size) if data_format == 'channels_first': X_test = np.transpose(X_test, (0, 1, 3, 4, 2)) X_hat = np.transpose(X_hat, (0, 1, 3, 4, 2)) X_hat = np.squeeze(X_hat, axis=-1) X_test = np.squeeze(X_test, axis=-1) # Xhat_filename = 'Xhat.npy' Xtest_filename = 'Xtest.npy' mse_videos_filename = 'mse_videos.json' mse_frame_filename = 'mse_frame.json' mse_prev_frame_filename = 'mse_prev_frame.json' mse_err_prev_frame_filename = 'mse_err_prev_frame.json' overall_mse_filename = 'predictions.txt' mse_videos_sd_filename = 'mse_videos_sd.json' mse_frame_sd_filename = 'mse_frame_sd.json' mse_prev_frame_sd_filename = 'mse_prev_frame_sd.json' mse_err_prev_frame_sd_filename = 'mse_err_prev_frame_sd.json' psnr_frame_filename = 'psnr_frame.json' psnr_prev_frame_filename = 'psnr_prev_frame.json' pred_save_dir = 'prediction_plots' err_save_dir = 'error_plots' err_prev_save_dir = 'prev_frame_plots' err_model_prev_save_dir = 'model_prev_frame_plots' sd_save_dir = 'sd_plots' sd_prev_save_dir = 'sd_prev_frame_plots' psnr_save_dir = 'psnr_plots' now = datetime.now folder_now = now().strftime("%Y_%m_%d-%H%M") if not os.path.exists(RESULTS_SAVE_DIR): os.mkdir(RESULTS_SAVE_DIR) if not os.path.exists(os.path.join(RESULTS_SAVE_DIR, subdir_test)): os.mkdir(os.path.join(RESULTS_SAVE_DIR, subdir_test)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now)): os.mkdir(os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, pred_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, pred_save_dir)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_save_dir)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_prev_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_prev_save_dir)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_model_prev_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_model_prev_save_dir)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, psnr_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, psnr_save_dir)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, sd_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, sd_save_dir)) if not os.path.exists( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, sd_prev_save_dir)): os.mkdir( os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, sd_prev_save_dir)) Xhat_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, Xhat_filename) Xtest_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, Xtest_filename) mse_videos_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_videos_filename) mse_frame_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_frame_filename) mse_prev_frame_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_prev_frame_filename) mse_err_prev_frame_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_err_prev_frame_filename) mse_videos_sd_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_videos_sd_filename) mse_frame_sd_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_frame_sd_filename) mse_prev_frame_sd_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_prev_frame_sd_filename) mse_err_prev_frame_sd_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, mse_err_prev_frame_sd_filename) psnr_frame_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, psnr_frame_filename) psnr_prev_frame_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, psnr_prev_frame_filename) overall_mse_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, overall_mse_filename) pred_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, pred_save_dir) err_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_save_dir) err_prev_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_prev_save_dir) err_model_prev_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, err_model_prev_save_dir) sd_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, sd_save_dir) sd_prev_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, sd_prev_save_dir) psnr_save_path = os.path.join(RESULTS_SAVE_DIR, subdir_test, folder_now, psnr_save_dir) if subdir_test == 'total': im_list, source_list = get_test_splits(['UCSDped1', 'UCSDped2']) else: im_list, source_list = get_test_splits([subdir_test]) if subdir_test == 'total': im_list.sort( key=lambda x: os.path.basename(os.path.dirname(x)) + '_' + os.path. basename(os.path.dirname(os.path.dirname(os.path.dirname(x))))) else: im_list.sort() source_list.sort() curr_location = 0 possible_starts = defaultdict(list) while curr_location < len(im_list) - nt + 1: if source_list[curr_location] == source_list[curr_location + nt - 1]: possible_starts[source_list[curr_location]].append(curr_location) curr_location += nt else: curr_location += 1 mse_videos = dict() mse_model_frame = defaultdict(list) mse_prev_frame = defaultdict(list) mse_err_prev_frame = defaultdict(list) mse_videos_sd = dict() mse_model_frame_sd = defaultdict(list) mse_prev_frame_sd = defaultdict(list) mse_err_prev_frame_sd = defaultdict(list) psnr_model_frame = defaultdict(list) psnr_prev_frame = defaultdict(list) i = 0 for k, v in sorted(possible_starts.items()): n_mini_clips = len(v) mse_model_video = np.mean((X_test[i:i + n_mini_clips, 1:] - X_hat[i:i + n_mini_clips, 1:])**2).item() mse_prev_video = np.mean((X_test[i:i + n_mini_clips, :-1] - X_test[i:i + n_mini_clips, 1:])**2).item() mse_err_prev_video = np.mean( (X_hat[i:i + n_mini_clips, 1:-1] - X_hat[i:i + n_mini_clips, 2:])**2).item() mse_model_video_sd = np.std((X_test[i:i + n_mini_clips, 1:] - X_hat[i:i + n_mini_clips, 1:])).item() mse_prev_video_sd = np.std((X_test[i:i + n_mini_clips, :-1] - X_test[i:i + n_mini_clips, 1:])).item() mse_err_prev_video_sd = np.std((X_hat[i:i + n_mini_clips, 1:-1] - X_hat[i:i + n_mini_clips, 2:])).item() for j in range(n_mini_clips): for z in range(1, nt): mse_model_frame[k].append( np.mean( (X_test[i + j, z, :] - X_hat[i + j, z, :])**2).item()) mse_prev_frame[k].append( np.mean((X_test[i + j, z - 1, :] - X_test[i + j, z, :])**2).item()) mse_model_frame_sd[k].append( np.std((X_test[i + j, z, :] - X_hat[i + j, z, :])).item()) mse_prev_frame_sd[k].append( np.std((X_test[i + j, z - 1, :] - X_test[i + j, z, :])).item()) psnr_model_frame[k].append( psnr(np.mean( (X_test[i + j, z, :] - X_hat[i + j, z, :])**2))) psnr_prev_frame[k].append( psnr( np.mean((X_test[i + j, z - 1, :] - X_test[i + j, z, :])**2))) if z > 1: mse_err_prev_frame[k].append( np.mean((X_hat[i + j, z - 1, :] - X_test[i + j, z, :])**2).item()) mse_err_prev_frame_sd[k].append( np.std((X_hat[i + j, z - 1, :] - X_test[i + j, z, :])).item()) mse_videos[k] = (mse_model_video, mse_prev_video, mse_err_prev_video) mse_videos_sd[k] = (mse_model_video_sd, mse_prev_video_sd, mse_err_prev_video_sd) i += n_mini_clips mse_model = np.mean( (X_test[:, 1:] - X_hat[:, 1:])**2) # look at all timesteps except the first mse_prev = np.mean((X_test[:, :-1] - X_test[:, 1:])**2) mse_model_sd = np.std( (X_test[:, 1:] - X_hat[:, 1:])) # look at all timesteps except the first mse_prev_sd = np.std((X_test[:, :-1] - X_test[:, 1:])) with open(mse_videos_path, 'w') as fp: json.dump(mse_videos, fp, sort_keys=True, indent=4) with open(mse_frame_path, 'w') as fp: json.dump(mse_model_frame, fp, sort_keys=True, indent=4) with open(mse_prev_frame_path, 'w') as fp: json.dump(mse_prev_frame, fp, sort_keys=True, indent=4) with open(mse_err_prev_frame_path, 'w') as fp: json.dump(mse_err_prev_frame, fp, sort_keys=True, indent=4) with open(mse_videos_sd_path, 'w') as fp: json.dump(mse_videos_sd, fp, sort_keys=True, indent=4) with open(mse_frame_sd_path, 'w') as fp: json.dump(mse_model_frame_sd, fp, sort_keys=True, indent=4) with open(mse_prev_frame_sd_path, 'w') as fp: json.dump(mse_prev_frame_sd, fp, sort_keys=True, indent=4) with open(mse_err_prev_frame_sd_path, 'w') as fp: json.dump(mse_err_prev_frame_sd, fp, sort_keys=True, indent=4) with open(psnr_frame_path, 'w') as fp: json.dump(psnr_model_frame, fp, sort_keys=True, indent=4) with open(psnr_prev_frame_path, 'w') as fp: json.dump(psnr_prev_frame, fp, sort_keys=True, indent=4) #np.save(Xhat_path, X_hat) #np.save(Xtest_path, X_test) # Compare MSE of PredNet predictions vs. using last frame. Write results to prediction_scores.txt f = open(overall_mse_path, 'w') f.write("Model MSE: %f\n" % mse_model) f.write("Previous Frame MSE: %f\n" % mse_prev) f.write("Model SDE: %f\n" % mse_model_sd) f.write("Previous Frame SDE: %f\n" % mse_prev_sd) f.close() compare_results(pred_save_path, X_test, X_hat, nt, n_plot) make_error_plot(mse_model_frame, err_save_path) make_error_plot(mse_prev_frame, err_prev_save_path) make_error_plot(mse_err_prev_frame, err_model_prev_save_path) make_error_plot(mse_model_frame_sd, sd_save_path, 2) make_error_plot(mse_prev_frame_sd, sd_prev_save_path, 2) make_error_plot(psnr_model_frame, psnr_save_path)