def main(id): img_path = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/Imgs/RoadAccidents/' weights_dir = '/home/lq/Documents/Thesis/Thesis/weight' model_weight_filename = os.path.join(weights_dir, 'sports1M_weights_tf.h5') # model_weight_filename = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/adam_temporal_1m/weights/weights.16.hdf5' N_classes = 2 batch_size = 16 epochs = 32 input_shape = (16, 120, 160, 3) windows_length = 16 #clip model = c3d_temportal_model(input_shape, N_classes) def my_l2(y_true, y_pred): loss = tf.nn.l2_loss(y_pred) return tf.reduce_mean(loss) #loss # Setting the Learning rate multipliers LR_mult_dict = {} LR_mult_dict['conv1'] = 1 LR_mult_dict['conv2'] = 1 LR_mult_dict['conv3a'] = 1 LR_mult_dict['conv3b'] = 1 LR_mult_dict['conv4a'] = 1 LR_mult_dict['conv4b'] = 1 LR_mult_dict['conv5a'] = 1 LR_mult_dict['conv5b'] = 1 LR_mult_dict['fc6'] = 1 LR_mult_dict['fc7'] = 1 LR_mult_dict['fc8'] = 10 # Setting up optimizer base_lr = 0.000005 adam = Adam(lr=base_lr, decay=0.00005, multipliers=LR_mult_dict) # sgd = SGD(lr=base_lr, decay=0.00005, multipliers=LR_mult_dict) opt = adam loss_weights = [1, 0.001] convLayers = ['conv1', 'conv2', 'conv3a', 'conv3b', 'conv4a', 'conv4b'] #, 'conv5a', 'conv5b'] for layer in convLayers: model.get_layer(layer).trainable = False model.compile(loss=['categorical_crossentropy', my_l2], loss_weights=loss_weights, optimizer=opt, metrics=['accuracy']) print('loading weight: {}'.format(model_weight_filename)) model.load_weights(model_weight_filename, by_name=True, skip_mismatch=True, reshape=True) model.summary() from dataUtil import load_train_data, load_val_data train_AS_windows, train_A_windows, train_BG_windows = load_train_data( ) # load train data N_train_samples = len(train_AS_windows) * 2 N_train_iterations = N_train_samples // batch_size val_AS_windows, val_A_windows, val_BG_windows = load_val_data( ) # load val data N_val_samples = len(val_A_windows) * 2 # N_val_samples = len(val_AS_windows) << 1 N_val_iterations = N_val_samples // batch_size # #################################### print("--#train AS windows: " + str(len(train_AS_windows)) + " #train A windows: " + str(len(train_A_windows)) + " #train BG windows: " + str(len(train_BG_windows))) # ################################## result_dir = '/media/lq/C13E-1ED0/dataset/UCF_Crimes/results/adam_temporal_{}/'.format( id) best_weight_dir = result_dir + 'weights' best_weight_name = best_weight_dir + '/weights.{epoch:02d}.hdf5' if not os.path.isdir(best_weight_dir): os.makedirs(best_weight_dir) if not os.path.exists(best_weight_dir): os.makedirs(result_dir) desp = result_dir + 'desp.txt' with open(desp, 'w') as f: f.write('batch size: {}\nbase_lr: {} \ntrain_samples:{} \n '.format( batch_size, base_lr, N_train_samples)) f.write('loss_weight:{}\n'.format(loss_weights)) f.write('init_weight: {}\n'.format(model_weight_filename)) f.write("batch: {}\n".format(train_batch_generator)) # callbacks csv_logger = CSVLogger(result_dir + '/log.csv', separator=',') checkpointer = ModelCheckpoint(filepath=best_weight_name, verbose=1, save_best_only=False, save_weights_only=True) # NAME = "THUMOS-{}".format(int(time.time())) log_dir = os.path.join(result_dir, 'log') tbCallBack = callbacks.TensorBoard(log_dir=log_dir, batch_size=batch_size, histogram_freq=0, write_graph=True, write_images=True) train_generator = train_batch_generator(train_AS_windows, train_A_windows, train_BG_windows, windows_length, batch_size, N_train_iterations, N_classes, img_path) val_generator = val_batch_generator([], val_A_windows, val_BG_windows, windows_length, batch_size, N_val_iterations, N_classes, img_path) history = model.fit_generator( train_generator, steps_per_epoch=N_train_iterations, epochs=epochs, callbacks=[csv_logger, tbCallBack, checkpointer], validation_data=val_generator, validation_steps=N_val_iterations, verbose=1) plot_history(history, result_dir)
def train(self, iterations=5000, batch_size=16, save_it=[2500], id=0): #load the real data train_AS_windows, train_A_windows, train_BG_windows = load_train_data( ) # load train data val_AS_windows, val_A_windows, val_BG_windows = load_val_data( ) # load val data #real image generator for discriminator (AS + non AS) disc_batch_generator = batch_generator_AS_A_BG_2_1_1( AS_windows=train_AS_windows, A_windows=train_A_windows, BG_windows=train_BG_windows, windows_length=self.length, batch_size=batch_size, N_iterations=iterations, N_classes=self.n_classes + 1, img_path=self.image_path) #real image generator for generator (AS only) gen_batch_generator = batch_generator_AS(AS_windows=train_AS_windows, windows_length=self.length, batch_size=batch_size, N_iterations=iterations, N_classes=self.n_classes + 1, img_path=self.image_path) #logs result_dir = './results/gan_{}/'.format(id) weight_dir = os.path.join(result_dir, 'weights') if not os.path.isdir(weight_dir): os.makedirs(weight_dir) if not os.path.exists(weight_dir): os.makedirs(result_dir) log_dir = os.path.join(result_dir, 'logs') desp = os.path.join(result_dir, 'desp.txt') with open(desp, 'w') as f: f.write("c3d weights: {}\n".format(self.c3d_weights)) f.write("disc_optimizer: {}\n".format( self.disc_optimizer.get_config())) f.write("gen_optimizer: {}\n".format( self.gen_optimizer.get_config())) callback = callbacks.TensorBoard(log_dir=log_dir, batch_size=batch_size, histogram_freq=0, write_graph=True, write_images=True) callback.set_model(self.GAN) # loss_names = ['disc_train_loss_real', 'disc_train_acc_real', 'disc_train_loss_fake', 'disc_train_acc_fake', 'gen_train_loss'] loss_names = ['disc_train_loss', 'disc_train_acc', 'gen_train_loss'] for cnt in tqdm(range(iterations)): '''discriminator''' #Sample random points in the latent space random_latent_vectors = np.random.standard_normal( size=(batch_size // 2, self.latent_dim)) #Decode them to fake images generated_features = self.G.predict(random_latent_vectors) #real images real_images, real_labels = next(disc_batch_generator) real_features = self.fixed_c3d.predict(real_images) fake_labels = np.ones(batch_size // 2) * ( self.n_classes) #n_classes=21, 0=>BG, 1-20=>actions, 21=>fake fake_labels = np_utils.to_categorical(fake_labels, self.n_classes + 1) combined_features = np.concatenate( [generated_features, real_features]) labels = np.concatenate([fake_labels, real_labels]) # Add random noise to the labels - important trick! # labels += 0.05 * np.random.random(labels.shape) d_loss, d_acc = self.D.train_on_batch(combined_features, labels) # d_loss_real, d_acc_real = self.D.train_on_batch(real_features, real_labels) # d_loss_fake, d_acc_fake = self.D.train_on_batch(generated_features, fake_labels) '''generator (via the gan model, where the discriminator weights are frozen)''' random_latent_vectors = np.random.standard_normal( size=(batch_size, self.latent_dim)) real_AS_images, real_AS_labels = next(gen_batch_generator) real_AS_features = self.fixed_c3d.predict(real_AS_images) g_loss = self.GAN.train_on_batch( [random_latent_vectors, real_AS_features], [real_AS_labels]) #the labels are not used #tensorboard log # logs = [d_loss_real, d_acc_real, d_loss_fake, d_acc_fake, g_loss] logs = [d_loss, d_acc, g_loss] write_log(callback, loss_names, logs, cnt) if cnt in save_it: self.save_weights(weight_dir, cnt) tqdm.write( 'iteration: {}, [Discriminator :: d_loss: {}, d_acc: {}], [ Generator :: loss: {}]' .format(cnt, d_loss, d_acc, g_loss)) self.save_weights(weight_dir, iterations) print('done')
def main(id): from data import videoPaths as path img_path = path.VALIDATION_IMAGES_PATH weights_dir = '/home/lq/Documents/Thesis/Thesis/weight' model_weight_filename = os.path.join(weights_dir, 'sports1M_weights_tf.h5') N_classes = 20 + 1 batch_size = 24 epochs = 16 # input_shape = (16,112,112,3) windows_length = 16 model = c3d_model.get_model() # Setting the Learning rate multipliers LR_mult_dict = {} LR_mult_dict['conv1'] = 1 LR_mult_dict['conv2'] = 1 LR_mult_dict['conv3a'] = 1 LR_mult_dict['conv3b'] = 1 LR_mult_dict['conv4a'] = 1 LR_mult_dict['conv4b'] = 1 LR_mult_dict['conv5a'] = 1 LR_mult_dict['conv5b'] = 1 LR_mult_dict['fc6'] = 1 LR_mult_dict['fc7'] = 1 LR_mult_dict['fc8'] = 10 # Setting up optimizer base_lr = 0.00001 adam = Adam(lr=base_lr, decay=0.00005, multipliers=LR_mult_dict) opt = adam # convLayers = ['conv1','conv2','conv3a','conv3b','conv4a'] # for layer in convLayers: # model.get_layer(layer).trainable = False model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) model.load_weights(model_weight_filename, by_name=True, skip_mismatch=True, reshape=True) ###################### # for layer in model.layers: # weights = layer.get_weights() # print(layer) # l = model.get_layer(name='fc8') # list = l.get_weights() # for l in list: # print(l.shape) # print(l[10]) # return 0 ##################### model.summary() from dataUtil import load_train_data, load_val_data train_AS_windows, train_A_windows, train_BG_windows = load_train_data( ) # load train data # N_A_samples = len(train_A_windows) # N_batch_groups = N_A_samples // (batch_size//2 + batch_size//4) # N_train_iterations = N_batch_groups * 2 #N_train_samples = len(train_AS_windows) *2 << 1 # half AS, half non-AS N_train_samples = len(train_AS_windows) * 2 N_train_iterations = N_train_samples // batch_size val_AS_windows, val_A_windows, val_BG_windows = load_val_data( ) # load val data N_val_samples = len(val_AS_windows) * 2 # N_val_samples = len(val_AS_windows) << 1 N_val_iterations = N_val_samples // batch_size # #################################### print("--#train AS windows: " + str(len(train_AS_windows)) + " #train A windows: " + str(len(train_A_windows)) + " #train BG windows: " + str(len(train_BG_windows))) print("-N_val_samples:" + str(N_val_samples) + "\n--#val AS windows: " + str(len(val_AS_windows)) + " #val A windows: " + str(len(val_A_windows)) + " #val BG windows: " + str(len(val_BG_windows))) # ################################## result_dir = './results/adam_c3d_{}/'.format(id) best_weight_dir = result_dir + 'weights' best_weight_name = best_weight_dir + '/weights.{epoch:02d}-{val_loss:.3f}.hdf5' if not os.path.isdir(best_weight_dir): os.makedirs(best_weight_dir) if not os.path.exists(best_weight_dir): os.makedirs(result_dir) desp = result_dir + 'desp.txt' with open(desp, 'w') as f: f.write( 'batch size: {}\nbase_lr: {} \ntrain_samples:{} \nval_samples:{}\n ' .format(batch_size, base_lr, N_train_samples, N_val_samples)) f.write('init_weiht: {}'.format(model_weight_filename)) f.write('batch_generator: {}'.format(train_batch_generator)) # callbacks csv_logger = CSVLogger(result_dir + '/log.csv', separator=',') checkpointer = ModelCheckpoint(filepath=best_weight_name, verbose=1, save_best_only=False, save_weights_only=True) # NAME = "THUMOS-{}".format(int(time.time())) log_dir = os.path.join(result_dir, 'log') tbCallBack = callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=True) train_generator = train_batch_generator(train_AS_windows, train_A_windows, train_BG_windows, windows_length, batch_size, N_train_iterations, N_classes, img_path) val_generator = val_batch_generator(val_AS_windows, val_A_windows, val_BG_windows, windows_length, batch_size, N_val_iterations, N_classes, img_path) history = model.fit_generator( train_generator, steps_per_epoch=N_train_iterations, epochs=epochs, callbacks=[csv_logger, tbCallBack, checkpointer], validation_data=val_generator, validation_steps=N_val_iterations, verbose=1) plot_history(history, result_dir)