def train(model = model(),checkpoint = "latest.hdf5"): dataset = build_dataset() callback = TensorBoard("_tensor_board") callback.set_model(model) labels = interested_words for i in range(10000): save_model(model, "checkpoint.model") print("Chunk "+str(i)+" of 10000...") X,Y = get_batch(dataset,batchsize=1000) for j in range(10): logs = model.train_on_batch(np.array(X),np.array(Y)) print("loss:",logs) write_log(callback, ["training loss"], [logs], i*10+j) X,Y = get_batch(dataset,batchsize=100,batchtype="test") results = model.predict(X) accuracy = 0 for result,actual in zip(results,Y): #print("running test") x =np.argmax(result) j =np.argmax(actual) try: print("expected "+labels[j]," got "+labels[x]) except: pass if x == j: accuracy += 1 write_log(callback,["test accuracy"],[accuracy],i)
optimizer = Adam(lr=1e-5) optimizer_classifier = Adam(lr=1e-5) model_rpn.compile(optimizer=optimizer, loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors)]) model_classifier.compile(optimizer=optimizer_classifier, loss=[losses.class_loss_cls, losses.class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mae') # Tensorboard log폴더 생성 log_path = './logs' if not os.path.isdir(log_path): os.mkdir(log_path) # Tensorboard log모델 연결 callback = TensorBoard(log_path) callback.set_model(model_all) epoch_length = 1000 num_epochs = int(options.num_epochs) iter_num = 0 train_step = 0 losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf class_mapping_inv = {v: k for k, v in class_mapping.items()} print('Starting training')
def create_keras_callbacks( model, modelname, output_dir, ): """ creates callbacks for keras training process creates: + tensorboard callback: logs are written to <output_dir>/logs/<modelname> + checkpoint model: saves best validation score model to <output_dir>/<modelname>.hdf5 + early stopping: stops training when validation hasn't improved for 25 epochs + learning rate schedular: reduces learning rate by half every 20 epochs returns: list of callbacks which can be passed directly to keras.fit TODO: TFRunMetaData: captures GPU usage, memory stats etc, but requires at least series 10 GPU WeightWriter: custom callback to output weights; needs integrating ValidationOutput: custom callback to output results of validation steps; needs integrating """ tensorboard_dir = join(output_dir, "logs", modelname) logger.info("tensorboard logs writing to '%s'" % tensorboard_dir) checkpoint_filepath = join(output_dir, modelname + ".hdf5") logger.info("writing checkpoint file to '%s'" % checkpoint_filepath) # weight_dir = join(output_dir, modelname, "weights") # logger.info("weightwriter writing to '%s'" % weight_dir) # validation_dir = join(output_dir, model_name, "validation") # logger.info("validation data writing to '%s'" % validation_dir) for dir in [output_dir, tensorboard_dir]: try: makedirs(dir) except FileExistsError: logger.warning("'%s' already exists" % dir) continue except OSError: logger.exception("creating '%s' failed with OSError" % dir) continue tensorboard_callback = TensorBoardCallback(log_dir=tensorboard_dir, histogram_freq=0, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) tensorboard_callback.set_model(model) return [ tensorboard_callback, ModelCheckpointCallback(filepath=checkpoint_filepath, verbose=0, save_best_only=True), EarlyStoppingCallback(monitor="val_loss", min_delta=0.00000001, patience=25, mode="min"), LearningRateScheduler(step_decay(20, 0.5)) #ReduceLROnPlateauCallback( # monitor="val_loss", # factor=0.5, # patience=4, # mode="min", # epsilon=0.01, # min_lr=0.0000001 #), #TFRunMetaData(tensorboard_callback), #WeightWriter(weight_dir), #ValidationOutput( # validation_dir, # kwargs["validation_generator"], # kwargs["validation_steps"] #) ]
def __init__(self, data=(), nb_class=16, opts='adam', lossfunc='sparse_categorical_crossentropy', model='dnn_1', batch_size=64, val_data=(), learning_rate=0.001, savepath=None, callback_type=None, epoch=100, restore=False, logger=None): self.model = model self.x_train, self.y_train = data self.nb_class = nb_class self.opts = opts self.lossfunc = lossfunc self.batch_size = batch_size self.x_val, self.y_val = val_data self.learning_rate = learning_rate self.savepath = savepath self.callback_type = callback_type self.epoch = epoch self.restore = restore input_shape = self.x_train.shape[1] nb_class = self.nb_class # model choose if self.model == 'dnn_1': model = dnn_1(input_shape, nb_class) elif self.model == 'bilstm': model = bilstm(input_shape, nb_class) elif self.model == 'advanced_dnn': model = sample_dnn(input_shape, nb_class) elif self.model == 'rwn': model = RWN(self, input_shape) elif self.model == 'semi_gan': model = SemiGan(self, input_shape) else: pass # loss funtion choose if self.lossfunc == 'focal_loss': lossfunc = focal_loss(alpha=1) elif self.lossfunc == 'binary_crossentropy': lossfunc = 'binary_crossentropy' elif self.lossfunc == 'hinge': lossfunc = 'hinge' elif self.lossfunc == 'squared_hinge': lossfunc = 'squared_hinge' elif self.lossfunc == 'categorical_crossentropy': lossfunc = 'categorical_crossentropy' elif self.lossfunc == 'sparse_categorical_crossentropy': lossfunc = 'sparse_categorical_crossentropy' elif self.lossfunc == 'kulback_leibler_divergence': lossfunc = 'kulback_leibler_divergence' else: pass # optimizer choose if self.opts == 'adam': opts = Adam(lr=self.learning_rate, decay=1e-6) elif self.opts == 'rmsprop': opts = RMSprop(lr=self.learning_rate, epsilon=1.0) elif self.opts == 'adagrad': opts = Adam(lr=self.learning_rate, decay=1e-6, adagrad=True) elif self.opts == 'sgd': opts = SGD(lr=self.learning_rate, momentum=0.9, decay=1e-6, nestrov=True) elif self.opts == 'amsgrad': opts = Adam(lr=self.learning_rate, decay=1e-6, amsgrad=True) elif self.opts == 'adagrad': opts = Adagrad(lr=self.learning_rate, decay=1e-6) elif self.opts == 'nadam': opts = Nadam(lr=self.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004) elif self.opts == 'adadelta': opts = Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0) elif self.opts == 'radam': opts = RectifiedAdam(lr=1e-3, total_steps=self.epoch, warmup_proportion=0.1, min_lr=1e-6) elif self.opts == 'lookahead': radam = RectifiedAdam() opts = Lookahead(radam, sync_period=6, slow_step_size=0.5) elif self.opts == 'lazyadam': opts = LazyAdam(lr=self.learning_rate) elif self.opts == 'conditionalgradient': opts = ConditionalGradient(lr=0.99949, lambda_=203) else: pass print("loss func is {}".format(lossfunc)) auc = tf.keras.metrics.AUC() recall = tf.keras.metrics.Recall() precision = tf.keras.metrics.Precision() model.compile(optimizer=opts, loss=lossfunc, metrics=['acc', auc, recall, precision]) # x_train = self.x_train # x_val = self.x_val # y_train = self.y_train # y_val = self.y_val x_train = np.asarray(self.x_train) x_val = np.asarray(self.x_val) y_train = np.asarray(self.y_train) y_val = np.asarray(self.y_val) MODEL_SAVE_FOLDER_PATH = os.path.join(self.savepath, self.model) if not os.path.exists(MODEL_SAVE_FOLDER_PATH): os.mkdir(MODEL_SAVE_FOLDER_PATH) def lrdropping(self): initial_lrate = self.learning_rate drop = 0.9 epochs_drop = 3.0 lrate = initial_lrate * math.pow( drop, math.floor((1 + epoch) / epochs_drop)) return lrate callbacks = [] if 'checkpoint' in self.callback_type: checkpoint = ModelCheckpoint(os.path.join( MODEL_SAVE_FOLDER_PATH, 'checkpoint-{epoch:02d}.h5'), monitor='val_auc', save_best_only=False, mode='max') callbacks.append(checkpoint) else: pass if 'elarystopping' in self.callback_type: earlystopping = EarlyStopping(monitor='val_auc', patience=5, verbose=1, mode='max') callbacks.append(earlystopping) else: pass if 'tensorboard' in self.callback_type: logger.info("tensorboard path : {}".format(MODEL_SAVE_FOLDER_PATH)) tensorboard = TensorBoard(log_dir=os.path.join( "tensorboard", MODEL_SAVE_FOLDER_PATH), histogram_freq=0, write_graph=True, write_images=True) tensorboard.set_model(model) train_summary_writer = tf.summary.create_file_writer( os.path.join("tensorboard", MODEL_SAVE_FOLDER_PATH)) callbacks.append(tensorboard) else: pass if 'rateschedule' in self.callback_type: lrd = LearningRateScheduler(lrdropping) callbacks.append(lrd) else: pass if 'interval_check' in self.callback_type: inter = IntervalEvaluation( validation_data=(x_val, y_val), interval=1, savedir=os.path.join(self.savepath, self.model), file='Evaluation_{}.csv'.format(self.model), logger=logger) callbacks.append(inter) else: pass if self.opts == 'conditionalgradient': def frobenius_norm(m): total_reduce_sum = 0 for i in range(len(m)): total_reduce_sum = total_reduce_sum + tf.math.reduce_sum( m[i]**2) norm = total_reduce_sum**0.5 return norm CG_frobenius_norm_of_weight = [] CG_get_weight_norm = LambdaCallback( on_epoch_end=lambda batch, logs: CG_frobenius_norm_of_weight. append(frobenius_norm(model.trainable_weights).np())) cgnorm = CG_frobenius_norm_of_weight callbacks.append(cgnorm) else: pass model_json = model.to_json() with open(os.path.join(MODEL_SAVE_FOLDER_PATH, 'model.json'), 'w') as json_file: json_file.write(model_json) if self.restore: chkp = last_checkpoint(MODEL_SAVE_FOLDER_PATH) model.load_weight(chkp) init_epoch = int(os.path.basename(chkp).split('-')[1]) print("================== restore checkpoint ==================") else: init_epoch = 0 print("================== restore failed ==================") cw = class_weight.compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train) logger.info("callback is {}".format(callbacks)) hist = model.fit(x=x_train, y=y_train, epochs=self.epoch, verbose=1, validation_data=(x_val, y_val), shuffle=True, callbacks=callbacks, class_weight=cw)
is_show_detail=True, is_adapt=False) #========================================== # training data shaper #========================================== data_shaper = shaper.Shape_data(LEFT_CONTEXT, RIGHT_CONTEXT, TRUNCATE_GRAD, NUMBER_OF_SKIP_FRAME ) #========================================== # set tensorboard #========================================== callback = TensorBoard(LOG_PATH) callback.set_model(mask_estimator) #========================================== # get features #========================================== feature_extractor = feature.Feature(SAMPLING_FREQUENCY, FFTL, SHIFT) noise_generator = augment.Generate_random_noise(noise_list, SAMPLING_FREQUENCY) reverbarent_generator = augment.RIR_convolve(SAMPLING_FREQUENCY) #========================================== # go training #========================================== utterance_count = 0 TRIM = np.int(0.05 * SAMPLING_FREQUENCY) # beginning and ending of uttearnce is not used for training val_loss = np.array([]) test_loss = np.array([])
def train_vsrganplus(self, epochs=None, batch_size=16, modelname=None, datapath_train=None, datapath_validation=None, steps_per_validation=1000, datapath_test=None, workers=4, max_queue_size=10, first_epoch=0, print_frequency=1, crops_per_image=2, log_weight_frequency=None, log_weight_path='./model/', log_tensorboard_path='./data/logs/', log_tensorboard_update_freq=10, log_test_frequency=500, log_test_path="./images/samples/", media_type='i'): """Train the VSRGANplus network :param int epochs: how many epochs to train the network for :param str modelname: name to use for storing model weights etc. :param str datapath_train: path for the image files to use for training :param str datapath_test: path for the image files to use for testing / plotting :param int print_frequency: how often (in epochs) to print progress to terminal. Warning: will run validation inference! :param int log_weight_frequency: how often (in epochs) should network weights be saved. None for never :param int log_weight_path: where should network weights be saved :param int log_test_frequency: how often (in epochs) should testing & validation be performed :param str log_test_path: where should test results be saved :param str log_tensorboard_path: where should tensorflow logs be sent """ # Create data loaders train_loader = DataLoader(datapath_train, batch_size, self.height_hr, self.width_hr, self.upscaling_factor, crops_per_image, media_type, self.channels, self.colorspace) # Validation data loader validation_loader = None if datapath_validation is not None: validation_loader = DataLoader(datapath_validation, batch_size, self.height_hr, self.width_hr, self.upscaling_factor, crops_per_image, 'i', self.channels, self.colorspace) test_loader = None if datapath_test is not None: test_loader = DataLoader(datapath_test, 1, self.height_hr, self.width_hr, self.upscaling_factor, 1, 'i', self.channels, self.colorspace) # Use several workers on CPU for preparing batches enqueuer = OrderedEnqueuer(train_loader, use_multiprocessing=True, shuffle=True) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() # Callback: tensorboard if log_tensorboard_path: tensorboard = TensorBoard(log_dir=os.path.join( log_tensorboard_path, modelname), histogram_freq=0, write_graph=True, update_freq=log_tensorboard_update_freq) tensorboard.set_model(self.vsrganplus) else: print( ">> Not logging to tensorboard since no log_tensorboard_path is set" ) # Learning rate scheduler def lr_scheduler(epoch, lr): factor = 0.1 decay_step = [50000, 100000, 200000, 300000] if epoch in decay_step and epoch: return lr * factor return lr lr_scheduler_gan = LearningRateScheduler(lr_scheduler, verbose=1) lr_scheduler_gan.set_model(self.vsrganplus) lr_scheduler_gen = LearningRateScheduler(lr_scheduler, verbose=0) lr_scheduler_gen.set_model(self.generator) lr_scheduler_dis = LearningRateScheduler(lr_scheduler, verbose=0) lr_scheduler_dis.set_model(self.discriminator) lr_scheduler_ra = LearningRateScheduler(lr_scheduler, verbose=0) lr_scheduler_ra.set_model(self.ra_discriminator) # Callback: format input value def named_logs(model, logs): """Transform train_on_batch return value to dict expected by on_batch_end callback""" result = {} for l in zip(model.metrics_names, logs): result[l[0]] = l[1] return result # Shape of output from discriminator disciminator_output_shape = list(self.ra_discriminator.output_shape) disciminator_output_shape[0] = batch_size disciminator_output_shape = tuple(disciminator_output_shape) # VALID / FAKE targets for discriminator real = np.ones(disciminator_output_shape) fake = np.zeros(disciminator_output_shape) # Each epoch == "update iteration" as defined in the paper print_losses = {"GAN": [], "D": []} start_epoch = datetime.datetime.now() # Random images to go through #idxs = np.random.randint(0, len(train_loader), epochs) # Loop through epochs / iterations for epoch in range(first_epoch, int(epochs) + first_epoch): lr_scheduler_gan.on_epoch_begin(epoch) lr_scheduler_ra.on_epoch_begin(epoch) lr_scheduler_dis.on_epoch_begin(epoch) lr_scheduler_gen.on_epoch_begin(epoch) print("\nEpoch {}/{}:".format(epoch + 1, epochs + first_epoch)) # Start epoch time if epoch % print_frequency == 0: start_epoch = datetime.datetime.now() # Train discriminator self.discriminator.trainable = True self.ra_discriminator.trainable = True imgs_lr, imgs_hr = next(output_generator) generated_hr = self.generator.predict(imgs_lr) real_loss = self.ra_discriminator.train_on_batch( [imgs_hr, generated_hr], real) #print("Real: ",real_loss) fake_loss = self.ra_discriminator.train_on_batch( [generated_hr, imgs_hr], fake) #print("Fake: ",fake_loss) discriminator_loss = 0.5 * np.add(real_loss, fake_loss) # Train generator self.discriminator.trainable = False self.ra_discriminator.trainable = False #for _ in tqdm(range(10),ncols=60,desc=">> Training generator"): imgs_lr, imgs_hr = next(output_generator) gan_loss = self.vsrganplus.train_on_batch([imgs_lr, imgs_hr], [imgs_hr, real, imgs_hr]) # Callbacks logs = named_logs(self.vsrganplus, gan_loss) tensorboard.on_epoch_end(epoch, logs) # Save losses print_losses['GAN'].append(gan_loss) print_losses['D'].append(discriminator_loss) # Show the progress if epoch % print_frequency == 0: g_avg_loss = np.array(print_losses['GAN']).mean(axis=0) d_avg_loss = np.array(print_losses['D']).mean(axis=0) print(">> Time: {}s\n>> GAN: {}\n>> Discriminator: {}".format( (datetime.datetime.now() - start_epoch).seconds, ", ".join([ "{}={:.4f}".format(k, v) for k, v in zip( self.vsrganplus.metrics_names, g_avg_loss) ]), ", ".join([ "{}={:.4f}".format(k, v) for k, v in zip( self.discriminator.metrics_names, d_avg_loss) ]))) print_losses = {"GAN": [], "D": []} # Run validation inference if specified if datapath_validation: validation_losses = self.generator.evaluate_generator( validation_loader, steps=steps_per_validation, use_multiprocessing=workers > 1, workers=workers) print(">> Validation Losses: {}".format(", ".join([ "{}={:.4f}".format(k, v) for k, v in zip( self.generator.metrics_names, validation_losses) ]))) # If test images are supplied, run model on them and save to log_test_path if datapath_test and epoch % log_test_frequency == 0: plot_test_images(self.generator, test_loader, datapath_test, log_test_path, epoch, modelname, channels=self.channels, colorspace=self.colorspace) # Check if we should save the network weights if log_weight_frequency and epoch % log_weight_frequency == 0: # Save the network weights self.save_weights(os.path.join(log_weight_path, modelname))
stage1_gen = build_stage1_generator(embedding_compressor_model) stage1_gen.compile(loss='binary_crossentropy', optimizer=gen_optimizer) stage1_disc = build_stage1_discriminator(embedding_compressor_model) stage1_disc.compile(loss=tf.nn.sigmoid_cross_entropy_with_logits, optimizer=disc_optimizer) adversarial_model = build_adversarial_model(stage1_gen, stage1_disc) adversarial_model.compile( loss=[tf.nn.sigmoid_cross_entropy_with_logits, KL_loss], optimizer=gen_optimizer, metrics=None) tensorboard = TensorBoard(log_dir='logs/'.format(time.time())) tensorboard.set_model(stage1_gen) tensorboard.set_model(stage1_disc) # tensorboard.set_model(ca_model) # tensorboard.set_model(embedding_compressor_model) real_labels = np.ones((batch_size, 1), dtype=np.float32) * 0.9 fake_labels = np.zeros((batch_size, 1), dtype=np.float32) + 0.1 summary_writer = tf.summary.create_file_writer('./logs') for epoch in range(epochs): print('===========================================') print('Epoch is:', epoch) print('Number of batches:', int(X_train.shape[0] / batch_size)) gen_losses = []
class GAN(): def __init__(self, model_yaml, train_yaml): """ Args: model_yaml: dictionnary with the model parameters train_yaml: dictionnary the tran parameters """ self.sigma_val = 0 self.model_yaml = model_yaml self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) if "dict_band_x" not in train_yaml: self.dict_band_X = None self.dict_band_label = None self.dict_rescale_type = None else: self.dict_band_X = train_yaml["dict_band_x"] self.dict_band_label = train_yaml["dict_band_label"] self.dict_rescale_type = train_yaml["dict_rescale_type"] self.s1bands = train_yaml["s1bands"] self.s2bands = train_yaml["s2bands"] # self.latent_dim = 100 # PATH self.model_name = model_yaml["model_name"] self.model_dir = train_yaml["training_dir"] + self.model_name + "/" self.this_training_dir = self.model_dir + "training_{}/".format( train_yaml["training_number"]) self.saving_image_path = self.this_training_dir + "saved_training_images/" self.saving_logs_path = self.this_training_dir + "logs/" self.checkpoint_dir = self.this_training_dir + "checkpoints/" self.previous_checkpoint = train_yaml["load_model"] # TRAIN PARAMETER self.normalization = train_yaml["normalization"] self.epoch = train_yaml["epoch"] self.batch_size = train_yaml["batch_size"] # self.sess = sess self.learning_rate = train_yaml["lr"] self.fact_g_lr = train_yaml["fact_g_lr"] self.beta1 = train_yaml["beta1"] self.val_directory = train_yaml["val_directory"] self.fact_s2 = train_yaml["s2_scale"] self.fact_s1 = train_yaml["s1_scale"] self.data_X, self.data_y, self.scale_dict_train = load_data( train_yaml["train_directory"], x_shape=model_yaml["input_shape"], label_shape=model_yaml["dim_gt_image"], normalization=self.normalization, dict_band_X=self.dict_band_X, dict_band_label=self.dict_band_label, dict_rescale_type=self.dict_rescale_type, fact_s2=self.fact_s2, fact_s1=self.fact_s1, s2_bands=self.s2bands, s1_bands=self.s1bands, lim=train_yaml["lim_train_tile"]) self.val_X, self.val_Y, scale_dict_val = load_data( self.val_directory, x_shape=model_yaml["input_shape"], label_shape=model_yaml["dim_gt_image"], normalization=self.normalization, dict_band_X=self.dict_band_X, dict_band_label=self.dict_band_label, dict_rescale_type=self.dict_rescale_type, dict_scale=self.scale_dict_train, fact_s2=self.fact_s2, fact_s1=self.fact_s1, s2_bands=self.s2bands, s1_bands=self.s1bands, lim=train_yaml["lim_val_tile"]) print("Loading the data done dataX {} dataY {}".format( self.data_X.shape, self.data_y.shape)) self.gpu = train_yaml["n_gpu"] self.num_batches = self.data_X.shape[0] // self.batch_size self.model_yaml = model_yaml self.im_saving_step = train_yaml["im_saving_step"] self.w_saving_step = train_yaml["weights_saving_step"] self.val_metric_step = train_yaml["metric_step"] # REDUCE THE DISCRIMINATOR PERFORMANCE self.val_lambda = train_yaml["lambda"] self.real_label_smoothing = tuple(train_yaml["real_label_smoothing"]) self.fake_label_smoothing = tuple(train_yaml["fake_label_smoothing"]) self.sigma_init = train_yaml["sigma_init"] self.sigma_step = train_yaml['sigma_step'] self.sigma_decay = train_yaml["sigma_decay"] self.ite_train_g = train_yaml["train_g_multiple_time"] self.max_im = 10 self.strategy = tf.distribute.MirroredStrategy() print('Number of devices: {}'.format( self.strategy.num_replicas_in_sync)) self.buffer_size = self.data_X.shape[0] self.global_batch_size = self.batch_size * self.strategy.num_replicas_in_sync with self.strategy.scope(): self.d_optimizer = Adam(self.learning_rate, self.beta1) self.g_optimizer = Adam(self.learning_rate * self.fact_g_lr, self.beta1) self.build_model() self.model_writer = tf.summary.create_file_writer( self.saving_logs_path) #self.strategy = tf.distribute.MirroredStrategy() def build_model(self): # strategy = tf.distribute.MirroredStrategy() # print('Number of devices: {}'.format(strategy.num_replicas_in_sync)) # We use the discriminator self.discriminator = self.build_discriminator(self.model_yaml) self.discriminator.compile(loss='binary_crossentropy', optimizer=self.d_optimizer, metrics=['accuracy']) self.generator = self.build_generator(self.model_yaml, is_training=True) print("Input G") g_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2], self.data_X.shape[3]), name="g_build_model_input_data") G = self.generator(g_input) print("G", G) # For the combined model we will only train the generator self.discriminator.trainable = False D_input = tf.concat([G, g_input], axis=-1) print("INPUT DISCRI ", D_input) # The discriminator takes generated images as input and determines validity D_output_fake = self.discriminator(D_input) # print(D_output) # The combined model (stacked generator and discriminator) # TO TRAIN WITH MULTIPLE GPU self.combined = Model(g_input, [D_output_fake, G], name="Combined_model") self.combined.compile(loss=['binary_crossentropy', L1_loss], loss_weights=[1, self.val_lambda], optimizer=self.g_optimizer) print("[INFO] combined model loss are : ".format( self.combined.metrics_names)) def build_generator(self, model_yaml, is_training=True): def build_resnet_block(input, id=0): """Define the ResNet block""" x = Conv2D(model_yaml["dim_resnet"], model_yaml["k_resnet"], padding=model_yaml["padding"], strides=tuple(model_yaml["stride"]), name="g_block_{}_conv1".format(id))(input) x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_block_{}_bn1".format(id))(x) x = ReLU(name="g_block_{}_relu1".format(id))(x) x = Dropout(rate=model_yaml["do_rate"], name="g_block_{}_do".format(id))(x) x = Conv2D(model_yaml["dim_resnet"], model_yaml["k_resnet"], padding=model_yaml["padding"], strides=tuple(model_yaml["stride"]), name="g_block_{}_conv2".format(id))(x) x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_block_{}_bn2".format(id))(x) x = Add(name="g_block_{}_add".format(id))([x, input]) x = ReLU(name="g_block_{}_relu2".format(id))(x) return x img_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2], self.data_X.shape[3]), name="g_input_data") if model_yaml["last_activation"] == "tanh": print("use tanh keras") last_activ = lambda x: tf.keras.activations.tanh(x) else: last_activ = model_yaml["last_activation"] x = img_input for i, param_lay in enumerate( model_yaml["param_before_resnet"] ): # build the blocks before the Resnet Blocks x = Conv2D(param_lay[0], param_lay[1], strides=tuple(model_yaml["stride"]), padding=model_yaml["padding"], name="g_conv{}".format(i))(x) x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_{}_bn".format(i))(x) x = ReLU(name="g_{}_lay_relu".format(i))(x) for j in range(model_yaml["nb_resnet_blocs"]): # add the Resnet blocks x = build_resnet_block(x, id=j) for i, param_lay in enumerate(model_yaml["param_after_resnet"]): x = Conv2D(param_lay[0], param_lay[1], strides=tuple(model_yaml["stride"]), padding=model_yaml["padding"], name="g_conv_after_resnetblock{}".format(i))(x) x = BatchNormalization( momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_after_resnetblock{}_bn2".format(i))(x) x = ReLU(name="g_after_resnetblock_relu_{}".format(i))(x) # The last layer x = Conv2D(model_yaml["last_layer"][0], model_yaml["last_layer"][1], strides=tuple(model_yaml["stride"]), padding=model_yaml["padding"], name="g_final_conv", activation=last_activ)(x) model_gene = Model(img_input, x, name="Generator") model_gene.summary() return model_gene def build_discriminator(self, model_yaml, is_training=True): discri_input = Input(shape=tuple([256, 256, 12]), name="d_input") if model_yaml["d_activation"] == "lrelu": d_activation = lambda x: tf.nn.leaky_relu( x, alpha=model_yaml["lrelu_alpha"]) else: d_activation = model_yaml["d_activation"] if model_yaml["add_discri_noise"]: x = GaussianNoise(self.sigma_val, input_shape=self.model_yaml["dim_gt_image"], name="d_GaussianNoise")(discri_input) else: x = discri_input for i, layer_index in enumerate(model_yaml["dict_discri_archi"]): layer_val = model_yaml["dict_discri_archi"][layer_index] layer_key = model_yaml["layer_key"] layer_param = dict(zip(layer_key, layer_val)) pad = layer_param["padding"] vpadding = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]]) # the last dimension is 12 x = tf.pad( x, vpadding, model_yaml["discri_opt_padding"], name="{}_padding_{}".format( model_yaml["discri_opt_padding"], layer_index)) # the type of padding is defined the yaml, # more infomration in https://www.tensorflow.org/api_docs/python/tf/pad # # x = ZeroPadding2D( # padding=(layer_param["padding"], layer_param["padding"]), name="d_pad_{}".format(layer_index))(x) x = Conv2D(layer_param["nfilter"], layer_param["kernel"], padding="valid", activation=d_activation, strides=(layer_param["stride"], layer_param["stride"]), name="d_conv{}".format(layer_index))(x) if i > 0: x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="d_bn{}".format(layer_index))(x) # x = Flatten(name="flatten")(x) # for i, dlayer_idx in enumerate(model_yaml["discri_dense_archi"]): # dense_layer = model_yaml["discri_dense_archi"][dlayer_idx] # x = Dense(dense_layer, activation=d_activation, name="dense_{}".format(dlayer_idx))(x) if model_yaml["d_last_activ"] == "sigmoid": x_final = tf.keras.layers.Activation('sigmoid', name="d_last_activ")(x) else: x_final = x model_discri = Model(discri_input, x_final, name="discriminator") model_discri.summary() return model_discri def produce_noisy_input(self, input, sigma_val): if self.model_yaml["add_discri_white_noise"]: # print("[INFO] On each batch GT label we add Gaussian Noise before training discri on labelled image") new_gt = GaussianNoise(sigma_val, input_shape=self.model_yaml["dim_gt_image"], name="d_inputGN")(input) if self.model_yaml["add_relu_after_noise"]: new_gt = tf.keras.layers.Activation( lambda x: tf.keras.activations.tanh(x), name="d_before_activ")(new_gt) else: new_gt = input return new_gt def define_callback(self): # Define Tensorboard callbacks self.g_tensorboard_callback = TensorBoard( log_dir=self.saving_logs_path, histogram_freq=0, batch_size=self.batch_size, write_graph=True, write_grads=True) self.g_tensorboard_callback.set_model(self.combined) def train_gpu(self): valid = np.ones( (self.batch_size, 30, 30, 1)) # because of the shape of the discri fake = np.zeros((self.batch_size, 30, 30, 1)) print("valid shape {}".format(valid.shape)) if self.previous_checkpoint is not None: print("LOADING the model from step {}".format( self.previous_checkpoint)) start_epoch = int(self.previous_checkpoint) + 1 self.load_from_checkpoint(self.previous_checkpoint) else: # create_safe_directory(self.saving_logs_path) create_safe_directory(self.saving_image_path) train_dataset = tf.data.Dataset.from_tensor_slices( (self.data_X, self.data_y)).shuffle(self.batch_size).batch( self.global_batch_size) train_dist_dataset = self.strategy.experimental_distribute_dataset( train_dataset) def train(self): # Adversarial ground truths valid = np.ones((self.global_batch_size, 30, 30, 1)) # because of the shape of the discri fake = np.zeros((self.global_batch_size, 30, 30, 1)) #print("valid shape {}".format(valid.shape)) if self.previous_checkpoint is not None: print("LOADING the model from step {}".format( self.previous_checkpoint)) start_epoch = int(self.previous_checkpoint) + 1 self.load_from_checkpoint(self.previous_checkpoint) else: # create_safe_directory(self.saving_logs_path) create_safe_directory(self.saving_image_path) start_epoch = 0 train_dataset = tf.data.Dataset.from_tensor_slices( (self.data_X, self.data_y)).shuffle(self.batch_size).batch( self.global_batch_size) # loop for epoch sigma_val = self.sigma_init # dict_metric={"epoch":[],"d_loss_real":[],"d_loss_fake":[],"d_loss":[],"g_loss":[]} d_loss_real = [100, 100] # init losses d_loss_fake = [100, 100] d_loss = [100, 100] l_val_name_metrics, l_val_value_metrics = [], [] start_time = time.time() for epoch in range(start_epoch, self.epoch): # print("starting epoch {}".format(epoch)) for idx, (batch_input, batch_gt) in enumerate(train_dataset): #print(batch_input) ## TRAIN THE DISCRIMINATOR d_noise_real = random.uniform( self.real_label_smoothing[0], self.real_label_smoothing[1]) # Add noise on the loss d_noise_fake = random.uniform( self.fake_label_smoothing[0], self.fake_label_smoothing[1]) # Add noise on the loss # Create a noisy gt images batch_new_gt = self.produce_noisy_input(batch_gt, sigma_val) # Generate a batch of new images # print("Make a prediction") gen_imgs = self.generator.predict( batch_input) # .astype(np.float32) D_input_real = tf.concat([batch_new_gt, batch_input], axis=-1) D_input_fake = tf.concat([gen_imgs, batch_input], axis=-1) print("shape d train") print(valid.shape, D_input_fake.shape) d_loss_real = self.discriminator.train_on_batch( D_input_real, d_noise_real * valid) d_loss_fake = self.discriminator.train_on_batch( D_input_fake, d_noise_fake * fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) g_loss = self.combined.train_on_batch(batch_input, [valid, batch_gt]) # Plot the progress print("%d iter %d [D loss: %f, acc.: %.2f%%] [G loss: %f %f]" % (epoch, self.num_batches * epoch + idx, d_loss[0], 100 * d_loss[1], g_loss[0], g_loss[1])) if epoch % self.im_saving_step == 0 and idx < self.max_im: # to save some generated_images gen_imgs = self.generator.predict(batch_input) save_images(gen_imgs, self.saving_image_path, ite=idx) # LOGS to print in Tensorboard if epoch % self.val_metric_step == 0: l_val_name_metrics, l_val_value_metrics = self.val_metric() name_val_metric = [ "val_{}".format(name) for name in l_val_name_metrics ] name_logs = self.combined.metrics_names + [ "g_loss_tot", "d_loss_real", "d_loss_fake", "d_loss_tot", "d_acc_real", "d_acc_fake", "d_acc_tot" ] val_logs = g_loss + [ g_loss[0] + 100 * g_loss[1], d_loss_real[0], d_loss_fake[0], d_loss[0], d_loss_real[1], d_loss_fake[1], d_loss[1] ] # The metrics #print(type(batch_gt),type(gen_imgs)) l_name_metrics, l_value_metrics = compute_metric( batch_gt.numpy(), gen_imgs) assert len(val_logs) == len( name_logs ), "The name and value list of logs does not have the same lenght {} vs {}".format( name_logs, val_logs) write_log_tf2( self.model_writer, name_logs + l_name_metrics + name_val_metric + ["time_in_sec"], val_logs + l_value_metrics + l_val_value_metrics + [start_time - time.time()], epoch) if epoch % self.sigma_step == 0: # update simga sigma_val = sigma_val * self.sigma_decay # save the models if epoch % self.w_saving_step == 0: self.save_model(epoch) def save_model(self, step): print("Saving model at {} step {}".format(self.checkpoint_dir, step)) checkpoint_dir = self.checkpoint_dir if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) if not os.path.isfile("{}model_generator.yaml".format( self.checkpoint_dir)): gene_yaml = self.generator.to_yaml() with open("{}model_generator.yaml".format(self.checkpoint_dir), "w") as yaml_file: yaml_file.write(gene_yaml) if not os.path.isfile("{}model_combined.yaml".format( self.checkpoint_dir)): comb_yaml = self.combined.to_yaml() with open("{}model_combined.yaml".format(self.checkpoint_dir), "w") as yaml_file: yaml_file.write(comb_yaml) if not os.path.isfile("{}model_discri.yaml".format( self.checkpoint_dir)): discri_yaml = self.discriminator.to_yaml() with open("{}model_discri.yaml".format(self.checkpoint_dir), "w") as yaml_file: yaml_file.write(discri_yaml) self.generator.save_weights("{}model_gene_i{}.h5".format( self.checkpoint_dir, step)) self.discriminator.save_weights("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)) self.combined.save_weights("{}model_combined_i{}.h5".format( self.checkpoint_dir, step)) def load_from_checkpoint(self, step): assert os.path.isfile("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)), "No file at {}".format("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)) self.discriminator.load_weights("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)) self.generator.load_weights("{}model_gene_i{}.h5".format( self.checkpoint_dir, step)) self.combined.load_weights("{}model_combined_i{}.h5".format( self.checkpoint_dir, step)) def load_generator(self, path_yaml, path_weight): # load YAML and create model yaml_file = open(path_yaml, 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model = model_from_yaml(loaded_model_yaml) # load weights into new model loaded_model.load_weights(path_weight) print("Loaded model from disk") return loaded_model def val_metric(self): test_dataset = tf.data.Dataset.from_tensor_slices( (self.val_X, self.val_Y)).batch(self.val_X.shape[0]) #test_dist_dataset = self.strategy.experimental_distribute_dataset(test_dataset) for i, (x, y) in enumerate(test_dataset): #print("eval on {}".format(i)) val_pred = self.generator.predict(x) #print("type {} {}".format(type(y),type(val_pred))) label = y return compute_metric(label.numpy(), val_pred) def predict_on_iter(self, batch, path_save, l_image_id=None, un_rescale=True): """given an iter load the model at this iteration, returns the a predicted_batch but check if image have been saved at this directory :param dataset: :param batch could be a string : path to the dataset or an array corresponding to the batch we are going to predict on """ if type(batch) == type( "u" ): # the param is an string we load the bathc from this directory #print("We load our data from {}".format(batch)) l_image_id = find_image_indir(batch + XDIR, "npy") batch, _ = load_data(batch, x_shape=self.model_yaml["input_shape"], label_shape=self.model_yaml["dim_gt_image"], normalization=self.normalization, dict_band_X=self.dict_band_X, dict_band_label=self.dict_band_label, dict_rescale_type=self.dict_rescale_type, dict_scale=self.scale_dict_train, fact_s2=self.fact_s2, fact_s1=self.fact_s1, s2_bands=self.s2bands, s1_bands=self.s1bands, clip_s2=False) else: if l_image_id is None: print("We defined our own index for image name") l_image_id = [i for i in range(batch.shape[0])] assert len(l_image_id) == batch.shape[ 0], "Wrong size of the name of the images is {} should be {} ".format( len(l_image_id), batch.shape[0]) if os.path.isdir(path_save): print( "[INFO] the directory where to store the image already exists") data_array, path_tile, _ = load_from_dir( path_save, self.model_yaml["dim_gt_image"]) return data_array else: create_safe_directory(path_save) batch_res = self.generator.predict(batch) # if un_rescale: # remove the normalization made on the data # _, batch_res, _ = rescale_array(batch, batch_res, dict_group_band_X=self.dict_band_X, # dict_group_band_label=self.dict_band_label, # dict_rescale_type=self.dict_rescale_type, # dict_scale=self.scale_dict_train, invert=True, fact_scale2=self.fact_s2, # fact_scale1=self.fact_s1,clip_s2=False) assert batch_res.shape[0] == batch.shape[ 0], "Wrong prediction should have shape {} but has shape {}".format( batch_res.shape, batch.shape) if path_save is not None: # we store the data at path_save for i in range(batch_res.shape[0]): np.save( "{}_image_{}".format(path_save, l_image_id[i].split("/")[-1]), batch_res[i, :, :, :]) return batch_res
class AGZeroModel: def __init__(self, N, batch_size=32, archive_fit_samples=64, use_tpu=None, log_path='logs/tensorboard'): self.N = N self.batch_size = batch_size self.model = None self.archive_fit_samples = archive_fit_samples self.position_archive = [] self.tpu_grpc_url = use_tpu tpu_name_environ_key = 'TPU_NAME' # Check has server got TPU if use_tpu is not False and tpu_name_environ_key in os.environ: tpu_name = os.environ[tpu_name_environ_key].strip() if tpu_name != "": self.is_tpu = True self.tpu_grpc_url = TPUClusterResolver( tpu=[os.environ[tpu_name_environ_key]]).get_master() # TODO write an if condition to validate and resolve the TPU url provided self.__loss_functions = [ 'categorical_crossentropy', 'binary_crossentropy' ] self.model_name = time.strftime('GM{0}-%y%m%dT%H%M%S').format('%02d' % N) # print(self.model_name) log_path = os.path.join(log_path, self.model_name) if not os.path.exists(log_path): os.makedirs(log_path) self.callback = TensorBoard(log_path) def create(self): bn_axis = 3 N = self.N position = Input((N, N, 6)) resnet = ResNet(n_stages=N) resnet.create(N, N, 6) x = resnet.model(position) dist = Conv2D(2, (1, 1))(x) dist = BatchNormalization(axis=bn_axis)(dist) dist = Activation('relu')(dist) dist = Flatten()(dist) dist = Dense(N * N + 1, activation='softmax', name='distribution')(dist) res = Conv2D(1, (1, 1))(x) res = BatchNormalization(axis=bn_axis)(res) res = Activation('relu')(res) res = Flatten()(res) res = Dense(256, activation='relu')(res) res = Dense(1, activation='sigmoid', name='result')(res) self.model = Model(position, [dist, res]) self.model.compile(Adam(lr=2e-2), self.__loss_functions) self.callback.set_model(self.model) # check if TPU available if self.tpu_grpc_url is not None: self.model = tf.contrib.tpu.keras_to_tpu_model( self.model, strategy=tf.contrib.tpu.TPUDistributionStrategy( tf.contrib.cluster_resolver.TPUClusterResolver( self.tpu_grpc_url))) self.model.summary() def fit_game(self, X_positions, result): X_posres = [] for pos, dist in X_positions: X_posres.append((pos, dist, result)) result = -result if len(self.position_archive) >= self.archive_fit_samples: archive_samples = random.sample(self.position_archive, self.archive_fit_samples) else: # initial case archive_samples = self.position_archive self.position_archive.extend(X_posres) # I'm going to some lengths to avoid the potentially overloaded + operator X_fit_samples = list(itertools.chain(X_posres, archive_samples)) self.__fit_model(X_fit_samples, self.batch_size) def retrain_position_archive(self, batch_size=None): self.__fit_model(self.position_archive, batch_size if batch_size else self.batch_size * 8) def reduce_position_archive(self, ratio=0.5): try: self.position_archive = random.sample( self.position_archive, int(len(self.position_archive) * ratio)) except: pass def __fit_model(self, X_fit_samples, batch_size): batch_no = 1 X, y_dist, y_res = [], [], [] X_shuffled = random.sample(X_fit_samples, len(X_fit_samples)) X_shuffled = create_batches(X_shuffled, batch_size) for batch in X_shuffled: for pos, dist, res in batch: X.append(pos) y_dist.append(dist) y_res.append(float(res) / 2 + 0.5) logs = self.model.train_on_batch( np.array(X), [np.array(y_dist), np.array(y_res)]) self.write_log(self.__loss_functions, logs, batch_no) batch_no += 1 X, y_dist, y_res = [], [], [] def write_log(self, names, logs, batch_no): for name, value in zip(names, logs): summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = value summary_value.tag = name self.callback.writer.add_summary(summary, batch_no) self.callback.writer.flush() def predict(self, X_positions): dist, res = self.model.predict(X_positions) res = np.array([r[0] * 2 - 1 for r in res]) return [dist, res] def save(self, snapshot_id, save_archive=False): self.model.save_weights('%s.weights.h5' % (snapshot_id, )) if save_archive: joblib.dump(self.position_archive, '%s.archive.joblib' % (snapshot_id, ), compress=5) def load(self, snapshot_id): self.model.load_weights('%s.weights.h5' % (snapshot_id, )) pos_fname = '%s.archive.joblib' % (snapshot_id, ) try: self.position_archive = joblib.load(pos_fname) except: print('Warning: Could not load position archive %s' % (pos_fname, )) def unload_pos_archive(self): self.position_archive = [] def load_pos_archive(self, archive_file): try: print('Attempting to load position archive %s' % (archive_file, )) self.position_archive = joblib.load(archive_file) print('Successfully loaded position archive %s' % (archive_file, )) return True except: import traceback traceback.print_exc() print('Warning: Could not load position archive %s' % (archive_file, )) return False def load_averaged(self, weights, log=None): new_weights = [] loaded_weights = [] for weight in weights: self.model.load_weights(weight) loaded_weights.append(self.model.get_weights()) print("Read weight: {0}".format(weight)) if log is not None: log("Read weight: {0}".format(weight), self.model_name) if len(loaded_weights) > 0: for weights_list_tuple in zip(*loaded_weights): new_weights.append([ np.array(weights_).mean(axis=0) for weights_ in zip(*weights_list_tuple) ]) self.model.set_weights(new_weights) else: print( "No weights to load. Initializing the model with random weights!" )
class NeuralNetworkLayout(object): def __init__(self,gtrate,dtrate,gbeta1,dbeta1,dimensions,noise,echeck,overrides,layoutDir='layouts',fileDir='out'): self.dir_output=fileDir self.gan_training_rate=gtrate self.d_training_rate=dtrate self.gan_beta1=gbeta1 self.d_beta1=dbeta1 self.dimensionality=dimensions self.noise=noise self.epochCheck=echeck self.logdir=self.dir_output+'/model/logs/' self.layout_dir=layoutDir self.overrides=overrides self.d=None self.g=None self.gan=None def compileComponent(self,verbose=False,component_type=None): if(component_type=='discriminator'): stream = open(self.layout_dir+"/discriminator_layout.yaml","r+") elif(component_type=='generator'): stream = open(self.layout_dir+"/generator_layout.yaml","r+") data = yaml.load(stream, Loader=yaml.FullLoader) g1_instructions= data['join'][0] g2_instructions= data['join'][1] gc_instructions = data['layers'] if(component_type=='discriminator'): g_in_1 = Input((1,)) elif(component_type=='generator'): g_in_1 = Input((self.noise,)) g_in_2=[] for i in range(self.dimensionality): g_in_2.append(Input((1,))) g1 = (g_in_1) for layer_info in g1_instructions['layers']: if(self.overrides.get('activation')): activation = self.overrides.get('activation') else: activation = layer_info.get('activation') if(component_type=='discriminator' and self.overrides.get('d_nodes')): units = self.overrides.get('d_nodes') elif(component_type=='generator' and self.overrides.get('g_nodes')): units = self.overrides.get('g_nodes') else: units = layer_info.get('nodes') if(self.overrides.get('dropout_amount')): rate = self.overrides.get('dropout_amount') else: rate = layer_info.get('dropout_amount') if(self.overrides.get('leaky_amount')): alpha = self.overrides.get('leaky_amount') else: alpha = layer_info.get('leaky_amount') if(layer_info['layer_type']=='dense'): g1=Dense(units=units,activation=activation)(g1) elif(layer_info['layer_type']=='dropout'): g1=Dropout(rate=rate)(g1) elif(layer_info['layer_type']=='selu'): g1=LeakyReLU(alpha=alpha)(g1) elif(layer_info['layer_type']=='batchnorm'): g1=BatchNormalization()(g1) g2=[] for i in range(self.dimensionality): g2_current = (g_in_2[i]) for layer_info in g2_instructions['layers']: if(self.overrides.get('activation')): activation = self.overrides.get('activation') else: activation = layer_info.get('activation') if(component_type=='discriminator' and self.overrides.get('d_nodes')): units = self.overrides.get('d_nodes') elif(component_type=='generator' and self.overrides.get('g_nodes')): units = self.overrides.get('g_nodes') else: units = layer_info.get('nodes') if(self.overrides.get('dropout_amount')): rate = self.overrides.get('dropout_amount') else: rate = layer_info.get('dropout_amount') if(self.overrides.get('leaky_amount')): alpha = self.overrides.get('leaky_amount') else: alpha = layer_info.get('leaky_amount') if(layer_info['layer_type']=='dense'): g2_current=Dense(units=units,activation=activation)(g2_current) elif(layer_info['layer_type']=='dropout'): g2_current=Dropout(rate=rate)(g2_current) elif(layer_info['layer_type']=='selu'): g2_current=LeakyReLU(alpha=alpha)(g2_current) elif(layer_info['layer_type']=='batchnorm'): g2_current=BatchNormalization()(g2_current) g2.append(g2_current) gc = Concatenate()([g1]+g2) for layer_info in gc_instructions: if(self.overrides.get('activation')): activation = self.overrides.get('activation') else: activation = layer_info.get('activation') if(component_type=='discriminator' and self.overrides.get('d_nodes')): units = self.overrides.get('d_nodes') elif(component_type=='generator' and self.overrides.get('g_nodes')): units = self.overrides.get('g_nodes') else: units = layer_info.get('nodes') if(self.overrides.get('dropout_amount')): rate = self.overrides.get('dropout_amount') else: rate = layer_info.get('dropout_amount') if(self.overrides.get('leaky_amount')): alpha = self.overrides.get('leaky_amount') else: alpha = layer_info.get('leaky_amount') if(layer_info['layer_type']=='dense'): gc=Dense(units=units,activation=activation)(gc) elif(layer_info['layer_type']=='dropout'): gc=Dropout(rate=rate)(gc) elif(layer_info['layer_type']=='selu'): gc=LeakyReLU(alpha=alpha)(gc) elif(layer_info['layer_type']=='batchnorm'): gc=BatchNormalization()(gc) if(component_type=='generator'): gc = Dense(1, activation="sigmoid")(gc) gc = Model(name="Generator", inputs=[g_in_1]+g_in_2, outputs=[gc]) elif(component_type=='discriminator'): gc = Dense(2, activation="softmax")(gc) gc = Model(name="Discriminator", inputs=[g_in_1]+g_in_2, outputs=[gc]) gc.compile(loss="categorical_crossentropy", optimizer=Adam(self.d_training_rate, beta_1=self.d_beta1), metrics=["accuracy"]) if(verbose): gc.summary() return gc def compileGAN(self,verbose=False): self.d=self.compileComponent(verbose,component_type='discriminator') self.g=self.compileComponent(verbose,component_type='generator') hyper_in=[] for i in range(self.dimensionality): hyper_in.append(Input((1,))) noise_in = Input((self.noise,)) inputs=[noise_in]+hyper_in inputs2=[self.g(inputs)]+hyper_in gan_out = self.d(inputs2) gan = Model(inputs, gan_out, name="GAN") self.d.trainable = False gan.compile(loss="categorical_crossentropy", optimizer=Adam(self.gan_training_rate, beta_1=self.gan_beta1), metrics=["accuracy"]) if(verbose): gan.summary() self.gan=gan def visualiseData(self,save=True): pass def saveHyperParameters(self): if not exists(self.dir_output+'/layouts/'): makedirs(self.dir_output+'/layouts/') copy_tree('./layouts/', self.dir_output+'/layouts/') if not exists(self.dir_output+'/model/'): makedirs(self.dir_output+'/model/') makedirs(self.dir_output+'/model/logs/') self.tensorboard = TensorBoard( log_dir=self.logdir, histogram_freq=self.epochCheck, batch_size=self.noise, write_graph=True, write_grads=True ) self.tensorboard.set_model(self.gan) with open(self.dir_output+'/model/hyperparameters.gan4ds',"w") as f: allParams=[a for a in dir(self) if not a.startswith('__') and not callable(getattr(self, a))] for param in allParams: f.writelines(param+"\t"+str(getattr(self,param))+"\n")
class OperatorNetwork: def __init__(self, x_batch_size, mask_batch_size, tensorboard_logs_dir="", add_mopt_perf_metric=True, use_early_stopping=True): self.batch_size = mask_batch_size * x_batch_size self.mask_batch_size = mask_batch_size self.x_batch_size = x_batch_size self.losses_per_sample = None # self.losses_per_sample = [] self.tr_loss_history = [] self.te_loss_history = [] self.tf_logs = tensorboard_logs_dir self.epoch_counter = 0 self.add_mopt_perf_metric = add_mopt_perf_metric self.useEarlyStopping = use_early_stopping def create_dense_model(self, input_shape, dense_arch, last_activation="linear"): self.x_shape = input_shape self.y_shape = dense_arch[-1] input_data_layer = Input(shape=input_shape) x = Flatten()(input_data_layer) input_mask_layer = Input(shape=input_shape) mask = Flatten()(input_mask_layer) # x = K.concatenate([x,mask]) x = tf.keras.layers.Concatenate(axis=1)([x, mask]) for units in dense_arch[:-1]: x = Dense(units, activation="sigmoid")(x) x = Dense(dense_arch[-1], activation=last_activation)(x) self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x) print("Object network model built:") #self.model.summary() def create_1ch_conv_model(self, input_shape, image_shape, filter_sizes, kernel_sizes, dense_arch, padding, last_activation="softmax"): # only for grayscale self.x_shape = input_shape self.y_shape = dense_arch[-1] input_data_layer = Input(shape=input_shape) in1 = Reshape(target_shape=(1,) + image_shape)(input_data_layer) input_mask_layer = Input(shape=input_shape) in2 = Reshape(target_shape=(1,) + image_shape)(input_mask_layer) x = tf.keras.layers.Concatenate(axis=1)([in1, in2]) for i in range(len(filter_sizes)): x = Conv2D(filters=filter_sizes[i], kernel_size=kernel_sizes[i], data_format="channels_first", activation="relu", padding=padding)(x) x = MaxPool2D(pool_size=(2, 2), padding=padding, data_format="channels_first")(x) x = Flatten()(x) for units in dense_arch[:-1]: x = Dense(units, activation="relu")(x) x = Dense(dense_arch[-1], activation=last_activation)(x) self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x) print("Object network model built:") #self.model.summary() def create_2ch_conv_model(self, input_shape, image_shape, filter_sizes, kernel_sizes, dense_arch, padding, last_activation="softmax"): # only for grayscale self.x_shape = input_shape self.y_shape = dense_arch[-1] input_data_layer = Input(shape=input_shape) ch_data = Reshape(target_shape=(1,) + image_shape)(input_data_layer) input_mask_layer = Input(shape=input_shape) ch_mask = Reshape(target_shape=(1,) + image_shape)(input_mask_layer) for i in range(len(filter_sizes)): ch_data = Conv2D(filters=filter_sizes[i], kernel_size=kernel_sizes[i], data_format="channels_last", activation="relu", padding=padding)(ch_data) ch_data = MaxPool2D(pool_size=(2, 2), padding=padding, data_format="channels_last")(ch_data) ch_mask = Conv2D(filters=filter_sizes[i], kernel_size=kernel_sizes[i], data_format="channels_last", activation="relu", padding=padding)(ch_mask) ch_mask = MaxPool2D(pool_size=(2, 2), padding=padding, data_format="channels_last")(ch_mask) ch_mask = Flatten()(ch_mask) ch_data = Flatten()(ch_data) x = tf.keras.layers.Concatenate(axis=1)([ch_mask, ch_data]) for units in dense_arch[:-1]: x = Dense(units, activation="relu")(x) x = Dense(dense_arch[-1], activation=last_activation)(x) self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x) print("Object network model built:") #self.model.summary() def create_batch(self, x, masks, y): """ x = [[1,2],[3,4]] -> [[1,2],[1,2],[1,2],[3,4],[3,4],[3,4]] masks = [[0,0],[1,0],[1,1]] -> [[0,0],[1,0],[1,1],[0,0],[1,0],[1,1]] y = [1,3] -> [1 ,1 ,1 ,3 ,3 ,3 ] """ # assert len(masks) == self.mask_size x_prim = np.repeat(x, len(masks), axis=0) y_prim = np.repeat(y, len(masks), axis=0) masks_prim = np.tile(masks, (len(x), 1)) x_prim *= masks_prim # MASKING # assert len(x_prim) == self.batch_size return x_prim, masks_prim, y_prim def named_logs(self, model, logs, mode="train"): result = {} try: iterator = iter(logs) except TypeError: logs = [logs] metricNames = (mode + "_" + i for i in model.metrics_names) for l in zip(metricNames, logs): result[l[0]] = l[1] return result def compile_model(self, loss_per_sample, combine_losses, combine_mask_losses, metrics=None): self.mask_loss_combine_function = combine_mask_losses if self.add_mopt_perf_metric is True: if metrics is None: metrics = [self.get_mopt_perf_metric()] else: metrics.append(self.get_mopt_perf_metric()) def logging_loss_function(y_true, y_pred): losses = loss_per_sample(y_true, y_pred) self.losses_per_sample = losses return combine_losses(losses) self.model.compile(loss=logging_loss_function, optimizer='nadam', metrics=metrics, run_eagerly=True) if self.tf_logs != "": log_path = './logs' self.tb_clbk = TensorBoard(self.tf_logs) self.tb_clbk.set_model(self.model) def get_per_mask_loss(self, used_target_shape=None): if used_target_shape is None: used_target_shape = (self.x_batch_size, self.mask_batch_size) losses = tf.reshape(self.losses_per_sample, used_target_shape) # losses = np.apply_along_axis(self.mask_loss_combine_function,0,losses) losses = self.mask_loss_combine_function(losses) return losses def get_per_mask_loss_with_custom_batch(self, losses, new_x_batch_size, new_mask_batch_size): losses = np.reshape(losses, newshape=(new_x_batch_size, new_mask_batch_size)) losses = np.apply_along_axis(self.mask_loss_combine_function, 0, losses) return losses def train_one(self, x, masks, y): x_prim, masks_prim, y_prim = self.create_batch(x, masks, y) curr_loss = self.model.train_on_batch(x=[x_prim, masks_prim], y=y_prim) self.tr_loss_history.append(curr_loss) self.epoch_counter += 1 if self.tf_logs != "": self.tb_clbk.on_epoch_end(self.epoch_counter, self.named_logs(self.model, curr_loss)) return x_prim, masks_prim, y_prim def validate_one(self, x, masks, y): x_prim, masks_prim, y_prim = self.create_batch(x, masks, y) # print("ON: x: "+str(x_prim)) # print("ON: m: "+str(masks_prim)) # print("ON: y_true: "+str(y_prim)) curr_loss = self.model.test_on_batch(x=[x_prim, masks_prim], y=y_prim) self.te_loss_history.append(curr_loss) if self.tf_logs != "": self.tb_clbk.on_epoch_end(self.epoch_counter, self.named_logs(self.model, curr_loss, "val")) if self.useEarlyStopping is True: self.check_ES() # print("ON y_pred:" +str(np.squeeze(y_pred))) # print("ON loss per sample:" +str(np.squeeze(self.losses_per_sample.numpy()))) return x_prim, masks_prim, y_prim, self.losses_per_sample.numpy() def test_one(self, x, masks, y): x_prim, masks_prim, y_prim = self.create_batch(x, masks, y) curr_loss = self.model.test_on_batch(x=[x_prim, masks_prim], y=y_prim) self.te_loss_history.append(curr_loss) return curr_loss def get_mopt_perf_metric(self): # used_target_shape = (self.x_batch_size,self.mask_batch_size) def m_opt_loss(y_pred, y_true): if (self.losses_per_sample.shape[0] % self.mask_batch_size != 0): # when testing happens, not used anymore return 0.0 else: # for training and validation batches losses = tf.reshape(self.losses_per_sample, (-1, self.mask_batch_size)) self.last_m_opt_perf = np.mean(losses[:, int(0.5 * self.mask_batch_size)]) return self.last_m_opt_perf return m_opt_loss def set_early_stopping_params(self, starting_epoch, patience_batches=800, minimize=True): self.ES_patience = patience_batches self.ES_minimize = minimize if (minimize is True): self.ES_best_perf = 1000000.0 else: self.ES_best_perf = -1000000.0 self.ES_best_epoch = starting_epoch self.ES_stop_training = False self.ES_start_epoch = starting_epoch self.ES_best_weights = None return def check_ES(self, ): if self.epoch_counter >= self.ES_start_epoch: if self.ES_minimize is True: if self.last_m_opt_perf < self.ES_best_perf: self.ES_best_perf = self.last_m_opt_perf self.ES_best_epoch = self.epoch_counter self.ES_best_weights = self.model.get_weights() else: if self.last_m_opt_perf > self.ES_best_perf: self.ES_best_perf = self.last_m_opt_perf self.ES_best_epoch = self.epoch_counter self.ES_best_weights = self.model.get_weights() # print("ES patience left: "+str(self.epoch_counter-self.ES_best_epoch)) if (self.epoch_counter - self.ES_best_epoch > self.ES_patience): self.ES_stop_training = True
class AngleEstimator(Model): def __init__(self, number_sides: int = 2, number_epochs: int = 1000, batch_size: int = 32, optimizer: Optimizer = None, tensorboard_dir: Path = None): super().__init__() if optimizer is None: optimizer = Adam(learning_rate=tf.Variable(0.1, dtype=tf.float64), beta_1=tf.Variable(0.9, dtype=tf.float64), beta_2=tf.Variable(0.999, dtype=tf.float64), epsilon=tf.Variable(1e-7, dtype=tf.float64)) optimizer.iterations self.optimizer = optimizer self.number_sides = number_sides self.input_layer = Dense(2, activation=linear, use_bias=False, dtype=tf.float64) self.hidden_layer = Dense(number_sides + 10, activation=linear, use_bias=False, dtype=tf.float64) self.output_layer = Dense(1, activation=linear, use_bias=False, dtype=tf.float64) self.batch_size = batch_size self.number_epochs = number_epochs self.tensorboard_dir = tensorboard_dir self.build((None, 2)) self._setup_tensorboard_callback() def _setup_tensorboard_callback(self): if self.tensorboard_dir is not None: self.tensorboard_callback = TensorBoard(self.tensorboard_dir) self.tensorboard_callback.set_model(self) def call(self, inputs): x = self.input_layer(inputs) x = self.hidden_layer(x) y = self.output_layer(x) return y def _train_step(self, inputs, output): with tf.GradientTape() as g: z = self.call(inputs) loss = mse(output, z) gradients = g.gradient(loss, self.trainable_variables) self.optimizer.apply_gradients(zip(gradients, self.trainable_variables)) return tf.reduce_sum(loss) def get_side_angles(self, x_y, z): data = np.concatenate((x_y, np.reshape(z, (-1, 1))), axis=1) transformer = MinMaxTransformer(column_wise=False) transformer.transform(data) x_y, z = data[:, 0:2], data[:, 2] input_splits, output_splits = create_train_test_splits(x_y, z, ratio=0.0) buffer_size = input_splits[0].shape[0] dataset = Dataset.from_tensor_slices( (input_splits[0], output_splits[0])).shuffle(buffer_size=buffer_size) dataset = dataset.batch(self.batch_size) steps_per_epoch = max(buffer_size // self.batch_size, 1) self._train(dataset=dataset, steps_per_epoch=steps_per_epoch) def _train(self, dataset, steps_per_epoch): for epoch in range(self.number_epochs): for (batch, (input, target)) in enumerate(dataset.take(steps_per_epoch)): loss = self._train_step(input, target) print("Epoch: {} Batch: {} Loss: {}".format( epoch, batch, loss)) def plot_3d_prediction_plot(self, data, plane_steps: int = 50): transformer = MinMaxTransformer(column_wise=False) transformer.transform(data) plane_input = self._get_prediction_array(x_range=(np.min(data[:, 0]), np.max(data[:, 0])), y_range=(np.min(data[:, 1]), np.max(data[:, 1])), steps=plane_steps) z_hat = self.call(plane_input).numpy() plane_data = np.concatenate((plane_input, z_hat), axis=1) return get_3d_scatter_with_prediction_planes(data, plane_data) def _get_prediction_array(self, x_range: Tuple, y_range: Tuple, steps: int = 50): prediction_array = np.zeros((steps**2, 2)) x_stepsize = self._get_step_size(x_range, steps) y_stepsize = self._get_step_size(y_range, steps) x = x_range[0] k = 0 for i in range(steps): y = y_range[0] for j in range(steps): prediction_array[k, 0] = x prediction_array[k, 1] = y k += 1 y += y_stepsize x += x_stepsize return prediction_array @staticmethod def _get_step_size(axis_range: Tuple, steps: int): return (axis_range[1] - axis_range[0]) / steps
def train(self, z_noise=None): start_time = time.time() # log writer logdir = self.log_dir + '/' + self.model_dir tensorboard_callback = TensorBoard(log_dir=logdir) tensorboard_callback.set_model(self.gan) scalar_names = ['d_loss', 'g_loss'] for epoch in range(self.start_epoch, self.epoch): # get batch data for idx in range(self.start_batch_id, self.iteration): # train generator noise = np.random.normal(0, 1, (self.batch_size, self.z_dim)) y_gen = np.ones(self.batch_size) g_loss = self.gan.train_on_batch(noise, y_gen) # train discriminator half_batch = self.batch_size // 2 real_x, _ = next(self.train_generator) while real_x.shape[0] != half_batch: real_x, _ = next(self.train_generator) noise = np.random.normal(0, 1, (half_batch, self.z_dim)) fake_x = self.generator.predict(noise) real_y = np.ones(half_batch) real_y[:] = 0.9 fake_y = np.zeros(half_batch) d_loss_real = self.discriminator.train_on_batch(real_x, real_y) d_loss_fake = self.discriminator.train_on_batch(fake_x, fake_y) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # write summary # self.write_log(tensorboard_callback, scalar_names, [d_loss, g_loss], self.counter) # display training status self.counter += 1 print("Epoch: [%2d] [%5d/%5d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" \ % (epoch, idx, self.iteration, time.time() - start_time, d_loss, g_loss)) # save training results for every n steps if type(z_noise) is np.ndarray: sample_num = z_noise.shape[0] else: sample_num = self.sample_num z_noise = np.random.normal(0, 1, (sample_num, self.z_dim)) if np.mod(idx+1, self.print_freq) == 0: sample_imgs = self.generator.predict(z_noise) manifold = int(np.ceil(np.sqrt(sample_num))) save_images_plt(sample_imgs, [manifold, manifold], f'{self.sample_dir}/{self.model_name}_train_{epoch:02d}_{idx+1:05d}', mode='sample') if np.mod(idx+1, self.save_freq) == 0: self.save(self.checkpoint_dir, self.counter) # After an epoch, start_batch_id is set to zero # non-zero value is only for the first epoch after loading pre-trained model self.start_batch_id = 0 # save model self.save(self.checkpoint_dir, self.counter) # save model for the final step self.save(self.checkpoint_dir, self.counter)
earlystop.set_model(model) earlystop.on_train_begin() modelcheckpoint = ModelCheckpoint(filepath="weights/", monitor="val_loss", verbose=1, save_best_only=True) modelcheckpoint.set_model(model) modelcheckpoint.on_train_begin() reduce_lr = ReduceLROnPlateau(monitor="val_loss", patience=10, verbose=1) reduce_lr.set_model(model) reduce_lr.on_train_begin() tensorboard = TensorBoard(log_dir="logs/") tensorboard.set_model(model) tensorboard.on_train_begin() epochs = 3 train_logs_dict = {} test_logs_dict = {} for epoch in range(epochs): training_acc, testing_acc, training_loss, testing_loss = [], [], [], [] print("\nStart of epoch %d" % (epoch + 1, )) # Iterate over the batches of the dataset. modelcheckpoint.on_epoch_begin(epoch) earlystop.on_epoch_begin(epoch) reduce_lr.on_epoch_begin(epoch) tensorboard.on_epoch_begin(epoch) for x_batch_train, y_batch_train in get_batch(batch_size, x_train, y_train):
class GAN_train(object): def __init__(self, dataset_common_folder_path, dataset): self.dataset_folder_path = dataset_common_folder_path + dataset + "_noglasses" self.dataset_path_images = self.dataset_folder_path + '/natural/face_before_inpainting' self.dataset_path_GAN = self.dataset_folder_path + '/natural/GAN' self.dataset_path_GAN_model = self.dataset_path_GAN + '/model' self.dataset_path_GAN_samples = self.dataset_path_GAN + '/samples' if not os.path.exists(self.dataset_path_GAN): os.mkdir(self.dataset_path_GAN) if not os.path.exists(self.dataset_path_GAN_model): os.mkdir(self.dataset_path_GAN_model) if not os.path.exists(self.dataset_path_GAN_samples): os.mkdir(self.dataset_path_GAN_samples) print(self.dataset_path_images) # Training image setting self.data = glob(os.path.join(self.dataset_path_images, 'face_*.png')) self.num_total_data = len(self.data) print(self.num_total_data) sample_idx = np.random.randint(0, self.num_total_data, size=1) self.x_train = PRL_data_image_load(self.data, sample_idx=sample_idx) print(self.x_train.shape) self.dataset = dataset self.img_rows = self.x_train.shape[1] self.img_cols = self.x_train.shape[2] self.channel = self.x_train.shape[3] # For GAN self.noise_dim = 100 self.GAN = LSGAN_Model(self.img_rows, self.img_cols, self.channel, self.noise_dim, dataset) self.discriminator = self.GAN.discriminator() self.generator = self.GAN.generator() self.discriminator_cost = self.GAN.discriminator_model( self.discriminator) self.adversarial_cost = self.GAN.adversarial_model( self.generator, self.discriminator) # For tensorboard callbacks now = datetime.now() log_path = "tensorboard/GAN_" + now.strftime("%Y%m%d-%H%M%S") self.callback = TensorBoard(log_path) self.callback.set_model(self.adversarial_cost) self.train_names = ['d_loss_real', 'd_loss_fake', 'd_loss', 'a_loss'] def train(self, num_epoch=2000, batch_size=256, save_interval=0): # Initial Update Data Generation -------------------------------------- sample_noise_input = None if save_interval > 0: # sample_noise_input = np.random.normal(0.0, 1.0, size=[36, self.noise_dim]) sample_noise_input = np.random.uniform(-1.0, 1.0, size=[36, self.noise_dim]) sample_idx = np.random.permutation(self.num_total_data) sample_idx = sample_idx[0:batch_size] images_train = PRL_data_image_load(self.data, sample_idx=sample_idx) # noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim]) noise = np.random.uniform(-1.0, 1.0, size=[batch_size, self.noise_dim]) images_fake = self.generator.predict(noise) # Initial Update Discriminator set_trainability(self.discriminator, True) d_loss_real = self.discriminator_cost.train_on_batch( images_train, np.ones([batch_size, 1])) d_loss_fake = self.discriminator_cost.train_on_batch( images_fake, np.zeros([batch_size, 1])) d_loss = d_loss_real + d_loss_fake # TRAINING STEPS ------------------------------------------------------ print('========= Main LSGAN Training ==========') num_batch = self.num_total_data // batch_size for e in range(num_epoch): shuffled_sample_idx = np.random.permutation(self.num_total_data) for b in tqdm(range(num_batch)): batch_sample_idx = shuffled_sample_idx[b * batch_size:(b + 1) * batch_size] images_train = PRL_data_image_load(self.data, sample_idx=batch_sample_idx) # noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim]) noise = np.random.uniform(-1.0, 1.0, size=[batch_size, self.noise_dim]) images_fake = self.generator.predict(noise) # Update Discriminator set_trainability(self.discriminator, True) d_loss_real = self.discriminator_cost.train_on_batch( images_train, np.ones([batch_size, 1])) d_loss_fake = self.discriminator_cost.train_on_batch( images_fake, np.zeros([batch_size, 1])) d_loss = d_loss_real + d_loss_fake # Update Generator y = np.ones([batch_size, 1]) # noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim]) noise = np.random.uniform(-1.0, 1.0, size=[batch_size, self.noise_dim]) set_trainability(self.discriminator, False) a_loss = self.adversarial_cost.train_on_batch(noise, y) # noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim]) noise = np.random.uniform(-1.0, 1.0, size=[batch_size, self.noise_dim]) set_trainability(self.discriminator, False) a_loss = self.adversarial_cost.train_on_batch(noise, y) # Log messages write_log(self.callback, 'D_Loss', d_loss, e * num_batch + b) write_log(self.callback, 'A_Loss', a_loss, e * num_batch + b) if save_interval > 0: if (e + 1) % save_interval == 0: log_mesg = "Epoch %d [D loss real: %f, acc real: %f]" % ( e + 1, d_loss_real[0], d_loss_real[1]) log_mesg = "%s: [D loss fake: %f, acc fake: %f]" % ( log_mesg, d_loss_fake[0], d_loss_fake[1]) log_mesg = "%s: [D loss: %f, acc: %f]" % ( log_mesg, d_loss[0], d_loss[1]) log_mesg = "%s [A loss: %f, acc: %f]" % ( log_mesg, a_loss[0], a_loss[1]) print(log_mesg) GAN_plot_images(generator=self.generator, x_train=self.x_train, dataset=self.dataset, save2file=True, samples=sample_noise_input.shape[0], noise=sample_noise_input, step=(e + 1), folder_path=self.dataset_path_GAN_samples) GAN_plot_images(generator=self.generator, x_train=self.x_train, dataset=self.dataset, save2file=False, samples=sample_noise_input.shape[0], noise=sample_noise_input, step=(e + 1)) # Save trained models self.adversarial_cost.save( self.dataset_path_GAN_model + "/GAN_" + str(e + 1) + "_" + self.dataset + "_forganECCV_adversarial_model_uniform.h5") self.discriminator.save(self.dataset_path_GAN_model + "/GAN_" + str(e + 1) + "_" + self.dataset + "_forganECCV_discriminator_uniform.h5") self.generator.save(self.dataset_path_GAN_model + "/GAN_" + str(e + 1) + "_" + self.dataset + "_forganECCV_generator_uniform.h5")
class Agent(Portfolio): def __init__(self, state_dim, balance, is_eval=False, model_name=""): super().__init__(balance=balance) self.model_type = 'DQN' self.state_dim = state_dim self.action_dim = 3 # hold, buy, sell self.memory = deque(maxlen=100) self.buffer_size = 60 self.gamma = 0.95 self.epsilon = 1.0 # initial exploration rate self.epsilon_min = 0.01 # minimum exploration rate self.epsilon_decay = 0.995 # decrease exploration rate as the agent becomes good at trading self.is_eval = is_eval self.model = load_model('saved_models/{}.h5'.format(model_name)) if is_eval else self.model() self.tensorboard = TensorBoard(log_dir='./logs/DQN_tensorboard', update_freq=90) self.tensorboard.set_model(self.model) def model(self): model = Sequential() model.add(Dense(units=64, input_dim=self.state_dim, activation='relu')) model.add(Dense(units=32, activation='relu')) model.add(Dense(units=8, activation='relu')) model.add(Dense(self.action_dim, activation='softmax')) model.compile(loss='mse', optimizer=Adam(lr=0.001)) return model def reset(self): self.reset_portfolio() self.epsilon = 1.0 # reset exploration rate def remember(self, state, actions, reward, next_state, done): self.memory.append((state, actions, reward, next_state, done)) def act(self, state): if not self.is_eval and np.random.rand() <= self.epsilon: return random.randrange(self.action_dim) options = self.model.predict(state) return np.argmax(options[0]) def experience_replay(self): # retrieve recent buffer_size long memory mini_batch = [] l = len(self.memory) for i in range(l - self.buffer_size + 1, l): mini_batch.append(self.memory[i]) for state, actions, reward, next_state, done in mini_batch: if not done: Q_target_value = reward + self.gamma * np.amax(self.model.predict(next_state)[0]) else: Q_target_value = reward next_actions = self.model.predict(state) next_actions[0][np.argmax(actions)] = Q_target_value history = self.model.fit(state, next_actions, epochs=1, verbose=0) if self.epsilon > self.epsilon_min: self.epsilon *= self.epsilon_decay return history.history['loss'][0]
class DQNAgent: def __init__(self, state_size, action_space, train=True): self.t = 0 self.max_Q = 0 self.train = train # Get size of state and action self.state_size = state_size self.action_space = action_space self.action_size = action_space # These are hyper parameters for the DQN self.discount_factor = 0.99 self.learning_rate = 1e-4 if self.train: self.epsilon = 0.5 self.initial_epsilon = 0.5 else: self.epsilon = 1e-6 self.initial_epsilon = 1e-6 self.epsilon_min = 0.02 self.batch_size = 64 self.train_start = 100 self.explore = 10000 # Create replay memory using deque self.memory = deque(maxlen=10000) # Create main model and target model if MODEL_TYPE == 1: self.model = self.build_transfer_model_has_batchnorm() self.target_model = self.build_transfer_model_has_batchnorm() elif MODEL_TYPE == 2: self.model = self.build_transfer_model_has_no_batchnorm() self.target_model = self.build_transfer_model_has_no_batchnorm() elif MODEL_TYPE == 3: self.model = self.build_transfer_model_has_batchnorm_with_canny() self.target_model = self.build_transfer_model_has_batchnorm_with_canny( ) elif MODEL_TYPE == 4: self.model = self.build_dc_model_without_transfer() self.target_model = self.build_dc_model_without_transfer() else: self.model = self.build_og_dc_model_from_repo() self.target_model = self.build_og_dc_model_from_repo() # Copy the model to target model # --> initialize the target model so that the parameters of model & target model to be same self.update_target_model() self.tensorboard = TensorBoard(log_dir='./logs/', histogram_freq=0, write_graph=True, write_grads=True) self.tensorboard.set_model(self.model) def build_transfer_model_has_batchnorm(self): source_model = load_model('./files/my_model_new.h5') model = Sequential() for layer in source_model.layers[:-1]: if 'batch' not in layer.name: model.add(layer) for layer in model.layers: layer.trainable = True # model.add(Dense(512, activation="relu", name='dense')) # model.add(Dense(512, activation="relu", name='dense_1')) model.add(Dense(15, activation="linear", name='dense_2')) adam = Adam(lr=self.learning_rate) model.compile(loss='mse', optimizer=adam) return model def build_transfer_model_has_no_batchnorm(self): source_model = load_model('./files/my_model_gs.h5') model = Sequential() for layer in source_model.layers[:-3]: model.add(layer) for layer in model.layers: layer.trainable = True model.add(Dense(512, activation="relu", name='dense')) model.add(Dense(512, activation="relu", name='dense_1')) model.add(Dense(15, activation="linear", name='dense_2')) # print(model.summary()) adam = Adam(lr=self.learning_rate) model.compile(loss='mse', optimizer=adam) return model def build_transfer_model_has_batchnorm_with_canny(self): # source_model = load_model('./files/my_model_canny.h5') # model = Sequential() # for layer in source_model.layers[:-3]: # model.add(layer) # for layer in model.layers: # layer.trainable = True # model.add(Dense(512, activation="relu", name='dense')) # model.add(Dense(512, activation="relu", name='dense_1')) # model.add(Dense(15, activation="linear", name='dense_2')) # # print(model.summary()) # adam = Adam(lr=self.learning_rate) # model.compile(loss='mse', optimizer=adam) # return model # # def build_og_dc_model_from_repo(self): # model = Sequential() # model.add(Conv2D(24, (5, 5), strides=(2, 2), padding="same", # input_shape=(IMG_ROWS, IMG_COLS, IMG_CHANNELS))) # 80*80*4 # model.add(Activation('relu')) # model.add(Conv2D(32, (5, 5), strides=(2, 2), padding="same")) # model.add(Activation('relu')) # model.add(Conv2D(64, (5, 5), strides=(2, 2), padding="same")) # model.add(Activation('relu')) # model.add(Conv2D(64, (3, 3), strides=(2, 2), padding="same")) # model.add(Activation('relu')) # model.add(Conv2D(64, (3, 3), strides=(1, 1), padding="same")) # model.add(Activation('relu')) # model.add(Flatten()) # model.add(Dense(512)) # model.add(Activation('relu')) # # # 15 categorical bins for Steering angles # model.add(Dense(15, activation="linear")) # # adam = Adam(lr=self.learning_rate) # model.compile(loss='mse', optimizer=adam) # # return model return def build_dc_model_without_transfer(self): Input_1 = Input(shape=(64, 64, 3), name='Input_1') Convolution2D_1 = Conv2D(4, kernel_size=3, padding='same', activation='relu')(Input_1) Convolution2D_2 = Conv2D(4, kernel_size=3, padding='same', activation='relu')(Convolution2D_1) # Convolution2D_2 = BatchNormalization()(Convolution2D_2) MaxPooling2D_1 = MaxPooling2D()(Convolution2D_2) Convolution2D_5 = Conv2D(8, kernel_size=3, padding='same', activation='relu')(MaxPooling2D_1) Convolution2D_6 = Conv2D(8, kernel_size=3, padding='same', activation='relu')(Convolution2D_5) # Convolution2D_6 = BatchNormalization()(Convolution2D_6) MaxPooling2D_2 = MaxPooling2D()(Convolution2D_6) Convolution2D_7 = Conv2D(16, kernel_size=3, padding='same', activation='relu')(MaxPooling2D_2) Convolution2D_8 = Conv2D(16, kernel_size=3, padding='same', activation='relu')(Convolution2D_7) Convolution2D_11 = Conv2D(16, kernel_size=3, padding='same', activation='relu')(Convolution2D_8) # Convolution2D_11 = BatchNormalization()(Convolution2D_11) MaxPooling2D_3 = MaxPooling2D()(Convolution2D_11) Convolution2D_9 = Conv2D(32, kernel_size=3, padding='same', activation='relu')(MaxPooling2D_3) Convolution2D_10 = Conv2D(32, kernel_size=3, padding='same', activation='relu')(Convolution2D_9) Convolution2D_12 = Conv2D(16, kernel_size=3, padding='same', activation='relu')(Convolution2D_10) # Convolution2D_12 = BatchNormalization()(Convolution2D_12) MaxPooling2D_4 = MaxPooling2D(name='MaxPooling2D_4')(Convolution2D_12) Convolution2D_13 = Conv2D(32, kernel_size=3, padding='same', activation='relu')(MaxPooling2D_4) Convolution2D_14 = Conv2D(32, kernel_size=3, padding='same', activation='relu')(Convolution2D_13) Convolution2D_16 = Conv2D(16, kernel_size=3, padding='same', activation='relu')(Convolution2D_14) # Convolution2D_16 = BatchNormalization()(Convolution2D_16) MaxPooling2D_5 = MaxPooling2D(name='MaxPooling2D_5')(Convolution2D_16) Flatten_1 = Flatten()(MaxPooling2D_5) Dense_1 = Dense(512, activation='relu')(Flatten_1) # Dropout_1 = Dropout(0.2)(Dense_1) Dense_2 = Dense(512, activation='relu')(Dense_1) # Dropout_2 = Dropout(0.2)(Dense_2) Dense_3 = Dense(15, activation='linear')(Dense_2) model = Model([Input_1], [Dense_3]) model.compile(optimizer='adam', loss='mse') return model def rgb2gray(self, rgb): ''' take a numpy rgb image return a new single channel image converted to greyscale ''' return np.dot(rgb[..., :3], [0.299, 0.587, 0.114]) def process_image(self, obs): # obs = self.rgb2gray(obs) obs = cv2.resize(obs, (IMG_ROWS, IMG_COLS)) return obs def process_image_for_canny(self, obs): # obs = self.rgb2gray(obs) obs = cv2.resize(obs, (IMG_ROWS, IMG_COLS)) obs1 = cv2.Canny(obs, 100, 200) return obs1 def update_target_model(self): self.target_model.set_weights(self.model.get_weights()) # Get action from model using epsilon-greedy policy def get_action(self, s_t): if np.random.rand() <= self.epsilon: return self.action_space.sample()[0] else: print('Max Q') # print("Return Max Q Prediction") q_value = self.model.predict(s_t) # Convert q array to steering value return linear_unbin(q_value[0]) def replay_memory(self, state, action, reward, next_state, done): self.memory.append((state, action, reward, next_state, done)) def update_epsilon(self): if self.epsilon > self.epsilon_min: self.epsilon -= (self.initial_epsilon - self.epsilon_min) / self.explore def named_logs(self, model, logs): result = {} for l in zip(model.metrics_names, logs): result[l[0]] = l[1] return result def train_replay(self, ep_num): if len(self.memory) < self.train_start: return batch_size = min(self.batch_size, len(self.memory)) minibatch = random.sample(self.memory, batch_size) state_t, action_t, reward_t, state_t1, terminal = zip(*minibatch) state_t = np.concatenate(state_t) state_t1 = np.concatenate(state_t1) targets = self.model.predict(state_t) self.max_Q = np.max(targets[0]) target_val = self.model.predict(state_t1) target_val_ = self.target_model.predict(state_t1) for i in range(batch_size): if terminal[i]: targets[i][action_t[i]] = reward_t[i] else: a = np.argmax(target_val[i]) targets[i][action_t[i]] = reward_t[i] + \ self.discount_factor * (target_val_[i][a]) logs = self.model.train_on_batch(state_t, targets) return logs # self.tensorboard.on_epoch_end(ep_num, self.named_logs(self.model, [logs])) def update_tensorboard(self, e, log, rew): self.tensorboard.on_epoch_end(e, self.named_logs(self.model, [rew])) def load_model(self, name): self.model.load_weights(name) # Save the model which is under training def save_model(self, name): self.model.save_weights(name)
class MLP: def __init__(self, x_batch_size, mask_batch_size, tensorboard_logs_dir="", add_mopt_perf_metric=True, use_early_stopping=True): self.batch_size = mask_batch_size * x_batch_size self.mask_batch_size = mask_batch_size self.x_batch_size = x_batch_size self.losses_per_sample = None self.tr_loss_history = [] self.te_loss_history = [] self.tf_logs = tensorboard_logs_dir self.epoch_counter = 0 self.add_mopt_perf_metric = add_mopt_perf_metric self.useEarlyStopping = use_early_stopping def create_dense_model(self, input_shape, dense_arch, last_activation="linear"): self.x_shape = input_shape self.y_shape = dense_arch[-1] input_data_layer = Input(shape=input_shape) x = Flatten()(input_data_layer) input_mask_layer = Input(shape=input_shape) mask = Flatten()(input_mask_layer) x = tf.keras.layers.Concatenate(axis=1)([x, mask]) for units in dense_arch[:-1]: x = Dense(units, activation="sigmoid")(x) x = Dense(dense_arch[-1], activation=last_activation)(x) self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x) # self.model.summary() def create_batch(self, x, masks, y): """ x = [[1,2],[3,4]] -> [[1,2],[1,2],[1,2],[3,4],[3,4],[3,4]] masks = [[0,0],[1,0],[1,1]] -> [[0,0],[1,0],[1,1],[0,0],[1,0],[1,1]] y = [1,3] -> [1 ,1 ,1 ,3 ,3 ,3 ] """ x_prim = np.repeat(x, len(masks), axis=0) y_prim = np.repeat(y, len(masks), axis=0) masks_prim = np.tile(masks, (len(x), 1)) x_prim *= masks_prim return x_prim, masks_prim, y_prim def named_logs(self, model, logs, mode="train"): result = {} try: iterator = iter(logs) except TypeError: logs = [logs] metricNames = (mode + "_" + i for i in model.metrics_names) for l in zip(metricNames, logs): result[l[0]] = l[1] return result def compile_model(self, loss_per_sample, combine_losses, combine_mask_losses, metrics=None): self.mask_loss_combine_function = combine_mask_losses if self.add_mopt_perf_metric is True: if metrics is None: metrics = [self.get_mopt_perf_metric()] else: metrics.append(self.get_mopt_perf_metric()) def logging_loss_function(y_true, y_pred): losses = loss_per_sample(y_true, y_pred)[:, 0] # print(losses) self.losses_per_sample = losses return combine_losses(losses) self.model.compile(loss=logging_loss_function, optimizer='nadam', metrics=metrics, run_eagerly=True) if self.tf_logs != "": log_path = './logs' self.tb_clbk = TensorBoard(self.tf_logs) self.tb_clbk.set_model(self.model) def get_per_mask_loss(self, used_target_shape=None): if used_target_shape is None: used_target_shape = (self.x_batch_size, self.mask_batch_size) losses = tf.reshape(self.losses_per_sample, used_target_shape) losses = self.mask_loss_combine_function(losses) return losses def get_per_mask_loss_with_custom_batch(self, losses, new_x_batch_size, new_mask_batch_size): losses = np.reshape(losses, newshape=(new_x_batch_size, new_mask_batch_size)) losses = np.apply_along_axis(self.mask_loss_combine_function, 0, losses) return losses def train_one(self, x, masks, y): x_prim, masks_prim, y_prim = self.create_batch(x, masks, y) curr_loss = self.model.train_on_batch(x=[x_prim, masks_prim], y=y_prim) self.tr_loss_history.append(curr_loss) self.epoch_counter += 1 if self.tf_logs != "": self.tb_clbk.on_epoch_end(self.epoch_counter, self.named_logs(self.model, curr_loss)) return x_prim, masks_prim, y_prim def test_one(self, x, masks, y): x_prim, masks_prim, y_prim = self.create_batch(x, masks, y) feature = self.model.predict(x=[x_prim, masks_prim]) return feature def get_mopt_perf_metric(self): def m_opt_loss(y_pred, y_true): if self.losses_per_sample.shape[0] % self.mask_batch_size != 0: return 0.0 else: losses = tf.reshape(self.losses_per_sample, (-1, self.mask_batch_size)) self.last_m_opt_perf = np.mean( losses[:, int(0.5 * self.mask_batch_size)]) return self.last_m_opt_perf return m_opt_loss def set_early_stopping_params(self, starting_epoch, patience_batches=800, minimize=True): self.ES_patience = patience_batches self.ES_minimize = minimize if minimize is True: self.ES_best_perf = 1000000.0 else: self.ES_best_perf = -1000000.0 self.ES_best_epoch = starting_epoch self.ES_stop_training = False self.ES_start_epoch = starting_epoch self.ES_best_weights = None return
#Define the adversarial model gan_model = networks.define_adversarial_model(genA2B, genB2A, discA, discB, train_optimizer, lambda_cyc=lambda_cyc, lambda_idt=lambda_idt) #Setup the tensorboard to store and visualise losses tensorboard = TensorBoard(log_dir="logs\{}".format(time.time()), write_images=True, write_grads=True, write_graph=True) tensorboard.set_model(genA2B) tensorboard.set_model(genB2A) tensorboard.set_model(discA) tensorboard.set_model(discB) print("Batch Size: {}".format(batch_size)) print("Num of ResNet Blocks: {}".format(num_resnet_blocks)) print( "Starting training for {0} epochs with lambda_cyc = {1}, lambda_idt = {2}, num_resnet_blocks = {3}" .format(epochs, lambda_cyc, lambda_idt, num_resnet_blocks)) #Start training for epoch in range(epochs): print("Epoch:{}".format(epoch)) start_time = time.time() dis_losses = [] gen_losses = []
class Agent(Portfolio): def __init__(self, state_dim, balance, is_eval=False, model_name=""): super().__init__(balance=balance) self.model_type = 'DQN' self.state_dim = state_dim self.action_dim = 3 # hold, buy, sell self.memory = deque(maxlen=100) self.buffer_size = 60 self.tau = 0.0001 self.gamma = 0.95 self.epsilon = 1.0 # initial exploration rate self.epsilon_min = 0.01 # minimum exploration rate self.epsilon_decay = 0.995 # decrease exploration rate as the agent becomes good at trading self.is_eval = is_eval self.model = load_model( f'saved_models/{model_name}.h5') if is_eval else self.model() self.model_target = load_model( f'saved_models/{model_name}_target.h5') if is_eval else self.model self.model_target.set_weights(self.model.get_weights( )) # hard copy model parameters to target model parameters self.tensorboard = TensorBoard(log_dir='./logs/DDQN_tensorboard', update_freq=90) self.tensorboard.set_model(self.model) def update_model_target(self): model_weights = self.model.get_weights() model_target_weights = self.model_target.get_weights() for i in range(len(model_weights)): model_target_weights[i] = self.tau * model_weights[i] + ( 1 - self.tau) * model_target_weights[i] self.model_target.set_weights(model_target_weights) def model(self): model = Sequential() model.add(Dense(units=64, input_dim=self.state_dim, activation='relu')) model.add(Dense(units=32, activation='relu')) model.add(Dense(units=8, activation='relu')) model.add(Dense(self.action_dim, activation='softmax')) model.compile(loss='mse', optimizer=Adam(lr=0.001)) return model def reset(self): self.reset_portfolio() self.epsilon = 1.0 def remember(self, state, actions, reward, next_state, done): self.memory.append((state, actions, reward, next_state, done)) def act(self, state): if not self.is_eval and np.random.rand() <= self.epsilon: return random.randrange(self.action_dim) options = self.model.predict(state) return np.argmax(options[0]) def experience_replay(self): # sample random buffer_size long memory mini_batch = random.sample(self.memory, self.buffer_size) for state, actions, reward, next_state, done in mini_batch: Q_expected = reward + (1 - done) * self.gamma * np.amax( self.model_target.predict(next_state)[0]) next_actions = self.model.predict(state) next_actions[0][np.argmax(actions)] = Q_expected history = self.model.fit(state, next_actions, epochs=1, verbose=0) self.update_model_target() if self.epsilon > self.epsilon_min: self.epsilon *= self.epsilon_decay return history.history['loss'][0]
# create model input_tensor = Input(shape=(img_height, img_widht, 3)) input_rois = Input(shape=(None, 5)) model_body = get_model_body(input_tensor, net='vgg16', trainable=True) # rpn model rpn_model = get_rpn_model(model_body, anchors_num) # rfcn model rfcn_model = get_rfcn_model(model_body, input_rois) log_path = './logs' if not os.path.exists(log_path): os.mkdir(log_path) rpn_callback = TensorBoard(os.path.join(log_path, '000')) rpn_callback.set_model(rpn_model) rfcn_callback = TensorBoard(os.path.join(log_path, '000')) rfcn_callback.set_model(rfcn_model) rpn_model_path = os.path.join(log_path, 'rpn_weights.h5') rfcn_model_path = os.path.join(log_path, 'rfcn_model_weights.h5') if os.path.exists(rpn_model_path): rpn_model.load_weights(rpn_model_path) if os.path.exists(rfcn_model_path): rfcn_model.load_weights(rfcn_model_path) rpn_optimizer = Adam(lr=1e-5) rfcn_optimizer = Adam(lr=1e-5) rpn_model.compile(optimizer=rpn_optimizer, loss=[rpn_cls_loss, rpn_reg_loss])
embedding_dim=50, feed_forward_units=50, head_num=1, block_num=2, dropout_rate=0.2) optimizer = AdamOptimizer(0.001) tbcb = TensorBoard(log_dir='/logs', histogram_freq=1, write_graph=True, write_grads=True, write_images=True, embeddings_freq=1) loss_history = [] cos_loss_history = [] T = 0.0 t0 = time.time() tbcb.set_model(model) tbcb.on_train_begin() f = open('log.txt', 'w') try: for epoch in range(200): tbcb.on_epoch_begin(epoch) cos_loss = CosineSimilarity() for step in (range(num_batch)): tbcb.on_train_batch_begin(step) print('========== step: {:03d} / {:03d} ============\r'.format(step, num_batch), end='')
class Trainer_adv: """Class object to setup and carry the training. Takes as input a generator that produces SR images. Conditionally, also a discriminator network and a feature extractor to build the components of the perceptual loss. Compiles the model(s) and trains in a GANS fashion if a discriminator is provided, otherwise carries a regular ISR training. Args: generator: Keras model, the super-scaling, or generator, network. discriminator: Keras model, the discriminator network for the adversarial component of the perceptual loss. feature_extractor: Keras model, feature extractor network for the deep features component of perceptual loss function. lr_train_dir: path to the directory containing the Low-Res images for training. hr_train_dir: path to the directory containing the High-Res images for training. lr_valid_dir: path to the directory containing the Low-Res images for validation. hr_valid_dir: path to the directory containing the High-Res images for validation. learning_rate: float. loss_weights: dictionary, use to weigh the components of the loss function. Contains 'generator' for the generator loss component, and can contain 'discriminator' and 'feature_extractor' for the discriminator and deep features components respectively. logs_dir: path to the directory where the tensorboard logs are saved. weights_dir: path to the directory where the weights are saved. dataname: string, used to identify what dataset is used for the training session. weights_generator: path to the pre-trained generator's weights, for transfer learning. weights_discriminator: path to the pre-trained discriminator's weights, for transfer learning. n_validation:integer, number of validation samples used at training from the validation set. flatness: dictionary. Determines determines the 'flatness' threshold level for the training patches. See the TrainerHelper class for more details. lr_decay_frequency: integer, every how many epochs the learning rate is reduced. lr_decay_factor: 0 < float <1, learning rate reduction multiplicative factor. Methods: train: combines the networks and triggers training with the specified settings. """ def __init__( self, generator, discriminator, feature_extractor, lr_train_dir, hr_train_dir, lr_valid_dir, hr_valid_dir, loss_weights={ 'generator': 1.0, 'discriminator': 0.003, 'feature_extractor': 1 / 12 }, log_dirs={ 'logs': 'logs', 'weights': 'weights' }, fallback_save_every_n_epochs=2, dataname=None, weights_generator=None, weights_discriminator=None, n_validation=None, flatness={ 'min': 0.0, 'increase_frequency': None, 'increase': 0.0, 'max': 0.0 }, learning_rate={ 'initial_value': 0.0004, 'decay_frequency': 100, 'decay_factor': 0.5 }, adam_optimizer={ 'beta1': 0.9, 'beta2': 0.999, 'epsilon': None }, losses={ 'generator': 'mae', 'discriminator': 'binary_crossentropy', 'feature_extractor': 'mse', }, metrics={'generator': 'PSNR_Y'}, ): self.generator = generator self.discriminator = discriminator self.feature_extractor = feature_extractor self.scale = generator.scale self.lr_patch_size = generator.patch_size self.learning_rate = learning_rate self.loss_weights = loss_weights self.weights_generator = weights_generator self.weights_discriminator = weights_discriminator self.adam_optimizer = adam_optimizer self.dataname = dataname self.flatness = flatness self.n_validation = n_validation self.losses = losses self.log_dirs = log_dirs self.metrics = metrics if self.metrics['generator'] == 'PSNR_Y': self.metrics['generator'] = PSNR_Y elif self.metrics['generator'] == 'PSNR': self.metrics['generator'] = PSNR self._parameters_sanity_check() self.mag_num = len(hr_train_dir) if int(np.log2(self.scale)) != self.mag_num: print('scale does not match hr_train_dir') return self.model = self._combine_networks() self.settings = {} self.settings['training_parameters'] = locals() self.settings['training_parameters'][ 'lr_patch_size'] = self.lr_patch_size self.settings = self.update_training_config(self.settings) self.logger = get_logger(__name__) self.helper = TrainerHelper( generator=self.generator, weights_dir=log_dirs['weights'], logs_dir=log_dirs['logs'], lr_train_dir=lr_train_dir, feature_extractor=self.feature_extractor, discriminator=self.discriminator, dataname=dataname, weights_generator=self.weights_generator, weights_discriminator=self.weights_discriminator, fallback_save_every_n_epochs=fallback_save_every_n_epochs, ) self.train_dh = {} for index in range(self.mag_num): self.train_dh[index] = DataHandler( lr_dir=lr_train_dir, hr_dir=hr_train_dir[index], patch_size=self.lr_patch_size, scale=int(np.exp2((index + 1))), n_validation_samples=None, ) self.valid_dh = {} for index in range(self.mag_num): if index == 0: self.valid_dh[index] = DataHandler( lr_dir=lr_valid_dir, hr_dir=hr_valid_dir[index], patch_size=self.lr_patch_size, scale=int(np.exp2((index + 1))), n_validation_samples=32) else: self.valid_dh[index] = DataHandler( lr_dir=lr_valid_dir, hr_dir=hr_valid_dir[index], patch_size=self.lr_patch_size, scale=int(np.exp2((index + 1))), n_validation_samples=32, index=self.valid_dh[0].index) #self.discriminator.model.summary() #self.generator.model.summary() #self.feature_extractor.model.summary() self.model.summary() def _parameters_sanity_check(self): """ Parameteres sanity check. """ if self.discriminator: assert self.lr_patch_size * self.scale == self.discriminator.patch_size self.adam_optimizer #if self.feature_extractor: # assert self.lr_patch_size * self.scale == self.feature_extractor.patch_size check_parameter_keys( self.learning_rate, needed_keys=['initial_value'], optional_keys=['decay_factor', 'decay_frequency'], default_value=None, ) check_parameter_keys( self.flatness, needed_keys=[], optional_keys=['min', 'increase_frequency', 'increase', 'max'], default_value=0.0, ) check_parameter_keys( self.adam_optimizer, needed_keys=['beta1', 'beta2'], optional_keys=['epsilon'], default_value=None, ) check_parameter_keys(self.log_dirs, needed_keys=['logs', 'weights']) def _combine_networks(self): """ Constructs the combined model which contains the generator network, as well as discriminator and geature extractor, if any are defined. """ losses = [] loss_weights = [] lr = Input(shape=(self.lr_patch_size, ) * 2 + (3, )) sr = self.generator.model(lr) outputs = sr losses.extend([self.losses['generator']] * len(sr)) loss_weights.extend([self.loss_weights['generator']] * len(sr)) final_sr = sr[-1] n = 0 if self.discriminator: self.discriminator.model.trainable = False validity = self.discriminator.model(final_sr) outputs.append(validity) n = 1 losses.append(self.losses['discriminator']) loss_weights.append(self.loss_weights['discriminator']) if self.feature_extractor: self.feature_extractor.model.trainable = False for i in range(len(sr) - n): sr_feats = self.feature_extractor.model(sr[i]) outputs.extend([*sr_feats]) losses.extend([self.losses['feature_extractor']] * len(sr_feats)) loss_weights.extend([ self.loss_weights['feature_extractor'] / len(sr_feats) * (1 + i / 3.0) ] * len(sr_feats)) combined = Model(inputs=lr, outputs=outputs) # https://stackoverflow.com/questions/42327543/adam-optimizer-goes-haywire-after-200k-batches-training-loss-grows optimizer = Adam( beta_1=self.adam_optimizer['beta1'], beta_2=self.adam_optimizer['beta2'], lr=self.learning_rate['initial_value'], epsilon=self.adam_optimizer['epsilon'], ) combined.compile(loss=losses, loss_weights=loss_weights, optimizer=optimizer, metrics=self.metrics) return combined def _lr_scheduler(self, epoch): """ Scheduler for the learning rate updates. """ n_decays = epoch // self.learning_rate['decay_frequency'] lr = self.learning_rate['initial_value'] * ( self.learning_rate['decay_factor']**n_decays) # no lr below minimum control 10e-7 return max(1e-7, lr) def _flatness_scheduler(self, epoch): if self.flatness['increase']: n_increases = epoch // self.flatness['increase_frequency'] else: return self.flatness['min'] f = self.flatness['min'] + n_increases * self.flatness['increase'] return min(self.flatness['max'], f) def _load_weights(self): """ Loads the pretrained weights from the given path, if any is provided. If a discriminator is defined, does the same. """ if self.weights_generator: self.model.get_layer('generator').load_weights( str(self.weights_generator)) if self.discriminator: if self.weights_discriminator: self.model.get_layer('discriminator').load_weights( str(self.weights_discriminator)) self.discriminator.model.load_weights( str(self.weights_discriminator)) def _format_losses(self, prefix, losses, model_metrics): """ Creates a dictionary for tensorboard tracking. """ return dict(zip([prefix + m for m in model_metrics], losses)) def update_training_config(self, settings): """ Summarizes training setting. """ _ = settings['training_parameters'].pop('weights_generator') _ = settings['training_parameters'].pop('self') _ = settings['training_parameters'].pop('generator') _ = settings['training_parameters'].pop('discriminator') _ = settings['training_parameters'].pop('feature_extractor') settings['generator'] = {} settings['generator']['name'] = self.generator.name settings['generator']['parameters'] = self.generator.params settings['generator']['weights_generator'] = self.weights_generator _ = settings['training_parameters'].pop('weights_discriminator') if self.discriminator: settings['discriminator'] = {} settings['discriminator']['name'] = self.discriminator.name settings['discriminator'][ 'weights_discriminator'] = self.weights_discriminator else: settings['discriminator'] = None if self.discriminator: settings['feature_extractor'] = {} settings['feature_extractor']['name'] = self.feature_extractor.name settings['feature_extractor'][ 'layers'] = self.feature_extractor.layers_to_extract else: settings['feature_extractor'] = None return settings def train(self, epochs, steps_per_epoch, batch_size, monitored_metrics): """ Carries on the training for the given number of epochs. Sends the losses to Tensorboard. Args: epochs: how many epochs to train for. steps_per_epoch: how many batches epoch. batch_size: amount of images per batch. monitored_metrics: dictionary, the keys are the metrics that are monitored for the weights saving logic. The values are the mode that trigger the weights saving ('min' vs 'max'). """ self.settings['training_parameters'][ 'steps_per_epoch'] = steps_per_epoch self.settings['training_parameters']['batch_size'] = batch_size starting_epoch = self.helper.initialize_training( self) # load_weights, creates folders, creates basename self.tensorboard = TensorBoard( log_dir=str(self.helper.callback_paths['logs'])) self.tensorboard.set_model(self.model) # validation data validation_set = {} top_left_fronts = {} transform_before = [] for index in range(self.mag_num): if index == 0: validation_set[ index], top_left_fronts, transform_before = self.valid_dh[ index].get_validation_set(batch_size) else: validation_set[index], _, _ = self.valid_dh[ index].get_validation_set(batch_size, top_left_fronts, transform_before) y_validation = [ validation_set[index]['hr'] for index in range(self.mag_num) ] if self.discriminator: discr_out_shape = list( self.discriminator.model.outputs[0].shape)[1:4] valid = np.ones([batch_size] + discr_out_shape) fake = np.zeros([batch_size] + discr_out_shape) validation_valid = np.ones( [len(validation_set[self.mag_num - 1]['hr'])] + discr_out_shape) y_validation.append(validation_valid) if self.feature_extractor: for index in range(self.mag_num): validation_feats = self.feature_extractor.model.predict( validation_set[index]['hr']) y_validation.extend([*validation_feats]) for epoch in range(starting_epoch, epochs): self.logger.info('Epoch {e}/{tot_eps}'.format(e=epoch, tot_eps=epochs)) K.set_value(self.model.optimizer.lr, self._lr_scheduler(epoch=epoch)) self.logger.info('Current learning rate: {}'.format( K.eval(self.model.optimizer.lr))) flatness = self._flatness_scheduler(epoch) if flatness: self.logger.info( 'Current flatness treshold: {}'.format(flatness)) epoch_start = time() for step in tqdm(range(steps_per_epoch)): batch = {} top_left_front = {} temp = {} idx = None transform_before = [] for index in range(self.mag_num): if index == 0: batch[ index], top_left_front, idx, transform_before = self.train_dh[ index].get_batch(batch_size, flatness=flatness) else: batch[index], temp, idx, _ = self.train_dh[ index].get_batch(batch_size, idx, flatness=flatness, top_left_front=top_left_front, transform_before=transform_before) y_train = [batch[index]['hr'] for index in range(self.mag_num)] training_losses = {} ## Discriminator training if self.discriminator: sr = self.generator.model.predict(batch[0]['lr']) d_loss_real = self.discriminator.model.train_on_batch( batch[self.mag_num - 1]['hr'], valid) d_loss_fake = self.discriminator.model.train_on_batch( sr[-1], fake) d_loss_fake = self._format_losses( 'train_d_fake_', d_loss_fake, self.discriminator.model.metrics_names) d_loss_real = self._format_losses( 'train_d_real_', d_loss_real, self.discriminator.model.metrics_names) training_losses.update(d_loss_real) training_losses.update(d_loss_fake) y_train.append(valid) ## Generator training if self.feature_extractor: for index in range(self.mag_num): hr_feats = self.feature_extractor.model.predict( batch[index]['hr']) y_train.extend([*hr_feats]) model_losses = self.model.train_on_batch( batch[0]['lr'], y_train) model_losses = self._format_losses('train_', model_losses, self.model.metrics_names) training_losses.update(model_losses) self.tensorboard.on_epoch_end(epoch * steps_per_epoch + step, training_losses) self.logger.debug('Losses at step {s}:\n {l}'.format( s=step, l=training_losses)) elapsed_time = time() - epoch_start self.logger.info('Epoch {} took {:10.1f}s'.format( epoch, elapsed_time)) validation_losses = self.model.evaluate(validation_set[0]['lr'], y_validation, batch_size=batch_size) validation_losses = self._format_losses('val_', validation_losses, self.model.metrics_names) if epoch == starting_epoch: remove_metrics = [] for metric in monitored_metrics: if (metric not in training_losses) and ( metric not in validation_losses): msg = ' '.join([ metric, 'is NOT among the model metrics, removing it.' ]) self.logger.error(msg) remove_metrics.append(metric) for metric in remove_metrics: _ = monitored_metrics.pop(metric) # should average train metrics end_losses = {} end_losses.update(validation_losses) end_losses.update(training_losses) self.helper.on_epoch_end( epoch=epoch, losses=end_losses, generator=self.model.get_layer('generator'), discriminator=self.discriminator, metrics=monitored_metrics, ) self.tensorboard.on_epoch_end(epoch, validation_losses) self.tensorboard.on_train_end(None)
def train(self, x_train, y_train, batch_size, epochs): # Select sampled real dataset (label O, balanced class) x_train, y_train, x_super, y_super = self.select_supervised_sampled( x_train, y_train) x_train, y_train = x_train.values, y_train.values # Calculate the number of batches per training epoch bat_per_epo = int(x_train.shape[0] / batch_size) # Calulate the number of trainging iterations n_steps = bat_per_epo * epochs half_batch = batch_size // 2 tensorboard = TensorBoard(log_dir="./logs", histogram_freq=3, write_graph=True, write_images=True) tensorboard.set_model(self.gan_model) train_log_dir = "./logs" train_summary_writer = tf.summary.create_file_writer(train_log_dir) metrics = {} print('epochs=%d, batch_size=%d, 1/2=%d, b/e=%d, steps=%d' % (epochs, batch_size, half_batch, bat_per_epo, n_steps)) for step in range(n_steps): # update supervised discriminator (c) [Xsup_real, labels], _ = self.generate_real_samples([x_super, y_super], half_batch) metrics["c_loss"], metrics["c_acc"], metrics["c_recall"], metrics[ "c_precision"] = self.c_model.train_on_batch( Xsup_real, labels) with train_summary_writer.as_default(): tf.summary.scalar("c_loss", metrics["c_loss"], step=step) tf.summary.scalar("c_acc", metrics["c_acc"], step=step) tf.summary.scalar("c_recall", metrics["c_recall"], step=step) tf.summary.scalar("c_precision", metrics["c_precision"], step=step) # update unsupervised discriminator (d) [X_real, _], y_real = self.generate_real_samples([x_train, y_train], half_batch) d_loss1 = self.d_model.train_on_batch(X_real, y_real) X_fake, y_fake = self.generate_fake_samples(half_batch) d_loss2 = self.d_model.train_on_batch(X_fake, y_fake) # update generator (g) z_input = np.random.randn(self.random_noise * half_batch) z_input = z_input.reshape(half_batch, self.random_noise) X_gan = z_input y_gan = np.ones((X_gan.shape[0], 1)) g_loss = self.gan_model.train_on_batch(X_gan, y_gan) if step % 10 == 0: print( '>%d, c[loss: %.3f, acc: %.3f, recall : %.3f, precision : %.3f], d[loss1: %.3f, loss2: %.3f], g_loss[%.3f]' % (step + 1, metrics["c_loss"], metrics["c_acc"] * 100, metrics["c_recall"] * 100, metrics["c_precision"] * 100, d_loss1, d_loss2, g_loss))
def train(train_lbld_trios, val_lbls_trios, network, weights, model_path, n_epochs, init_lr, optmzr_name, imagenet=False, freeze_until=None): """Training function: train a model of type 'network' over the data. Args: network (str): String identifying the network architecture to use. weights (str): Path string to a .cpkt weights file. model_path (str): Path string to a directory to save models in. n_epochs (int): Integer representing the number of epochs to run training. """ # Create a folder for saving trained models if os.path.isdir(model_path) is False: logging.info("Creating a folder to save models at: " + str(model_path)) os.mkdir(model_path) starting_epoch = 0 if network == 'SiameseNetTriplet': siamese_net = SiameseNetTriplet((128, 128, 3), arch='resnet18', sliding=True, imagenet=imagenet, freeze_until=freeze_until) optimizer = Adam(lr=0.0006) model = siamese_net.build_model() loss_model = siamese_net.loss_model single_model = siamese_net.single_model if weights: print("Loading model at: " + str(weights)) starting_epoch = int(weights.split('-')[1]) + 1 model.load_weights(weights) model.compile(loss=triplet_loss, optimizer=optimizer, metrics=[cos_sim_pos, cos_sim_neg]) # Load data and create data generator: train_ds = tf.data.Dataset.from_generator( image_trio_generator, args=[train_lbld_trios, True, False, False, imagenet], output_types=((tf.float32, tf.float32, tf.float32), tf.float32, tf.string), output_shapes=(((TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3)), (1, None), (3))) batched_train_ds = train_ds.batch(BATCH_SIZE) # shuffle(10000).batch valid_ds = tf.data.Dataset.from_generator( image_trio_generator, args=[val_lbld_trios, False, False, False, imagenet], output_types=((tf.float32, tf.float32, tf.float32), tf.float32, tf.string), # output_shapes=(((TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3)), (1, None))) output_shapes=(((TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3)), (1, None), (3))) batched_valid_ds = valid_ds.batch(BATCH_SIZE) logdir = "logs/scalars/" + datetime.now().strftime("%Y%m%d-%H%M%S") tensorboard_callback = TensorBoard(log_dir=logdir, histogram_freq=0, batch_size=BATCH_SIZE, write_graph=True, write_grads=True) tensorboard_callback.set_model(model) def named_logs(metrics_names, logs): result = {} for l in zip(metrics_names, logs): result[l[0]] = l[1] return result # Train model: steps_per_epoch = len(train_lbld_trios) // BATCH_SIZE val_steps_per_epoch = len( val_lbld_trios) // BATCH_SIZE # steps_per_epoch//3 best_val_loss = 1000 best_train_loss = 1000 batched_train_iter = iter(batched_train_ds) batched_val_iter = iter(batched_valid_ds) # batched_train_iter = batched_train_ds.make_one_shot_iterator() # batched_val_iter = batched_valid_ds.make_one_shot_iterator() for epoch in range(starting_epoch, n_epochs): cumm_csim_pos_tr = 0 cumm_csim_neg_tr = 0 cumm_tr_loss = 0 cumm_csim_pos_val = 0 cumm_csim_neg_val = 0 cumm_val_loss = 0 print('Epoch #' + str(epoch) + ':') for step in tqdm(range(steps_per_epoch)): train_inputs, train_y, train_seq_ids = next(batched_train_iter) #train_inputs, train_y = batched_train_iter.get_next() # tinrain_x1 = train_inputs['input_1'] # train_x2 = train_inputs['input_2'] train_x1 = train_inputs[0] train_x2 = train_inputs[1] train_x3 = train_inputs[2] # train_y = train_y['output'] X_dict = {} seq_ids = [] for idx, row in enumerate(train_seq_ids): for idx2, class_id in enumerate(row): class_id = class_id.numpy().decode('utf8') if class_id not in seq_ids: seq_ids.append(class_id) if class_id in X_dict: X_dict[class_id].append(train_inputs[idx2][idx]) else: X_dict[class_id] = [] X_dict[class_id].append(train_inputs[idx2][idx]) triplets = get_batch_hard(model, train_inputs, seq_ids, BATCH_SIZE) loss, csim_pos_tr, csim_neg_tr = model.train_on_batch( [triplets[0], triplets[1], triplets[2]], train_y[:BATCH_SIZE // 2]) cumm_tr_loss += loss cumm_csim_pos_tr += csim_pos_tr cumm_csim_neg_tr += csim_neg_tr cumm_csim_pos_tr = cumm_csim_pos_tr / steps_per_epoch cumm_csim_neg_tr = cumm_csim_neg_tr / steps_per_epoch cumm_tr_loss = cumm_tr_loss / steps_per_epoch # evaluate for step in tqdm(range(val_steps_per_epoch)): valid_inputs, val_y, val_seq_ids = next(batched_val_iter) # valid_inputs, valid_y = batched_val_iter.get_next() valid_x1 = valid_inputs[0] valid_x2 = valid_inputs[1] valid_x3 = valid_inputs[2] val_loss, csim_pos_val, csim_neg_val = model.test_on_batch( [valid_x1, valid_x2, valid_x3], val_y) cumm_val_loss += val_loss cumm_csim_pos_val += csim_pos_val cumm_csim_neg_val += csim_neg_val cumm_csim_pos_val = cumm_csim_pos_val / val_steps_per_epoch cumm_csim_neg_val = cumm_csim_neg_val / val_steps_per_epoch cumm_val_loss = cumm_val_loss / val_steps_per_epoch print('Training loss: ' + str(cumm_tr_loss)) print('Validation loss: ' + str(cumm_val_loss)) print('* Cosine sim positive (train) for this epoch: %0.2f' % (cumm_csim_pos_tr)) print('* Cosine sim negative (train) for this epoch: %0.2f' % (cumm_csim_neg_tr)) print('* Cosine sim positive (valid) for this epoch: %0.2f' % (cumm_csim_pos_val)) print('* Cosine sim negative (valid) for this epoch: %0.2f' % (cumm_csim_neg_val)) metrics_names = [ 'tr_loss', 'tr_csim_pos', 'tr_csim_neg', 'val_loss', 'val_csim_pos', 'val_csim_neg' ] tensorboard_callback.on_epoch_end( epoch, named_logs(metrics_names, [ cumm_tr_loss, cumm_csim_pos_tr, cumm_csim_neg_tr, cumm_val_loss, cumm_csim_pos_val, cumm_csim_neg_val ])) model_filepath = os.path.join( model_path, "model-{epoch:03d}-{val_loss:.4f}.hdf5".format( epoch=epoch, val_loss=cumm_val_loss)) if cumm_val_loss < best_val_loss * 1.5: if cumm_val_loss < best_val_loss: best_val_loss = cumm_val_loss model.save(model_filepath) # OR model.save_weights() print("Best model w/ val loss {} saved to {}".format( cumm_val_loss, model_filepath)) tensorboard_callback.on_train_end(None) return
def train(self, epochs=10, batch_size=1, print_interval=50): # Tensorboard callback train_callback = TensorBoard(os.path.join(self.save_folder, 'train')) train_callback.set_model(self.cycle_vae) val_callback = TensorBoard(os.path.join(self.save_folder, 'val')) val_callback.set_model(self.cycle_vae) metrics_names = [ 'discri_loss_source', 'discri_accuracy_source', 'discri_loss_targ', 'discri_accuracy_targ', 'g_total_loss', 'g_reconst', 'g_kl', 'g_cyclic_reconst', 'g_cyclic_kl', 'g_gan', 'g_vgg' ] # Dataset X_day_train, X_night_train = load_day_and_night('train') self.X_day_val, self.X_night_val = load_day_and_night('val') n_train = len(X_day_train) n_val = len(self.X_day_val) print('Training examples: {}'.format(n_train)) print('Validation examples: {}'.format(n_val)) discri_dim = self.D_source.output.get_shape()[1] valid = np.ones((batch_size, discri_dim)) fake = np.zeros((batch_size, discri_dim)) val_loss_min = float('inf') iter_nb = 0 for e in range(epochs): n_iter = n_train // batch_size indices = np.random.permutation(n_train) for i in range(n_iter): idx = np.random.choice(indices, size=batch_size, replace=False) #indices[i*batch_size : (i+1)*batch_size] source_batch = load_batch(X_day_train[idx], self.image_size) targ_batch = load_batch(X_night_train[idx], self.image_size) # Discriminator d_loss_source, d_loss_targ = self.discri_train_on_batch( source_batch, targ_batch, valid, fake) # Generator g_loss = self.cycle_vae.train_on_batch( [source_batch, targ_batch], [valid, valid]) if i % print_interval == 0: print('Iteration {:4d}: Training loss:'.format(i)) logs = self.get_logs(d_loss_source, d_loss_targ, g_loss) print( 'Discri source loss: {:.2f}, accuracy: {:.1f}%'.format( logs[0], logs[1])) print('Discri targ loss: {:.2f}, accuracy: {:.1f}%'.format( logs[2], logs[3])) print('Gen loss: {:.2f}'.format(logs[4])) print('Reconst: {:.2f}, kl: {:.2f}, cyclic_reconst: {:.2f}, cyclic_kl: {:.2f},gan: {:.2f}, vgg: {:.2f}'\ .format(*logs[5:])) write_log(train_callback, metrics_names, logs, iter_nb) if i % 100 == 0: self.predict(e + 1, iter=iter_nb, save_fig=True) print('Figure saved.') iter_nb += 1 # Calculate validation loss val_loss = 0.0 n_iter_val = n_val // batch_size indices = np.arange(n_val) logs = np.zeros(len(metrics_names)) for i in range(n_iter_val): idx = indices[i * batch_size:(i + 1) * batch_size] source_batch = load_batch(self.X_day_val[idx], self.image_size) targ_batch = load_batch(self.X_night_val[idx], self.image_size) # Discriminator d_loss_source, d_loss_targ = self.discri_train_on_batch( source_batch, targ_batch, valid, fake) # Generator g_loss = self.cycle_vae.train_on_batch( [source_batch, targ_batch], [valid, valid]) logs += self.get_logs(d_loss_source, d_loss_targ, g_loss) val_loss += g_loss[0] val_loss /= n_iter_val logs /= n_iter_val print('\n Epoch {} - Validation loss: {:.2f}'.format( e + 1, val_loss)) print('Discri source loss: {:.2f}, accuracy: {:.1f}%'.format( logs[0], logs[1])) print('Discri targ loss: {:.2f}, accuracy: {:.1f}%'.format( logs[2], logs[3])) print('Gen loss: {:.2f}'.format(logs[4])) print('Reconst: {:.2f}, kl: {:.2f}, cyclic_reconst: {:.2f}, cyclic_kl: {:.2f},gan: {:.2f}, vgg: {:.2f}' \ .format(*logs[5:])) write_log(val_callback, metrics_names, logs, iter_nb) if val_loss < val_loss_min: val_loss_min = val_loss print('Saving model\n') weights_filename = os.path.join( self.save_folder, 'cycle_vae.epoch{:02d}-val_loss{:.2f}.h5'.format( e + 1, val_loss)) self.cycle_vae.save_weights(weights_filename)
def train(self, epochs, batch_size=1, sample_interval=50): start_time = datetime.datetime.now() self.data_loader.preprocess() tensorboard = TensorBoard(log_dir="logs/".format(time.time())) tensorboard.set_model(self.generator) tensorboard.set_model(self.discriminator) # Checpoint management start_epoch = 0 #print("##### BEFORE", self.discriminator.get_weights()) if self.ckpt_manager.latest_checkpoint: start_epoch = int(self.ckpt_manager.latest_checkpoint.split('-')[-1]) print("Checkpoint found. Starting at epoch {}".format(start_epoch)) # restoring the latest checkpoint in checkpoint_path self.ckpt.restore(self.ckpt_manager.latest_checkpoint) #print("##### AFTER", self.discriminator.get_weights()) for epoch in tqdm.trange(start_epoch, epochs): # ---------------------- # Train Discriminator # ---------------------- # Sample images and their conditioning counterparts imgs_hr, imgs_lr = self.data_loader.load_data(batch_size) # From low res. image generate high res. version fake_hr = self.generator.predict(imgs_lr) valid = np.ones((batch_size,) + self.disc_patch) fake = np.zeros((batch_size,) + self.disc_patch) # Train the discriminators (original images = real / generated = Fake) d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid) d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) print("d_loss:", d_loss) # ------------------ # Train Generator # ------------------ # Sample images and their conditioning counterparts imgs_hr, imgs_lr = self.data_loader.load_data(batch_size) # The generators want the discriminators to label the generated images as real valid = np.ones((batch_size,) + self.disc_patch) # Extract ground truth image features using pre-trained VGG19 model image_features = self.vgg.predict(imgs_hr) # Train the generators g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr], [valid, image_features]) print("g_loss:", g_loss) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print ("%d time: %s" % (epoch, elapsed_time)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) if epoch % 2000 == 0: # Save models #self.generator.save_weights("{}/generator-{}.h5".format(CHECKPOINT_PATH, epoch)) #self.discriminator.save_weights("{}/discriminator-{}.h5".format(CHECKPOINT_PATH, epoch)) ckpt_save_path = self.ckpt_manager.save() print ('Saving checkpoint for epoch {} at {}'.format(epoch+1, ckpt_save_path))
features = vgg(generated_high_resolution_images) # Make the discriminator network as non-trainable discriminator.trainable = False # Get the probability of generated high-resolution images probs = discriminator(generated_high_resolution_images) # Create and compile an adversarial model adversarial_model = Model([input_low_resolution, input_high_resolution], [probs, features]) adversarial_model.compile(loss=['binary_crossentropy', 'mse'], loss_weights=[1e-3, 1], optimizer=common_optimizer) # Add Tensorboard tensorboard = TensorBoard(log_dir="logs/training") train_summary_writer = tf.summary.create_file_writer("logs/training") tensorboard.set_model(generator) tensorboard.set_model(discriminator) for epoch in range(epochs): print("Epoch:{}".format(epoch)) """ Train the discriminator network """ # Sample a batch of images high_resolution_images, low_resolution_images = sample_images(data_dir=data_dir, batch_size=batch_size, low_resolution_shape=low_resolution_shape, high_resolution_shape=high_resolution_shape) # Normalize images high_resolution_images = high_resolution_images / 127.5 - 1.
class FIR_Network: def __init__(self, mask_batch_size, tensorboard_logs_dir=""): self.batch_size = mask_batch_size self.mask_batch_size = mask_batch_size self.tr_loss_history = [] self.te_loss_history = [] self.y_pred_std_history = [] self.y_true_std_history = [] self.tf_logs = tensorboard_logs_dir self.epoch_counter = 0 self.data_masks = None self.data_targets = None self.best_performing_mask = None self.sample_weights = None def create_dense_model(self, input_shape, dense_arch): input_mask_layer = Input(shape=input_shape) x = Flatten()(input_mask_layer) for i in range(len(dense_arch[:-1])): x = Dense(dense_arch[i], activation="sigmoid")(x) x = Dense(dense_arch[-1], activation="linear")(x) self.model = Model(inputs=[input_mask_layer], outputs=x) # self.model.summary() def named_logs(self, model, logs): result = {} try: iterator = iter(logs) except TypeError: logs = [logs] for l in zip(model.metrics_names, logs): result[l[0]] = l[1] return result def compile_model(self): self.model.compile(loss='mae', optimizer='adam', metrics=[ self.get_y_std_metric(True), self.get_y_std_metric(False) ]) if self.tf_logs != "": self.tb_clbk = TensorBoard(self.tf_logs) self.tb_clbk.set_model(self.model) def train_one(self, epoch_number, apply_weights): if apply_weights == False: curr_loss = self.model.train_on_batch(x=self.data_masks, y=self.data_targets) else: curr_loss = self.model.train_on_batch( x=self.data_masks, y=self.data_targets, sample_weight=self.sample_weights) self.best_performing_mask = self.data_masks[np.argmin( self.data_targets, axis=0)] self.tr_loss_history.append(curr_loss) self.epoch_counter = epoch_number if self.tf_logs != "": self.tb_clbk.on_epoch_end(self.epoch_counter, self.named_logs(self.model, curr_loss)) self.data_masks = None self.data_targets = None def append_data(self, x, y): if self.data_masks is None: self.data_masks = x self.data_targets = y else: self.data_masks = np.concatenate([self.data_masks, x], axis=0) self.data_targets = tf.concat([self.data_targets, y], axis=0) def test_one(self, x, y): y_pred = self.model.predict(x=x) curr_loss = self.model.test_on_batch(x=x, y=y) self.te_loss_history.append(curr_loss) return curr_loss def predict(self, x): y_pred = self.model.predict(x=x) return y_pred def get_y_std_metric(self, ifpred=True): def y_pred_std_metric(y_true, y_pred): y_pred_std = K.std(y_pred) self.y_pred_std_history.append(y_pred_std) return y_pred_std def y_true_std_metric(y_true, y_pred): y_true_std = K.std(y_true) self.y_true_std_history.append(y_true_std) return y_true_std if ifpred: return y_pred_std_metric else: return y_true_std_metric
def _run(game, network_params, memory_params, ops): """Sets up and runs the gaming simulation. Initializes TensorFlow, the training agent, and the game environment. The agent plays the game from the starting state for a number of episodes set by the user. Args: args: The arguments from the command line parsed by_parse_arguments. """ # Setup TensorBoard Writer. trial_id = json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '') output_path = ops.job_dir if not trial_id else ops.job_dir + '/' tensorboard = TensorBoard(log_dir=output_path) hpt = hypertune.HyperTune() graph = tf.Graph() with graph.as_default(): env = gym.make(game) agent = _create_agent(env, network_params, memory_params) rewards = [] tensorboard.set_model(agent.policy) def _train_or_evaluate(print_score, training=False): """Runs a gaming simulation and writes results for tensorboard. Args: print_score (bool): True to print a score to the console. training (bool): True if the agent is training, False to eval. Returns: loss if training, else reward for evaluating. """ reward = _play(agent, env, training) if print_score: print( 'Train - ', 'Episode: {}'.format(episode), 'Total reward: {}'.format(reward), ) return reward for episode in range(1, ops.episodes + 1): print_score = ops.print_rate and episode % ops.print_rate == 0 get_summary = ops.eval_rate and episode % ops.eval_rate == 0 rewards.append(_train_or_evaluate(print_score, training=True)) if get_summary: avg_reward = sum(rewards) / len(rewards) summary = {'eval_reward': avg_reward} tensorboard.on_epoch_end(episode, summary) hpt.report_hyperparameter_tuning_metric( hyperparameter_metric_tag='avg_reward', metric_value=avg_reward, global_step=episode) print( 'Eval - ', 'Episode: {}'.format(episode), 'Average Reward: {}'.format(avg_reward), ) rewards = [] tensorboard.on_train_end(None) _record_video(env, agent, output_path) agent.policy.save(output_path, save_format='tf')
def find_best_lr(path_desc='', combined_lobes=False, records_add=''): min_lr = 1e-7 max_lr = 1e-1 num_classes = 9 if combined_lobes: num_classes = 6 max_conv_blocks = 3 iteration = 0 change_background = True for weighted_loss in [True, False]: for atrous in [True, False]: for layer in [2, 3]: for growth_rate in [4]: for filters in [8]: for num_conv_blocks in [ 2 ]: # Always start with 1 before downsampling for conv_lambda in [1]: base_path, morfeus_drive = return_paths() run_data = { 'layers': layer, 'max_conv_blocks': max_conv_blocks, 'filters': filters, 'num_conv_blocks': num_conv_blocks, 'conv_lambda': conv_lambda, 'growth_rate': growth_rate, 'atrous': atrous } layers_dict = get_layers_dict_dense( **run_data, pool=(2, 4, 4), num_classes=num_classes) things = [ 'change_background{}'.format( change_background), 'weighted_loss_{}'.format(weighted_loss), 'atrous_{}'.format(atrous), 'layers{}'.format(layer), 'max_conv_blocks_{}'.format( max_conv_blocks), 'filters_{}'.format(filters), 'num_conv_blocks_{}'.format( num_conv_blocks), 'conv_lambda_{}'.format(conv_lambda), 'growth_rate_{}'.format(growth_rate), '{}_Iteration'.format(iteration) ] out_path = os.path.join( morfeus_drive, path_desc, 'DenseNet') if records_add != '': things = ['Gaussian'] + things for thing in things: out_path = os.path.join(out_path, thing) if os.path.exists(out_path): print('already done') continue os.makedirs(out_path) print(out_path) model = my_UNet( layers_dict=layers_dict, image_size=(None, None, None, 1), mask_output=True, out_classes=num_classes, explictly_defined=True).created_model k = TensorBoard(log_dir=out_path, profile_batch=0, write_graph=True) k.set_model(model) k.on_train_begin() lr_opt = tf.keras.optimizers.Adam base_path, morfeus_drive, train_generator, validation_generator = return_generators( combined_lobes=combined_lobes, records_add=records_add, change_background=change_background) loss = tf.keras.losses.CategoricalCrossentropy( from_logits=False) LearningRateFinder( epochs=10, model=model, metrics=[ 'categorical_accuracy', MeanDSC(num_classes=num_classes) ], out_path=out_path, optimizer=lr_opt, loss=loss, steps_per_epoch=len(train_generator), train_generator=train_generator.data_set, lower_lr=min_lr, high_lr=max_lr) tf.keras.backend.clear_session() return None # repeat!