def test_CallbackValData(): np.random.seed(1337) (X_train, y_train), (X_test, y_test) = get_data_callbacks() y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) model = Sequential() model.add(Dense(num_hidden, input_dim=input_dim, activation='relu')) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) cbk = callbacks.LambdaCallback(on_train_end=lambda x: 1) model.fit(X_train, y_train, batch_size=batch_size, validation_data=(X_test, y_test), callbacks=[cbk], epochs=1) cbk2 = callbacks.LambdaCallback(on_train_end=lambda x: 1) train_generator = data_generator(X_train, y_train, batch_size) model.fit_generator(train_generator, len(X_train), epochs=1, validation_data=(X_test, y_test), callbacks=[cbk2]) # callback validation data should always have x, y, and sample weights assert len(cbk.validation_data) == len(cbk2.validation_data) == 3 assert cbk.validation_data[0] is cbk2.validation_data[0] assert cbk.validation_data[1] is cbk2.validation_data[1] assert cbk.validation_data[2].shape == cbk2.validation_data[2].shape
def test_CallbackValData(): np.random.seed(1337) (X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples, num_test=test_samples, input_shape=(input_dim, ), classification=True, num_classes=num_classes) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) model = Sequential() model.add(Dense(num_hidden, input_dim=input_dim, activation='relu')) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) cbk = callbacks.LambdaCallback(on_train_end=lambda x: 1) model.fit(X_train, y_train, batch_size=batch_size, validation_data=(X_test, y_test), callbacks=[cbk], epochs=1) def data_generator(train): if train: max_batch_index = len(X_train) // batch_size else: max_batch_index = len(X_test) // batch_size i = 0 while 1: if train: yield (X_train[i * batch_size:(i + 1) * batch_size], y_train[i * batch_size:(i + 1) * batch_size]) else: yield (X_test[i * batch_size:(i + 1) * batch_size], y_test[i * batch_size:(i + 1) * batch_size]) i += 1 i = i % max_batch_index cbk2 = callbacks.LambdaCallback(on_train_end=lambda x: 1) model.fit_generator(data_generator(True), len(X_train), epochs=1, validation_data=(X_test, y_test), callbacks=[cbk2]) # callback validation data should always have x, y, and sample weights assert len(cbk.validation_data) == len(cbk2.validation_data) == 3 assert cbk.validation_data[0] is cbk2.validation_data[0] assert cbk.validation_data[1] is cbk2.validation_data[1] assert cbk.validation_data[2].shape == cbk2.validation_data[2].shape
def run(self, model_file, load_weights=True, save_weights=True): model_path = os.path.split(model_file)[0] if not os.path.exists(model_path): os.makedirs(model_path) # 载入模型 if os.path.exists(model_file) and load_weights: self.vae.load_weights(model_file, skip_mismatch=True) tensor_board = callbacks.TensorBoard(log_dir=self.summary_path, histogram_freq=0, update_freq=1000) test_callback = callbacks.LambdaCallback(on_epoch_end=lambda epoch, logs: self.test(epoch + 1)) # 测试 self.test(0) # 训练:fit_generator self.vae.fit_generator(generator=self.my_generator( self.train_sketch_x, self.train_x_img, self.train_x_img, self.train_sketch_x, self.batch_size), epochs=self.max_epoch, verbose=2, callbacks=[tensor_board, test_callback], steps_per_epoch=len(self.train_sketch_x) // self.batch_size) # 测试 self.test(self.max_epoch) # 保存模型 if save_weights: self.vae.save_weights(model_file) pass
def _callbacks(self): def lambdaCallbackFunc(epoch, _): print(K.eval(self._model.optimizer.lr)) if (epoch + 1) % MODEL_SAVE_PERIOD == 0: self._save(epoch + 1) with open(self._model_config_file_path(), mode='w', encoding='utf-8') as f: dic = {} dic['epoch'] = epoch + 1 dic['lr'] = K.eval(self._model.optimizer.lr) json.dump(dic, f, cls=NumpyEncoder) def learningRateSchedulerFunc(epoch): return LR * (0.5**(epoch // 8)) return [ # callbacks.ReduceLROnPlateau(monitor='loss', factor=LR_REDUCE_FACTOR, patience=LR_REDUCE_PATIENCE, epsilon=LR_REDUCE_EPSILON), # callbacks.ModelCheckpoint(os.path.join(proj_path, '{epoch:d}.hdf5'), period=MODEL_SAVE_PERIOD), # callbacks.LambdaCallback(on_epoch_end=lambda epoch, logs: self._model.save_weights(os.path.join(model_dir, 'epoch_{}.hdf5'.format(epoch)))), callbacks.LearningRateScheduler(schedule=learningRateSchedulerFunc ), callbacks.LambdaCallback(on_epoch_end=lambdaCallbackFunc), callbacks.TensorBoard(log_dir=self._model_dir(), batch_size=BATCH_SIZE) ]
def test_LambdaCallback(): (X_train, y_train), (X_test, y_test) = get_test_data(nb_train=train_samples, nb_test=test_samples, input_shape=(input_dim, ), classification=True, nb_class=nb_class) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) model = Sequential() model.add(Dense(nb_hidden, input_dim=input_dim, activation='relu')) model.add(Dense(nb_class, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) # Start an arbitrary process that should run during model training and be terminated after training has completed. def f(): while True: pass p = multiprocessing.Process(target=f) p.start() cleanup_callback = callbacks.LambdaCallback( on_train_end=lambda logs: p.terminate()) cbks = [cleanup_callback] model.fit(X_train, y_train, batch_size=batch_size, validation_data=(X_test, y_test), callbacks=cbks, nb_epoch=5) p.join() assert not p.is_alive()
def fit_model(flow_steps, potential_fn, learning_rate=1e-5, parameter_updates=500000, batch_size=100, optimizer='rmsprop', use_temperature=True, min_temperature=0.01, max_temperature=1.0, temperature_steps=10000): """Performs the building and training of the model to test.""" input_layer = layers.Input(shape=(1,)) # Random input output_layer = Parameters(flow_steps, 2)(input_layer) # Flow parameters output_layer = flows.PlanarFlow(flow_steps, 2, # 2 latent dimensions activation=activations.tanh)(output_layer) model = models.Model(inputs=input_layer, outputs=output_layer) # Add likelihood loss, with temperature for first parameters updates temperature = backend.variable(1.0) model.add_loss(temperature * backend.mean(potential_fn(output_layer))) # Finish model model.compile(optimizer=getattr(optimizers, optimizer)(lr=learning_rate)) model.summary() # If using temperature, then add callback to update it the_callbacks = None if use_temperature: def set_temperature(batch, _): """Set temperature for this batch.""" if batch <= temperature_steps: temp_step = (max_temperature - min_temperature) / temperature_steps backend.set_value(temperature, min_temperature + temp_step * batch) backend.set_value(temperature, min_temperature) the_callbacks = [callbacks.LambdaCallback( on_batch_begin=set_temperature)] # Train the model with random input, that is discarded model.fit(x=np.ones(batch_size * parameter_updates), epochs=1, batch_size=batch_size, callbacks=the_callbacks) return model
def atm_progress_callback(framework, purpose): if framework == "keras": return callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: ATM.report({ "name": "progress", 'purpose': purpose, 'progress': (epoch + 1) / ATM.props["epochs"], 'loss': logs['loss'], 'finished': False, 'gpu' : tf.config.experimental.list_physical_devices('GPU') }), on_train_end=lambda logs: ATM.report({ "name": "progress", 'purpose': purpose, 'finished': True, 'gpu' : tf.config.experimental.list_physical_devices('GPU') }) ) return None;
def train(self, model_file, csv_log_file, load_weights=True, save_weights=True): model_path = os.path.split(model_file)[0] if not os.path.exists(model_path): os.makedirs(model_path) # 载入模型 if os.path.exists(model_file) and load_weights: self.sketch_model.load_weights(model_file, skip_mismatch=True) # Callback tensor_board = callbacks.TensorBoard(log_dir=self.summary_path, histogram_freq=0, update_freq=1000) test_callback = callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: self._test(epoch + 1)) csv_logger = callbacks.CSVLogger(csv_log_file) # 测试 self._test("first") # 训练:输入和输出在Model中定义 # 没有数据增强 # self.sketch_model.fit(x={'inputs': self.sketch_train_image}, y=[self.sketch_train_label], # batch_size=self.batch_size, epochs=self.max_epoch, # verbose=2, callbacks=[tensor_board, csv_logger, test_callback]) # 数据增强 data_gen = preprocessing.image.ImageDataGenerator( horizontal_flip=True, vertical_flip=True, rotation_range=30, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.2) data_gen.fit(self.sketch_train_image) self.sketch_model.fit_generator( generator=data_gen.flow(self.sketch_train_image, self.sketch_train_label, batch_size=self.batch_size), epochs=self.max_epoch, verbose=2, callbacks=[tensor_board, csv_logger, test_callback], steps_per_epoch=len(self.sketch_train_image) // self.batch_size) # 测试 self._test("final") # 保存模型 if save_weights: self.sketch_model.save_weights(model_file) pass
def get_callbacks(): """ :return: A list of `keras.callbacks.Callback` instances to apply during training. """ return [ callbacks.ModelCheckpoint(model_checkpoint, monitor='val_acc', verbose=1, save_best_only=True), callbacks.EarlyStopping(monitor='val_loss', patience=12, verbose=1), callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=2, verbose=1), callbacks.LambdaCallback(on_epoch_end=on_epoch_end), callbacks.TensorBoard(log_dir=tensor_board_logs, histogram_freq=4, write_graph=True, write_images=True) ]
def CallbackList(self, outputdir): filepath = os.path.join(outputdir, "model{epoch:03d}-{val_accuracy:.3f}.hdf5") Checkpoint = callbacks.ModelCheckpoint(filepath, monitor='val_accuracy', save_best_only=True, mode='max', period=1, save_weights_only=True) batch_print_callback = callbacks.LambdaCallback( on_epoch_end=lambda batch, logs: print( 'Epoch[%d] Train-accuracy=%f Epoch[%d] Validation-accuracy=%f' % (batch, logs['accuracy'], batch, logs['val_accuracy']))) self.callbacklist = [Checkpoint, batch_print_callback]
def init_callbacks(self): self.callbacks.append( CosineAnnealingCallback( lr_min=self.config.trainer.lr_min, lr_max=self.config.trainer.lr_max, run_initial=self.config.trainer.run_initial, run_mult=self.config.trainer.run_mult, n_batches=self.n_batches, )) if self.config.trainer.tensorboard_enabled: self.callbacks.append( callbacks.LambdaCallback( on_epoch_begin=lambda epoch, logs: self.set_epoch(epoch), on_batch_begin=lambda batch, logs: self.log_lr(batch), ))
def run(self, model_file, load_weights=True, save_weights=True): model_path = os.path.split(model_file)[0] if not os.path.exists(model_path): os.makedirs(model_path) # 载入模型 if os.path.exists(model_file) and load_weights: self.vae.load_weights(model_file, skip_mismatch=True) tensor_board = callbacks.TensorBoard(log_dir=self.summary_path, histogram_freq=0, update_freq=1000) test_callback = callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: self.test(epoch + 1)) # 测试 self.test(0) # 训练 self.vae.fit(x={ 'sketch_features': self.train_sketch_x, 'image_features': self.train_x_img }, y=[self.train_x_img, self.train_sketch_x], batch_size=self.batch_size, epochs=self.max_epoch, verbose=2, callbacks=[tensor_board, test_callback]) # 测试 self.test(self.max_epoch) # 保存模型 if save_weights: self.vae.save_weights(model_file) pass
def train(s, stream, epochs=200, callback=(lambda x: x), debug=False): features, labels = zip(*s._format_stream(stream)) features, labels = np.array(features), np.array(labels) if not s._model: num_features = len(features[0]) s._model = Sequential([ Dense(num_features**2, input_dim=num_features), Dense(num_features * 2), Dense(num_features) ]) s._compile() print("Training. Please wait...") prog = 1.0 / epochs best_callback = callbacks.ModelCheckpoint(os.path.join( s._path, s.best), save_best_only=True, monitor="acc") # stop_callback = callbacks.EarlyStopping(monitor="loss") epoch_callback = callbacks.LambdaCallback( on_epoch_end=lambda x, y: callback(x * prog, y["acc"])) res = s._model.fit( features, labels, shuffle=True, # validation_split=0.5, epochs=epochs, callbacks=[best_callback, epoch_callback], #, stop_callback], verbose=1 if debug else 0) s.save_state() # print(res.history["acc"]) return s
def callback_save_net(self): return(kc.LambdaCallback(on_batch_end= lambda batch, logs: self.__batch_save_net(batch, logs)))
epochs = rundef.num_epochs - epoch_start else: epoch_start = 0 epochs = rundef.num_epochs """Callbacks""" scheduler = rundef.scheduler # Scheduler class can vary tensorboard = call.TensorBoard(**rundef.tensorboard_params) checkpointer = call.ModelCheckpoint(**rundef.checkpointer_params) evaluate_agent = EvaluateAgentCallback(**{ 'tb_callback': tensorboard, **rundef.evaluator_params }) # Added to address OOM Error - not sure if needed anymore # See: https://github.com/keras-team/keras/issues/3675 garbage_collection = call.LambdaCallback( on_epoch_end=lambda epoch, logs: gc.collect()) """ Model train code """ print("Begin Training Model") rundef.model.fit_generator( rundef.train_gen, epochs=rundef.num_epochs, steps_per_epoch=rundef.steps_per_epoch, verbose=1, # 0 in notebook, verbose doesn't slow down training, we checked callbacks=[ tensorboard, #scheduler, evaluate_agent, checkpointer, garbage_collection, ], shuffle=True, # Note this only works because we removed step parameters
def train(_run, train_cnf, generator_cnf): batch_size = train_cnf["batch_size"] max_epochs = train_cnf["max_epochs"] patience = train_cnf["patience"] loss = train_cnf["loss"] learning_rate = train_cnf["learning_rate"] beta_1 = train_cnf["beta_1"] beta_2 = train_cnf["beta_2"] dset = train_cnf["dset"] base_dir = train_cnf["base_dir"] fig_save_freq = train_cnf["fig_save_freq"] fig_pairs = train_cnf["fig_pairs"] # Setup environment (logging directory etc) # Just creates the figures and models dirs experiment_name = 'unetfocal/exp' + str(_run._id) experiment_dir = os.path.join(base_dir, 'experiments', experiment_name) general_utils.setup_logging(experiment_dir) figures_dir = os.path.join(experiment_dir, 'figures') models_dir = os.path.join(experiment_dir, 'models') # Load data x_train, y_train, x_val, y_val, x_test, y_test = data_utils.load_data(dset, base_dir) img_dim = x_train.shape[-3:] unet_upsampling_model = models.unet_upsampling(img_dim=img_dim, **generator_cnf) try: # Create optimizers optimizer = Adam(lr=learning_rate, beta_1=beta_1, beta_2=beta_2) class ImgPlotter(callbacks.Callback): def __init__(self, x_train, y_train, x_val, y_val, figures_dir, fig_save_freq): super(ImgPlotter, self).__init__() self.data = { 'train': (x_train, y_train), 'val': (x_val, y_val), } self.figures_dir = figures_dir self.fig_save_freq = fig_save_freq def on_epoch_end(self, epoch, logs=None): if epoch % self.fig_save_freq == 0: for batch_set in ['train', 'val']: data_utils.plot_generated_batch(*self.data[batch_set], self.model, batch_set, self.figures_dir, epoch) train_idx = np.random.choice(x_train.shape[0], fig_pairs, replace=False) val_idx = np.random.choice(x_val.shape[0], fig_pairs, replace=False) imgplotter_cb = ImgPlotter(x_train[train_idx], y_train[train_idx], x_val[val_idx], y_val[val_idx], figures_dir, fig_save_freq) # Create the logging callback # The metrics are logged in the run's metrics and at heartbeat events # every 10 secs they get written to mongodb def metrics_log(epoch, logs): for metric_name, metric_value in logs.items(): # The validation set keys have val_ prepended to the metric, # add train_ to the training set keys if 'val' not in metric_name: metric_name = 'train_' + metric_name _run.log_scalar(metric_name, metric_value, epoch) metrics_log_cb = callbacks.LambdaCallback(on_epoch_end=metrics_log) model_filepath = os.path.join(models_dir, 'weights.best.hdf5') callbacks_list = [callbacks.EarlyStopping( monitor='val_loss', patience=patience), callbacks.ModelCheckpoint( filepath=model_filepath, monitor='val_loss', save_best_only=True, save_weights_only=True), imgplotter_cb, metrics_log_cb] if loss == 'focal_loss': unet_upsampling_model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['binary_accuracy', precision, recall, fmeasure]) unet_upsampling_model.fit(x_train, y_train, epochs=2, batch_size=batch_size, validation_data=(x_val, y_val), callbacks=callbacks_list, verbose=2) unet_upsampling_model.compile(loss=[focal_loss(alpha=.25, gamma=2)], optimizer=optimizer, metrics=['binary_accuracy', precision, recall, fmeasure]) unet_upsampling_model.fit(x_train, y_train, epochs=max_epochs, batch_size=batch_size, validation_data=(x_val, y_val), callbacks=callbacks_list, initial_epoch=2, verbose=2) elif loss == 'binary_crossentropy': unet_upsampling_model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['binary_accuracy', precision, recall, fmeasure]) unet_upsampling_model.fit(x_train, y_train, epochs=max_epochs, batch_size=batch_size, validation_data=(x_val, y_val), callbacks=callbacks_list, verbose=2) # Clear GPU + RAM K.clear_session() del unet_upsampling_model except KeyboardInterrupt: pass
print('loss:', loss, 'accuracy:', acc) print('######') print() #定义训练结束后的操作 def train_end_operation(): print('GAME OVER!') #自定义回调函数 #每轮结束后的回调函数 epoch_print_callback = callb.LambdaCallback( #定义在每轮结束是操作 on_epoch_end=lambda epoch, #执行操作 logs: epoch_end_operation()) #训练结束后的回调函数 train_end_callback = callb.LambdaCallback( on_train_end=lambda logs: train_end_operation()) #需要执行的回调函数列表 callback_list = [epoch_print_callback, train_end_callback] #训练模型 model.fit(x_train, y_train, validation_split=0.3, batch_size=60, epochs=3,
def prepare_callbacks_for_training(self, model_instance, eval_params, use_custom_eval=True): """ Prepare Keras Callbacks for model training Returns a list of keras callbacks """ training_CB = [] if eval_params is None: monitor, mon_mode = 'val_acc', 'max' else: X_val, Y_val, val_classes, train_distribution, \ ms_classes, fs_classes, X_val_many, Y_val_many, X_val_few, Y_val_few = eval_params evaluate_specific_params = (train_distribution, ms_classes, fs_classes) # Set the monitor (metric) for validation. # This is used for early-stopping during development. monitor, mon_mode = None, None if use_custom_eval: if UserArgs.train_dist == "dragon": monitor, mon_mode = 'val_wgtAcc', 'max' else: monitor, mon_mode = 'val_perClassAcc', 'max' training_CB += [ callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: logs.update( DragonTrainer.training_evaluation( model_instance, (X_val, X_val_many, X_val_few), (Y_val, Y_val_many, Y_val_few), (val_classes, ms_classes, fs_classes ), evaluate_specific_params))) ] else: monitor, mon_mode = 'val_har_acc', 'max' training_CB += [ callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: logs.update( DragonTrainer.training_evaluation( model_instance, (X_val, X_val_many, X_val_few), (Y_val, Y_val_many, Y_val_few), (val_classes, ms_classes, fs_classes ), evaluate_specific_params))) ] print(f'monitoring = {monitor}') # Save a model checkpoint only when monitor indicates that the best performance so far training_CB += [ callbacks.ModelCheckpoint(monitor=monitor, mode=mon_mode, save_best_only=True, filepath=os.path.join( self.training_dir, 'best-checkpoint'), verbose=UserArgs.verbose) ] # Set an early stopping callback training_CB += [ callbacks.EarlyStopping(monitor=monitor, mode=mon_mode, patience=UserArgs.patience, verbose=UserArgs.verbose, min_delta=UserArgs.min_delta) ] # Log training history to CSV training_CB += [ callbacks.CSVLogger(os.path.join(self.training_dir, 'training_log.csv'), separator='|', append=True) ] # Flush stdout buffer on every epoch training_CB += [ callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: sys.stdout.flush()) ] return training_CB
def train(train_list_fname='benchmark_RELEASE/dataset/train.txt', val_list_fname='benchmark_RELEASE/dataset/val.txt', img_root='benchmark_RELEASE/dataset/img', mask_root='benchmark_RELEASE/dataset/pngs', weights_path='converted/dilation8_pascal_voc.npy', batch_size=2, learning_rate=0.0001): # Create image generators for the training and validation sets. Validation has # no data augmentation. transformer_train = RandomTransformer(horizontal_flip=True, vertical_flip=True) datagen_train = SegmentationDataGenerator(transformer_train) transformer_val = RandomTransformer(horizontal_flip=False, vertical_flip=False) datagen_val = SegmentationDataGenerator(transformer_val) train_desc = 'lr{:.0e}-bs{:03d}'.format(learning_rate, batch_size) checkpoints_folder = 'trained/' + train_desc try: os.makedirs(checkpoints_folder) except OSError: shutil.rmtree(checkpoints_folder, ignore_errors=True) os.makedirs(checkpoints_folder) model_checkpoint = callbacks.ModelCheckpoint( checkpoints_folder + '/ep{epoch:02d}-vl{val_loss:.4f}.hdf5', monitor='loss') tensorboard_cback = callbacks.TensorBoard( log_dir='{}/tboard'.format(checkpoints_folder), histogram_freq=0, write_graph=False, write_images=False) csv_log_cback = callbacks.CSVLogger( '{}/history.log'.format(checkpoints_folder)) reduce_lr_cback = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, verbose=1, min_lr=0.05 * learning_rate) model = add_softmax(get_frontend(500, 500)) load_weights(model, weights_path) model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizers.SGD(lr=learning_rate, momentum=0.9), metrics=['accuracy']) # Build absolute image paths def build_abs_paths(basenames): img_fnames = [os.path.join(img_root, f) + '.jpg' for f in basenames] mask_fnames = [os.path.join(mask_root, f) + '.png' for f in basenames] return img_fnames, mask_fnames train_basenames = [l.strip() for l in open(train_list_fname).readlines()] val_basenames = [l.strip() for l in open(val_list_fname).readlines()][:500] train_img_fnames, train_mask_fnames = build_abs_paths(train_basenames) val_img_fnames, val_mask_fnames = build_abs_paths(val_basenames) skipped_report_cback = callbacks.LambdaCallback( on_epoch_end=lambda a, b: open( '{}/skipped.txt'.format(checkpoints_folder), 'a').write( '{}\n'.format(datagen_train.skipped_count))) print("Inside build_abs_paths") model.fit_generator( datagen_train.flow_from_list(train_img_fnames, train_mask_fnames, shuffle=True, batch_size=batch_size, img_target_size=(500, 500), mask_target_size=(16, 16)), samples_per_epoch=len(train_basenames), nb_epoch=20, validation_data=datagen_val.flow_from_list(val_img_fnames, val_mask_fnames, batch_size=8, img_target_size=(500, 500), mask_target_size=(16, 16)), nb_val_samples=len(val_basenames), callbacks=[ model_checkpoint, tensorboard_cback, csv_log_cback, reduce_lr_cback, skipped_report_cback, ])
model.add(Conv1D(32, 3, border_mode='same', activation='relu') ) #border_mode='same')) # now output [batch_size, n_steps, 32] model.add(MaxPooling1D()) model.add(Flatten()) model.add(Dense( n_class, activation='softmax')) #output layer, output prob for every class model.summary() #change learning rate #opt = optimizers.RMSprop(lr = 0.1) #too big opt = optimizers.RMSprop(lr=0.01) #model.compile(optimizer='rmsprop', model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) batch_print_callback = callbacks.LambdaCallback( on_batch_begin=lambda batch, logs: print(batch)) model.fit(x_train, y_train, epochs=10, batch_size=1500, callbacks=[batch_print_callback]) model.save('models/ecg_rnn_3class.h5') model.evaluate(x_test, y_test, batch_size=1000) y_predict_prob = model.predict(x_test, batch_size=1000) y_predict_class = np.argmax(y_predict_prob, axis=1)
metrics=['accuracy']) if not args.eval: # Callbacks checkpoint = callbacks.ModelCheckpoint(args.save_dir + '/weights-{epoch:02d}.h5', save_best_only=True, save_weights_only=True, verbose=1) csv_logger = callbacks.CSVLogger(args.save_dir + '/log.csv') lr_reduce = callbacks.ReduceLROnPlateau(factor=0.9, monitor='val_loss', mode='min', verbose=1, patience=5) copy_weights = callbacks.LambdaCallback( on_batch_begin=lambda batch, logs: backbone_fix.copy_weights()) callbacks = [checkpoint, lr_reduce, csv_logger, copy_weights] train_model.fit_generator(generator=generator, steps_per_epoch=int(y_train.shape[0] / args.batch_size), epochs=args.epochs, validation_data=[x_test, y_test], callbacks=callbacks, verbose=1) train_model.save_weights(args.save_dir + '/trained_model.h5') # --- evaluation # compute statistics on the test dataset
def callback_epoch_end(self): self.start_time = time.time() return(kc.LambdaCallback(on_epoch_begin = lambda epoch, logs: self.__set_start_time(epoch, logs), on_epoch_end= lambda epoch, logs: self.__epoch_logging(epoch, logs)))
def smooth_curve(points, factor=0.9): smoothed_points = [] for point in points: if smoothed_points: previous = smoothed_points[-1] smoothed_points.append(previous * factor + point * (1 - factor)) else: smoothed_points.append(point) return smoothed_points #Print the current "epoch" during training: def epoca_feedback(epoch, logs): print(epoch) #Define the callbacks list (a keras functionality): callbacks_list = [callbacks.LambdaCallback(on_epoch_end=epoca_feedback)] #Import the generated data: train_data = np.load('train_helal_samples_n.npy') train_targets = np.load('train_helal_labels_n.npy') test_data = np.load('test_helal_samples_n.npy') test_targets = np.load('test_helal_labels_n.npy') #Normalize the data: #INPUT mean = train_data.mean() train_data -= mean std = train_data.std() train_data /= std test_data -= mean test_data /= std
def prepare_callbacks_for_training(model, common_args): """ Prepare Keras Callbacks for model training Returns a list of keras callbacks """ training_CB = [] # Note that checkpoint is saved after metrics eval and *before* # categorical zero-shot transfer update. I.e. For reflecting the metric, # it is not saved with the latest custom updated leaf weights if common_args.is_dev: # Set the monitor (metric) for validation. # This is used for early-stopping during development. monitor, mon_mode = model.monitor, model.mon_mode print(f'monitor = {monitor}') # Save a model checkpoint only when monitor indicates that the best # performance so far training_CB += [ callbacks.ModelCheckpoint(monitor=monitor, mode=mon_mode, save_best_only=True, filepath=get_file('best-checkpoint'), verbose=common_args.verbose) ] # Set an early stopping callback training_CB += [ callbacks.EarlyStopping(monitor=monitor, mode=mon_mode, patience=common_args.patience, verbose=common_args.verbose, min_delta=common_args.min_delta) ] # An option to dump results to tensorboard if common_args.tensorboard_dump: training_CB += [ callbacks.TensorBoard( log_dir=os.path.expanduser(common_args.train_dir)) ] # Log training history to CSV training_CB += [ callbacks.CSVLogger(get_file('csv-log'), separator='|', append=is_alternating_train_config()) ] # Touch a progress indicator file on every epoch end training_CB += [ callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: ml_utils.touch(get_file('touch'))) ] # Flush stdout buffer on every epoch training_CB += [ callbacks.LambdaCallback( on_epoch_end=lambda epoch, logs: sys.stdout.flush()) ] return training_CB
def callback_epoch_begin(self): return(kc.LambdaCallback(on_epoch_begin= lambda epoch, logs: self.__epoch_logging(epoch, logs)))
def callback_batch_end(self): return(kc.LambdaCallback(on_batch_begin = lambda epoch, logs: self.__set_start_time(epoch, logs), on_batch_end= lambda batch, logs: self.__batch_logging(batch, logs)))
def callback_batch_begin(self): return(kc.LambdaCallback(on_batch_begin= lambda batch, logs: self.__batch_logging(batch, logs)))
def train(): global train_list_fname global val_list_fname global img_root global mask_root global weights_path global batch_size global learning_rate global modeltype train_data_gen = SegmentationDataGenerator( RandomTransformer(horizontal_flip=True, vertical_flip=True)) val_data_gen = SegmentationDataGenerator( RandomTransformer(horizontal_flip=True, vertical_flip=True)) trained_log = '{}-lr{:.0e}-bs{:03d}'.format( time.strftime("%Y-%m-%d %H:%M"), learning_rate, batch_size) checkpoints_folder = 'trained_log/' + trained_log try: os.makedirs(checkpoints_folder) except OSError: shutil.rmtree(checkpoints_folder, ignore_errors=True) os.makedirs(checkpoints_folder) model_checkpoint = callbacks.ModelCheckpoint( checkpoints_folder + '/ep{epoch:02d}-vl{val_loss:.4f}.hdf5', monitor='loss') model_tensorboard = callbacks.TensorBoard( log_dir='{}/tboard'.format(checkpoints_folder), histogram_freq=0, write_graph=False, write_images=False) model_csvlogger = callbacks.CSVLogger( '{}/history.log'.format(checkpoints_folder)) model_reducelr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, verbose=1, min_lr=0.05 * learning_rate) model = add_softmax(dilated_frontend(500, 500)) #load_weights(model, weights_path) model.compile(optimizer=optimizers.SGD(lr=learning_rate, momentum=0.9), loss='sparse_categorical_crossentropy', metrics=['accuracy']) train_basenames = [l.strip() for l in open(train_list_fname).readlines()] val_basenames = [l.strip() for l in open(val_list_fname).readlines()][:500] train_img_fnames, train_mask_fnames = build_abs_paths(train_basenames) val_img_fnames, val_mask_fnames = build_abs_paths(val_basenames) model_skipped = callbacks.LambdaCallback(on_epoch_end=lambda a, b: open( '{}/skipped.txt'.format(checkpoints_folder), 'a').write('{}\n'.format( train_data_gen.skipped_count))) model.fit_generator( train_data_gen.flow_from_list(train_img_fnames, train_mask_fnames, shuffle=True, batch_size=batch_size, img_target_size=(500, 500), mask_target_size=(16, 16)), steps_per_epoch=(len(train_basenames) / batch_size), epochs=50, validation_data=val_data_gen.flow_from_list(val_img_fnames, val_mask_fnames, batch_size=8, img_target_size=(500, 500), mask_target_size=(16, 16)), validation_steps=(len(val_basenames) / 8), callbacks=[ model_checkpoint, model_tensorboard, model_csvlogger, model_reducelr, model_skipped ])
def start(): model = yolo() model.summary() num_of_epochs = 100 nb_conv = 23 weight_reader = WeightReader(pre_trained_weights) weight_reader.reset() generator_config = { 'IMAGE_H': IMAGE_H, 'IMAGE_W': IMAGE_W, 'GRID_H': GRID_H, 'GRID_W': GRID_W, 'BOX': BOX, 'LABELS': LABELS, 'CLASS': CLASS, 'ANCHORS': ANCHORS, 'BATCH_SIZE': BATCH_SIZE, 'TRUE_BOX_BUFFER': TRUE_BOX_BUFFER, } for i in range(1, nb_conv + 1): conv_layer = model.get_layer('conv_' + str(i)) if i < nb_conv: norm_layer = model.get_layer('norm_' + str(i)) size = np.prod(norm_layer.get_weights()[0].shape) beta = weight_reader.read_bytes(size) gamma = weight_reader.read_bytes(size) mean = weight_reader.read_bytes(size) var = weight_reader.read_bytes(size) weights = norm_layer.set_weights([gamma, beta, mean, var]) if len(conv_layer.get_weights()) > 1: bias = weight_reader.read_bytes( np.prod(conv_layer.get_weights()[1].shape)) kernel = weight_reader.read_bytes( np.prod(conv_layer.get_weights()[0].shape)) kernel = kernel.reshape( list(reversed(conv_layer.get_weights()[0].shape))) kernel = kernel.transpose([2, 3, 1, 0]) conv_layer.set_weights([kernel, bias]) else: kernel = weight_reader.read_bytes( np.prod(conv_layer.get_weights()[0].shape)) kernel = kernel.reshape( list(reversed(conv_layer.get_weights()[0].shape))) kernel = kernel.transpose([2, 3, 1, 0]) conv_layer.set_weights([kernel]) # Get last convolutional layer layer = model.layers[-4] weights = layer.get_weights() new_kernel = np.random.normal(size=weights[0].shape) / (GRID_H * GRID_W) new_bias = np.random.normal(size=weights[1].shape) / (GRID_H * GRID_W) layer.set_weights([new_kernel, new_bias]) train_imgs, seen_train_labels = parse_annotation(train_annot_folder, train_image_folder, labels=LABELS) train_batch = BatchGenerator(train_imgs, generator_config) val_imgs, seen_val_labels = parse_annotation(val_annot_folder, val_image_folder, labels=LABELS) prev_model = get_pretrained_model() model.load_weights(prev_model) valid_batch = BatchGenerator(val_imgs, generator_config, jitter=False) early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=3, mode='min', verbose=1) checkpoint = callbacks.ModelCheckpoint(temp_training_model, monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) onfinish = callbacks.LambdaCallback( on_train_end=lambda logs: save_checkpoint(logs, temp_training_model, prev_model)) dirs = listdir(path.expanduser(tensorboard_log)) arr_log = [log for log in dirs if 'wr_' in log] tb_counter = len(arr_log) + 1 tensorboard = callbacks.TensorBoard( log_dir=path.expanduser(tensorboard_log) + 'wr_' + '_' + str(tb_counter), histogram_freq=0, write_graph=True, write_images=False, ) optimizer = optimizers.Adam( lr=0.5e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0, ) model.compile(loss=custom_loss, optimizer=optimizer) model.fit_generator( generator=train_batch.get_generator(), steps_per_epoch=train_batch.get_dateset_size(), epochs=num_of_epochs, verbose=1, validation_data=valid_batch.get_generator(), validation_steps=valid_batch.get_dateset_size(), callbacks=[early_stop, checkpoint, tensorboard, onfinish], max_queue_size=3, )
def callback_save_result(self): return(kc.LambdaCallback(on_episode_end= lambda epoch, logs: self.__epoch_save(epoch, logs)))