model_pre.load_weights('/home/ucl/cp3/fbury/storage/MoMEMtaModelNN/4lay50_inv_trainmass/DYweight.h5') # load weights print ('[INFO] Loading pre-stage DNN weights : Success') except: sys.exit('Could not find pre-stage DNN model') DNN.get_layer("L1").set_weights(model_pre.get_layer('dense_1').get_weights()) DNN.get_layer("L2").set_weights(model_pre.get_layer('dense_2').get_weights()) DNN.get_layer("L3").set_weights(model_pre.get_layer('dense_3').get_weights()) DNN.get_layer("Mjj").set_weights(model_pre.get_layer('dense_4').get_weights()) DNN.get_layer("Mll").set_weights(model_pre.get_layer('dense_5').get_weights()) DNN.get_layer("Mlljj").set_weights(model_pre.get_layer('dense_6').get_weights()) print (model_pre.get_config()) # Callbacks # early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20, verbose=1, mode='min') csv_logger = CSVLogger(path_out+'training.log') reduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, verbose=1, mode='min', epsilon=0.0001, cooldown=0, min_lr=0.0001) checkpoint = ModelCheckpoint(path_out+args.label+'weight.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode='min', period=1) Callback_list = [csv_logger,checkpoint,early_stopping,reduceLR] # Fit # epoch = 200 batch = 1000 DNN.fit(train_inputs,{'Mlljj':train_targets[:,0], 'Mjj':train_targets[:,1], 'Mll':train_targets[:,2]}, sample_weight={'Mlljj':train_weight, 'Mjj':train_weight, 'Mll':train_weight}, epochs=epoch, batch_size=batch, verbose=2, validation_data=(test_inputs,[test_targets[:,0], test_targets[:,1], test_targets[:,2]],[test_weight, test_weight, test_weight]), callbacks=Callback_list) # Save model # model_json = DNN.to_json() with open(path_out+args.label+'model.json', "w") as json_file: json_file.write(model_json)
model = Sequential() model.add(Embedding(max_features, embedding_vecor_length, input_length=max_len)) model.add(LSTM(128)) model.add(Dropout(0.1)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint( filepath="logs/lstm/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='val_acc', mode='max') csv_logger = CSVLogger('logs/lstm/training_set_lstmanalysis.csv', separator=',', append=False) model.fit(X_train, y_train, batch_size=32, nb_epoch=1000, validation_split=0.33, shuffle=True, callbacks=[checkpointer, csv_logger]) score, acc = model.evaluate(X_test, y_test, batch_size=32) print('Test score:', score) print('Test accuracy:', acc)
finetune_model.summary() num_train_images = utils.get_num_files(TRAIN_DIR) num_val_images = utils.get_num_files(VAL_DIR) def lr_decay(epoch): if epoch % 20 == 0 and epoch != 0: lr = K.get_value(model.optimizer.lr) K.set_value(model.optimizer.lr, lr / 2) print("LR changed to {}".format(lr / 2)) return K.get_value(model.optimizer.lr) learning_rate_schedule = LearningRateScheduler(lr_decay) # setup checkpoints csv_logger = CSVLogger(os.path.join(OUT_DIR, "log.csv"), append=True, separator=";") latest_filepath = os.path.join(OUT_DIR, args.model + "_model_latest.h5") latest_checkpoint = ModelCheckpoint(latest_filepath, monitor="accuracy", verbose=1) best_filepath = os.path.join(OUT_DIR, args.model + "_model_best.h5") best_checkpoint = ModelCheckpoint( best_filepath, monitor="val_accuracy", verbose=1, save_best_only=True, mode="max", ) change_transfer_strategy = FixedThenFinetune(args.from_epoch + args.num_epochs)
def train_model(model: Model, dataset_id, method, proto_num, dataset_prefix, nb_iterations=100000, batch_size=128, val_subset=None, cutoff=None, normalize_timeseries=False, opt='Adam', learning_rate=1e-3, early_stop=False, balance_classes=True): X_train1, X_train2, y_train, X_test1, X_test2, y_test, is_timeseries = load_dataset( dataset_id, method, proto_num, normalize_timeseries=normalize_timeseries) #calculate num of batches nb_epochs = math.ceil(nb_iterations * (batch_size / X_train1.shape[0])) if balance_classes == True: classes = np.arange(0, nb_classes(dataset_id)) le = LabelEncoder() y_ind = le.fit_transform(y_train.ravel()) recip_freq = len(y_train) / (len(le.classes_) * np.bincount(y_ind).astype(np.float64)) class_weight = recip_freq[le.transform(classes)] print("Class weights : ", class_weight) y_train = to_categorical(y_train, nb_classes(dataset_id)) y_test = to_categorical(y_test, nb_classes(dataset_id)) reduce_lr = ReduceLROnPlateau(monitor='loss', patience=math.ceil(nb_epochs / 20), mode='auto', factor=1 / 2., cooldown=0, min_lr=learning_rate / 10., verbose=2) model_checkpoint = ModelCheckpoint( "./weights/%s_%s_%s_weights.h5" % (dataset_prefix, method, str(proto_num)), verbose=2, monitor='loss', save_best_only=True, save_weights_only=True) tensorboard = TensorBoard(log_dir='./logs/%s_%s_%s' % (dataset_prefix, method, str(proto_num)), batch_size=batch_size) csv_logger = CSVLogger('./logs/%s_%s_%s.csv' % (dataset_prefix, method, str(proto_num))) if early_stop: early_stopping = EarlyStopping(monitor='loss', patience=500, mode='auto', verbose=2, restore_best_weights=True) callback_list = [ model_checkpoint, early_stopping, tensorboard, csv_logger ] else: callback_list = [model_checkpoint, tensorboard, csv_logger] if opt == 'SGD': optm = SGD(lr=learning_rate, momentum=0.9, decay=5e-4) elif opt == 'Nadam': optm = Nadam(lr=learning_rate) elif opt == 'Adam_decay': optm = Adam(lr=learning_rate, decay=9. / nb_iterations) else: optm = Adam(lr=learning_rate) model.compile(optimizer=optm, loss='categorical_crossentropy', metrics=['accuracy']) if balance_classes: model.fit([X_train1, X_train2], y_train, batch_size=batch_size, epochs=nb_epochs, callbacks=callback_list, class_weight=class_weight, verbose=2, validation_data=([X_test1, X_test2], y_test)) else: model.fit([X_train1, X_train2], y_train, batch_size=batch_size, epochs=nb_epochs, callbacks=callback_list, verbose=2, validation_data=([X_test1, X_test2], y_test))
def main(run_dir, data_dir, nb_epoch, early_stopping_patience, desired_sample_rate, fragment_length, batch_size, fragment_stride, nb_output_bins, keras_verbose, _log, seed, _config, debug, learn_all_outputs, train_only_in_receptive_field, _run, use_ulaw): if run_dir is None: run_dir = os.path.join( 'models', datetime.datetime.now().strftime('run_%Y-%m-%d_%H:%M:%S')) _config['run_dir'] = run_dir print_config(_run) _log.info('Running with seed %d' % seed) if not debug: if os.path.exists(run_dir): raise EnvironmentError('Run with seed %d already exists' % seed) os.mkdir(run_dir) checkpoint_dir = os.path.join(run_dir, 'checkpoints') json.dump(_config, open(os.path.join(run_dir, 'config.json'), 'w')) _log.info('Loading data...') data_generators, nb_examples = dataset.generators( data_dir, desired_sample_rate, fragment_length, batch_size, fragment_stride, nb_output_bins, learn_all_outputs, use_ulaw) _log.info('Building model...') model = build_model(fragment_length) _log.info(model.summary()) optim = make_optimizer() _log.info('Compiling Model...') loss = objectives.categorical_crossentropy all_metrics = [ metrics.categorical_accuracy, metrics.categorical_mean_squared_error ] if train_only_in_receptive_field: loss = skip_out_of_receptive_field(loss) all_metrics = [skip_out_of_receptive_field(m) for m in all_metrics] model.compile(optimizer=optim, loss=loss, metrics=all_metrics) # TODO: Consider gradient weighting making last outputs more important. callbacks = [ ReduceLROnPlateau(patience=early_stopping_patience / 2, cooldown=early_stopping_patience / 4, verbose=1), EarlyStopping(patience=early_stopping_patience, verbose=1), ] if not debug: callbacks.extend([ ModelCheckpoint(os.path.join( checkpoint_dir, 'checkpoint.{epoch:05d}-{val_loss:.3f}.hdf5'), save_best_only=True), CSVLogger(os.path.join(run_dir, 'history.csv')), ]) if not debug: os.mkdir(checkpoint_dir) _log.info('Starting Training...') model.fit_generator(data_generators['train'], nb_examples['train'], nb_epoch=nb_epoch, validation_data=data_generators['test'], nb_val_samples=nb_examples['test'], callbacks=callbacks, verbose=keras_verbose)
# 1. define the network model = Sequential() model.add(Dense(1024, input_dim=41, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(768, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(1)) model.add(Activation('sigmoid')) # try using different optimizers and different optimizer configs model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint( filepath="kddresults/dnn3layer/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='loss') csv_logger = CSVLogger('kddresults/dnn3layer_csv/training_set_dnnanalysis.csv', separator=',', append=False) model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=100, callbacks=[checkpointer, csv_logger]) model.save("kddresults/dnn3layer/dnn3layer_model.hdf5")
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['acc']) # Fit Network ## Including fooling in naming if args.fooling == 1: fooling = "fooling" else: fooling = "no-fooling" ## Implement callbacks #### CSVLogger date = str(datetime.datetime.now().year) + "-" + str( datetime.datetime.now().month) + "-" + str(datetime.datetime.now().day) csv_name = date + "_{}-{}_{}_ratio-{}_{}".format( args.src_lang, args.tgt_lang, args.optimizer, args.ratio, fooling) csv_path = os.path.join("./training_logs/csv/", csv_name) logger = CSVLogger(filename=csv_path) #### Model Checkpoint cp_path = "./models/monitoring/" + date + "_{}-{}_{}_ratio-{}_{}_cp".format( args.src_lang, args.tgt_lang, args.optimizer, args.ratio, fooling) checkpoint = ModelCheckpoint(filepath=cp_path, monitor="val_loss", save_best_only=True, save_weights_only=False, verbose=1) callbacks_list = [logger, checkpoint] if args.earlystopping: callbacks_list.append( EarlyStopping(patience=args.earlystopping_patience, monitor="val_loss"))
if n_gpu > 1: with tf.device('/cpu:0'): model = Net5(im_size,1) model.load_weights('model_128.h5') parallel_model = multi_gpu_model(model, gpus=n_gpu) else: model = Net5(im_size, 1) model.load_weights('model_128.h5') print(model.summary()) parallel_model = model modelcheck = ModelCheckpoint('model_256_best.h5', monitor='val_loss', save_best_only=True) lr_scheduler = LearningRateScheduler(scheduler256) csv = CSVLogger('mode256.csv') parallel_model.compile(optimizer='sgd',loss=loss, metrics=['acc']) parallel_model.fit_generator( train_generator, steps_per_epoch=nb_train_samples // batch_size, epochs=epochs, validation_data=test_generator, validation_steps=nb_test_samples // batch_size, class_weight= class_weight, callbacks=[modelcheck,lr_scheduler,csv], use_multiprocessing=True, workers=cpus) model.save_weights('model_256.h5')
def training_code(): #titanic-complete3.arff är som det vi fick men: #har attributen normaliserade mellan 0 till 1 #Filter:ordinal to numeric, #cabin och embarked attributet är borttagna #Gör om arff filen till numpy arrays. raw_data = loadarff('titanic-complete3.arff') df_data = pd.DataFrame(raw_data[0]) arr = df_data.to_numpy() #Survival attributet blev en string, fixas här numbers = [] x = 0 for i in range(arr.size//6): for word in arr[x,5].split(): if word.isdigit(): numbers.append(int(word)) x += 1 y=0 for i in range(len(numbers)): arr[y,5]= numbers[y] y+=1 arr = arr.astype("float32") # X_input är de 5 attributen # Y_output är de respektive matchande klasstillhörighetern X_input = arr[:,0:5] Y_output = arr[:,5] num_folds = 10 acc_per_fold = [] loss_per_fold = [] # initialiserar vikterna np.random.seed(42) skf = StratifiedKFold(n_splits=num_folds, shuffle=True) fold_no = 1 visualo = [] visuaacc = [] num_of_epochs = 2000 #genomför cross validation och tränar modellen for train_index, test_index in skf.split(X_input, Y_output): # One hot encoding X_train, X_test = X_input[train_index], X_input[test_index] Y_train, Y_test = np_utils.to_categorical(Y_output[train_index], 2), np_utils.to_categorical(Y_output[test_index], 2) #bygger min mlp model = Sequential() model.add(Dense(4,input_dim = (5), activation="relu")) model.add(Dense(2, activation="sigmoid")) #Adam är en form av SGD model.compile(optimizer="Adam", loss="binary_crossentropy", metrics = ["accuracy"]) print('------------------------------------------------------------------------') print(f'Training for fold {fold_no} ...') csv_logger = CSVLogger("training_mlp.log", append=True, separator=";") history = model.fit(x=X_train, y=Y_train, epochs=num_of_epochs, callbacks=[csv_logger], verbose=0) model.save("Titanic_mlp.h5") scores = model.evaluate(X_test, Y_test, verbose=0) print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%') acc_per_fold.append(scores[1] * 100) loss_per_fold.append(scores[0]) fold_no += 1 visualo.append(history.history["loss"]) visuaacc.append(history.history["accuracy"]) print('------------------------------------------------------------------------') print('Score per fold') for i in range(0, len(acc_per_fold)): print('------------------------------------------------------------------------') print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%') print('------------------------------------------------------------------------') print('Average scores for all folds:') print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})') print(f'> Loss: {np.mean(loss_per_fold)}') print('------------------------------------------------------------------------') #plottar träningen med snittet för varje epok per k_fold herek = np.asarray(visualo, dtype=np.float32) loss_visu = np.sum(herek, axis=0) plt.plot(loss_visu/num_folds, label='Binary Crossentropy (loss)') herek2 = np.asarray(visuaacc, dtype=np.float32) acc_visu2 = np.sum(herek2, axis=0) plt.plot(acc_visu2/num_folds, label='accuracy/100') plt.title('training progress for titanic_mlp') plt.ylabel('value') plt.xlabel('No. epoch') plt.legend(loc="upper left") plt.show()
def init_callbacks(self, n_epochs, n_batches, **kwargs): from keras.callbacks import TerminateOnNaN, EarlyStopping, \ ReduceLROnPlateau, CSVLogger, TensorBoard from validation_checkpoint import ValidationCheckpoint from clr_callback import CyclicLR print('Initializing model callbacks') use_tensorboard = kwargs.pop('use_tensorboard', False) val_monitor = kwargs.pop('monitor', 'val_loss') callbacks = kwargs.pop('callbacks', []) # strides to test/save model during training test_period = kwargs.pop('test_period', 1) save_period = kwargs.pop('save_period', 0) # 0 = disable warmup_epoch = kwargs.pop('warmup_epoch', 3) random_state = kwargs.pop('random_state', 42) verbose = kwargs.pop('verbose', 1) test_ids = kwargs.pop('test_ids', []) step_lr = kwargs.pop('step_lr', None) clr_mult = kwargs.pop('clr_mult', 4) # total epochs before CLR changes signs # exit early if the last [stop_early] test scores are all worse than the best early_stop = int(n_epochs * 0.2) stop_early = kwargs.pop('stop_early', None) or early_stop stop_delta = optparams['stop_delta'] save_preds = kwargs.pop('save_preds', True) save_model = kwargs.pop('save_model', True) model_dir = self.model_dir initial_monitor = self.start_monitor initial_epoch = self.start_epoch # configure callbacks val_mode = 'auto' ctimestr = epoch2str(gettime()) train_logf = pathjoin( model_dir, 'training_log_%s_pid%d.csv' % (ctimestr, self.pid)) # if pathexists(train_logf) and pathsize(train_logf) != 0: # #ctimestr = epoch2str(gettime()) # #ctimestr = epoch2str(pathctime(train_logf)) # ctimestr = '1' # logf_base,logf_ext = splitext(train_logf) # old_logf = logf_base+'_'+ctimestr+logf_ext # print('Backing up existing log file "%s" to "%s"'%(train_logf,old_logf)) # os.rename(train_logf,old_logf) self.val_monitor = val_monitor self.save_preds = save_preds self.save_model = save_model self.save_period = save_period self.test_period = test_period self.stop_early = stop_early self.stop_delta = stop_delta self.test_ids = test_ids self.val_cb = ValidationCheckpoint(val_monitor=val_monitor, save_best_preds=save_preds, save_best_model=save_model, model_dir=model_dir, mode=val_mode, pid=self.pid, initial_monitor=initial_monitor, initial_epoch=initial_epoch, warmup_epoch=warmup_epoch, save_period=save_period, test_period=test_period, test_ids=test_ids, verbose=verbose) #self.val_cb = ModelCheckpoint(model_iterf,monitor=val_monitor,mode=val_mode, period=save_epoch, # save_best_only=True, save_weights_only=False, # verbose=False) step_lr = step_lr or int(n_batches * clr_mult) self.lr_cb = CyclicLR(base_lr=optparams['lr_min'], max_lr=optparams['lr_max'], step_size=step_lr) # else: # step_lr = step_lr or min(100,int(n_epochs*0.01)) # self.lr_cb = ReduceLROnPlateau(monitor=val_monitor, # mode=val_mode, # patience=step_lr, # min_lr=optparams['lr_min'], # factor=optparams['reduce_lr'], # epsilon=optparams['tol'], # verbose=verbose) self.es_cb = EarlyStopping(monitor=val_monitor, mode=val_mode, patience=stop_early, min_delta=stop_delta, verbose=verbose) self.tn_cb = TerminateOnNaN() self.cv_cb = CSVLogger(filename=train_logf, append=True) self.callbacks = callbacks + [ self.val_cb, self.lr_cb, self.es_cb, self.tn_cb, self.cv_cb ] if self.backend == 'tensorflow' and use_tensorboard: tb_batch_size = 32 tb_histogram_freq = 1 tb_embeddings_freq = 0 tb_log_dir = pathjoin(model_dir, 'tb_logs_pid%d' % self.pid) if not pathexists(tb_log_dir): os.makedirs(tb_log_dir) self.tb_cb = TensorBoard(log_dir=tb_log_dir, histogram_freq=tb_histogram_freq, batch_size=tb_batch_size, write_graph=True, write_grads=True, write_images=True, embeddings_freq=tb_embeddings_freq, embeddings_layer_names=None, embeddings_metadata=None) self.callbacks.append(self.tb_cb) elif self.backend != 'tensorflow' and use_tensorboard: print('Cannot use tensorboard with backend "%s"' % self.backend) use_tensorboard = False print('Initialized %d callbacks:' % len(self.callbacks), str(self.callbacks))
y_train = load_images(output_train) y_valid = load_images(output_valid) ch_p = ModelCheckpoint( '/home/polina/1tb/ss_prediction/centroid_90/redone/models_steps/step3/model1/best_weights.h5', save_best_only=True, save_weights_only=True, monitor='val_loss', mode='min') model.compile(loss=comb_loss, optimizer=optimizers.Adagrad(lr=0.03), metrics=[f_mera]) csv_logger = CSVLogger( '/home/polina/1tb/ss_prediction/centroid_90/redone/models_steps/step3/model1/training.log' ) model.fit(x_train, y_train, validation_data=(x_valid, y_valid), batch_size=batch_size, epochs=epochs, verbose=2, shuffle=True, callbacks=[csv_logger, ch_p]) model.save_weights( '/home/polina/1tb/ss_prediction/centroid_90/redone/models_steps/step3/model1/weights.h5' )
sample_factor=sample_factor, batch_size=batch_size, dims=dims, levels=train_levels) sampleset_size_validn = math.ceil( len(test_true_list) / sample_factor) + len(test_true_list) steps_per_epoch_validn = math.ceil(sampleset_size_validn / batch_size) checkpointer = ModelCheckpoint( checkpoint_dir + date + '_weights_' + saves_name + '_{epoch:02d}--{categorical_accuracy:.4f}--{val_loss:.4f}.hdf5', monitor='categorical_accuracy', save_weights_only=True, save_best_only=True) csvlogger = CSVLogger(log_dir + 'fit.log', append=True) if load_weights != None: model.load_weights(load_weights) print('Loaded weights from {}'.format(load_weights)) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['categorical_accuracy']) history = model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=epochs, verbose=1, callbacks=[checkpointer, csvlogger], validation_data=validn_generator, validation_steps=steps_per_epoch_validn,
model.add(Dense(num_classes)) model.add(Activation('softmax')) model.summary() # Compile the model model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # Callbacks callbacks = [] modelcheckpoint = ModelCheckpoint('dnn/weights.{epoch:03d}-{val_acc:.5f}.h5', monitor='val_acc', save_best_only=True) callbacks.append(modelcheckpoint) csv_logger = CSVLogger('dnn_log.csv', separator=',', append=False) callbacks.append(csv_logger) # Fit the model if isValid: model.fit_generator(train_gen.flow(X_train, Y_train, batch_size=batch_size), steps_per_epoch=10*X_train.shape[0]//batch_size, epochs=epochs, callbacks=callbacks, validation_data=(X_valid, Y_valid)) else: model.fit_generator(train_gen.flow(X_train, Y_train, batch_size=batch_size), steps_per_epoch=10*X_train.shape[0]//batch_size, epochs=epochs, callbacks=callbacks)
metrics=['accuracy']) model.summary() patience = 50 filepath = './ensemble/Model.{epoch:02d}-{val_acc:.4f}.hdf5' log_file_path = './ensemble/training.log' model_checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='auto') early_stop = EarlyStopping('val_acc', patience=patience) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(patience / 4), verbose=1) csv_logger = CSVLogger(log_file_path, append=False) datagen = ImageDataGenerator(featurewise_center=False, featurewise_std_normalization=False, rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, zoom_range=.1, horizontal_flip=True, data_format='channels_last') datagen.fit(Train_x) model.fit_generator( datagen.flow(Train_x, Train_y, batch_size=batch_size), samples_per_epoch=Train_x.shape[0],
mms=mms) ctc_x, ctc_y = next(train_batch) for name, value in ctc_x.items(): print(name, value.shape) for name, value in ctc_y.items(): print(name, value.shape) checkpointer = ModelCheckpoint(model_path, verbose=1, save_best_only=False, save_weights_only=True, period=1) csv_to_log = CSVLogger(join(path, "logger_0627.csv")) lr_change = ReduceLROnPlateau(monitor="loss", factor=0.5, patience=1, min_lr=0.000, epsilon=0.1, verbose=1) tfboard = TensorBoard(log_dir=log_path) if os.path.exists(log_path): shutil.rmtree(log_path) callback_list = [checkpointer, tfboard, lr_change] history = model.fit_generator(train_batch, steps_per_epoch=40, epochs=100,
metrics_callback = keras_utils.ReadmissionMetrics( train_data=train_raw, val_data=val_raw, target_repl=(args.target_repl_coef > 0), batch_size=args.batch_size, verbose=args.verbose) # make sure save directory exists dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) saver = ModelCheckpoint(path, verbose=1, period=args.save_every) if not os.path.exists('keras_logs'): os.makedirs('keras_logs') csv_logger = CSVLogger(os.path.join('keras_logs', model.final_name + '.csv'), append=True, separator=';') print("==> training") model.fit(x=train_raw[0], y=train_raw[1], validation_data=val_raw, nb_epoch=n_trained_chunks + args.epochs, initial_epoch=n_trained_chunks, callbacks=[metrics_callback, saver, csv_logger], shuffle=True, verbose=args.verbose, batch_size=args.batch_size) elif args.mode == 'test':
#classWeight = compute_class_weight(TRAINDATA) trainGenerator = myGenerator(TRAINDATA, batchSize=BATCHSIZE, n_classes=N_CLASS) testGenerator = myGenerator(TESTDATA, shuffle=False, batchSize=BATCHSIZE, n_classes=N_CLASS) #tensorBoard = TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=BATCHSIZE, write_graph=True, # write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, # embeddings_metadata=None) filepath = '../model/weigh-' + str( BATCHSIZE) + '-resnet-noweighted-fc-{epoch:02d}-{val_acc:.2f}.hdf5' checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max') csv_logger = CSVLogger('resnet_' + str(BATCHSIZE) + '_noweighted_fc.log') classifier.fit_generator(trainGenerator, samples_per_epoch=int(np.floor(TRAINSIZE / BATCHSIZE)), epochs=EPOCHS, verbose=VERBOSE, class_weight=None, validation_data=testGenerator, validation_steps=int(np.floor(TESTSIZE / BATCHSIZE)), workers=1, callbacks=[csv_logger, checkpoint])
model = Sequential() model.add(Flatten()) model.add(Dense(256)) model.add(Activation('relu')) model.add(Dense(256)) model.add(Dropout(0.50)) model.add(Activation('relu')) model.add(Dense(hasy_tools.n_classes, activation='softmax')) # Compile model model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # Fit the model csv_logger = CSVLogger('log.csv', append=True, separator=';') checkpointer = ModelCheckpoint(filepath='checkpoint.h5', verbose=1, period=10, save_best_only=True) model.fit(data['x_train'], data['y_train'], validation_data=(data['x_val'], data['y_val']), epochs=500, batch_size=128, callbacks=[csv_logger, checkpointer]) # Serialize model model.save('model.h5') # evaluate the model
def tune_model(): # Build the Inception V3 network. base_model = inception_v3.InceptionV3(include_top=False, weights='imagenet', pooling='avg') print('Model loaded.') # build a classifier model to put on top of the convolutional model top_input = Input(shape=base_model.output_shape[1:]) top_output = Dense(5, activation='softmax')(top_input) top_model = Model(top_input, top_output) # Note that it is necessary to start with a fully-trained classifier, # including the top classifier, in order to successfully do fine-tuning. top_model.load_weights(top_model_weights_path) # add the model on top of the convolutional base model = Model(inputs=base_model.inputs, outputs=top_model(base_model.outputs)) # Set all layers up to 'mixed8' to non-trainable (weights will not be updated) last_train_layer = model.get_layer(name='mixed8') for layer in model.layers[:model.layers.index(last_train_layer)]: layer.trainable = False # Compile the model with a SGD/momentum optimizer and a very slow learning rate. model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1e-4, momentum=0.9), metrics=['accuracy']) # Prepare data augmentation configuration train_datagen = ImageDataGenerator( preprocessing_function=inception_v3.preprocess_input, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) test_datagen = ImageDataGenerator( preprocessing_function=inception_v3.preprocess_input) train_generator = train_datagen.flow_from_directory( train_data_dir, target_size=(img_height, img_width), batch_size=batch_size, class_mode='categorical') validation_generator = test_datagen.flow_from_directory( validation_data_dir, target_size=(img_height, img_width), batch_size=batch_size, class_mode='categorical') loss = model.evaluate_generator(validation_generator, nb_validation_samples // batch_size) print('Model validation performance before fine-tuning:', loss) csv_logger = CSVLogger(output_dir + 'model_tuning.csv') # fine-tune the model model.fit_generator(train_generator, steps_per_epoch=nb_train_samples // batch_size, epochs=tune_epochs, validation_data=validation_generator, validation_steps=nb_validation_samples // batch_size, workers=4, callbacks=[csv_logger]) model.save_weights(tuned_weights_path)
x, y = load_mnist() elif args.dataset == 'usps': x, y = load_usps('data/usps') # define the model model = CAE(input_shape=x.shape[1:], filters=[32, 64, 128, 10]) plot_model(model, to_file=args.save_dir + '/%s-pretrain-model.png' % args.dataset, show_shapes=True) model.summary() # compile the model and callbacks optimizer = 'adam' model.compile(optimizer=optimizer, loss='mse') from keras.callbacks import CSVLogger csv_logger = CSVLogger(args.save_dir + '/%s-pretrain-log.csv' % args.dataset) # begin training t0 = time() model.fit(x, x, batch_size=args.batch_size, epochs=args.epochs, callbacks=[csv_logger]) print('Training time: ', time() - t0) model.save(args.save_dir + '/%s-pretrain-model-%d.h5' % (args.dataset, args.epochs)) # extract features feature_model = Model(inputs=model.input, outputs=model.get_layer(name='embedding').output)
def main(encoder_name, resnet_layers, top_filters, reduction_layers, neurons, dropout, name): mapping = {'DIPG': 0, 'MB': 1, 'EP': 2} train = None with open('train.txt', 'r') as fp: train = [x.strip() for x in fp.readlines()] test = None with open('test.txt', 'r') as fp: test = [x.strip() for x in fp.readlines()] key = name train_generator = DataGenerator(train, mapping, 17, 3, True, True) test_generator = DataGenerator(test, mapping, len(test), 3, True, True) K.clear_session() model = None if os.path.exists(os.path.join('models', '{}.h5'.format(key))): model = load_model(os.path.join('models', '{}.h5'.format(key))) print('Loading model from history...') else: model = classifier(encoder_name, resnet_layers, top_filters, reduction_layers, neurons, dropout, name) model.compile('adadelta', 'categorical_crossentropy', [categorical_accuracy]) print('Created new model...') initial_epoch = None e = None if os.path.exists('clf-epochs.json'): e = json.load(open('clf-epochs.json')) if key in e: initial_epoch = e[key]['epochs'] else: e[key] = {'trained': 'n'} initial_epoch = -1 else: e = {key: {'trained': 'n'}} initial_epoch = -1 if not os.path.exists(os.path.join('logs', '{}_log'.format(key))): os.mkdir(os.path.join('logs', '{}_log'.format(key))) if e[key]['trained'] == 'n': epoch_cbk = EpochCallback(key) checkpoint = ModelCheckpoint(filepath=os.path.join('models', '{}.h5'.format(key)), monitor='val_loss', verbose=1, save_best_only=True, mode='min') earlystop = EarlyStopping(monitor='val_loss', patience=10, mode='min') tensorboard = TensorBoard(log_dir=os.path.join('logs', '{}_log'.format(key)), batch_size=13, write_graph=False, write_grads=False, write_images=True) csvlogger = CSVLogger(filename=os.path.join('logs', '{}.csv'.format(key))) history = model.fit_generator(generator=train_generator, epochs=100, verbose=1, callbacks=[epoch_cbk, csvlogger, earlystop, checkpoint, tensorboard], validation_data=test_generator, use_multiprocessing=True, workers=4, initial_epoch=initial_epoch + 1) e = json.load(open('clf-epochs.json')) e[key]['trained'] = 'y' json.dump(e, open('clf-epochs.json', 'w')) eval_results = model.evaluate_generator(generator=test_generator) d = {} if os.path.exists('clf-scores.json'): d = json.load(open('clf-scores.json')) d[key] = eval_results[1] json.dump(d, open('clf-scores.json', 'w'))
def train(data_type, seq_length, model, saved_model=None, class_limit=None, image_shape=None, load_to_memory=False, batch_size=32, nb_epoch=100): # Helper: Save the model. checkpointer = ModelCheckpoint( filepath=os.path.join('data', 'checkpoints', model + '-' + data_type + \ '.{epoch:03d}-{val_loss:.3f}.hdf5'), verbose=1, save_best_only=True) # Helper: TensorBoard tb = TensorBoard(log_dir=os.path.join('data', 'logs', model)) # Helper: Stop when we stop learning. early_stopper = EarlyStopping(patience=10) # Helper: Save results. timestamp = time.time() csv_logger = CSVLogger(os.path.join('data', 'logs', model + '-' + 'training-' + \ str(timestamp) + '.log')) # Get the data and process it. if image_shape is None: data = DataSet(seq_length=seq_length, class_limit=class_limit) else: data = DataSet(seq_length=seq_length, class_limit=class_limit, image_shape=image_shape) # Get samples per epoch. # Multiply by 0.7 to attempt to guess how much of data.data is the train set. steps_per_epoch = (len(data.data) * 0.7) // batch_size if load_to_memory: # Get data. X, y = data.get_all_sequences_in_memory('train', data_type) X_test, y_test = data.get_all_sequences_in_memory('test', data_type) else: # Get generators. generator = data.frame_generator(batch_size, 'train', data_type) val_generator = data.frame_generator(batch_size, 'test', data_type) # Get the model. rm = ResearchModels(len(data.classes), model, seq_length, saved_model) # Fit! if load_to_memory: # Use standard fit. rm.model.fit(X, y, batch_size=batch_size, validation_data=(X_test, y_test), verbose=1, callbacks=[tb, early_stopper, csv_logger, checkpointer], epochs=nb_epoch) else: # Use fit generator. rm.model.fit_generator( generator=generator, steps_per_epoch=steps_per_epoch, epochs=nb_epoch, verbose=1, callbacks=[tb, early_stopper, csv_logger, checkpointer], validation_data=val_generator, validation_steps=50, workers=4)
model.compile(Adam(lr=learn), loss='binary_crossentropy', metrics=[dice_coef,'accuracy']) return model K.clear_session() model = ResNet50() model.summary() from keras.callbacks import ModelCheckpoint, TensorBoard, CSVLogger mc = ModelCheckpoint('model_loss', monitor='val_loss', save_best_only=True) #tb = TensorBoard(write_images=True) cv = CSVLogger('/content/drive/My Drive/Colab/results/latest/smriti_resnet_test1_l3.csv',append=True) from google.colab import drive drive.mount('/content/drive') os.listdir('/content/drive/My Drive/Colab/results/best/') epochs =160 bestdc=0 total=0 for i in range(epochs): print(i) history= model.fit_generator(datagen.flow(train_X, train_y, batch_size=4), nb_epoch=1, samples_per_epoch=len(train_X), callbacks=[cv,mc]) #history= model.fit(train_X, train_y, batch_size=4, nb_epoch=1, validation_data=(valid_X,valid_y), callbacks=[cv,mc]) total=0 for j in range(8):
def train(self, epochs, learning_rate, batch_size, wmap, vbal, model_name, new_ex): if ".hdf5" in model_name: model_name = model_name.split(".hdf5")[0] else: pass print("Loading data") if wmap == False: imgs_train, imgs_mask_train = self.load_data(wmap=wmap, vbal=vbal) else: imgs_train, imgs_mask_train, img_weights = self.load_data( wmap=wmap, vbal=vbal) print("Loading data done") model = self.get_mitosegnet(wmap, learning_rate) print("Got MitoSegNet") print(self.path + os.sep + model_name) if os.path.isfile(self.path + os.sep + model_name + ".hdf5"): model.load_weights(self.path + os.sep + model_name + ".hdf5") print("Loading weights") else: print( "No previously optimized weights were loaded. Proceeding without" ) # Set network weights saving mode. # save previously established network weights (saving model after every epoch) print('Fitting model...') if new_ex == "New": first_ep = 0 model_name = model_name + "_" + str(self.img_rows) + "_" elif new_ex == "Finetuned_New": first_ep = 0 else: if os.path.isfile(self.path + os.sep + model_name + 'training_log.csv'): prev_csv = True prev_csv_file = pd.read_csv(self.path + os.sep + model_name + 'training_log.csv') first_ep = len(prev_csv_file) if prev_csv_file.shape[1] > 7: prev_csv_file = prev_csv_file.drop( prev_csv_file.columns[[0]], axis=1) else: prev_csv = False csv_logger = CSVLogger(self.path + os.sep + model_name + 'training_log.csv') tensorboard = TensorBoard(log_dir=self.path + os.sep + "logs/{}".format(time())) # Set callback functions to early stop training and save the best model so far callbacks = [ EarlyStopping(monitor='val_loss', patience=20), ModelCheckpoint(filepath=self.path + os.sep + model_name + ".hdf5", monitor='val_loss', verbose=1, save_best_only=True), csv_logger, tensorboard ] if wmap == True: x = [imgs_train, img_weights] else: x = imgs_train print( "\nCopy the line below into the terminal, press enter and click on the link to evaluate the training " "performance:\n\ntensorboard --logdir=" + self.path + os.sep + "logs/\n") ### ECG see if this solves # https://github.com/tensorflow/tensorflow/issues/34944 tf.config.experimental_run_functions_eagerly(True) ### model.fit(x=x, y=imgs_mask_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_split=0.2, shuffle=True, callbacks=callbacks) csv_file = pd.read_csv(self.path + os.sep + model_name + 'training_log.csv') if new_ex == "New" or new_ex == "Finetuned_New": csv_file["epoch"] = list(range(1, len(csv_file) + 1)) last_ep = len(csv_file) if new_ex == "Existing" and prev_csv == True: frames = [prev_csv_file, csv_file] merged = pd.concat(frames, names=[]) merged["epoch"] = list(range(1, len(merged) + 1)) last_ep = len(merged) merged.to_csv(self.path + os.sep + model_name + 'training_log.csv') if new_ex == "New": info_file = open( self.path + os.sep + model_name + str(first_ep) + "-" + str(last_ep) + "_train_info.txt", "w") info_file.write("Learning rate: " + str(learning_rate) + "\nBatch size: " + str(batch_size) + "\nClass balance weight factor: " + str(vbal)) info_file.close() K.clear_session()
model.summary() epochs = 1000 batch_size = 16 checkpointer = ModelCheckpoint( filepath=os.path.join('//data/d14122793/human_protein_atlas_image_classification', 'checkpoints', 'Training-' + \ '.{epoch:03d}-{val_loss:.3f}.hdf5'), verbose=1, save_best_only=True) early_stopper = EarlyStopping(patience=50) # Helper: Save results. timestamp = time.time() csv_logger = CSVLogger( os.path.join('.', 'logs', 'training-' + str(timestamp) + '.log')) # split and suffle data np.random.seed(2018) indexes = np.arange(train_dataset_info.shape[0]) np.random.shuffle(indexes) train_indexes = indexes[:27500] valid_indexes = indexes[27501:] # create train and valid datagens train_generator = data_gen.create_train(train_dataset_info[train_indexes], batch_size, (299, 299, 3)) validation_generator = data_gen.create_train(train_dataset_info[valid_indexes], 100, (299, 299, 3), augument=False)
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() DATASET = args.dataset MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker paths, pose_label = load_data(DATASET) n_fold = 1 losses = {"pose_prediction": custom_mse_pose} metrics = {"pose_prediction": pose_metric} print('[K-FOLD] Started...') kf = KFold(n_splits=10, shuffle=True, random_state=1) kf_split = kf.split(pose_label) for train_idx, test_idx in kf_split: model = None if MODEL == 'vggFace': model = Net(MODEL,1,5,8,2,8) model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL,1,5,8,2,8) train_paths = paths[train_idx] train_label = pose_label[train_idx] test_paths = paths[test_idx] test_label = pose_label[test_idx] print(len(train_paths),len(test_paths)) model.summary() weights_path = './train_weights/pose/{}/{}/'.format(DATASET,model.name) logs_path = './train_log/pose/{}/{}/'.format(DATASET,model.name) if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) model_names = weights_path + '.{epoch:02d}-{val_pose_metric: 0.4f}.hdf5' csv_name = logs_path + '{}.log'.format(n_fold) board_name = logs_path + '{}'.format(n_fold) checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True) csvlogger=CSVLogger(csv_name) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE/2), verbose=1) tensorboard = TensorBoard(log_dir=board_name,batch_size=BATCH_SIZE) callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard] if MODEL == 'ssrnet': callbacks = [ # ModelCheckpoint( # "train_weight/{}-{val_gender_prediction_binary_accuracy:.4f}-{val_age_prediction_mean_absolute_error:.4f}.h5".format( # MODEL), # verbose=1, save_best_only=True, save_weights_only=True), ModelCheckpoint(MODEL, 'val_loss', verbose=1,save_best_only=True), CSVLogger('train_log/{}-{}.log'.format(MODEL, n_fold)), DecayLearningRate([30, 60])] model.compile(optimizer='adam', loss=losses, metrics = metrics) model.fit_generator( DataGenerator(model, train_paths, train_label, BATCH_SIZE), validation_data=DataGenerator(model, test_paths, test_label, BATCH_SIZE), epochs=EPOCH, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks ) n_fold += 1 del train_paths, train_roll,train_pictch,train_yaw del test_paths, test_roll,test_pitch,test_yaw
x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 print('x_train shape:', x_train.shape) print('y_train shape:', y_train.shape) print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') print('-' * 30) print('Creating and compiling model...') print('-' * 30) model = getinceptionv3() csv_logger = CSVLogger('log-inceptionv3.csv') model_checkpoint = ModelCheckpoint('weights-inceptionv3.h5', monitor='acc', save_best_only=True) gen = keras.preprocessing.image.ImageDataGenerator(rotation_range=15., width_shift_range=0.15, height_shift_range=0.15, shear_range=0.4, zoom_range=0.4, channel_shift_range=0.5, horizontal_flip=True, vertical_flip=False) batch_size = 32 train_steps = int(x_train.shape[0] / batch_size) + 1
loss = 'binary_crossentropy' embedding_dim = 768 claims_input, pred_label = training.lstm_model(max_seq_length, embedding_dim, nb_classes) model = Model([claims_input], pred_label) print(model.summary()) print("labels ndim ", labels.shape) print(claims_sents_vec.shape) early_stopping = EarlyStopping(monitor='val_loss', patience=2) model.compile(optimizer=optimizers.Adam(), loss=loss, metrics=['accuracy']) csv_logger = CSVLogger("bert_training.log") model_path = 'models_fever_full/claim_classifier_models/model_bert_fever_full_binary_claim_classifierAcc74.h5' history = model.fit({'claims': claims_sents_vec}, labels, epochs=60, batch_size=64, validation_split=0.12, callbacks=[ early_stopping, metrics, csv_logger, ModelCheckpoint(filepath=model_path, monitor='val_loss', save_best_only=True) ]) dataset_name = test_dataset_name test_lstm_model = testModel(dataset_name, model_path, nb_classes) test_lstm_model.get_results_on_test_data()
test_labels = np_utils.to_categorical(test_labels, nb_classes=5) model = conv_model(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) if validation_switch: val_cot = 1 kf = KFold(train_data.shape[0], n_folds) for train_index, val_index in kf: X_train, X_val = train_data[train_index], train_data[val_index] y_train, y_val = train_labels[train_index], train_labels[ val_index] csv_logger = CSVLogger('epoch' + str(nb_epoch) + '_' + str(cot) + '_val_' + str(val_cot) + '.csv') val_cot += 1 model.fit(X_train, y_train, nb_epoch=nb_epoch, batch_size=16, verbose=1, validation_data=(X_val, y_val), callbacks=[csv_logger]) csv_logger = CSVLogger('epoch' + str(nb_epoch) + '_' + str(cot) + '.csv') model.fit(train_data, train_labels,
def __init__(self, *args, **kws): KerasCSVLogger.__init__(self, *args, **kws) if self.filename.startswith("gs://"): self.on_train_begin = self._gcp_on_train_begin
y_train = to_categorical(y_train, classes) y_test = to_categorical(y_test, classes) #building model nn5 = Sequential() nn5.add(Dense(1000, activation='relu', input_shape=(784,))) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(1000, activation ='relu')) nn5.add(Dense(classes, activation='softmax')) nn5.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) print(nn5.summary()) csv_logger = CSVLogger('training2.log', separator=',', append=False) nn5_history = nn5.fit(X_train, y_train, epochs = 50, verbose=1, validation_data=(X_test, y_test), callbacks=[csv_logger]) nn5_score = nn5.evaluate(X_test, y_test) print('Test score:', nn5_score[0]) print('Test accuracy:', nn5_score[1]) plot_model_history(nn5_history)