os.environ["CUDA_VISIBLE_DEVICES"] = params.gpu import tensorflow as tf config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = params.mem set_session(tf.Session(config=config)) model = ResnetBuilder.build_resnet_cifar10(params.ly, use_bn=params.bn) sgd = SGD(momentum=0.9) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) dir_name = "e" + datetime.now().strftime("%m%d-%H-%M-%S") + \ "l" + str(params.ly) + \ "_ing" csv_logger = CSVLogger(os.path.join(dir_name, 'log.csv')) lr_scheduler = LearningRateScheduler(lambda x: 0.05 if x < 60 else 0.03 if x < 80 else 0.005) board = TensorBoard(log_dir=dir_name, histogram_freq=0, write_graph=False, write_images=False) checker = ModelCheckpoint(filepath=os.path.join(dir_name, "weights.{epoch:02d}-{val_loss:.2f}.hdf5"), period=20) if not os.path.exists(dir_name): os.mkdir(dir_name) with open(os.path.join(dir_name, 'config'), 'w') as wf: wf.write("-----" + str(params) + dir_name) callbacks = [lr_scheduler, csv_logger, board, checker] datagen_train = ImageDataGenerator( width_shift_range=params.shift, height_shift_range=params.shift, ) datagen_test = ImageDataGenerator() model.fit_generator(datagen_train.flow(x_train, y_train, batch_size=params.bs), steps_per_epoch=x_train.shape[0] / params.bs, validation_data=datagen_test.flow(x_test, y_test, batch_size=params.bs), validation_steps=x_test.shape[0] / params.bs,
def train_model_from_images(network, model_train_params, train_data_dir, validation_data_dir, verbose=0, tb_logs=False, csv_log=False, early_stopping=False, save_to_s3=False): """ Train and validate the Convolutional Neural Network (CNN) model by reading data from image folders. """ # dimensions of our images. img_width, img_height = network.input_dim[1:] loss = model_train_params['loss'] optimizer = model_train_params['optimizer'] metrics = model_train_params['metrics'] batch_size = model_train_params['batch_size'] nb_epochs = model_train_params['nb_epochs'] nb_train_samples = model_train_params['nb_train_samples'] nb_validation_samples = model_train_params['nb_validation_samples'] train_datagen = ImageDataGenerator(rescale=1.0/255) train_generator = \ train_datagen.flow_from_directory(train_data_dir, target_size=(img_width, img_height), batch_size=100, classes=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], class_mode='categorical', seed=131) total_samples = 30000 image_set = np.empty([0, 3, img_width, img_height]) label_set = np.empty([0, 10]) for i in range(total_samples/batch_size): (X, y) = train_generator.next() image_set = np.concatenate((image_set, X), axis=0) label_set = np.concatenate((label_set, y), axis=0) # this is the augmentation configuration we will use for training train_datagen = ImageDataGenerator(rescale=1.0/255, samplewise_center=False, featurewise_center=True) train_datagen.fit(image_set) # this is the augmentation configuration we will use for testing: # only rescaling val_datagen = ImageDataGenerator(rescale=1.0/255, samplewise_center=False, featurewise_center=True) val_datagen.fit(image_set) train_generator = \ train_datagen.flow_from_directory(train_data_dir, target_size=(img_width, img_height), batch_size=batch_size, classes=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], class_mode='categorical') validation_generator = \ val_datagen.flow_from_directory(validation_data_dir, target_size=(img_width, img_height), batch_size=batch_size, classes=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], class_mode='categorical') if optimizer == 'sgd': optimizer = SGD(lr=model_train_params['lr'], momentum=model_train_params['momentum'], decay=model_train_params['decay'], nesterov=model_train_params['nesterov']) elif optimizer == 'adam': optimizer = Adam(lr=model_train_params['lr']) network.model.compile(loss=loss, metrics=metrics, optimizer=optimizer) callbacks = [] if tb_logs: tb_callback = TensorBoard(log_dir='./logs', histogram_freq=1, write_graph=True) callbacks = [tb_callback] if early_stopping: early_stopping = EarlyStopping(monitor='val_acc', patience=2, min_delta=0.005, verbose=0, mode='auto') callbacks.append(early_stopping) if csv_log: csv_filename = (network.name + '_lr_' + str(model_train_params['lr']) + '_l2weightdecay_' + str(model_train_params['decay']) + '.csv') csv_logger_callback = CSVLogger(filename=csv_filename, append=True) callbacks.append(csv_logger_callback) if save_to_s3: s3 = boto3.resource('s3') else: s3 = None def scheduler(epoch): init_lr = model_train_params['lr'] return init_lr * (0.9**epoch) change_lr = LearningRateScheduler(scheduler) callbacks.append(change_lr) # model_checkpoint_cb = ModelCheckpoint2S3(filepath=network.name + '_' + time.strftime("%x") + '.h5', # monitor='val_loss', # save_best_only=True, # verbose=0, # s3resource=s3, # s3filename=network.name + '_' + time.strftime("%x") + '.h5') model_checkpoint_cb = ModelCheckpoint(filepath=network.name + '.{epoch:02d}-{val_loss:.2f}.hdf5', monitor='val_loss') callbacks.append(model_checkpoint_cb) loss_history = LossHistory(filepath=network.name + '_lr_' + str(model_train_params['lr']) + '_' + time.strftime("%x").replace("/", "_") + "_trainloss.csv", s3resource=s3, s3filename=network.name + '_lr_' + str(model_train_params['lr']) + '_' + time.strftime("%x").replace("/", "_") + "_trainloss.csv") callbacks.append(loss_history) history = network.model.fit_generator(train_generator, samples_per_epoch=nb_train_samples, nb_epoch=nb_epochs, validation_data=validation_generator, nb_val_samples=nb_validation_samples, callbacks=callbacks, verbose=1) return history
#data augmentation datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, ) datagen.fit(x_train) #training batch_size = 64 opt_rms = keras.optimizers.rmsprop(lr=0.001, decay=1e-6) model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy']) model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\ steps_per_epoch=x_train.shape[0] // batch_size,epochs=125,\ verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)]) #save to disk model_json = model.to_json() with open('model.json', 'w') as json_file: json_file.write(model_json) model.save_weights('model.h5') #testing scores = model.evaluate(x_test, y_test, batch_size=128, verbose=1) print('\nTest result: %.3f loss: %.3f' % (scores[1] * 100, scores[0]))
time_loss_weights[0] = 0 prednet = PredNet(stack_sizes, R_stack_sizes, A_filt_sizes, Ahat_filt_sizes, R_filt_sizes, output_mode='error', return_sequences=True) inputs = Input(shape=(nt,) + input_shape) errors = prednet(inputs) # errors will be (batch_size, nt, nb_layers) errors_by_time = TimeDistributed(Dense(1, trainable=False), weights=[layer_loss_weights, np.zeros(1)], trainable=False)(errors) # calculate weighted error by layer errors_by_time = Flatten()(errors_by_time) # will be (batch_size, nt) final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) # weight errors by time model = Model(inputs=inputs, outputs=final_errors) model.compile(loss='mean_absolute_error', optimizer='adam') train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True) val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val) lr_schedule = lambda epoch: 0.001 if epoch < 75 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs callbacks = [LearningRateScheduler(lr_schedule)] if save_model: if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) callbacks.append(ModelCheckpoint(filepath=weights_file, monitor='val_loss', save_best_only=True)) history = model.fit_generator(train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, validation_data=val_generator, validation_steps=N_seq_val / batch_size) #history=model.fit(train, if save_model: json_string = model.to_json() with open(json_file, "w") as f: f.write(json_string)
def train(): KITTI_train_gen = KITTILoader(subset='training') dim_avg, dim_cnt = KITTI_train_gen.get_average_dimension() #### #dim_avg = {"Car": np.array([1.64473343, 1.68501193, 4.19985967])} new_data = orientation_confidence_flip(KITTI_train_gen.image_data, dim_avg) model = nn.network() #model.load_weights('model00000006.hdf5') early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=10, mode='min', verbose=1) checkpoint = callbacks.ModelCheckpoint('model{epoch:08d}.hdf5', monitor='val_loss', verbose=1, save_best_only=False, mode='min', period=1) tensorboard = callbacks.TensorBoard(log_dir='logs/', histogram_freq=0, write_graph=True, write_images=False) all_examples = len(new_data) trv_split = int(cfg().split * all_examples) # train val split train_gen = data_gen(new_data[:trv_split]) valid_gen = data_gen(new_data[trv_split:all_examples]) print("READY FOR TRAINING") train_num = int(np.ceil(trv_split / cfg().batch_size)) valid_num = int(np.ceil((all_examples - trv_split) / cfg().batch_size)) #gen_flow = gen_flow_for_two_inputs(X_train, X_angle_train, y_train) # choose the minimizer to be sgd # minimizer = optimizer.SGD(lr=0.0001, momentum = 0.9) minimizer = Adam(lr=0.0001) #minimizer = SGD(lr=0.0001) # multi task learning model.compile( optimizer=minimizer, #minimizer, loss={ 'dimensions': 'mean_squared_error', 'orientation': orientation_loss, 'confidence': 'categorical_crossentropy' }, loss_weights={ 'dimensions': 2., 'orientation': 1., 'confidence': 4. }) print("####################################################") print(K.get_value(model.optimizer.lr)) # Tambahan aing def scheduler(epoch): if epoch % 50 == 0 and epoch != 0: lr = K.get_value(model.optimizer.lr) K.set_value(model.optimizer.lr, lr * .8) print("lr changed to {}".format(lr * .8)) print("lr = ", K.get_value(model.optimizer.lr)) return K.get_value(model.optimizer.lr) lr_sched = LearningRateScheduler(scheduler) #print(asdadsad) # d:0.0088 o:0.0042, c:0.0098 # steps_per_epoch=train_num, # validation_steps=valid_num, # callbacks=[early_stop, checkpoint, tensorboard], model.fit_generator(generator=train_gen, steps_per_epoch=train_num, epochs=500, verbose=1, validation_data=valid_gen, validation_steps=valid_num, shuffle=True, callbacks=[checkpoint, tensorboard, lr_sched], max_queue_size=3)
batch_size = 5 epochs = 40 model_type = "large_full" #### Train #### # Callbacks checkpoint = ModelCheckpoint('output/icnet_' + model_type + '_{epoch:03d}_{categorical_accuracy:.3f}.h5', mode='max') tensorboard = TensorBoard( batch_size=batch_size, log_dir="./logs/ICNet/" + model_type + "/{}/".format(strftime("%Y-%m-%d-%H-%M-%S", gmtime()))) lr_decay = LearningRateScheduler(PolyDecay(0.01, 0.9, epochs).scheduler) # Generators train_generator = utils.generator(df=utils.load_train_data(), batch_size=batch_size, resize_shape=(configs.img_width, configs.img_height), crop_shape=(configs.img_width, configs.img_height), n_classes=34, training=True) val_generator = utils.generator(df=utils.load_val_data(configs.val_label_path), batch_size=1, resize_shape=(configs.img_width, configs.img_height),
#opt = keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=decay_rate) opt = keras.optimizers.Adam(lr=0.3, beta_1=0.3, beta_2=0.999, epsilon=1e-08) model.compile(optimizer=opt,loss='categorical_crossentropy',metrics=['accuracy',f1]) # hyperparameter tuning filepath=" Result binary weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') #early = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=10) reduce1 = keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', verbose=1,factor=0.33, patience=2,min_lr=0.000001 ) lrate = LearningRateScheduler(step_decay, verbose=1) #callbacks_list = [checkpoint,early,reduce1] #callbacks_list = [checkpoint,reduce1,lrate] callbacks_list = [checkpoint,reduce1] #callbacks_list = [checkpoint,lrate] # data augmentation train_datagen = ImageDataGenerator(
def train_(): X, y = read_train_data() train_img, test_img, train_mask, test_mask = train_test_split( X, y, test_size=0.2, random_state=1) train_img, val_img, train_mask, val_mask = train_test_split(train_img, train_mask, test_size=0.2, random_state=1) print(train_img.shape, train_mask.shape) print('-' * 30) print('UNET FOR MASK SEGMENTATION.') print('-' * 30) model = get_unet(IMG_WIDTH=256, IMG_HEIGHT=256, IMG_CHANNELS=1) model_checkpoint = ModelCheckpoint(model_name + ".hdf5", monitor='loss', save_best_only=False) model.summary() model_json = model.to_json() with open("{}.json".format(model_name), "w") as json_file: json_file.write(model_json) print('...Fitting model...') print('-' * 30) change_lr = LearningRateScheduler(step_decay) tensorboard = TensorBoard(log_dir="logs/{}".format(model_name)) datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 30, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range= 0.3, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.3, # randomly shift images vertically (fraction of total height) zoom_range=0.3, shear_range=0., horizontal_flip=True, # randomly flip images vertical_flip=True, # randomly flip images fill_mode='constant', dim_ordering='tf') model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef]) # Fit the model on the batches generated by datagen.flow(). model.fit_generator( datagen.flow(train_img, train_mask, batch_size=8), validation_data=(val_img, val_mask), samples_per_epoch=train_img.shape[0], nb_epoch=10, callbacks=[model_checkpoint, change_lr, tensorboard], ) score = model.evaluate(test_img, test_mask, batch_size=16) model.save_weights('seg_weight.h5')
# f.write(s) model_check = ModelCheckpoint(model_name, save_best_only=True, verbose=1) def get_lr(epoch): if epoch < 50: return 0.8e-3 elif epoch < 100: return 5e-4 elif epoch < 300: return 1e-4 else: return 5e-5 lr = LearningRateScheduler(get_lr) rms = RMSprop(lr=1e-5) print ('compiling...') model.compile(optimizer=rms, loss='mape', metrics=[]) model.fit(X_train, Y_train, batch_size=128, nb_epoch=100, sample_weight=None, validation_data=(x_val, y_val), callbacks=[model_check, lr]) # model.load_weights(model_name) # pred = test_pred.reshape((5, 9, 66)) # with codecs.open('result/test_3937.csv', 'w') as f: # for id, date in enumerate([22, 24, 26, 28, 30]): # for it, time_idx in enumerate(predict_idx): # if (date, time_idx) in [(24, 46), (28, 46)]: # pass # else:
def NNModel(input_array: np.ndarray, output_array: np.ndarray, n_layers: int, n_neurons: int, model_name: str, normal_out: bool = True, nn_type: str = "normal", scalar: str = "stardardize") -> float: print("Starting: " + model_name) if normal_out: if nn_type == "normal": folder_name = "Heston" elif nn_type == "tanh": folder_name = "HestonTanh" else: folder_name = "HestonMix" else: if nn_type == "normal": folder_name = "Heston_non_normal" elif nn_type == "tanh": folder_name = "Heston_non_normal_tanh" else: folder_name = "Heston_non_normal_mix" if not os.path.exists("Models2/" + folder_name): os.makedirs("Models2/" + folder_name) # checking file name no = 0 for i in range(1, 100): saveString = "Models2/" + folder_name + "/" + model_name + "_" + str( i) + ".h5" no = i if os.path.isfile(saveString) == False: break model_path = "Models2/" + folder_name + "/" + model_name + "_" + str( no) + ".h5" X_train, X_test, Y_train, Y_test = train_test_split(input_array, output_array, test_size=0.3, random_state=42) if scalar == "stardardize": norm_features = StandardScaler() else: norm_features = MinMaxScaler() if normal_out: norm_labels = StandardScaler() Y_train = norm_labels.fit_transform(Y_train) Y_test = norm_labels.transform(Y_test) X_train_norm = norm_features.fit_transform(X_train) X_test_norm = norm_features.transform(X_test) if nn_type == "normal": model = nng.NN_generator(n_layers, n_neurons, np.shape(input_array)[1], np.shape(output_array)[1]) elif nn_type == "tanh": model = nng.NN_generator_tanh(n_layers, n_neurons, np.shape(input_array)[1], np.shape(output_array)[1]) else: model = nng.NN_generator_mix(n_layers, n_neurons, np.shape(input_array)[1], np.shape(output_array)[1]) adam = Adam() model.compile( loss='mean_squared_error', #mean squared error optimizer=adam) callbacks_list = [ LearningRateScheduler(lr_schedule, verbose=0), ModelCheckpoint(model_path, monitor="val_loss", save_best_only=True) ] start_time = time.time() model.fit(X_train_norm, Y_train, epochs=100, batch_size=1024, verbose=0, callbacks=callbacks_list, validation_split=0.1, shuffle=True) stop_time = time.time() score = model.evaluate(X_test_norm, Y_test, verbose=2) # Saving model model.save(model_path) # Saving normalization parameters joblib.dump( norm_features, "Models2/" + folder_name + "/" + model_name + "_norm_features_" + str(no) + ".pkl") if normal_out: joblib.dump( norm_labels, "Models2/" + folder_name + "/" + model_name + "_norm_labels_" + str(no) + ".pkl") # Appending test score to file with open("Models2/" + folder_name + "/HestonModels.txt", "a") as output_file: output_file.write("\n") output_file.write(model_name + " has a score of: " + str(score) + ", and took a total time of: " + str(stop_time - start_time)) print("Stopping: " + model_name) return score
def NNModelNext(data_set: list, folder: str, model_name: str, n_layers: int, n_neurons: int, nn_type: str, output_scaling: str, input_scaling: str, include_loss: bool = False) -> float: model_save = "Models5/" + folder + "/" + model_name + "_" + str( n_layers) + "_" + str(n_neurons) + ".h5" model_path = "Models5/" + folder + "/" if not os.path.exists(model_path): os.makedirs(model_path) X_train = data_set[0] X_test = data_set[1] Y_train = data_set[2] Y_test = data_set[3] if input_scaling == "standardize": norm_features = StandardScaler() normal_in = True elif input_scaling == "normalize": norm_features = MinMaxScaler() normal_in = True else: normal_in = False if output_scaling == "standardize": norm_labels = StandardScaler() normal_out = True elif output_scaling == "normalize": norm_labels = MinMaxScaler() normal_out = True else: normal_out = False if normal_in: X_train = norm_features.fit_transform(X_train) X_test = norm_features.transform(X_test) ### saving feature normalization if it doesn't exists. joblib.dump(norm_features, model_path + "norm_feature.pkl") if normal_out: Y_train = norm_labels.fit_transform(Y_train) Y_test = norm_labels.transform(Y_test) joblib.dump(norm_labels, model_path + "norm_labels.pkl") if nn_type == "normal": model = nng.NN_generator(n_layers, n_neurons, np.shape(X_train)[1], np.shape(Y_train)[1]) elif nn_type == "tanh": model = nng.NN_generator_tanh(n_layers, n_neurons, np.shape(X_train)[1], np.shape(Y_train)[1]) elif nn_type == "mix": model = nng.NN_generator_mix(n_layers, n_neurons, np.shape(X_train)[1], np.shape(Y_train)[1]) elif nn_type == "regularization": model = nng.NN_generator_regul(n_layers, n_neurons, np.shape(X_train)[1], np.shape(Y_train)[1]) elif nn_type == "dropput": model = nng.NN_generator_dropout(n_layers, n_neurons, np.shape(X_train)[1], np.shape(Y_train)[1]) else: model = nng.NN_generator_mix_noise(n_layers, n_neurons, np.shape(X_train)[1], np.shape(Y_train)[1]) adam = Adam() model.compile( loss='mean_squared_error', #mean squared error optimizer=adam) callbacks_list = [ LearningRateScheduler(lr_schedule, verbose=0), ModelCheckpoint(model_save, monitor="val_loss", save_best_only=True) ] if (model_name.find("mat") != -1): n_batch_size = 1024 * 5 elif (model_name.find("single") != -1): n_batch_size = 1024 * 25 elif np.shape(X_train)[0] > 140000: n_batch_size = 1024 * 2 else: n_batch_size = 1024 start_time = time.time() loss = model.fit(X_train, Y_train, epochs=100, batch_size=n_batch_size, verbose=0, callbacks=callbacks_list, validation_split=0.1, shuffle=True) stop_time = time.time() score = model.evaluate(X_test, Y_test, verbose=2) if score > 0.7: #if overfitting, save that model print("overfit, saving overfit model") model.save(model_save) if not os.path.exists(model_path + "/HestonModels.txt"): with open(model_path + "/HestonModels.txt", "w") as output_file: pass # Appending test score to file with open(model_path + "/HestonModels.txt", "a") as output_file: output_file.write("\n") output_file.write(model_save + " has a score of: " + str(score) + ", and took a total time of: " + str(stop_time - start_time)) print("Done with: ", model_save) if include_loss: return loss, model else: return score
nt, batch_size=batch_size, N_seq=N_seq_val) lr_schedule = lambda epoch: 0.001 if epoch < 100 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs from keras.callbacks import Callback class ImageCallback(Callback): not_print = 'False' def on_epoch_end(self, epoch, logs): print("EPOCH_CHECK", epoch, logs['loss']) callbacks = [LearningRateScheduler(lr_schedule), ImageCallback()] save_model = 'True' if save_model: if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR) callbacks.append( ModelCheckpoint(filepath=weights_file, monitor='loss', save_best_only=True)) #callbacks.append(ModelCheckpoint(filepath=weights_file2, monitor='val_loss', save_best_only=True)) history = model.fit_generator(train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, verbose=0,
initial_lrate = 0.01 save_weights = True # epoch是10的整数倍时,学习率重置初始值,连续两个整十数之间呈指数下降 def Exponentialdecay(epoch): drop = 0.9 yushu = epoch % 100 if (yushu == 0): lrate = initial_lrate else: lrate = initial_lrate * math.pow(drop, yushu) return lrate mylrate = LearningRateScheduler(Exponentialdecay) optimal = SGD(lr=initial_lrate, momentum=0.9, decay=0.0, nesterov=False) # Adam(lr = 0.002) if (activation == 'sigmoid'): UnetModel.compile(optimizer=optimal, loss='binary_crossentropy', metrics=['accuracy']) # sgd,rmsprop ['accuracy'] else: UnetModel.compile(optimizer=optimal, loss='categorical_crossentropy', metrics=['accuracy']) model_checkpoint = ModelCheckpoint(pretrained_weights, monitor='acc', verbose=1,
model = create_model() if os.path.exists(WEIGHTS): model.load_weights(WEIGHTS) if MODE == 1: model_checkpoint = ModelCheckpoint(filepath=WEIGHTS, monitor='loss', verbose=1, save_best_only=True) reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.5, patience=10) scheduler = LearningRateScheduler(learning_scheduler) model.fit( Y_channel, UV_channel, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1, # validation_split=0.1, callbacks=[scheduler]) model.save_weights(WEIGHTS) elif MODE == 2: for i in range(0, 1000): y = Y_channel[i]
"airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck" ] # construct the image generator for data augmentation aug = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, fill_mode="nearest") # construct the set of callbacks figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())]) jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())]) callbacks = [ TrainingMonitor(figPath, jsonPath=jsonPath), LearningRateScheduler(poly_decay) ] # initialize the optimizer and model print("[INFO] compiling model...") opt = SGD(lr=INIT_LR, momentum=0.9) model = MiniGoogLeNet.build(width=32, height=32, depth=3, classes=10) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) # train the network print("[INFO] training network...") model.fit_generator(aug.flow(trainX, trainY, batch_size=64), validation_data=(testX, testY), steps_per_epoch=len(trainX) // 64,
def ResNet(trainset, valset, n=3, version=1, batch_size=32, epochs=200, data_augmentation=True, save_dir='./save_models'): if not trainset.with_labels: raise ModelTrainingError('The training dataset does not have labels.') if not valset.with_labels: raise ModelTrainingError( 'The validation dataset does not have labels.') # Training parameters #batch_size = 32 # orig paper trained all networks with batch_size=128 #epochs = 200 # Subtracting pixel mean improves accuracy # Model parameter # ---------------------------------------------------------------------------- # | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch # Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti # |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2) # ---------------------------------------------------------------------------- # ResNet20 | 3 (2)| 92.16 | 91.25 | ----- | ----- | 35 (---) # ResNet32 | 5(NA)| 92.46 | 92.49 | NA | NA | 50 ( NA) # ResNet44 | 7(NA)| 92.50 | 92.83 | NA | NA | 70 ( NA) # ResNet56 | 9 (6)| 92.71 | 93.03 | 93.01 | NA | 90 (100) # ResNet110 |18(12)| 92.65 | 93.39+-.16| 93.15 | 93.63 | 165(180) # ResNet164 |27(18)| ----- | 94.07 | ----- | 94.54 | ---(---) # ResNet1001| (111)| ----- | 92.39 | ----- | 95.08+-.14| ---(---) # --------------------------------------------------------------------------- # n = 3 # Model version # Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2) # version = 1 # Computed depth from supplied model parameter n if version == 1: depth = n * 6 + 2 elif version == 2: depth = n * 9 + 2 # Model name, depth and version model_type = 'ResNet%dv%d' % (depth, version) # Load the data. if trainset.in_memory: x_train, y_train = trainset.preprocess(trainset.images, trainset.labels, trainset.mean) else: pass if valset.in_memory: x_val, y_val = valset.preprocess(valset.images, valset.labels, trainset.mean) else: pass # Input image dimensions. input_shape = x_train.shape[1:] # If subtract pixel mean is enabled print('x_train shape:', x_train.shape) print(x_train.shape[0], 'train samples') print(x_val.shape[0], 'test samples') print('y_train shape:', y_train.shape) if version == 2: model = resnet_v2(input_shape=input_shape, depth=depth) else: model = resnet_v1(input_shape=input_shape, depth=depth) model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=lr_schedule(0)), metrics=['accuracy']) model.summary() print(model_type) # Prepare model model saving directory. save_dir = os.path.join(os.getcwd(), 'saved_models') model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) # Prepare callbacks for model saving and for learning rate adjustment. checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_acc', verbose=1, save_best_only=True) lr_scheduler = LearningRateScheduler(lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) callbacks = [checkpoint, lr_reducer, lr_scheduler] # Run training, with or without data augmentation. # Run training, with or without data augmentation. if not data_augmentation: print('Not using data augmentation.') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_val, y_val), shuffle=True, callbacks=callbacks) else: print('Using real-time data augmentation.') # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # epsilon for ZCA whitening zca_epsilon=1e-06, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # set range for random shear shear_range=0., # set range for random zoom zoom_range=0., # set range for random channel shifts channel_shift_range=0., # set mode for filling points outside the input boundaries fill_mode='nearest', # value used for fill_mode = "constant" cval=0., # randomly flip.py images horizontal_flip=True, # randomly flip.py images vertical_flip=False, # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_train) # Fit the model on the batches generated by datagen.flow(). model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), validation_data=(x_val, y_val), epochs=epochs, verbose=1, workers=4, callbacks=callbacks) # Score trained model. scores = model.evaluate(x_val, y_val, verbose=1) print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) return model
def train2(self): clean = [] adv = [] query = os.path.join("/home/natalia/clever", "**", "*.jpg") clean.extend(glob.glob(query, recursive=True)) query = os.path.join("/home/natalia/adv", "**", "*.jpg") adv.extend(glob.glob(query, recursive=True)) clean_set = [] adv_set = [] for c in clean: img = cv2.imread(c) clean_set.append(img) for a in adv: img = cv2.imread(a) adv_set.append(img) y_clean = [[0] for _ in range(len(clean_set))] y_adv = [[1] for _ in range(len(adv_set))] shuffle(clean_set) shuffle(adv_set) x_train = clean_set[:45000] + adv_set[:45000] y_train = y_clean[:45000] + y_adv[:45000] x_test = clean_set[45000:] adv_test = adv_set[45000:] for i, e in enumerate(x_test): cv2.imwrite("/home/natalia/test/clean_{}.jpg".format(str(i)), e) for i, e in enumerate(adv_test): cv2.imwrite("/home/natalia/test/adv_{}.jpg".format(str(i)), e) y_test = y_clean[45000:] y_train = keras.utils.to_categorical(y_train, 2) y_test = keras.utils.to_categorical(y_test, 2) x_train = np.array(x_train) y_train = np.array(y_train) x_test = np.array(x_test) y_test = np.array(y_test) x_train, x_test = self.color_preprocessing(x_train, x_test) print("Loaded dataset") model = self.build_detector() adam = optimizers.Adam(lr=0.0001, beta_1=0.99, beta_2=0.999) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) change_lr = LearningRateScheduler(self.scheduler) checkpoint = ModelCheckpoint(self.model_filename, monitor='val_loss', verbose=0, save_best_only=True, mode='auto') cbks = [change_lr, checkpoint] datagen = ImageDataGenerator(horizontal_flip=True, width_shift_range=0.125, height_shift_range=0.125, fill_mode='constant', cval=0.) datagen.fit(x_train) model.fit_generator(datagen.flow(x_train, y_train, batch_size=self.batch_size), steps_per_epoch=self.iterations, epochs=200, callbacks=cbks, validation_data=(x_test, y_test)) model.save( "/home/natalia/repos/cleverhans/my_tests/models/resnet_detector.h5" )
if not args.no_weights: print('Loading weights from ', os.path.abspath(args.weights)) model.load_weights(args.weights, by_name=True) # Define model callbacks. model_checkpoint = ModelCheckpoint(filepath='snapshot/fcn8s_resnet_epoch-{epoch:02d}_loss-{loss:.3f}_val_loss-{val_loss:.3f}.h5', monitor='loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=10) csv_logger = CSVLogger(filename='log/fcn8s_resnet_training_log.csv', separator=',', append=True) learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule, verbose=1) terminate_on_nan = TerminateOnNaN() callbacks = [model_checkpoint, csv_logger, learning_rate_scheduler, terminate_on_nan] train_generator = DataGenerator(split='train', batch_size=args.batch_size) val_generator = DataGenerator(split='minival', batch_size=1, shuffle=False) history = model.fit_generator(generator=train_generator, epochs=args.epochs, callbacks=callbacks, workers=1, validation_data=val_generator, initial_epoch=args.init_epoch)
print(model_type) # Prepare model model saving directory. save_dir = os.path.join(os.getcwd(), 'saved_models') model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) # Prepare callbacks for model saving and for learning rate adjustment. checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_acc', verbose=1, save_best_only=True) lr_scheduler = LearningRateScheduler(lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) callbacks = [checkpoint, lr_reducer, lr_scheduler] history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, callbacks=callbacks)
def model_train(model_name, train_data, valid_data, trainTemporal, validTemporal, topo_data, type): # set callbacks csv_logger = CSVLogger(PATH + '/' + MODELNAME + '_' + type + '.log') checkpointer_path = PATH + '/' + MODELNAME + '_' + type + '.h5' checkpointer = ModelCheckpoint(filepath=checkpointer_path, verbose=1, save_best_only=True) early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='auto') LearnRate = LearningRateScheduler(lambda epoch: LR) # data generator train_generator = data_generator(train_data, trainTemporal, topo_data, BATCHSIZE, TIMESTEP, model_name) val_generator = data_generator(valid_data, validTemporal, topo_data, BATCHSIZE, TIMESTEP, model_name) sep = (sum([len(x) for x in train_data]) - TIMESTEP * len(train_data)) * train_data[0].shape[1] // BATCHSIZE val_sep = (sum([len(x) for x in valid_data]) - TIMESTEP * len(valid_data)) * valid_data[0].shape[1] // BATCHSIZE # train model model = get_model_structure(model_name) # model = multi_gpu_model(model, gpus=2) # gpu parallel model.compile(loss=LOSS, optimizer=OPTIMIZER) model.fit_generator( train_generator, steps_per_epoch=sep, epochs=EPOCH, validation_data=val_generator, validation_steps=val_sep, callbacks=[csv_logger, checkpointer, LearnRate, early_stopping]) # compute mse val_nolabel_generator = test_generator(valid_data, validTemporal, topo_data, BATCHSIZE, TIMESTEP) val_predY = model.predict_generator(val_nolabel_generator, steps=val_sep) valY = get_test_true(valid_data, TIMESTEP, model_name) # mse scaled_valY = np.reshape(valY, ((sum([len(x) for x in valid_data]) - TIMESTEP * len(valid_data)), HEIGHT, WIDTH)) scaled_predValY = np.reshape(val_predY, ((sum([len(x) for x in valid_data]) - TIMESTEP * len(valid_data)), HEIGHT, WIDTH)) print('val scale shape: ', scaled_predValY.shape) val_scale_MSE = np.mean((scaled_valY - scaled_predValY)**2) print("Model val scaled MSE", val_scale_MSE) # rescale mse val_rescale_MSE = val_scale_MSE * MAX_VALUE**2 print("Model val rescaled MSE", val_rescale_MSE) # write record with open(PATH + '/' + MODELNAME + '_prediction_scores.txt', 'a') as wf: wf.write('train flow {} start time: {}\n'.format(type, StartTime)) wf.write('train flow {} end time: {}\n'.format( type, datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))) wf.write("Keras MSE on flow {} trainData, {}\n".format( type, val_scale_MSE)) wf.write("Rescaled MSE on flow {} trainData, {}\n".format( type, val_rescale_MSE)) return val_scale_MSE, val_rescale_MSE
def get_lrate(epoch, lr): ''' learning rate drop = 0.5 epochs_drop = 2.0 lrate = lr * math.pow(drop, math.floor((1+epoch)/epochs_drop)) ''' # try jumping out of local min shall there exists # if random.random() < .1: # return lr * 2 return min(.005, .25 / (epoch**1.25 + 1)) loss_type = 'mae' lr = LearningRateScheduler(get_lrate, verbose=0) # model.compile(optimizer=RMSprop(lr=.001), loss=loss_type) model.compile(optimizer=Adam(lr=.01), loss=loss_type) # model.compile(optimizer='adam', loss=loss_type) stoppu = EarlyStopping(min_delta=1e-8, patience=5, verbose=1, mode='min') btfu = ModelCheckpoint('vox_' + loss_type + '.hdf5', save_best_only=False, period=1) ''' # %% MIREX data_dir = '/home/tran.ngo.quang.ngoc/Downloads/MIR-1K_for_MIREX/Wavfile/' sample_rate, fmat = wavread(data_dir+'abjones_1_01.wav') # sample_rate, fmat = wavread(data_dir+'abjones_2_02.wav') master = fmat[:,0] * .5 + fmat[:,1] * .5
model.summary() epochs = 25 initial_lrate = 0.01 def decay(epoch, steps=100): initial_lrate = 0.01 drop = 0.96 epochs_drop = 8 lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop)) return lrate sgd = SGD(lr=initial_lrate, momentum=0.9, nesterov=False) lr_sc = LearningRateScheduler(decay, verbose=1) model.compile(loss=['categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy'], loss_weights=[1, 0.3, 0.3], optimizer=sgd, metrics=['accuracy']) history = model.fit(X_train, [y_train, y_train, y_train], validation_data=( X_test, [y_test, y_test, y_test]), epochs=epochs, batch_size=256, callbacks=[lr_sc]) if __name__ == "__main__": start()
valid_set_x, valid_set_y = dataset[1] test_set_x, test_set_y = dataset[2] print(train_set_x.shape) print(train_set_y.shape) print(test_set_x.shape) print(test_set_y.shape) train_set_x = train_set_x.reshape(-1, 32, 32, 1) test_set_x = test_set_x.reshape(-1, 32, 32, 1) # build network print(model.summary()) # set callback #tb_cb = TensorBoard(log_dir='./lenet', histogram_freq=0) change_lr = LearningRateScheduler(scheduler) ckpt = ModelCheckpoint('./output/ckpt.h5', save_weights_only=True, save_best_only=False, mode='auto', period=10) cbks = [change_lr, ckpt] # start train model.fit(train_set_x, train_set_y, batch_size=128, epochs=200, callbacks=cbks, validation_data=(test_set_x, test_set_y), shuffle=True)
print('Learning rate: ', lr) return lr ############################################################## ############################################################## input_shape = list(x_train.shape[1:]) n_classes = y_train.shape[1] n_train = x_train.shape[0] clean_index = np.array([(y_train_noisy[i, :] == y_train[i, :]).all() for i in range(n_train)]) # For tracking only, unused during training noisy_index = np.array([not i for i in clean_index]) INCV_lr_callback = LearningRateScheduler(INCV_lr_schedule) # Define optimizer and compile model optimizer = optimizers.Adam(lr=INCV_lr_schedule(0), beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model = create_model(input_shape=input_shape, classes=n_classes, name='INCV_ResNet32', architecture='ResNet32') model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) weights_initial = model.get_weights() # Print model architecture
def objective_function(params): nb_epochs = 200 start_time = time.time() depth = 28 width = 2 batch_size = 128 B_min = 2 B_max = 6 lr_initial_hp = float(np.exp(params[0])) decay_rate_factor_hp = float(np.exp(params[1])) l2_regular_weight_hp = float(np.exp(params[2])) momentum_hp = float(params[3]) model = wide_resnet(depth, width, l2_regular_weight_hp)(dset.shape, dset.output_size) model.compile(loss="categorical_crossentropy", optimizer=SGD(lr=lr_initial_hp, momentum=momentum_hp), metrics=["accuracy"]) # Create the data augmentation generator datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False) datagen.fit(x_train) schedule = StepDecay(initAlpha=lr_initial_hp, factor=decay_rate_factor_hp, dropEvery=40) callbacks = [LearningRateScheduler(schedule)] # Train the model global initial if initial < n_init_num: B = B_max else: B = np.random.uniform(low=B_min, high=B_max) initial = initial + 1 ImportanceTraining(model, presample=float(B)).fit_generator( datagen.flow(x_train, y_train, batch_size=batch_size), validation_data=(x_test, y_test), epochs=200, verbose=1, batch_size=batch_size, callbacks=callbacks, steps_per_epoch=int(np.ceil(float(len(x_train)) / batch_size))) loss, score = model.evaluate(x_test, y_test, verbose=1) c = time.time() - start_time print('Loss:', loss) print('Test error:', 1.0 - score) global obj_track obj_track.append(1 - score) global loss_track loss_track.append(loss) print("Obj Track: ", obj_track) print("Loss Track: ", loss_track) global iter_num iter_num = iter_num + 1 print("Iter num:", iter_num) global best_obj best_obj = min(best_obj, 1 - score) global best_loss best_loss = min(best_loss, loss) print("Best Error: ", best_obj) print("Best Loss:", best_loss) print("#######################") end_time = time.time() - start_time print("Time to run this hp:", end_time) print("#######################") return 1.0 - score
# Change `figPath` and `jsonPath` to correctly use the `FIG_PATH` # and `JSON_PATH` in the `tiny_imagenet_config`? # construct the set of callbacks figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())]) jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())]) fname = os.path.sep.join( [args["weights"], "weights-{epoch:03d}-{val_acc:.4f}.hdf5"]) checkpoint = ModelCheckpoint(fname, monitor="val_acc", mode="max", save_best_only=True, verbose=1) callbacks = [ TrainingMonitor(figPath, jsonPath=jsonPath), LearningRateScheduler(poly_decay), checkpoint ] # initialize the optimizer and model (ResNet-56) print("[INFO] compiling model...") model = ResNet.build(64, 64, 3, 2, (3, 4, 6), (64, 128, 256, 512), reg=0.0005, dataset="tiny_imagenet") opt = SGD(lr=INIT_LR, momentum=0.9) model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"]) # train the network print("[INFO] training network...")
# CSVLogger to save the training results in a csv file csv_logger = CSVLogger(out_dir + 'csv_log.csv', separator=';') def lr_scheduler(epoch, lr): if epoch == 15: return lr elif epoch == 25: return lr * .1 elif epoch == 35: return lr * .1 else: return lr lrs = LearningRateScheduler(lr_scheduler) # Callback to terminate on NaN loss (so terminate on error) NanLoss = TerminateOnNaN() callbacks = [checkpointer, csv_logger, NanLoss, lrs, es] # Train model model.fit_generator(adSeq, steps_per_epoch=None, epochs= 80, shuffle=True, callbacks=callbacks, verbose=1, validation_data=adSeq_val)
'{}'.format(exp_suffix) + '/') nb_epochs = epochs nb_cycles = 1 init_lr = 0.0005 def _cosine_anneal_schedule(t): cos_inner = np.pi * (t % (nb_epochs // nb_cycles)) cos_inner /= nb_epochs // nb_cycles cos_out = np.cos(cos_inner) + 1 return float(init_lr / 2 * cos_out) lr_schedule = LearningRateScheduler(_cosine_anneal_schedule, verbose=True) callbacks_list = [lr_schedule, tensorboard] # warm up model model = create_model(input_shape=(SIZE, SIZE, 3), n_out=28) POS_WEIGHT = 10 # multiplier for positive targets, needs to be tuned import tensorflow as tf import keras.backend.tensorflow_backend as tfb def weighted_binary_crossentropy(target, output): """ Weighted binary crossentropy between an output tensor and a target tensor. POS_WEIGHT is used as a multiplier
model.load_weights( '/home/AN35190/fer+/fer+_data/data_48x48/best/best.hdf5') model.layers.pop() model.outputs = [model.layers[-1].output] model.layers[-1].outbound_nodes = [] model.add(Dense(1, activation=last)) # Compile model optimizer = Adam(lr=ler) model.compile(loss='mse', optimizer=optimizer, metrics=['mse']) return model model = larger_model() model.summary() call = my_callback(file) lrate = LearningRateScheduler(step_decay) csv_logger = CSVLogger('log_' + file + '.csv', append=True, separator=';') model.fit_generator( batch_generator_train.my_generator_train(), epochs=500, validation_data=batch_generator_dev.my_generator_dev(), steps_per_epoch=int(52352 * 10 / batch), validation_steps=int(59175 / batch), callbacks=[call, csv_logger, TensorBoard(log_dir='./logs/' + str(file))], verbose=2) #247500 #train 297000/batch dev 49800/batch
if debug: print model_noNorm.output_shape current_lr = LEARNING_RATE_SCHEDULE[0] def lr_function(e): global current_lr if e in LEARNING_RATE_SCHEDULE: _current_lr = LEARNING_RATE_SCHEDULE[e] current_lr = _current_lr else: _current_lr = current_lr return _current_lr lr_callback = LearningRateScheduler(lr_function) if getWinSolWeights: w_load_worked = False for l in model_norm.layers: if debug: print '---' if debug: print len(l.get_weights()) l_weights = l.get_weights() if len(l_weights) == len(w_kSorted): if debug: for i in range(len(l_weights)): print type(l_weights[i]) print np.shape(l_weights[i]) if not np.shape(l_weights[i]) == np.shape(w_kSorted[i]): "somethings wrong with the loaded weight shapes" l.set_weights(w_kSorted)