def load_lsa16(data_dir, config, splits): """ Load lsa16 dataset. Args: data_dir (str): path of the directory with 'splits', 'data' subdirs. config (dict): general dict with program settings. splits (list): list of strings 'train'|'val'|'test' Returns (dict): dictionary with keys as splits and values as tf.Dataset """ DATASET_NAME = "lsa16" DATASET_PATH = "/develop/data/lsa16/data" data = hd.load(DATASET_NAME, DATASET_PATH) features = data[0] classes = data[1]['y'] uniqueClasses = np.unique(classes) x_train, x_test, y_train, y_test = train_test_split_balanced(features, classes, train_size=config['data.train_size'], test_size=config['data.test_size']) x_train, x_test = x_train / 255.0, x_test / 255.0 _, amountPerTrain = np.unique(y_train, return_counts=True) _, amountPerTest = np.unique(y_test, return_counts=True) train_datagen_args = dict(featurewise_center=True, featurewise_std_normalization=True, rotation_range=config['data.rotation_range'], width_shift_range=config['data.width_shift_range'], height_shift_range=config['data.height_shift_range'], horizontal_flip=config['data.horizontal_flip'], fill_mode='constant', cval=0) train_datagen = ImageDataGenerator(train_datagen_args) train_datagen.fit(x_train) test_datagen_args = dict(featurewise_center=True, featurewise_std_normalization=True, fill_mode='constant', cval=0) test_datagen = ImageDataGenerator(test_datagen_args) test_datagen.fit(x_train) w, h, c = list(map(int, config['model.x_dim'].split(','))) ret = {} for split in splits: # n_way (number of classes per episode) if split in ['val', 'test']: n_way = config['data.test_way'] else: n_way = config['data.train_way'] # n_support (number of support examples per class) if split in ['val', 'test']: n_support = config['data.test_support'] else: n_support = config['data.train_support'] # n_query (number of query examples per class) if split in ['val', 'test']: n_query = config['data.test_query'] else: n_query = config['data.train_query'] if split in ['val', 'test']: y = y_test x = x_test dg = train_datagen dg_args = train_datagen_args else: y = y_train x = x_train dg = test_datagen dg_args = test_datagen_args amountPerClass = amountPerTest if split in ['val', 'test'] else amountPerTrain i = np.argsort(y) x = x[i, :, :, :] if config['model.type'] in ['processed']: for index in i: x[index, :, :, :] = dg.apply_transform(x[index], dg_args) data = np.reshape(x, (len(uniqueClasses), amountPerClass[0], w, h, c)) data_loader = DataLoader(data, n_classes=len(uniqueClasses), n_way=n_way, n_support=n_support, n_query=n_query, x_dim=(w, h, c)) ret[split] = data_loader return ret
# if idx >= 1: # break print("folder :", folder) base_dir = TRAIN_DIR + '/' + folder + '/' # 각 폴더 디렉토리 'C:/data/LPD_competition/train/0' print("basedir :", base_dir) img_lst = natsorted(os.listdir(base_dir)) # 각 폴더 안에 있는 jpg 파일 리스트 print(img_lst) for i, f in enumerate(img_lst): # 각 폴더에 접근해서 # if i >= 1: # break print("f :", f) img_dir = base_dir + f # 'C:/data/LPD_competition/train/0/0.jpg' print("img_dir :", img_dir) img = np.expand_dims(image.load_img(img_dir, target_size=(DIMENSION, DIMENSION)), axis=0) # 각 이미지를 불어온다 print(img, "\n", img.shape) # cv_img = cv.imread(img_dir) # cv.imshow("whatever", cv_img) # cv.waitKey(0) # cv.destroyAllWindows() train_datagen.fit(img) #image gen for x, val in zip( train_datagen.flow(x=img, save_to_dir=base_dir, save_prefix='aug', shuffle=False), range(1)): pass print("base dir :", base_dir)
def get_data(problem, shards, rank, data_augmentation_level, n_batch_train, n_batch_test, n_batch_init, resolution): if problem == 'mnist': from tensorflow.keras.datasets import mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() y_train = np.reshape(y_train, [-1]) y_test = np.reshape(y_test, [-1]) # Pad with zeros to make 32x32 x_train = np.lib.pad(x_train, ((0, 0), (2, 2), (2, 2)), 'minimum') # Pad with zeros to make 32x23 x_test = np.lib.pad(x_test, ((0, 0), (2, 2), (2, 2)), 'minimum') # x_train = np.tile(np.reshape(x_train, (-1, 32, 32, 1)), (1, 1, 1, 3)) # x_test = np.tile(np.reshape(x_test, (-1, 32, 32, 1)), (1, 1, 1, 3)) x_train = np.reshape(x_train, (-1, 32, 32, 1)) x_test = np.reshape(x_test, (-1, 32, 32, 1)) elif problem == 'cifar10': from tensorflow.keras.datasets import cifar10 (x_train, y_train), (x_test, y_test) = cifar10.load_data() y_train = np.reshape(y_train, [-1]) y_test = np.reshape(y_test, [-1]) else: raise Exception() print('n_train:', x_train.shape[0], 'n_test:', x_test.shape[0]) # Shard before any shuffling x_train, y_train = shard((x_train, y_train), shards, rank) x_test, y_test = shard((x_test, y_test), shards, rank) print('n_shard_train:', x_train.shape[0], 'n_shard_test:', x_test.shape[0]) from tensorflow.keras.preprocessing.image import ImageDataGenerator datagen_test = ImageDataGenerator() if data_augmentation_level == 0: datagen_train = ImageDataGenerator() else: if problem == 'mnist': datagen_train = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1) elif problem == 'cifar10': if data_augmentation_level == 1: datagen_train = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1) elif data_augmentation_level == 2: datagen_train = ImageDataGenerator( width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, rotation_range=15, # degrees rotation zoom_range=0.1, shear_range=0.02, ) else: raise Exception() else: raise Exception() datagen_train.fit(x_train) datagen_test.fit(x_test) train_flow = datagen_train.flow(x_train, y_train, n_batch_train) test_flow = datagen_test.flow(x_test, y_test, n_batch_test, shuffle=False) def make_iterator(flow, resolution): def iterator(): x_full, y = flow.next() x_full = x_full.astype(np.float32) x = downsample(x_full, resolution) x = x_to_uint8(x) return x, y return iterator #init_iterator = make_iterator(train_flow, resolution) train_iterator = make_iterator(train_flow, resolution) test_iterator = make_iterator(test_flow, resolution) # Get data for initialization data_init = make_batch(train_iterator, n_batch_train, n_batch_init) return train_iterator, test_iterator, data_init
featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=10, # randomly rotate images in the range (degrees, 0 to 180) zoom_range = 0.2, shear_range = 0.2, width_shift_range=0.2, # randomly shift images horizontally (fraction of total width) height_shift_range=0.2, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) #model.fit_generator(datagen.flow(X_train, y_train,batch_size=32), steps_per_epoch=len(X_train) / 32, epochs=13, validation_data=val_datagen.flow(X_test, y_test, batch_size=32), validation_steps = len(y_train)/32, callbacks=[earlyStopping], shuffle=True) datagen.fit(X_train) val_datagen = ImageDataGenerator(rescale=1./255) # create base model from pretrained base_model = tf.keras.applications.MobileNetV2(input_shape=(96,96,3), include_top=False, weights='imagenet') # freeze the conv base_model base_model.trainable = False #base_model.summary() print(X.shape)
### dataset (train_x, train_y), (valid_x, valid_y) = tf.keras.datasets.cifar10.load_data() train_x = train_x.astype('float') train_y = np.squeeze(train_y) valid_x = valid_x.astype('float') valid_y = np.squeeze(valid_y) print('train_x shape : ', train_x.shape) print('train_y shape : ', train_y.shape) print('valid_x shape : ', valid_x.shape) print('valid_y shape : ', valid_y.shape) train_datagen = ImageDataGenerator(horizontal_flip=True, width_shift_range=0.1, height_shift_range=0.1) train_datagen.fit(train_x) valid_datagen = ImageDataGenerator() model = xenqore.apps.VGGNet13(mode=0, network_config=network_config, layer_config=layers_config, act_config=activations_config, saved_model='', classes=network_config.classes, input_shape=train_x.shape[1:]) model.summary() model.compile(optimizer=tf.keras.optimizers.Adam( learning_rate=network_config.initial_lr),
model.add(Flatten()) model.add(Dense(units=512, activation="relu") ) model.add(Dropout(0.5)) model.add(Dense(units=10,activation='softmax')) """ model.summary() #data augmentation datagen = ImageDataGenerator( rotation_range=20, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True ) datagen.fit(train_feature_normalize) #training batch_size = 64 #opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6) #model.compile(loss='categorical_crossentropy',optimizer='adam' , metrics=['accuracy']) opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6) model.compile(loss='categorical_crossentropy',optimizer=opt_rms , metrics=['accuracy']) try: model.load_weights("Cifar10_CNN_model_3.weight") print("載入模型參數成功,繼續訓練模型!") except : print("載入模型參數失敗,開始訓練一個新模型!")
model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy']) # featurewise_center = False, featurewise_std_normalization=False, img = ImageDataGenerator( rotation_range=25, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1, zoom_range=0.1, horizontal_flip=True, vertical_flip=False, dtype=np.float32) img.fit(x_train) # call back callbacks = [] file_path = model_save_path modelCheckpoint = ModelCheckpoint( file_path, monitor='val_accuracy', save_best_only=True, mode='max', save_weights_only=False) callbacks.append(modelCheckpoint)
def main(): print("Load and normalize Cifar-10 dataset") (X_train, y_train), (X_test, y_test) = load_and_normalize_cifar10() model_file_path, recipe_file_path = download_model_and_recipe(root_dir) print("Load pretrained model") base_model = tf.keras.models.load_model(model_file_path) base_model.summary() scores = base_model.evaluate(X_test, y_test, verbose=1) print("Test loss:", scores[0]) print("Test accuracy:", scores[1]) optimizer = tf.keras.optimizers.Adam() steps_per_epoch = math.ceil(len(X_train) / batch_size) # Enhance the model and optimizer for pruning using the manager loggers = TensorBoardLogger(log_dir=log_dir, update_freq=update_freq) manager = ScheduledModifierManager.from_yaml(recipe_file_path) model_for_pruning, optimizer, callbacks = manager.modify(base_model, optimizer, steps_per_epoch, loggers=loggers) callbacks.append(LossesAndMetricsLoggingCallback(loggers)) callbacks.append(model_checkpoint_callback()) # Compile the enhanced model model_for_pruning.compile( loss=tf.keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=["accuracy"], run_eagerly=True, ) # Run training and pruning, with or without data augmentation. if not data_augmentation: print("Not using data augmentation.") model_for_pruning.fit( X_train, y_train, batch_size=batch_size, validation_data=(X_test, y_test), shuffle=True, callbacks=callbacks, epochs=manager.max_epochs, ) else: print("Using real-time data augmentation.") # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # epsilon for ZCA whitening zca_epsilon=1e-06, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # set range for random shear shear_range=0.0, # set range for random zoom zoom_range=0.0, # set range for random channel shifts channel_shift_range=0.0, # set mode for filling points outside the input boundaries fill_mode="nearest", # value used for fill_mode = "constant" cval=0.0, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0, ) # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(X_train) # Fit the model on the batches generated by datagen.flow(). model_for_pruning.fit_generator( datagen.flow(X_train, y_train, batch_size=batch_size), validation_data=(X_test, y_test), epochs=manager.max_epochs, verbose=1, workers=4, callbacks=callbacks, ) # Erase pruning masks and export to ONNX model pruned_model = manager.finalize(model_for_pruning) exporter = ModelExporter(pruned_model, output_dir=pruned_model_dir) onnx_model_name = "pruned_resnet20_v1.onnx" exporter.export_onnx(name=onnx_model_name) print("Model exported to {}".format( os.path.join(pruned_model_dir, onnx_model_name)))
if data_augmentation: # Data augmentation data_gen_args = dict(rotation_range=90, width_shift_range=0.1, height_shift_range=0.1, shear_range=5, zoom_range=0.1, fill_mode='nearest', horizontal_flip=True, vertical_flip=True) image_datagen = ImageDataGenerator(**data_gen_args) segmentation_datagen = ImageDataGenerator(**data_gen_args) # Provide the same seed and keyword arguments to the fit and flow methods seed = 1 image_datagen.fit(dataset_train.x, augment=True, seed=seed) segmentation_datagen.fit(dataset_train.y, augment=True, seed=seed) # Combine generators into one which yields image and masks # Use option save_to_dir='img_flow' to save augmented images train_generator = zip( image_datagen.flow(dataset_train.x, batch_size=batch_size, seed=seed, sample_weight=sample_weight), segmentation_datagen.flow(dataset_train.y, batch_size=batch_size, seed=seed, sample_weight=sample_weight)) # Fit the model with a generator
def run(logger, options, session_log_file, logits_dir, models_dir): logger.info(cfg.student_train_spacer + "GENERIC MULTISTAGE" + cfg.student_train_spacer) with open(session_log_file, "w") as f: f.write("begin test: " + datetime.datetime.now().isoformat() + "\n") f.close() # load configuration file configuration = import_config(options.config_file_path) teacher_name = configuration['teacher_name'] epochs = configuration['epochs'] temperatures = configuration['temp_config'] alphas = configuration['alpha_config'] order_combinations = configuration['size_combinations'] # loading training data X_train, Y_train, X_test, Y_test = load_dataset.load_dataset_by_name( logger, cfg.dataset) # mean subtraction regularization if cfg.subtract_pixel_mean is True: x_train_mean = np.mean(X_train, axis=0) X_train -= x_train_mean X_test -= x_train_mean if cfg.use_fit_generator_student is True or cfg.use_fit_generator_teacher is True: # data generator for on the fly training data manipulation datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # epsilon for ZCA whitening zca_epsilon=1e-06, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # set range for random shear shear_range=0., # set range for random zoom zoom_range=0., # set range for random channel shifts channel_shift_range=0., # set mode for filling points outside the input boundaries fill_mode='nearest', # value used for fill_mode = "constant" cval=0., # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) datagen.fit(X_train) try: for order in order_combinations: for temp in temperatures: # clearing all saved teacher logits for alpha in alphas: tf.keras.backend.clear_session( ) # must clear the current session to free memory! K.clear_session( ) # must clear the current session to free memory! logger.info( "Clearing tensorflow/keras backend session and de-allocating remaining models..." ) model = None previousModel = None if teacher_name is not None: ssm = model_loader(logger, options.teacherModel) previousModel = ssm.get_loaded_model() teacher_name = options.teacherModel # creating experiment1 metadata experiment_result = { "experiment_results": [] } # empty space for our experiment1's data experiment_metadata = create_meta(cfg.dataset, teacher_name, epochs, temp, alpha, order) experiment_result['metadata'] = experiment_metadata # performing experiment on given size, alpha, and temperature combination for net_size in order: model = None # perform KD if there is a previously trained model to work with if previousModel is not None: model = knowledge_distillation_models.get_model( cfg.dataset, cfg.dataset_num_classes, X_train, net_size, ) logger.info( "loading soft targets for student training...") print("previous model to load logits for: %s" % str(previousModel)) teacher_train_logits, teacher_test_logits = get_pretrained_teacher_logits( logits_dir, previousModel, alpha, cfg.dataset, order) Y_train_new, Y_test_new = teacher_utils.convert_logits_to_soft_targets( temp, teacher_train_logits, teacher_test_logits, Y_train, Y_test) # # TODO remove next three lines # file_name = "/home/blakete/" + temp + "_" + previousModel + "_training_labels.npy" # filehandler = open(file_name, 'wb') # pickle.dump(Y_train_new, filehandler) # pickle.dump(Y_test_new, filehandler) if Y_train_new is None or Y_test_new is None: logger.info( "soft targets not loaded correctly!") else: logger.info("completed") # filehandler = open("mnist_10_soft_targets.pkl", 'wb') # pickle.dump(Y_train_new, filehandler) # pickle.dump(Y_test_new, filehandler) model = helper_util.apply_knowledge_distillation_modifications( logger, model, temp) # model = multi_gpu_model(model, gpus=4) optimizer = get_optimizer( cfg.student_optimizer) model.compile( optimizer=optimizer, loss=lambda y_true, y_pred: helper_util. knowledge_distillation_loss( logger, y_true, y_pred, alpha), metrics=[helper_util.acc]) logger.info( "training model...\norder:%s\nsize:%d\ntemp:%d\nalpha:%f" % (order, net_size, temp, alpha)) callbacks = [ EarlyStopping(monitor='val_acc', patience=50, min_delta=0.00007), # ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=4, min_lr=0.0001), ModelCheckpoint(cfg.checkpoint_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max') ] if cfg.use_fit_generator_student is True: model.fit(datagen.flow( X_train, Y_train_new, batch_size=cfg.student_batch_size), validation_data=(X_test, Y_test_new), epochs=epochs, verbose=1, callbacks=callbacks) else: model.fit( X_train, Y_train_new, batch_size=cfg.student_batch_size, epochs=epochs, verbose=1, callbacks=callbacks, validation_data=(X_test, Y_test_new)) # model = helper_util.revert_knowledge_distillation_modifications(logger, model) del model # train_score, val_score = helper_util.calculate_unweighted_score(logger, model, X_train, Y_train, # X_test, Y_test) model = knowledge_distillation_models.get_model( cfg.dataset, cfg.dataset_num_classes, X_train, net_size, ) # model.summary() # load best model from checkpoint for evaluation model.load_weights(cfg.checkpoint_path) optimizer = get_optimizer( cfg.student_optimizer) model.compile( optimizer=optimizer, loss= logloss, # the same as the custom loss function metrics=['accuracy']) train_score = model.evaluate(X_train, Y_train, verbose=0) val_score = model.evaluate(X_test, Y_test, verbose=0) result = create_result(net_size, temp, alpha, train_score, val_score) logger.info(result) experiment_result["experiment_results"].append( result) # # remove checkpoint of best model for new checkpoint # os.remove(cfg.checkpoint_path) # save the trained model the saved model directory saved_model(logger, cfg.dataset, net_size, alpha, val_score, order, model, models_dir) if order.index(net_size) < len(order) - 1: # save soft targets logger.info( "creating student training data...") Y_train_new, Y_test_new = teacher_utils.createStudentTrainingData( model, temp, X_train, Y_train, X_test, Y_test) save_pretrained_teacher_logits( logits_dir, net_size, alpha, Y_train_new, Y_test_new, cfg.dataset, order) logger.info("done.") else: logger.info( "skipping creation of student training data, we are @ target model..." ) # clear soft targets Y_train_new = None Y_test_new = None # set model to current net size to preserve in previousModel model = net_size # if no previously trained model, train the network else: # load the already created soft targets Y_train_new = None Y_test_new = None val_score = None teacher_train_logits, teacher_test_logits = get_pretrained_teacher_logits( logits_dir, net_size, alpha, cfg.dataset, order) # train network if not previously created logits if teacher_train_logits is None or teacher_test_logits is None: if os.path.isfile(cfg.checkpoint_path): logger.info("removing previous checkpoint") os.remove(cfg.checkpoint_path ) # remove previous checkpoint logger.info( "training teacher model...\norder:%s\nsize:%d\ntemp:%d\nalpha:%f" % (order, net_size, temp, alpha)) model = knowledge_distillation_models.get_model( cfg.dataset, cfg.dataset_num_classes, X_train, net_size, ) # model.summary() optimizer = get_optimizer( cfg.start_teacher_optimizer) model.compile( optimizer=optimizer, loss= logloss, # the same as the custom loss function metrics=['accuracy']) # train network and save model with bet validation accuracy to cfg.checkpoint_path callbacks = [ EarlyStopping(monitor='val_acc', patience=50, min_delta=0.00007), # ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=4, min_lr=0.0001), ModelCheckpoint(cfg.checkpoint_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max') ] if cfg.use_fit_generator_teacher is True: model.fit(datagen.flow( X_train, Y_train, batch_size=cfg.student_batch_size), validation_data=(X_test, Y_test), epochs=epochs, verbose=1, callbacks=callbacks) else: model.fit( X_train, Y_train, validation_data=(X_test, Y_test), batch_size=cfg.student_batch_size, epochs=epochs, verbose=1, callbacks=callbacks) # load best model from checkpoint for evaluation del model model = knowledge_distillation_models.get_model( cfg.dataset, cfg.dataset_num_classes, X_train, net_size, ) model.load_weights(cfg.checkpoint_path) optimizer = get_optimizer( cfg.start_teacher_optimizer) model.compile( optimizer=optimizer, loss= logloss, # the same as the custom loss function metrics=['accuracy']) # evaluate network train_score, val_score = helper_util.calculate_unweighted_score( logger, model, X_train, Y_train, X_test, Y_test) # save evaluation result = create_result(net_size, None, None, train_score, val_score) logger.info(result) experiment_result["experiment_results"].append( result) if len(order) != 1: logger.info( "creating student training data...") teacher_train_logits, teacher_test_logits = teacher_utils.createStudentTrainingData( model, temp, X_train, Y_train, X_test, Y_test) save_pretrained_teacher_logits( logits_dir, net_size, alpha, teacher_train_logits, teacher_test_logits, cfg.dataset, order) logger.info( "done creating student training data.") # save the trained model the saved model directory saved_model(logger, cfg.dataset, net_size, alpha, val_score, order, model, models_dir) # # remove checkpoint of best model for new checkpoint # os.remove(cfg.checkpoint_path) else: model = net_size # temporarily serialize model to load as teacher in following KD training to avoid errors del previousModel # free memory previousModel = net_size # previously trained model becomes teacher # appending experiment result to log file if os.path.isfile(session_log_file): open_type = 'a' else: open_type = 'w' with open(session_log_file, open_type) as f: f.write(json.dumps(experiment_result)) f.write("\n") f.close() # printing the results of training logger.info(cfg.student_train_spacer) # free model variables for next configuration iteration del model del previousModel logger.info('-- COMPLETE') except Exception: traceback.print_exc() error = traceback.format_exc() # error.upper() logging.error('Error encountered: %s' % error, exc_info=True)
#model.summary() # Compile the model opt = keras.optimizers.Adam(learning_rate=0.00001) ls = keras.losses.binary_crossentropy model.compile(loss=ls, optimizer=opt, metrics=['accuracy']) my_callbacks=[ callbacks.EarlyStopping(patience = 15, monitor='val_loss', mode = 'min'), callbacks.ModelCheckpoint(filepath='Task2_0707_AlexNet4_ash.h5', save_best_only=True) ] ###### Training x_tr, x_val, y_tr, y_val = train_test_split(x_train_value, y_train, test_size = 0.2, random_state = 1) aug = ImageDataGenerator(vertical_flip=True, horizontal_flip=True, rotation_range=90) aug.fit(x_tr) history = model.fit_generator(aug.flow(x_tr, y_tr, batch_size=64, shuffle=True), epochs = 50, verbose = 1, validation_data = (x_val, y_val), callbacks = my_callbacks) ###### Evaluation from sklearn.metrics import accuracy_score from sklearn.metrics import roc_auc_score from sklearn.metrics import confusion_matrix from sklearn.preprocessing import binarize import matplotlib.pyplot as plt plt.figure(figsize=(10,8)).suptitle('Loss Plot', fontsize = 20) plt.plot(history.history['loss'], label = 'train')
''' Images_load_length = Images.shape[0] train_images = Images[0:np.int(Images_load_length*train_ratio),:,:,:] test_images = Images[np.int(Images_load_length*train_ratio) :,:,:,:] train_labels = Labels[0:np.int( Images_load_length*train_ratio )] test_labels = Labels[np.int(Images_load_length*train_ratio) :] ''' # ============================================================================== # Data augment using keras data generator # ============================================================================== # datagen = ImageDataGenerator(horizontal_flip=True , vertical_flip=True , zoom_range = [1,1.2] , rotation_range=10, width_shift_range=0.1,height_shift_range=0.1) datagen = ImageDataGenerator(horizontal_flip=True , vertical_flip=True ) datagen.fit(train_images) # ============================================================================== # Define the neural network hierachy and classes # ============================================================================== model = Eff_net_B0_simplest(Net_file_path , 'All' , Checkpoint_network) # ============================================================================== # Define the learning variable and fit the model # ============================================================================== callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True, min_delta=0.0001) model.summary() model.compile(optimizer=Adam(lr=0.0001),loss = 'binary_crossentropy', metrics=['accuracy']) history = model.fit_generator(datagen.flow(train_images, train_labels, batch_size=32, shuffle=True) , epochs=EPOCHS, callbacks = [callback], validation_data=(test_images, test_labels), verbose = 1)
def MobileNet(data, settings, width_multiplier=1.0, q=None): model_type = 'MobileNetv2' batch_size = settings[0] epochs = settings[1] data_augmentation = settings[2] weight_precision = settings[5] activations_prescision = settings[6] x_train = data[0] y_train = data[1] x_test = data[2] y_test = data[3] x_train_mean = data[4] input_shape = data[5] num_classes = data[6] depth = 26 k = settings[3] """MobileNetv2 This function defines a MobileNetv2 architectures. # Arguments input_shape: An integer or tuple/list of 3 integers, shape of input tensor. k: Integer, number of classes. # Returns MobileNetv2 model. """ global nlay nlay = -1 inputs = Input(shape=input_shape) x = _conv_block(inputs, roundup(int(32 * width_multiplier)), (3, 3), strides=(2, 2), use_bias=False) nlay += 1 fix = 0 if width_multiplier - 1.3 < 0.01: fix = -2 x = _inverted_residual_block(x, roundup(int(16 * width_multiplier)), (3, 3), t=1, strides=1, n=1) x = _inverted_residual_block(x, roundup(int(24 * width_multiplier)), (3, 3), t=6, strides=2, n=2) x = _inverted_residual_block(x, roundup(int(32 * width_multiplier)), (3, 3), t=6, strides=2, n=3) x = _inverted_residual_block(x, roundup(int(64 * width_multiplier) + fix), (3, 3), t=6, strides=2, n=4) x = _inverted_residual_block(x, roundup(int(96 * width_multiplier)), (3, 3), t=6, strides=1, n=3) x = _inverted_residual_block(x, roundup(int(160 * width_multiplier)), (3, 3), t=6, strides=2, n=3) x = _inverted_residual_block(x, roundup(int(320 * width_multiplier)), (3, 3), t=6, strides=1, n=1) last_conv_size = max(1280, int(1280 * width_multiplier)) x = _conv_block(x, last_conv_size, (1, 1), strides=(1, 1), use_bias=False) x = GlobalAveragePooling2D()(x) x = Reshape((1, 1, last_conv_size))(x) x = Dropout(0.3, name='Dropout')(x) x = Conv2D(k, (1, 1), padding='same', name='logits', use_bias=True)(x) x = Activation('softmax', name='softmax')(x) outputs = Reshape((k,), name='out')(x) model = Model(inputs=inputs, outputs=outputs) #Quantize # a/w: 8/8, 16/16, 16/8 if q == True: model = QUANTIZER.apply_quantization(model, pruning_policy=None, weight_precision=weight_precision, activation_precision=activations_prescision, activation_margin=None) # plot_model(model, to_file='MobileNetv2.png', show_shapes=True) #-------------------------------------------------------------------------------- #Get Output num_layers = 156 all_layers = list() for layer_index in range(1, num_layers): all_layers.append(model.get_layer(name=None, index=layer_index).output) intermediate_layer_model_input = model.input intermediate_layer_model = Model(inputs=intermediate_layer_model_input, outputs=all_layers) #--------------------------------------------------------------------------------- model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=Settings.lr_schedule(0)), metrics=['accuracy']) model.summary() print(model_type) # Prepare model model saving directory. save_dir = os.path.join(os.getcwd(), 'saved_models') model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) # Prepare callbacks for model saving and for learning rate adjustment. checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_acc', verbose=1, save_best_only=True) lr_scheduler = LearningRateScheduler(Settings.lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) callbacks = [checkpoint, lr_reducer, lr_scheduler] # Run training, with or without data augmentation. if not data_augmentation: print('Not using data augmentation.') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, callbacks=callbacks) else: print('Using real-time data augmentation.') # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # epsilon for ZCA whitening zca_epsilon=1e-06, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # set range for random shear shear_range=0., # set range for random zoom zoom_range=0., # set range for random channel shifts channel_shift_range=0., # set mode for filling points outside the input boundaries fill_mode='nearest', # value used for fill_mode = "constant" cval=0., # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_train) # Fit the model on the batches generated by datagen.flow(). model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), validation_data=(x_test, y_test), epochs=epochs, verbose=1, workers=4, callbacks=callbacks) #---------------------------------------------------------- ''' data = x_test num_batches = data.shape[0] // batch_size for batch_idx in range(num_batches): start = batch_idx * batch_size end = start + batch_size intermediate_output = intermediate_layer_model.predict(data[start:end]) print("Intermediate result batch {}/{} done".format(batch_idx, num_batches)) print("Got intermediate, a random entry: {}".format(intermediate_output[0][0])) ''' # Score trained model. scores = model.evaluate(x_test, y_test, verbose=1) print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) #model.save("mobilenet_model.h5") return model
cifar10 = tf.keras.datasets.cifar10 (x_train, y_train), (x_test, y_test) = cifar10.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 imagetrain = ImageDataGenerator(rescale=1. / 1., rotation_range=45, width_shift_range=.15, height_shift_range=.15, horizontal_flip=False, zoom_range=0.5) #x_train, x_test = x_train / 255.0, x_test / 255.0 imagetrain.fit(x_train) class Baseline(Model): def __init__(self): super(Baseline, self).__init__() self.c1 = Conv2D(filters=6, kernel_size=(5, 5), padding='same') # 卷积层 self.b1 = BatchNormalization() # BN层 self.a1 = Activation('relu') # 激活层 self.p1 = MaxPool2D(pool_size=(2, 2), strides=2, padding='same') # 池化层 self.d1 = Dropout(0.2) # dropout层 self.flatten = Flatten() self.f1 = Dense(128, activation='relu') self.d2 = Dropout(0.2) self.f2 = Dense(10, activation='softmax')
def runExample(numEpochs, batchSize, dataAugmentation, directoryName, modelName): "This function is based off of the cifar10_cnn.py example shown in class" "Baseline test for this examples was: (numEpochs=100, NUM_CLASSES=10, batchSize=32, " "dataAugmentation=True)" "Create the save directory for this specific test/example" saveDir = os.path.join(SAVE_DIR, directoryName) "Set num_predictions to 20" num_predictions = 20 "Create tensorboard callback" tensorboard_callback = keras.callbacks.TensorBoard(log_dir=saveDir, histogram_freq=1, profile_batch = 10000000000) # The data, split between train and test sets: (x_train, y_train), (x_test, y_test) = cifar10.load_data() print('x_train shape:', x_train.shape) print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') # Convert class vectors to binary class matrices. y_train = keras.utils.to_categorical(y_train, NUM_CLASSES) y_test = keras.utils.to_categorical(y_test, NUM_CLASSES) model = Sequential() model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:])) model.add(Activation('relu')) model.add(Conv2D(32, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Conv2D(64, (3, 3), padding='same')) model.add(Activation('relu')) model.add(Conv2D(64, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(NUM_CLASSES)) model.add(Activation('softmax')) # initiate RMSprop optimizer opt = keras.optimizers.RMSprop(learning_rate=0.0001, decay=1e-6) # Let's train the model using RMSprop model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 if not dataAugmentation: print('Not using data augmentation.') model.fit(x_train, y_train, batch_size=batchSize, epochs=numEpochs, validation_data=(x_test, y_test), shuffle=True, callbacks=[tensorboard_callback]) # "Added tensorboard callback" else: print('Using real-time data augmentation.') # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening zca_epsilon=1e-06, # epsilon for ZCA whitening rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, shear_range=0., # set range for random shear zoom_range=0., # set range for random zoom channel_shift_range=0., # set range for random channel shifts # set mode for filling points outside the input boundaries fill_mode='nearest', cval=0., # value used for fill_mode = "constant" horizontal_flip=True, # randomly flip images vertical_flip=False, # randomly flip images # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) # Compute quantities required for feature-wise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_train) # Fit the model on the batches generated by datagen.flow(). "Add the tensorboard callback" model.fit_generator(datagen.flow(x_train, y_train, batch_size=batchSize), epochs=numEpochs, validation_data=(x_test, y_test), workers=4, callbacks=[tensorboard_callback]) # Save model and weights if not os.path.isdir(saveDir): os.makedirs(saveDir) model_path = os.path.join(saveDir, modelName) model.save(model_path) print('Saved trained model at %s ' % model_path) # Score trained model. scores = model.evaluate(x_test, y_test, verbose=1) print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) print()
def train(self): # mirrored_strategy = tf.distribute.MirroredStrategy() mirrored_strategy = tf.contrib.distribute.MirroredStrategy() with mirrored_strategy.scope(): model = Sequential() weight_decay = self.weight_decay model.add( Conv2D(64, (3, 3), padding='same', input_shape=self.x_shape, kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.3)) model.add( Conv2D(64, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.5)) model.add(Flatten()) model.add( Dense(512, kernel_regularizer=regularizers.l2(weight_decay))) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(self.num_classes)) model.add(Activation('softmax')) # optimization details learning_rate = 0.1 lr_decay = 1e-6 sgd = optimizers.SGD(lr=learning_rate, decay=lr_decay, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # training parameters batch_size = 128 maxepoches = 250 learning_rate = 0.1 lr_decay = 1e-6 lr_drop = 20 # The data, shuffled and split between train and test sets: (x_train, y_train), (x_test, y_test) = cifar10.load_data() x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train, x_test = self.normalize(x_train, x_test) y_train = keras.utils.to_categorical(y_train, self.num_classes) y_test = keras.utils.to_categorical(y_test, self.num_classes) def lr_scheduler(epoch): return learning_rate * (0.5**(epoch // lr_drop)) reduce_lr = keras.callbacks.LearningRateScheduler(lr_scheduler) # data augmentation datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 15, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) # randomly flip images # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_train) # training process in a for loop with learning rate drop every 25 epoches. ''' historytemp = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=maxepoches, validation_data=(x_test, y_test), callbacks=[reduce_lr], verbose=2) ''' for e in range(maxepoches): print('Epoch', e) batches = 0 for x_batch, y_batch in datagen.flow(x_train, y_train, batch_size=batch_size): model.fit(x_batch, y_batch) batches += 1 if batches >= len(x_train) / batch_size: # we need to break the loop by hand because # the generator loops indefinitely break model.save_weights('cifar10vgg.h5') return model
fashion_mnist = tf.keras.datasets.fashion_mnist # 导入数据集 (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() # 分出训练集和测试集 x_test = x_test / 255. x_train = x_train.reshape(x_train.shape[0], 28, 28, 1) x_test = x_test.reshape(x_test.shape[0], 28, 28, 1) image_gen_train = ImageDataGenerator( rescale=1. / 255, #归至0~1 rotation_range=45, #随机45度旋转 width_shift_range=.15, #宽度偏移 height_shift_range=.15, #高度偏移 horizontal_flip=True, #水平翻转 zoom_range=0.5 #将图像随机缩放到50% ) image_gen_train.fit(x_train) model = tf.keras.models.Sequential([ Conv2D(input_shape=(28, 28, 1), filters=32, kernel_size=(5, 5), padding='same'), # 卷积层 BatchNormalization(), # BN层 Activation('relu'), # 激活层 MaxPool2D(pool_size=(2, 2), strides=2, padding='same'), # 池化层 Dropout(0.2), # dropout层 Conv2D(64, kernel_size=(5, 5), padding='same'), BatchNormalization(), Activation('relu'), MaxPool2D(pool_size=(2, 2), strides=2, padding='same'), Dropout(0.2),
def train_and_save_network(attr=None): if attr is None: attr=sys.argv[1] assert (attr is not None) subject = f'cnn.py {attr} {datetime.datetime.now()}' subprocess.run('> nohup.out', shell=True) os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1' GPU = get_available_device() print('* * * device to be used:%s * * *'%GPU) globals().update(get_data(attr=attr)) model = create_model(x_train, y_train) callback = CallbackMailer(subject) if not data_augmentation: print('Not using data augmentation.') with tf.device(GPU): hist = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_valid, y_valid), shuffle=True, callbacks = [callback]) else: print('Using real-time data augmentation.') # preprocessing and realtime data augmentation datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening zca_epsilon=1e-06, # epsilon for ZCA whitening rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, shear_range=0., # set range for random shear zoom_range=0., # set range for random zoom channel_shift_range=0., # set range for random channel shifts # set mode for filling points outside the input boundaries fill_mode='nearest', cval=0., # value used for fill_mode = "constant" horizontal_flip=True, # randomly flip images vertical_flip=False, # randomly flip images # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) # Compute quantities required for feature-wise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_train) with tf.device(GPU): # Fit the model on the batches generated by datagen.flow(). hist = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), epochs=epochs, validation_data=(x_valid, y_valid)) # Save model and weights print('saving model... ',end='',flush=True) model_path = raw_model_path.format(attr=attr) model.save(model_path) print('done! >>> Saved trained model at {}'.format(model_path)) write_model_run(hist, model, attr, callback, subject) # Score trained model. scores = np.array(model.evaluate(x_test, y_test, verbose=1))#df.loc['test'].values print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) mail( subject = subject, message = "cnn.py is finished running: loss, accuracy = {}".format(scores.tolist()) )
dataset = load_dataset(validation_size=validation_size, test_size=test_size, split=False, img_size=[img_rows, img_cols]) datagen_train = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True) datagen_test = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True) datagen_train.fit(dataset['x_train']) datagen_test.fit(dataset['x_train']) def prot_div(left, right): if right == 0: return 0 else: return left / right def if_func(condition, state1, state2): if condition: return state1 else: return state2
print('Weight for class 0: {:.2f}'.format(weight_for_0)) print('Weight for class 1: {:.2f}'.format(weight_for_1)) print("#######################\n" "# DATA AUGMENTATION #\n" "#######################\n") datagen = ImageDataGenerator(horizontal_flip=hflips, vertical_flip=vflips, rotation_range=rotation_range, shear_range=k_shear_range, width_shift_range=k_w_shift_r, height_shift_range=k_h_shift_r, fill_mode='reflect') datagen.fit(x_train, augment=True, seed=seed) print("#################################\n" "# BUILD AND TRAIN THE NETWORK #\n" "#################################\n") optim = tf.keras.optimizers.Adam(learning_rate=learning_rate_value) # Create model model = create_oztel_model(optimizer=optim, loss=loss) # Check the network created model.summary(line_length=150) os.makedirs(char_dir, exist_ok=True) model_name = os.path.join(char_dir, "model_plot_" + job_identifier + ".png") plot_model(model, to_file=model_name, show_shapes=True, show_layer_names=True)
def ResNetTrainer(network, data, mypath, batchsize, maxepochs): datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # epsilon for ZCA whitening zca_epsilon=1e-06, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # set range for random shear shear_range=0., # set range for random zoom zoom_range=0., # set range for random channel shifts channel_shift_range=0., # set mode for filling points outside the input boundaries fill_mode='nearest', # value used for fill_mode = "constant" cval=0., # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, # set rescaling factor (applied before any other transformation) rescale=None, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). Xtrain, Ytrain, Xval, Yval, Xtest, Ytest, nclasses = data datagen.fit(Xtrain) epoch = 0 print("\nEvaluate network with no training:") TrainL0, TrainA0 = network.evaluate(Xtrain, Ytrain, batch_size=200, verbose=2) ValL0, ValA0 = network.evaluate(Xval, Yval, batch_size=200, verbose=2) TestL0, TestA0 = network.evaluate(Xtest, Ytest, batch_size=200, verbose=2) TrainLoss = np.asarray([TrainL0]) TrainAccuracy = np.asarray([TrainA0]) TestLoss = np.asarray([TestL0]) TestAccuracy = np.asarray([TestA0]) ValLoss = np.asarray([ValL0]) ValAccuracy = np.asarray([ValA0]) mask_perlayer, remaining, total = getpercentages(network) RemainingWeights = np.asarray([remaining]) maxtrainacc = 0 maxvalacc = 0 maxtestacc = 0 lr = 1e-3 while epoch < maxepochs: start_time = time.time() loss, metric = network.metrics_names if epoch == 80: lr = 1e-4 kb.set_value(network.optimizer.lr, lr) if epoch == 120: lr = 1e-5 kb.set_value(network.optimizer.lr, lr) if epoch == 160: lr = 1e-6 kb.set_value(network.optimizer.lr, lr) print('Standard learning rate: ', lr) fit_history = network.fit_generator(datagen.flow(Xtrain, Ytrain, batch_size=batchsize), validation_data=(Xtest, Ytest), epochs=1, verbose=1, workers=1, shuffle=True) TrainLoss = np.append(TrainLoss, fit_history.history[loss]) ValLoss = np.append(ValLoss, fit_history.history['val_loss']) TrainAccuracy = np.append(TrainAccuracy, fit_history.history[metric]) ValAccuracy = np.append(ValAccuracy, fit_history.history['val_' + metric]) maxtrainacc = max(maxtrainacc, TrainAccuracy[-1]) maxtestacc = max(maxtestacc, ValAccuracy[-1]) maxvalacc = max(maxvalacc, ValAccuracy[-1]) print("\nepoch {}/{}".format(epoch + 1, maxepochs)) print("batchsize =", batchsize) print("trn loss = {:.7f}".format(TrainLoss[-1])) print("val loss = {:.7f}".format(ValLoss[-1])) print("tst loss = {:.7f}".format(TestLoss[-1])) print("trn {} = {:.7f}, best {:.7f}".format(metric, TrainAccuracy[-1], maxtrainacc)) print("val {} = {:.7f}, best {:.7f}".format(metric, ValAccuracy[-1], maxvalacc)) print("tst {} = {:.7f}, best {:.7f}".format(metric, ValAccuracy[-1], maxtestacc)) mask_perlayer, remaining, total = getpercentages(network) RemainingWeights = np.append(RemainingWeights, remaining) epoch += 1 print("Output:", mypath) Logs = { "trainLoss": TrainLoss, "testLoss": ValLoss, "valLoss": ValLoss, "trainAccuracy": TrainAccuracy, "testAccuracy": ValAccuracy, "valAccuracy": ValAccuracy, "remainingWeights": RemainingWeights } file = open(mypath + "TrainLogs.pkl", "wb") pickle.dump(Logs, file) file.close() print("Execution time: {:.3f} seconds".format(time.time() - start_time)) print("=" * (len(mypath) + 8)) file = open(mypath + "Masks.pkl", "wb") pickle.dump(getmasks(network), file) file.close() W = [] P = 1 for l in range(1, len(network.layers)): w = network.layers[l].get_weights() # m = network.layers[l].get_mask() if isinstance(w, list): W.append([]) continue # print(w.shape) print("maxw=", np.max(w), "minw=", np.min(w)) # print("maxm=", np.max(m), "minm=", np.min(m)) # print(np.max(m), np.min(m)) P *= np.max(w) W.append(w) # print(w) print("product of all layer's weights:", P) file = open(mypath + "Weights.pkl", "wb") pickle.dump(W, file) file.close() return Logs
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], 1) X_validation = X_validation.reshape(X_validation.shape[0], X_validation.shape[1], X_validation.shape[2], 1) X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], 1) ############################### AUGMENTATAION OF IMAGES: TO MAKEIT MORE GENERIC dataGen = ImageDataGenerator( width_shift_range= 0.1, # 0.1 = 10% IF MORE THAN 1 E.G 10 THEN IT REFFERS TO NO. OF PIXELS EG 10 PIXELS height_shift_range=0.1, zoom_range=0.2, # 0.2 MEANS CAN GO FROM 0.8 TO 1.2 shear_range=0.1, # MAGNITUDE OF SHEAR ANGLE rotation_range=10) # DEGREES dataGen.fit(X_train) batches = dataGen.flow( X_train, y_train, batch_size=20 ) # REQUESTING DATA GENRATOR TO GENERATE IMAGES BATCH SIZE = NO. OF IMAGES CREAED EACH TIME ITS CALLED X_batch, y_batch = next(batches) # TO SHOW AGMENTED IMAGE SAMPLES fig, axs = plt.subplots(1, 15, figsize=(20, 5)) fig.tight_layout() for i in range(15): axs[i].imshow(X_batch[i].reshape(imageDimesion[0], imageDimesion[1])) axs[i].axis('off') plt.show() y_train = to_categorical(y_train, noOfClasses)
!ls X = cv2.imread('ferrari-spider-indian-theluxecafe.jpg') X = cv2.cvtColor(X, cv2.COLOR_BGR2RGB) plt.imshow(X) print(X.shape) IMAGE_SIZE = X.shape X = np.expand_dims(X, axis=0) print(X.shape) y = np.ndarray([1]) print(y.shape) from tensorflow.keras.preprocessing.image import ImageDataGenerator datagen = ImageDataGenerator(zoom_range = [1,6]) datagen.fit(X) batch_size = 10 X_batch = datagen.flow(X, batch_size=batch_size) print(X_batch) show(X_batch, batch_size)
#history = model.fit(cnnInput, y_train_hot_encoded, batch_size=256, epochs=50, shuffle=True, validation_split=0.1) model.summary() # Data augmentation # this is the augmentation configuration we will use for training train_datagen = ImageDataGenerator(width_shift_range=0.2, height_shift_range=0.2, zoom_range=[.6, 1], vertical_flip=True, horizontal_flip=True) train_generator = train_datagen.flow(cnnInput, y_train, batch_size=64, seed=11) valid_datagen = ImageDataGenerator() valid_generator = valid_datagen.flow(cnnValidation, y_valid, batch_size=64, seed=11) train_datagen.fit(cnnInput) valid_datagen.fit(cnnValidation) model.fit_generator(train_datagen.flow(cnnInput, y_train_hot_encoded, batch_size=64), shuffle=True, validation_data=valid_datagen.flow(cnnValidation, y_test_hot_encoded, batch_size=64), class_weight=class_weights, epochs=20)
# aug_image_batch, aug_label_batch = next(augmented_data_gen) val_batch, val_label_batch = next(val_data_gen) X_train = image_batch y_train = label_batch X_valid = val_batch y_valid = val_label_batch #data augmentation train_datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.15, height_shift_range=0.15, shear_range=0.15, zoom_range=0.15, horizontal_flip=True, ) train_datagen.fit(X_train) batch_size = 128 epochs = 50 early_stopping = EarlyStopping( monitor='val_accuracy', min_delta=0.00005, patience=11, verbose=1, restore_best_weights=True, ) lr_scheduler = ReduceLROnPlateau( monitor='val_accuracy', factor=0.5,
def get_augmented( X_train, Y_train, X_val=None, Y_val=None, batch_size=32, seed=0, data_gen_args=dict( rotation_range=10.0, # width_shift_range=0.02, height_shift_range=0.02, shear_range=5, # zoom_range=0.3, horizontal_flip=True, vertical_flip=False, fill_mode="constant", ), ): """[summary] Args: X_train (numpy.ndarray): [description] Y_train (numpy.ndarray): [description] X_val (numpy.ndarray, optional): [description]. Defaults to None. Y_val (numpy.ndarray, optional): [description]. Defaults to None. batch_size (int, optional): [description]. Defaults to 32. seed (int, optional): [description]. Defaults to 0. data_gen_args ([type], optional): [description]. Defaults to dict(rotation_range=10.0,# width_shift_range=0.02,height_shift_range=0.02,shear_range=5,# zoom_range=0.3,horizontal_flip=True,vertical_flip=False,fill_mode="constant",). Returns: [type]: [description] """ # Train data, provide the same seed and keyword arguments to the fit and flow methods X_datagen = ImageDataGenerator(**data_gen_args) Y_datagen = ImageDataGenerator(**data_gen_args) X_datagen.fit(X_train, augment=True, seed=seed) Y_datagen.fit(Y_train, augment=True, seed=seed) X_train_augmented = X_datagen.flow(X_train, batch_size=batch_size, shuffle=True, seed=seed) Y_train_augmented = Y_datagen.flow(Y_train, batch_size=batch_size, shuffle=True, seed=seed) train_generator = zip(X_train_augmented, Y_train_augmented) if not (X_val is None) and not (Y_val is None): # Validation data, no data augmentation, but we create a generator anyway X_datagen_val = ImageDataGenerator(**data_gen_args) Y_datagen_val = ImageDataGenerator(**data_gen_args) X_datagen_val.fit(X_val, augment=True, seed=seed) Y_datagen_val.fit(Y_val, augment=True, seed=seed) X_val_augmented = X_datagen_val.flow(X_val, batch_size=batch_size, shuffle=True, seed=seed) Y_val_augmented = Y_datagen_val.flow(Y_val, batch_size=batch_size, shuffle=True, seed=seed) # combine generators into one which yields image and masks val_generator = zip(X_val_augmented, Y_val_augmented) return train_generator, val_generator else: return train_generator
def train(): print('Loading and preprocessing train data...') print('-' * 30) imgs_train, imgs_mask_train = load_train_data() print('Loaded train images: ', imgs_train.shape, imgs_mask_train.shape) print('-' * 30) # Normalization of the train set #1 and #3 imgs_train = imgs_train.astype('float32') mean = np.mean(imgs_train) # mean for data centering std = np.std(imgs_train) # std for data normalization imgs_train -= mean imgs_train /= std imgs_mask_train = imgs_mask_train.astype('float32') # Normalization of the train set #2 and #4 # imgs_train = rescale_intensity(imgs_train, in_range=(0, 1)) # imgs_train = imgs_train.astype(np.float32) # imgs_mask_train = imgs_mask_train.astype(np.float32) print('Train test split') X_train, X_test, y_train, y_test = train_test_split(imgs_train, imgs_mask_train, test_size=0.1) print('-' * 30) print('Data Augmentation Start') data_gen_args = dict(shear_range=0.1, rotation_range=20, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.3, fill_mode='constant', horizontal_flip=True, vertical_flip=True, cval=0) image_datagen = ImageDataGenerator(**data_gen_args) mask_datagen = ImageDataGenerator(**data_gen_args) seed = 1 image_datagen.fit(X_train, augment=True, seed=seed) mask_datagen.fit(y_train, augment=True, seed=seed) image_generator = image_datagen.flow(X_train, batch_size=BATCH_SIZE) mask_generator = mask_datagen.flow(y_train, batch_size=BATCH_SIZE) train = zip(image_generator, mask_generator) # val = zip(X_test, y_test) print('-' * 30) print('Data Augmentation End') print('-' * 30) print('Creating and compiling model...') print('-' * 30) model = unet() #Saving the weights and the loss of the best predictions we obtained model_checkpoint = ModelCheckpoint( '/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_best.h5', monitor='val_loss', save_best_only=True, mode="min") print('Fitting model...') print('-' * 30) history = model.fit( train, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1, shuffle=True, validation_data=(X_test, y_test), # validation_split=0.1, steps_per_epoch=X_train.shape[0], callbacks=[model_checkpoint]) model.save( '/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_last.h5' ) # convert the history.history dict to a pandas DataFrame: hist_df = pd.DataFrame(history.history) # save to json: hist_json_file = '/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_history.json' with open(hist_json_file, mode='w') as f: hist_df.to_json(f) print("history saved") plt.plot(history.history['dice_coef']) plt.plot(history.history['val_dice_coef']) plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('Model dice coeff') plt.ylabel('Dice coeff') plt.xlabel('Epoch') plt.legend(['Train', 'Val', 'Loss', 'Val Loss'], loc='upper left') # save plot to file plt.savefig( '/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_plot.png' )
def prepare_data(trainX): #data augmentation datagen = ImageDataGenerator(width_shift_range=0.1, \ rotation_range=45, height_shift_range=0.1, horizontal_flip=True) datagen.fit(trainX) return datagen
# 7. example (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data() y_train = tf.keras.utils.to_categorical(y_train, num_classes=10) y_test = tf.keras.utils.to_categorical(y_test, num_classes=10) datagen = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True, vertical_flip=True, fill_mode="constant", cval=0) datagen.fit(x_train) # simple cnn inputs = tf.keras.Input(shape=(32, 32, 3)) x = tf.keras.layers.Conv2D(filters=16, kernel_size=(3, 3), activation='relu', input_shape=(32, 32, 3), strides=(1, 1), name='Conv2D_layer1')(inputs) x = (tf.keras.layers.MaxPooling2D((2, 2), name='Maxpooling1_2D'))(x) x = tf.keras.layers.Conv2D(filters=16, kernel_size=(3, 3), activation='relu', strides=(1, 1), name='Conv2D_layer2')(x)
def get_dataset(config): """Get dataset, either from ``.npz`` files or ``keras.ImageDataGenerator``. Returns Dictionaries with keys ``x_test`` and ``y_test`` if data set was loaded in ``.npz`` format, or with ``dataflow`` key if data will be loaded from ``.jpg``, ``.png``, or ``.bmp`` files by a ``keras.ImageDataGenerator``. Parameters ---------- config: configparser.ConfigParser Settings. Returns ------- normset: dict Used to normalized the network parameters. testset: dict Used to test the networks. """ testset = {} normset = try_get_normset_from_scalefacs(config) dataset_path = config.get('paths', 'dataset_path') is_testset_needed = config.getboolean('tools', 'evaluate_ann') or \ config.getboolean('tools', 'simulate') or \ config.getboolean('loihi', 'normalize_thresholds', fallback=False) is_normset_needed = config.getboolean('tools', 'normalize') and \ normset is None # _______________________________ Keras __________________________________# try: keras_dataset = config.get('input', 'keras_dataset') if keras_dataset: from keras_rewiring.utilities.load_dataset \ import load_and_preprocess_dataset num_to_test = config.getint('simulation', 'num_to_test') data = load_and_preprocess_dataset(keras_dataset) x_test, y_test = data['test'] testset = { 'x_test': x_test[:num_to_test], 'y_test': y_test[:num_to_test]} if is_normset_needed: normset = {'x_norm': x_test} return normset, testset except (NoOptionError, ImportError) as e: print("Warning:", e) # ________________________________ npz ___________________________________# if config.get('input', 'dataset_format') == 'npz': print("Loading data set from '.npz' files in {}.\n".format( dataset_path)) if is_testset_needed: num_to_test = config.getint('simulation', 'num_to_test') testset = { 'x_test': load_npz(dataset_path, 'x_test.npz')[:num_to_test], 'y_test': load_npz(dataset_path, 'y_test.npz')[:num_to_test]} assert testset, "Test set empty." if is_normset_needed: normset = {'x_norm': load_npz(dataset_path, 'x_norm.npz')} assert normset, "Normalization set empty." # ________________________________ jpg ___________________________________# elif config.get('input', 'dataset_format') in {'jpg', 'png'}: from tensorflow.keras.preprocessing.image import ImageDataGenerator print("Loading data set from ImageDataGenerator, using images in " "{}.\n".format(dataset_path)) # Transform str to dict datagen_kwargs = eval(config.get('input', 'datagen_kwargs')) dataflow_kwargs = eval(config.get('input', 'dataflow_kwargs')) # Get class labels class_idx_path = config.get('paths', 'class_idx_path') if class_idx_path != '': class_idx = json.load(open(os.path.abspath(class_idx_path))) dataflow_kwargs['classes'] = \ [class_idx[str(idx)][0] for idx in range(len(class_idx))] # Get proprocessing function if 'preprocessing_function' in datagen_kwargs: from snntoolbox.utils.utils import import_helpers helpers = import_helpers(datagen_kwargs['preprocessing_function'], config) datagen_kwargs['preprocessing_function'] = \ helpers.preprocessing_function dataflow_kwargs['directory'] = dataset_path if 'batch_size' not in dataflow_kwargs: dataflow_kwargs['batch_size'] = config.getint('simulation', 'batch_size') datagen = ImageDataGenerator(**datagen_kwargs) if (datagen.featurewise_center or datagen.featurewise_std_normalization or datagen.zca_whitening): # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied) rs = datagen_kwargs.get('rescale', None) x_orig = ImageDataGenerator(rescale=rs).flow_from_directory( **dataflow_kwargs).next()[0] datagen.fit(x_orig) if is_normset_needed: shuffle = dataflow_kwargs.get('shuffle') dataflow_kwargs['shuffle'] = True normset = { 'dataflow': datagen.flow_from_directory(**dataflow_kwargs)} dataflow_kwargs['shuffle'] = shuffle assert normset, "Normalization set empty." if is_testset_needed: testset = { 'dataflow': datagen.flow_from_directory(**dataflow_kwargs)} assert testset, "Test set empty." # _______________________________ aedat __________________________________# elif config.get('input', 'dataset_format') == 'aedat': if is_normset_needed: normset = {'x_norm': load_npz(dataset_path, 'x_norm.npz')} assert normset, "Normalization set empty." testset = {} return normset, testset