test_size=0.6, random_state=42) total_train = len(train_df) total_val = len(val_df) image_data_gen_train = TiffImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, rotation_range=20, width_shift_range=0.0, height_shift_range=0.0, brightness_range=(0.8, 1.1), shear_range=0.0, zoom_range=(0.9, 1.1), channel_shift_range=0.0, fill_mode='wrap', cval=0.0, horizontal_flip=True, vertical_flip=True, rescale=None, preprocessing_function=None, data_format='channels_last', validation_split=0.2, dtype='float32') image_data_gen_val = TiffImageDataGenerator(dtype='float32') train_data_gen = image_data_gen_train.image_generator_dataframe( train_df,
def main(): if len(sys.argv) != 2: config_file = 'config.ini' else: config_file = sys.argv[1] if not os.path.isfile(config_file): sys.exit('ERROR:\tThe config file %s was not found.' % config_file) sys.stdout.write('\nNot using GPU.\n') os.environ["CUDA_VISIBLE_DEVICES"] = "-1" if len(tf.config.experimental.list_physical_devices('GPU')): print('GPU found. Num GPUs Available: ', len(tf.config.experimental.list_physical_devices('GPU'))) print(tf.config.experimental.list_physical_devices('GPU')) if (len(tf.config.experimental.list_logical_devices('GPU'))): print('Logical GPU found. Num logical GPUs Available: ', len(tf.config.experimental.list_logical_devices('GPU'))) print(tf.config.experimental.list_logical_devices('GPU')) else: print("No GPU found") if len(tf.config.experimental.list_physical_devices('CPU')): print('Physical CPU found. Num Physical CPUs Available: ', len(tf.config.experimental.list_physical_devices('CPU'))) print(tf.config.experimental.list_physical_devices('CPU')) if (len(tf.config.experimental.list_logical_devices('CPU'))): print('Logical CPU found. Num logical CPUs Available: ', len(tf.config.experimental.list_logical_devices('CPU'))) print(tf.config.experimental.list_logical_devices('CPU')) else: print("No CPU found") config = configparser.ConfigParser() config.read(config_file) print("\nConfiguration file:\n") for section in config.sections(): print("Section: %s" % section) for options in config.options(section): print(" %s = %s" % (options, config.get(section, options))) ###### Paths WORKDIR = config['general']['workdir'] #WORKDIR = os.path.abspath(sys.argv[2]) sys.stdout.write('Project directory: %s\n' % WORKDIR) #SRC = os.path.join(WORKDIR, 'src') DATA = os.path.join(WORKDIR, 'data') RESULTS = os.path.join(WORKDIR, 'results') TRAIN_MULTIBAND = config['general']['train_multiband'] #TEST_MULTIBAND = os.path.join(DATA, 'test_multiband') image_catalog = pd.read_csv(os.path.join( DATA, 'catalog/image_catalog2.0train.csv'), comment='#', index_col=0) print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n") # Training parameters batch_size = config['trainparams'].getint( 'batch_size') # orig paper trained all networks with batch_size=128 epochs = config['trainparams'].getint('epochs') num_classes = 1 data_bias = 'none' # Model parameter # ---------------------------------------------------------------------------- # | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch # Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti # |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2) # ---------------------------------------------------------------------------- # ResNet20 | 3 (2)| 92.16 | 91.25 | ----- | ----- | 35 (---) # ResNet32 | 5(NA)| 92.46 | 92.49 | NA | NA | 50 ( NA) # ResNet44 | 7(NA)| 92.50 | 92.83 | NA | NA | 70 ( NA) # ResNet56 | 9 (6)| 92.71 | 93.03 | 93.01 | NA | 90 (100) # ResNet110 |18(12)| 92.65 | 93.39+-.16| 93.15 | 93.63 | 165(180) # ResNet164 |27(18)| ----- | 94.07 | ----- | 94.54 | ---(---) # ResNet1001| (111)| ----- | 92.39 | ----- | 95.08+-.14| ---(---) # --------------------------------------------------------------------------- n = config['trainparams'].getint('n') # Model version # Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2) version = config['trainparams'].getint('resnetversion') # Computed depth from supplied model parameter n if version == 1: depth = n * 6 + 2 elif version == 2: depth = n * 9 + 2 # Model name, depth and version # This is ok if we use weighted losses. lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'), index_col=0) dataframe_for_generator = build_generator_dataframe( lens_df, TRAIN_MULTIBAND) ###### Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below! train_df, val_df = train_test_split( dataframe_for_generator, test_size=config['trainparams'].getfloat('test_fraction'), random_state=42) total_train = len(train_df) total_val = len(val_df) print("The number of objects in the whole training sample is: ", total_train) print("The number of objects in the whole validation sample is: ", total_val) test_fraction = float(config["trainparams"]["test_fraction"]) print("The test fraction is: ", test_fraction) if config['trainparams']['subsample_train'] == 'total': subsample_train = total_train subsample_val = total_val else: try: subsample_train = int(config['trainparams']['subsample_train']) subsample_val = int(subsample_train * test_fraction / (1. - test_fraction)) except: raise ValueError('subsample_train should be \'total\' or int.') print("The number of objects in the training subsample is: ", subsample_train) print("The number of objects in the validation subsample is: ", subsample_val) train_steps_per_epoch = int(subsample_train // batch_size) val_steps_per_epoch = int(subsample_val // batch_size) print("The number of training steps is: ", train_steps_per_epoch) print("The number of validation steps is: ", val_steps_per_epoch) augment_train_data = bool(int(config['trainparams']['augment_train_data'])) ###### Create Tiff Image Data Generator objects for train and validation image_data_gen_train = TiffImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, rotation_range=0, width_shift_range=0.0, height_shift_range=0.0, brightness_range=(0.8, 1.1), shear_range=0.0, zoom_range=(0.9, 1.01), channel_shift_range=0.0, fill_mode='wrap', cval=0.0, horizontal_flip=True, vertical_flip=True, rescale=None, preprocessing_function=None, data_format='channels_last', dtype='float32') image_data_gen_val = TiffImageDataGenerator(dtype='float32') bands = [ config['bands'].getboolean('VIS0'), config['bands'].getboolean('NIR1'), config['bands'].getboolean('NIR2'), config['bands'].getboolean('NIR3') ] print("The bands are: ", bands) ###### Create generators for Images and Labels ratio = 0.5 train_data_gen = image_data_gen_train.prop_image_generator_dataframe( train_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=not (augment_train_data), ratio=ratio, bands=bands) val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=True, ratio=ratio, bands=bands) roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=subsample_val, validation=True, ratio=ratio, bands=bands) ###### Obtain model from the saving directory model_type = 'RN%dv%d' % (depth, version) model_name = '%s_Tr%i_Te%i_bs%i_ep%.03d_aug%i_VIS%i_NIR%i%i%i_DB%s_ratio%.01f_dropout.h5' % ( model_type, subsample_train, subsample_val, batch_size, epochs, int(augment_train_data), config['bands'].getint('VIS0'), config['bands'].getint('NIR1'), config['bands'].getint('NIR2'), config['bands'].getint('NIR3'), config['trainparams']['data_bias'], ratio) model = tf.keras.models.load_model(os.path.join(RESULTS, model_name)) model.summary() history_path = os.path.join(RESULTS, model_name.replace('h5', 'history')) ## History with open(history_path, 'rb') as file_pi: history = pickle.load(file_pi) ## Checkpoints save_dir = os.path.join(RESULTS, 'checkpoints/resnet/') if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) ### Plots fig, ax1 = plt.subplots(1, 1, figsize=(10, 5)) ax2 = ax1.twinx() ax1.plot(range(len(history['loss'])), history['val_loss'], label='Validation loss', marker='o', c='b') ax1.plot(range(len(history['loss'])), history['loss'], label='Training loss', marker='o', c='r') ax1.set_ylim([0, 1]) ax2.plot(range(len(history['loss'])), history['val_acc'], label='Validation accuracy', marker='o', c='b', ls='--', fillstyle='none') ax2.plot(range(len(history['loss'])), history['acc'], label='Training accuracy', marker='o', c='r', ls='--', fillstyle='none') ax1.set_xlabel('Epoch') ax1.legend(loc=(-0.1, 1)) ax2.legend(loc=(0.9, 1)) ax1.set_ylabel('Loss') ax2.set_ylabel('Accuracy') plt.gcf() plt.savefig(os.path.join( RESULTS, 'plots/' + os.path.basename(history_path).replace('.history', '.png')), dpi=200) plt.figure(2) train_auc = np.array(history['auc']) val_auc = np.array(history['val_auc']) plt.plot(np.arange(0, len(val_auc)), train_auc, 'ob', label='Train') plt.plot(np.arange(0, len(val_auc)), val_auc, 'or', label='Validation') plt.xlabel('Epochs') plt.ylabel('AUC') plt.ylim(0, 1) plt.legend() plt.savefig(os.path.join( RESULTS, 'plots/AUC_' + os.path.basename(history_path).replace('.history', '.png')), dpi=200) print("history keys:\n", history.keys()) ##Score #scores = model.evaluate_generator(val_data_gen, verbose=2, steps=val_steps_per_epoch) ##Roc curve images_val, labels_true = next(roc_val_data_gen) labels_score = model.predict(images_val, batch_size=subsample_val, verbose=2) fpr, tpr, thresholds = roc_curve(np.ravel(labels_true), np.ravel(labels_score)) print(fpr) print(tpr) plt.figure(3) plt.plot(fpr, tpr, 'or', label='Validation') plt.xlabel('FPR') plt.ylabel('TPR') plt.xlim(0, 1) plt.ylim(0, 1) plt.plot([0, 1], [0, 1]) plt.legend() plt.savefig(os.path.join( RESULTS, 'plots/ROCsklearn_' + os.path.basename(history_path).replace('.history', '.png')), dpi=200)
def main(): """Fit the model. If checkpoint does not exist in checkpoints or in the results directory, a new model is created and fitted according to parameters set in the config file. If a checkpoint or end model (in results dir) is found, it is loaded and the training is resumed according to values defined in the config file. By default, a checkpoint is preferred over an end model since it is assumed to be more recent (in case of manual stopping of training).""" if len(sys.argv) != 2: config_file = 'config_lesta_df.ini' else: config_file = sys.argv[1] if not os.path.isfile(config_file): sys.exit('ERROR:\tThe config file %s was not found.' % config_file) config = configparser.ConfigParser() config.read(config_file) print("\nConfiguration file:\n") for section in config.sections(): print("Section: %s" % section) for options in config.options(section): print(" %s = %s" % (options, config.get(section, options))) if not bool(config['general'].getboolean('use_gpu')): sys.stdout.write('\nNot using GPU.\n') os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Paths WORKDIR = config['general']['workdir'] sys.stdout.write('Project directory: %s\n' % WORKDIR) SRC = os.path.join(WORKDIR, 'src') DATA = os.path.join(WORKDIR, 'data') RESULTS = os.path.join(WORKDIR, 'results') TRAIN_MULTIBAND = config['general']['train_multiband'] TEST_MULTIBAND = os.path.join(DATA, 'test_multiband') # Catalog lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'), index_col=0) dataframe_for_generator = build_generator_dataframe( lens_df, TRAIN_MULTIBAND) # Extract data proportions for loss weighting n_lens_clean = len(lens_df[lens_df['is_lens'] == True]) n_nolens_clean = len(lens_df[lens_df['is_lens'] == False]) equal_class_coeff = np.array([n_lens_clean / n_nolens_clean, 1]) natural_class_coeff = np.array([1000 * n_lens_clean / n_nolens_clean, 1]) # Training parameters batch_size = config['trainparams'].getint('batch_size') epochs = config['trainparams'].getint('epochs') data_bias = config['trainparams']['data_bias'] test_fraction = config['trainparams'].getfloat('test_fraction') augment_train_data = bool(int(config['trainparams']['augment_train_data'])) kernel_size_1 = int(config['trainparams']['kernel_size_1']) kernel_size_2 = int(config['trainparams']['kernel_size_2']) dropout_config = config['trainparams'][ 'dropout_kind'] #Import dropout and check values are valid. if dropout_config == 'dropout': dropout_kind = Dropout elif dropout_config == 'spatialdropout': dropout_kind = SpatialDropout2D else: raise NotImplementedError( 'dropout_kind must be \'dropout\' or \'spatialdropout\'\nPlease check config file.' ) pool_size = int(config['trainparams']['pool_size']) bands = [ config['bands'].getboolean('VIS0'), config['bands'].getboolean('NIR1'), config['bands'].getboolean('NIR2'), config['bands'].getboolean('NIR3') ] print("The bands are: ", bands) binary = bool(int(config['general']['binary'])) ratio = float(config['trainparams']['lens_nolens_ratio']) # Split catalog in train and test (validation) sets. We used fixed state 42. train_df, val_df = train_test_split(dataframe_for_generator, test_size=test_fraction, random_state=42) total_train = len(train_df) total_val = len(val_df) print("The number of objects in the whole training sample is: ", total_train) print("The number of objects in the whole validation sample is: ", total_val) print("The test fraction is: ", test_fraction) if config['trainparams'][ 'subsample_train'] == 'total': #Import subsample size and check values are as expected. subsample_train = total_train subsample_val = total_val else: try: subsample_train = int(config['trainparams']['subsample_train']) subsample_val = int(subsample_train * test_fraction / (1. - test_fraction)) except: raise ValueError('subsample_train should be \'total\' or int.') print("The number of objects in the training subsample is: ", subsample_train) print("The number of objects in the validation subsample is: ", subsample_val) train_steps_per_epoch = int(subsample_train // batch_size) val_steps_per_epoch = int(subsample_val // batch_size) print("The number of training steps is: ", train_steps_per_epoch) print("The number of validation steps is: ", val_steps_per_epoch) # Create TiffImageDataGenerator objects to inherit random transformations from Keras' class. image_data_gen_train = TiffImageDataGenerator(featurewise_center=False, rotation_range=0, fill_mode='wrap', horizontal_flip=True, vertical_flip=True, preprocessing_function=None, data_format='channels_last', dtype='float32') image_data_gen_val = TiffImageDataGenerator(dtype='float32') # Create Generator objects from the initialized TiffImageDataGenerators. # To train train_data_gen = image_data_gen_train.prop_image_generator_dataframe( train_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=not (augment_train_data), ratio=ratio, bands=bands, binary=binary) # To validate val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=True, ratio=ratio, bands=bands, binary=binary) # To predict/evaluate roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=True, ratio=ratio, bands=bands, binary=binary) # To safely obtain image size temp_data_gen = image_data_gen_train.image_generator_dataframe( train_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=1, validation=True, bands=bands, binary=binary) # Obtain image size image, _ = next(temp_data_gen) input_shape = image[0].shape # Define correct bias to initialize (use if not forcing generator to load equal proportions of data) output_bias = tf.keras.initializers.Constant( np.log(n_lens_clean / n_nolens_clean)) # Path to save checkpoints model_type = 'lastro_cnn' save_dir = os.path.join(RESULTS, 'checkpoints/%s/' % model_type) model_name = '%s_Tr%i_Te%i_bs%i_ep%.03d_aug%i_VIS%i_NIR%i%i%i_DB%s_ratio%.01f_ks%i%i_ps%i_%s_%s.h5' % ( model_type, subsample_train, subsample_val, batch_size, epochs, int(augment_train_data), bands[0], bands[1], bands[2], bands[3], data_bias, ratio, kernel_size_1, kernel_size_2, pool_size, dropout_kind.__name__, os.path.basename(TRAIN_MULTIBAND)) # Create path of checkpoints if necessary if not os.path.isdir(save_dir): os.makedirs(save_dir) # Checkpoint file path checkpoint_filepath = os.path.join(save_dir, model_name) # Final model file path end_model_name = os.path.join(RESULTS, model_name) print("The model name is: ", model_name) history_path = os.path.join(RESULTS, model_name.replace('h5', 'history')) # Callbacks # Checkpoint callback to save every epoch for better resuming training cp_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_filepath, save_best_only=False, verbose=1, monitor='val_acc', save_freq='epoch') # Checkpoint best model cp_best_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_filepath.replace('.h5', '_BEST.h5'), save_best_only=True, verbose=1, monitor='val_acc') # Early stopping callback (currently using a very high patience to avoid it triggering) es_callback = tf.keras.callbacks.EarlyStopping( monitor='val_acc', #min_delta=0.1, patience=30, verbose=1, mode='auto', baseline=None, restore_best_weights=True) # Learning rate reducer callback. lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=20, min_lr=0.5e-6, monitor='val_acc', verbose=1, mode='auto') # Callback to save log to csv. It is probably better to use this than save-resume history. logger_callback = tf.keras.callbacks.CSVLogger(checkpoint_filepath.replace( '.h5', '.log'), separator=',', append=True) # Callback to resume history if history_path exists. history_callback = ResumeHistory(history_path) # Callback to use Tensorboard log_dir = os.path.join( RESULTS, "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) if not os.path.isdir(log_dir): os.mkdir(log_dir) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, profile_batch=0) # Define metrics for the model. metrics = [ keras.metrics.TruePositives(name='tp'), keras.metrics.FalsePositives(name='fp'), keras.metrics.TrueNegatives(name='tn'), keras.metrics.FalseNegatives(name='fn'), keras.metrics.BinaryAccuracy(name='acc'), keras.metrics.AUC(name='auc') ] # If there are no checkpoints or final models saved, compile a new one. if not os.path.isfile(checkpoint_filepath) and not os.path.isfile( end_model_name): model = build_lastro_model(kernel_size_1, kernel_size_2, pool_size, input_shape, dropout_kind) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=metrics) elif not os.path.isfile(checkpoint_filepath) and os.path.isfile( end_model_name): print('Loading existing model from result.') model = tf.keras.models.load_model(end_model_name) epochs = int(config['trainparams']['new_epochs']) learning_rate = config['trainparams']['learning_rate'] change_learning_rate(model, learning_rate) elif os.path.isfile(checkpoint_filepath): print('Loading existing model from checkpoint.') model = tf.keras.models.load_model(checkpoint_filepath) epochs = int(config['trainparams']['new_epochs']) learning_rate = config['trainparams']['learning_rate'] change_learning_rate(model, learning_rate) model.summary() # Define class weights for unevenly distributed (biased) dataset. if data_bias == 'natural': sys.stdout.write( 'Using natural data bias: 1000x more non lenses than lenses.\n') class_coeff = natural_class_coeff elif data_bias == 'none': sys.stdout.write( 'Using no data bias (simulate equal proportion among classes).\n') class_coeff = equal_class_coeff elif data_bias == 'raw': sys.stdout.write('Using the raw bias (no weights applied).\n') class_coeff = [1, 1] else: raise NotImplementedError( 'data_bias must be either natural, none or raw.') class_weights = {0: class_coeff[0], 1: class_coeff[1]} sys.stdout.write('Using loss weights: %s\n' % class_weights) # Fit the model and save the history callback. # Use multiprocessing True when using > 1 workers. (Seems to cause problems) # Expect tf update to use threadsafe_iter class. history = model.fit_generator( train_data_gen, steps_per_epoch=subsample_train // batch_size, epochs=epochs, validation_data=val_data_gen, validation_steps=subsample_val // batch_size, callbacks=[ cp_callback, es_callback, lr_reducer, cp_best_callback, history_callback, logger_callback, tensorboard_callback ], class_weight=class_weights, # use_multiprocessing=True, verbose=1, # workers=16 ) model.save(end_model_name) # If training finishes normally (Is not stopped by user), save final model. # Save complete history if the training was resumed. if history_callback.use_history_file_flag: with open(history_path, 'wb') as file_pi: pickle.dump(history_callback.complete_history, file_pi) else: with open(history_path, 'wb') as file_pi: pickle.dump(history.history, file_pi) # Score trained model. scores = model.evaluate_generator(val_data_gen, verbose=2, steps=val_steps_per_epoch) images_val, labels_true = next(roc_val_data_gen) labels_score = model.predict(images_val, batch_size=batch_size, verbose=2) fpr, tpr, thresholds = roc_curve(np.ravel(labels_true), np.ravel(labels_score)) auc = history.history['val_auc'][-1] acc = history.history['val_acc'][-1] # Save TPR and FPR metrics to plot ROC. np.savetxt(os.path.join(RESULTS, model_name.replace('h5', 'FPRvsTPR.dat')), np.array([fpr, tpr]).T, header='auc=%.3f\nacc=%.3f' % (auc, acc))
def main(): if len(sys.argv) != 2: config_file = 'config_resnet.ini' else: config_file = sys.argv[1] if not os.path.isfile(config_file): sys.exit('ERROR:\tThe config file %s was not found.' % config_file) if len(tf.config.experimental.list_physical_devices('GPU')): print('GPU found. Num GPUs Available: ', len(tf.config.experimental.list_physical_devices('GPU'))) print(tf.config.experimental.list_physical_devices('GPU')) if len(tf.config.experimental.list_logical_devices('GPU')): print('Logical GPU found. Num logical GPUs Available: ', len(tf.config.experimental.list_logical_devices('GPU'))) print(tf.config.experimental.list_logical_devices('GPU')) else: print("No GPU found") if len(tf.config.experimental.list_physical_devices('CPU')): print('Physical CPU found. Num Physical CPUs Available: ', len(tf.config.experimental.list_physical_devices('CPU'))) print(tf.config.experimental.list_physical_devices('CPU')) if len(tf.config.experimental.list_logical_devices('CPU')): print('Logical CPU found. Num logical CPUs Available: ', len(tf.config.experimental.list_logical_devices('CPU'))) print(tf.config.experimental.list_logical_devices('CPU')) else: print("No CPU found") config = configparser.ConfigParser() config.read(config_file) print("\nConfiguration file:\n") for section in config.sections(): print("Section: %s" % section) for options in config.options(section): print(" %s = %s" % (options, config.get(section, options))) if not bool(config['general'].getboolean('use_gpu')): sys.stdout.write('\nNot using GPU.\n') os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Paths WORKDIR = config['general']['workdir'] #WORKDIR = os.path.abspath(sys.argv[2]) sys.stdout.write('Project directory: %s\n' % WORKDIR) #SRC = os.path.join(WORKDIR, 'src') DATA = os.path.join(WORKDIR, 'data') RESULTS = os.path.join(WORKDIR, 'results') TRAIN_MULTIBAND = config['general']['train_multiband'] #TEST_MULTIBAND = os.path.join(DATA, 'test_multiband') image_catalog = pd.read_csv(os.path.join( DATA, 'catalog/image_catalog2.0train.csv'), comment='#', index_col=0) print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n") # Training parameters # orig paper trained all networks with batch_size=128 batch_size = config['trainparams'].getint('batch_size') epochs = config['trainparams'].getint('epochs') num_classes = 1 data_bias = config['trainparams']['data_bias'] # Model parameter # ---------------------------------------------------------------------------- # | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch # Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti # |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2) # ---------------------------------------------------------------------------- # ResNet20 | 3 (2)| 92.16 | 91.25 | ----- | ----- | 35 (---) # ResNet32 | 5(NA)| 92.46 | 92.49 | NA | NA | 50 ( NA) # ResNet44 | 7(NA)| 92.50 | 92.83 | NA | NA | 70 ( NA) # ResNet56 | 9 (6)| 92.71 | 93.03 | 93.01 | NA | 90 (100) # ResNet110 |18(12)| 92.65 | 93.39+-.16| 93.15 | 93.63 | 165(180) # ResNet164 |27(18)| ----- | 94.07 | ----- | 94.54 | ---(---) # ResNet1001| (111)| ----- | 92.39 | ----- | 95.08+-.14| ---(---) # --------------------------------------------------------------------------- n = config['trainparams'].getint('n') # Model version # Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2) version = config['trainparams'].getint('resnetversion') # Computed depth from supplied model parameter n if version == 1: depth = n * 6 + 2 elif version == 2: depth = n * 9 + 2 # Model name, depth and version model_type = 'RN%dv%d' % (depth, version) # Create the dataframe containing filenames and labels. # This is ok if we use weighted losses. lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'), index_col=0) dataframe_for_generator = build_generator_dataframe( lens_df, TRAIN_MULTIBAND) # Extract data proportions for loss weighting n_lens_clean = len(lens_df[lens_df['is_lens'] == True]) n_nolens_clean = len(lens_df[lens_df['is_lens'] == False]) equal_class_coeff = np.array([n_lens_clean / n_nolens_clean, 1]) natural_class_coeff = np.array([1000 * n_lens_clean / n_nolens_clean, 1]) # Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below! train_df, val_df = train_test_split( dataframe_for_generator, test_size=config['trainparams'].getfloat('test_fraction'), random_state=42) # before it was 42 total_train = len(train_df) total_val = len(val_df) print("The number of objects in the whole training sample is: ", total_train) print("The number of objects in the whole validation sample is: ", total_val) test_fraction = float(config["trainparams"]["test_fraction"]) print("The test fraction is: ", test_fraction) if config['trainparams']['subsample_train'] == 'total': subsample_train = total_train subsample_val = total_val else: try: subsample_train = int(config['trainparams']['subsample_train']) subsample_val = int(subsample_train * test_fraction / (1. - test_fraction)) except: raise ValueError('subsample_train should be \'total\' or int.') print("The number of objects in the training subsample is: ", subsample_train) print("The number of objects in the validation subsample is: ", subsample_val) train_steps_per_epoch = int(subsample_train // batch_size) val_steps_per_epoch = int(subsample_val // batch_size) print("The number of training steps is: ", train_steps_per_epoch) print("The number of validation steps is: ", val_steps_per_epoch) augment_train_data = bool(int(config['trainparams']['augment_train_data'])) # Create Tiff Image Data Generator objects for train and validation image_data_gen_train = TiffImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, #rotation_range=0, #width_shift_range=0.0, #height_shift_range=0.0, #brightness_range=(0.99, 1.01), #shear_range=0.0, #zoom_range=(0.99, 1.01), #channel_shift_range=0.0, #fill_mode='wrap', #cval=0.0, horizontal_flip=True, vertical_flip=True, rescale=None, preprocessing_function=None, data_format='channels_last', dtype='float32') image_data_gen_val = TiffImageDataGenerator(dtype='float32') bands = [ config['bands'].getboolean('VIS0'), config['bands'].getboolean('NIR1'), config['bands'].getboolean('NIR2'), config['bands'].getboolean('NIR3') ] print("The bands are: ", bands) # Create generators for Images and Labels ratio = float(config['trainparams']['lens_nolens_ratio']) binary = bool(int(config['general']['binary'])) train_data_gen = image_data_gen_train.prop_image_generator_dataframe( train_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=not (augment_train_data), ratio=ratio, bands=bands, binary=binary) val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=True, ratio=ratio, bands=bands, binary=binary) roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=subsample_val, validation=True, ratio=ratio, bands=bands, binary=binary) # Obtain the shape of the input data (train images) temp_data_gen = image_data_gen_train.image_generator_dataframe( train_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=batch_size, validation=True, bands=bands) image, _ = next(temp_data_gen) input_shape = image[0].shape # Create model if version == 2: model = myf.resnet_v2(input_shape=input_shape, depth=depth, num_classes=num_classes) else: model = myf.resnet_v1(input_shape=input_shape, depth=depth, num_classes=num_classes) # Define metrics for the model. metrics = [ keras.metrics.TruePositives(name='tp'), keras.metrics.FalsePositives(name='fp'), keras.metrics.TrueNegatives(name='tn'), keras.metrics.FalseNegatives(name='fn'), keras.metrics.BinaryAccuracy(name='acc'), keras.metrics.AUC(name='auc') ] model.compile( loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=myf.lr_schedule(0)), metrics=metrics) model.summary() model.get_config() # Prepare model model saving directory. save_dir = os.path.join(RESULTS, 'checkpoints/resnet/') model_name = '%s_Tr%i_Te%i_bs%i_ep%.03d_aug%i_VIS%i_NIR%i%i%i_DB%s_ratio%.01f_dropout.h5' % ( model_type, subsample_train, subsample_val, batch_size, epochs, int(augment_train_data), config['bands'].getint('VIS0'), config['bands'].getint('NIR1'), config['bands'].getint('NIR2'), config['bands'].getint('NIR3'), config['trainparams']['data_bias'], ratio) if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) print("The model name is: ", model_name) # Prepare callbacks for model saving and for learning rate adjustment. checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=filepath, monitor='val_acc', verbose=1, save_best_only=True) # save_weights_only=True lr_scheduler = tf.keras.callbacks.LearningRateScheduler(myf.lr_schedule) lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) callbacks = [checkpoint, lr_reducer, lr_scheduler] # Define class weights for unevenly distributed (biased) dataset. if data_bias == 'natural': sys.stdout.write( 'Using natural data bias: 1000x more non lenses than lenses.\n') class_coeff = natural_class_coeff elif data_bias == 'none': sys.stdout.write( 'Using no data bias (simulate equal proportion among classes).\n') class_coeff = equal_class_coeff elif data_bias == 'raw': sys.stdout.write('Using the raw bias (no weights applied).\n') class_coeff = [1., 1.] else: raise NotImplementedError('data_bias must be either natural or none.') class_weights = {0: class_coeff[0], 1: class_coeff[1]} sys.stdout.write('Using weights: %s\n' % class_weights) # Train the ResNet print('Train the ResNet using real-time data augmentation.') history = model.fit_generator(train_data_gen, steps_per_epoch=train_steps_per_epoch, epochs=epochs, validation_data=val_data_gen, validation_steps=val_steps_per_epoch, callbacks=callbacks, class_weight=class_weights, use_multiprocessing=True, verbose=2) # Score trained model. scores = model.evaluate_generator(val_data_gen, verbose=2, steps=val_steps_per_epoch) #images_val, labels_true = next(roc_val_data_gen) #labels_score = model.predict(images_val, batch_size=subsample_val, verbose=2) #fpr, tpr, thresholds = roc_curve(np.ravel(labels_true), np.ravel(labels_score)) # print(fpr) # print(tpr) model.save(os.path.join(RESULTS, model_name)) with open(os.path.join(RESULTS, model_name.replace('h5', 'history')), 'wb') as file_pi: pickle.dump(history.history, file_pi) print('Test loss:', scores[0]) print('Test accuracy:', scores[1])
def main(): if len(sys.argv) == 2: config_file = 'config_lastro.ini' model_name = sys.argv[1] elif len(sys.argv) == 3: config_file = sys.argv[1] model_name = sys.argv[2] if not os.path.isfile(config_file): sys.exit('ERROR:\tThe config file %s was not found.' % config_file) # Avoid using GPU to evaluate models. sys.stdout.write('\nNot using GPU.\n') os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Import configuration file config = configparser.ConfigParser() config.read(config_file) # Extract parameters from model name if 'train_multiband_bin' in model_name: datadir = 'train_multiband_bin' elif 'train_multiband_noclip_bin' in model_name: datadir = 'train_multiband_noclip_bin' else: datadir = 'train_multiband_noclip_bin' # Extract bands from filename bands = [] if 'VIS0' in model_name: bands.append(False) elif 'VIS1' in model_name: bands.append(True) if 'NIR000' in model_name: [bands.append(False) for i in range(3)] elif 'NIR111' in model_name: [bands.append(True) for i in range(3)] bands = list(np.array(bands).reshape(-1)) print("The bands are: ", bands) # Extract split ratio from filename for param in model_name.split('_'): if 'ratio' in param: ratio = float(param.replace('ratio', '')) # Paths WORKDIR = config['general']['workdir'] sys.stdout.write('Project directory: %s\n' % WORKDIR) DATA = os.path.join(WORKDIR, 'data') RESULTS = os.path.join(WORKDIR, 'results') TRAIN_MULTIBAND = os.path.join(DATA, datadir) image_catalog = pd.read_csv(os.path.join( DATA, 'catalog/image_catalog2.0train.csv'), comment='#', index_col=0) print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n") lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels_old.csv'), index_col=0) lens_df_new = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'), index_col=0) dataframe_for_generator = build_generator_dataframe( lens_df, TRAIN_MULTIBAND) # Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below! train_df, val_df = train_test_split( dataframe_for_generator, test_size=config['trainparams'].getfloat('test_fraction'), random_state=42) total_train = len(train_df) total_val = len(val_df) print(train_df) sys.exit(0) print("The number of objects in the whole training sample is: ", total_train) print("The number of objects in the whole validation sample is: ", total_val) test_fraction = float(config["trainparams"]["test_fraction"]) print("The test fraction is: ", test_fraction) if config['trainparams']['subsample_train'] == 'total': subsample_train = total_train subsample_val = total_val else: try: subsample_train = int(config['trainparams']['subsample_train']) subsample_val = int(subsample_train * test_fraction / (1. - test_fraction)) except: raise ValueError('subsample_train should be \'total\' or int.') print("The number of objects in the training subsample is: ", subsample_train) print("The number of objects in the validation subsample is: ", subsample_val) augment_train_data = bool(int(config['trainparams']['augment_train_data'])) # Create Tiff Image Data Generator objects for train and validation image_data_gen_train = TiffImageDataGenerator(featurewise_center=False, rotation_range=0, fill_mode='wrap', horizontal_flip=True, vertical_flip=True, preprocessing_function=None, data_format='channels_last', dtype='float32') image_data_gen_val = TiffImageDataGenerator(dtype='float32') # Create generators for Images and Labels roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe( val_df, directory=TRAIN_MULTIBAND, x_col='filenames', y_col='labels', batch_size=subsample_val, validation=True, ratio=ratio, bands=bands, binary=True) # Obtain model from the saving directory model_name_base = os.path.basename(model_name) model = tf.keras.models.load_model(model_name) model.summary() history_path = model_name.replace('h5', 'history') # Checkpoints dir save_dir = os.path.join(RESULTS, 'checkpoints/lastro_cnn/') if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name_base) # Plots # History if os.path.isfile(history_path): with open(history_path, 'rb') as file_pi: history = pickle.load(file_pi) fig, ax1 = plt.subplots(1, 1, figsize=(10, 5)) ax2 = ax1.twinx() ax1.plot( range(len(history['loss'])), history['val_loss'], label='Validation loss', # marker='o', c='b', lw=3) ax1.plot( range(len(history['loss'])), history['loss'], label='Training loss', # marker='o', c='r', lw=3) ax2.set_ylim([0.5, 1]) ax2.plot( range(len(history['loss'])), history['val_acc'], label='Validation accuracy', # marker='^', c='b', ls='--', fillstyle='none', lw=3) ax2.plot( range(len(history['loss'])), history['acc'], label='Training accuracy', # marker='^', c='r', ls='--', fillstyle='none', lw=3) ax1.set_xlabel('Epoch') ax1.legend(loc=(-0.1, 1)) ax2.legend(loc=(0.9, 1)) ax1.set_ylabel('Loss') ax2.set_ylabel('Accuracy') plt.gcf() plt.savefig(os.path.join( RESULTS, 'plots/' + os.path.basename(history_path).replace('.history', '.png')), dpi=200) # Roc curve images_val, labels_true = next(roc_val_data_gen) print(labels_true) labels_score = model.predict(images_val, batch_size=1, verbose=2, workers=16, use_multiprocessing=True) fpr, tpr, thresholds = roc_curve(np.ravel(labels_true), np.ravel(labels_score)) scores = model.evaluate(images_val, labels_true, batch_size=True, verbose=1, workers=16, use_multiprocessing=True) scores_dict = { metric: value for metric, value in zip(model.metrics_names, scores) } print(scores) print(model.metrics_names) acc = scores_dict['acc'] auc = scores_dict['auc'] np.savetxt(os.path.join(RESULTS, model_name_base.replace('h5', 'FPRvsTPR.dat')), np.array([fpr, tpr]).T, header='auc=%.3f\nacc=%.3f' % (auc, acc)) plt.figure(2) plt.xlabel('FPR') plt.ylabel('TPR') plt.xlim(0, 1) plt.ylim(0, 1) plt.plot([0, 1], [0, 1]) plt.legend() plt.plot(fpr, tpr, label='Validation\nAUC=%.3f\nACC=%.3f' % (auc, acc), lw=3) plt.xlabel('FPR') plt.ylabel('TPR') plt.xlim(0, 1) plt.ylim(0, 1) plt.plot([0, 1], [0, 1], lw=3) plt.legend() plt.savefig(os.path.join( RESULTS, 'plots/ROCsklearn_' + os.path.basename(model_name).replace('.h5', '.png')), dpi=200)
def main(): if len(sys.argv) == 2: config_file = 'config_lesta_df.ini' model_name = sys.argv[1] elif len(sys.argv) == 3: config_file = sys.argv[1] model_name = sys.argv[2] else: sys.exit( 'ERROR:\tUnexpected number of arguments.\nUSAGE:\t%s [CONFIG_FILE] MODEL_FILENAME' % sys.argv[0]) if not os.path.isfile(config_file): sys.exit('ERROR:\tThe config file %s was not found.' % config_file) if not os.path.isfile(model_name): sys.exit('ERROR:\tThe model file %s was not found.' % model_name) # Import configuration file config = configparser.ConfigParser() config.read(config_file) # Extract parameters from model name if 'train_multiband_bin' in model_name: datadir = 'train_multiband_bin' elif 'train_multiband_noclip_bin' in model_name: datadir = 'train_multiband_noclip_bin' else: datadir = 'train_multiband_noclip_bin' # Extract bands from filename bands = [ bool(int(config['bands']['VIS0'])), bool(int(config['bands']['NIR1'])), bool(int(config['bands']['NIR2'])), bool(int(config['bands']['NIR3'])) ] print("The bands are: ", bands) # Paths WORKDIR = config['general']['workdir'] sys.stdout.write('Project directory: %s\n' % WORKDIR) DATA = os.path.join(WORKDIR, 'data') RESULTS = os.path.join(WORKDIR, 'results') TRAIN_MULTIBAND = config['general']['train_multiband'] TEST_MULTIBAND = TRAIN_MULTIBAND.replace('train', 'test') image_catalog = pd.read_csv(os.path.join( DATA, 'catalog/image_catalog2.0train.csv'), comment='#', index_col=0) print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n") lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'), index_col=0) dataframe_for_generator = build_generator_dataframe( lens_df, TRAIN_MULTIBAND) print(dataframe_for_generator['filenames']) # Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below! train_df, val_df = train_test_split( dataframe_for_generator, test_size=config['trainparams'].getfloat('test_fraction'), random_state=42) total_train = len(train_df) total_val = len(val_df) print("The number of objects in the whole training sample is: ", total_train) print("The number of objects in the whole validation sample is: ", total_val) test_fraction = float(config["trainparams"]["test_fraction"]) print("The test fraction is: ", test_fraction) if config['trainparams']['subsample_train'] == 'total': subsample_train = total_train subsample_val = total_val else: try: subsample_train = int(config['trainparams']['subsample_train']) subsample_val = int(subsample_train * test_fraction / (1. - test_fraction)) except: raise ValueError('subsample_train should be \'total\' or int.') print("The number of objects in the training subsample is: ", subsample_train) print("The number of objects in the validation subsample is: ", subsample_val) # Create Tiff Image Data Generator objects for train and validation image_data_gen_val = TiffImageDataGenerator(dtype='float32') # Create generators for Images and Labels prediction_ids = [] test_data_gen = image_data_gen_val.generator_from_directory( directory=TEST_MULTIBAND, id_logger=prediction_ids, batch_size=1, bands=bands, binary=True) # Obtain model from the saving directory model_name_base = os.path.basename(model_name) model = tf.keras.models.load_model(model_name) model.summary() len_gen = len(os.listdir(TRAIN_MULTIBAND)) print(len_gen) predictions = model.predict_generator(test_data_gen, verbose=1, steps=len_gen) int_prediction_ids = np.array([get_file_id(fn) for fn in prediction_ids], dtype=int) np.savetxt(os.path.join(RESULTS, model_name_base.replace('.h5', 'predictions.dat')), np.array([ np.squeeze(int_prediction_ids).astype(int), np.squeeze(predictions) ]).T, fmt='%i %.5f')