def main():
if len(sys.argv) != 2:
config_file = 'config.ini'
else:
config_file = sys.argv[1]
if not os.path.isfile(config_file):
sys.exit('ERROR:\tThe config file %s was not found.' % config_file)
sys.stdout.write('\nNot using GPU.\n')
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
if len(tf.config.experimental.list_physical_devices('GPU')):
print('GPU found. Num GPUs Available: ',
len(tf.config.experimental.list_physical_devices('GPU')))
print(tf.config.experimental.list_physical_devices('GPU'))
if (len(tf.config.experimental.list_logical_devices('GPU'))):
print('Logical GPU found. Num logical GPUs Available: ',
len(tf.config.experimental.list_logical_devices('GPU')))
print(tf.config.experimental.list_logical_devices('GPU'))
else:
print("No GPU found")
if len(tf.config.experimental.list_physical_devices('CPU')):
print('Physical CPU found. Num Physical CPUs Available: ',
len(tf.config.experimental.list_physical_devices('CPU')))
print(tf.config.experimental.list_physical_devices('CPU'))
if (len(tf.config.experimental.list_logical_devices('CPU'))):
print('Logical CPU found. Num logical CPUs Available: ',
len(tf.config.experimental.list_logical_devices('CPU')))
print(tf.config.experimental.list_logical_devices('CPU'))
else:
print("No CPU found")
config = configparser.ConfigParser()
config.read(config_file)
print("\nConfiguration file:\n")
for section in config.sections():
print("Section: %s" % section)
for options in config.options(section):
print(" %s = %s" % (options, config.get(section, options)))
###### Paths
WORKDIR = config['general']['workdir']
#WORKDIR = os.path.abspath(sys.argv[2])
sys.stdout.write('Project directory: %s\n' % WORKDIR)
#SRC = os.path.join(WORKDIR, 'src')
DATA = os.path.join(WORKDIR, 'data')
RESULTS = os.path.join(WORKDIR, 'results')
TRAIN_MULTIBAND = config['general']['train_multiband']
#TEST_MULTIBAND = os.path.join(DATA, 'test_multiband')
image_catalog = pd.read_csv(os.path.join(
DATA, 'catalog/image_catalog2.0train.csv'),
comment='#',
index_col=0)
print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n")
# Training parameters
batch_size = config['trainparams'].getint(
'batch_size') # orig paper trained all networks with batch_size=128
epochs = config['trainparams'].getint('epochs')
num_classes = 1
data_bias = 'none'
# Model parameter
# ----------------------------------------------------------------------------
# | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch
# Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti
# |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)
# ----------------------------------------------------------------------------
# ResNet20 | 3 (2)| 92.16 | 91.25 | ----- | ----- | 35 (---)
# ResNet32 | 5(NA)| 92.46 | 92.49 | NA | NA | 50 ( NA)
# ResNet44 | 7(NA)| 92.50 | 92.83 | NA | NA | 70 ( NA)
# ResNet56 | 9 (6)| 92.71 | 93.03 | 93.01 | NA | 90 (100)
# ResNet110 |18(12)| 92.65 | 93.39+-.16| 93.15 | 93.63 | 165(180)
# ResNet164 |27(18)| ----- | 94.07 | ----- | 94.54 | ---(---)
# ResNet1001| (111)| ----- | 92.39 | ----- | 95.08+-.14| ---(---)
# ---------------------------------------------------------------------------
n = config['trainparams'].getint('n')
# Model version
# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)
version = config['trainparams'].getint('resnetversion')
# Computed depth from supplied model parameter n
if version == 1:
depth = n * 6 + 2
elif version == 2:
depth = n * 9 + 2
# Model name, depth and version
# This is ok if we use weighted losses.
lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'),
index_col=0)
dataframe_for_generator = build_generator_dataframe(
lens_df, TRAIN_MULTIBAND)
###### Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below!
train_df, val_df = train_test_split(
dataframe_for_generator,
test_size=config['trainparams'].getfloat('test_fraction'),
random_state=42)
total_train = len(train_df)
total_val = len(val_df)
print("The number of objects in the whole training sample is: ",
total_train)
print("The number of objects in the whole validation sample is: ",
total_val)
test_fraction = float(config["trainparams"]["test_fraction"])
print("The test fraction is: ", test_fraction)
if config['trainparams']['subsample_train'] == 'total':
subsample_train = total_train
subsample_val = total_val
else:
try:
subsample_train = int(config['trainparams']['subsample_train'])
subsample_val = int(subsample_train * test_fraction /
(1. - test_fraction))
except:
raise ValueError('subsample_train should be \'total\' or int.')
print("The number of objects in the training subsample is: ",
subsample_train)
print("The number of objects in the validation subsample is: ",
subsample_val)
train_steps_per_epoch = int(subsample_train // batch_size)
val_steps_per_epoch = int(subsample_val // batch_size)
print("The number of training steps is: ", train_steps_per_epoch)
print("The number of validation steps is: ", val_steps_per_epoch)
augment_train_data = bool(int(config['trainparams']['augment_train_data']))
###### Create Tiff Image Data Generator objects for train and validation
image_data_gen_train = TiffImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
zca_epsilon=1e-06,
rotation_range=0,
width_shift_range=0.0,
height_shift_range=0.0,
brightness_range=(0.8, 1.1),
shear_range=0.0,
zoom_range=(0.9, 1.01),
channel_shift_range=0.0,
fill_mode='wrap',
cval=0.0,
horizontal_flip=True,
vertical_flip=True,
rescale=None,
preprocessing_function=None,
data_format='channels_last',
dtype='float32')
image_data_gen_val = TiffImageDataGenerator(dtype='float32')
bands = [
config['bands'].getboolean('VIS0'), config['bands'].getboolean('NIR1'),
config['bands'].getboolean('NIR2'), config['bands'].getboolean('NIR3')
]
print("The bands are: ", bands)
###### Create generators for Images and Labels
ratio = 0.5
train_data_gen = image_data_gen_train.prop_image_generator_dataframe(
train_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=not (augment_train_data),
ratio=ratio,
bands=bands)
val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=True,
ratio=ratio,
bands=bands)
roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=subsample_val,
validation=True,
ratio=ratio,
bands=bands)
###### Obtain model from the saving directory
model_type = 'RN%dv%d' % (depth, version)
model_name = '%s_Tr%i_Te%i_bs%i_ep%.03d_aug%i_VIS%i_NIR%i%i%i_DB%s_ratio%.01f_dropout.h5' % (
model_type, subsample_train, subsample_val, batch_size, epochs,
int(augment_train_data), config['bands'].getint('VIS0'),
config['bands'].getint('NIR1'), config['bands'].getint('NIR2'),
config['bands'].getint('NIR3'), config['trainparams']['data_bias'],
ratio)
model = tf.keras.models.load_model(os.path.join(RESULTS, model_name))
model.summary()
history_path = os.path.join(RESULTS, model_name.replace('h5', 'history'))
## History
with open(history_path, 'rb') as file_pi:
history = pickle.load(file_pi)
## Checkpoints
save_dir = os.path.join(RESULTS, 'checkpoints/resnet/')
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)
### Plots
fig, ax1 = plt.subplots(1, 1, figsize=(10, 5))
ax2 = ax1.twinx()
ax1.plot(range(len(history['loss'])),
history['val_loss'],
label='Validation loss',
marker='o',
c='b')
ax1.plot(range(len(history['loss'])),
history['loss'],
label='Training loss',
marker='o',
c='r')
ax1.set_ylim([0, 1])
ax2.plot(range(len(history['loss'])),
history['val_acc'],
label='Validation accuracy',
marker='o',
c='b',
ls='--',
fillstyle='none')
ax2.plot(range(len(history['loss'])),
history['acc'],
label='Training accuracy',
marker='o',
c='r',
ls='--',
fillstyle='none')
ax1.set_xlabel('Epoch')
ax1.legend(loc=(-0.1, 1))
ax2.legend(loc=(0.9, 1))
ax1.set_ylabel('Loss')
ax2.set_ylabel('Accuracy')
plt.gcf()
plt.savefig(os.path.join(
RESULTS,
'plots/' + os.path.basename(history_path).replace('.history', '.png')),
dpi=200)
plt.figure(2)
train_auc = np.array(history['auc'])
val_auc = np.array(history['val_auc'])
plt.plot(np.arange(0, len(val_auc)), train_auc, 'ob', label='Train')
plt.plot(np.arange(0, len(val_auc)), val_auc, 'or', label='Validation')
plt.xlabel('Epochs')
plt.ylabel('AUC')
plt.ylim(0, 1)
plt.legend()
plt.savefig(os.path.join(
RESULTS, 'plots/AUC_' +
os.path.basename(history_path).replace('.history', '.png')),
dpi=200)
print("history keys:\n", history.keys())
##Score
#scores = model.evaluate_generator(val_data_gen, verbose=2, steps=val_steps_per_epoch)
##Roc curve
images_val, labels_true = next(roc_val_data_gen)
labels_score = model.predict(images_val,
batch_size=subsample_val,
verbose=2)
fpr, tpr, thresholds = roc_curve(np.ravel(labels_true),
np.ravel(labels_score))
print(fpr)
print(tpr)
plt.figure(3)
plt.plot(fpr, tpr, 'or', label='Validation')
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.plot([0, 1], [0, 1])
plt.legend()
plt.savefig(os.path.join(
RESULTS, 'plots/ROCsklearn_' +
os.path.basename(history_path).replace('.history', '.png')),
dpi=200)
def main():
if len(sys.argv) != 2:
config_file = 'config_resnet.ini'
else:
config_file = sys.argv[1]
if not os.path.isfile(config_file):
sys.exit('ERROR:\tThe config file %s was not found.' % config_file)
if len(tf.config.experimental.list_physical_devices('GPU')):
print('GPU found. Num GPUs Available: ',
len(tf.config.experimental.list_physical_devices('GPU')))
print(tf.config.experimental.list_physical_devices('GPU'))
if len(tf.config.experimental.list_logical_devices('GPU')):
print('Logical GPU found. Num logical GPUs Available: ',
len(tf.config.experimental.list_logical_devices('GPU')))
print(tf.config.experimental.list_logical_devices('GPU'))
else:
print("No GPU found")
if len(tf.config.experimental.list_physical_devices('CPU')):
print('Physical CPU found. Num Physical CPUs Available: ',
len(tf.config.experimental.list_physical_devices('CPU')))
print(tf.config.experimental.list_physical_devices('CPU'))
if len(tf.config.experimental.list_logical_devices('CPU')):
print('Logical CPU found. Num logical CPUs Available: ',
len(tf.config.experimental.list_logical_devices('CPU')))
print(tf.config.experimental.list_logical_devices('CPU'))
else:
print("No CPU found")
config = configparser.ConfigParser()
config.read(config_file)
print("\nConfiguration file:\n")
for section in config.sections():
print("Section: %s" % section)
for options in config.options(section):
print(" %s = %s" % (options, config.get(section, options)))
if not bool(config['general'].getboolean('use_gpu')):
sys.stdout.write('\nNot using GPU.\n')
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
# Paths
WORKDIR = config['general']['workdir']
#WORKDIR = os.path.abspath(sys.argv[2])
sys.stdout.write('Project directory: %s\n' % WORKDIR)
#SRC = os.path.join(WORKDIR, 'src')
DATA = os.path.join(WORKDIR, 'data')
RESULTS = os.path.join(WORKDIR, 'results')
TRAIN_MULTIBAND = config['general']['train_multiband']
#TEST_MULTIBAND = os.path.join(DATA, 'test_multiband')
image_catalog = pd.read_csv(os.path.join(
DATA, 'catalog/image_catalog2.0train.csv'),
comment='#',
index_col=0)
print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n")
# Training parameters
# orig paper trained all networks with batch_size=128
batch_size = config['trainparams'].getint('batch_size')
epochs = config['trainparams'].getint('epochs')
num_classes = 1
data_bias = config['trainparams']['data_bias']
# Model parameter
# ----------------------------------------------------------------------------
# | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch
# Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti
# |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)
# ----------------------------------------------------------------------------
# ResNet20 | 3 (2)| 92.16 | 91.25 | ----- | ----- | 35 (---)
# ResNet32 | 5(NA)| 92.46 | 92.49 | NA | NA | 50 ( NA)
# ResNet44 | 7(NA)| 92.50 | 92.83 | NA | NA | 70 ( NA)
# ResNet56 | 9 (6)| 92.71 | 93.03 | 93.01 | NA | 90 (100)
# ResNet110 |18(12)| 92.65 | 93.39+-.16| 93.15 | 93.63 | 165(180)
# ResNet164 |27(18)| ----- | 94.07 | ----- | 94.54 | ---(---)
# ResNet1001| (111)| ----- | 92.39 | ----- | 95.08+-.14| ---(---)
# ---------------------------------------------------------------------------
n = config['trainparams'].getint('n')
# Model version
# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)
version = config['trainparams'].getint('resnetversion')
# Computed depth from supplied model parameter n
if version == 1:
depth = n * 6 + 2
elif version == 2:
depth = n * 9 + 2
# Model name, depth and version
model_type = 'RN%dv%d' % (depth, version)
# Create the dataframe containing filenames and labels.
# This is ok if we use weighted losses.
lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'),
index_col=0)
dataframe_for_generator = build_generator_dataframe(
lens_df, TRAIN_MULTIBAND)
# Extract data proportions for loss weighting
n_lens_clean = len(lens_df[lens_df['is_lens'] == True])
n_nolens_clean = len(lens_df[lens_df['is_lens'] == False])
equal_class_coeff = np.array([n_lens_clean / n_nolens_clean, 1])
natural_class_coeff = np.array([1000 * n_lens_clean / n_nolens_clean, 1])
# Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below!
train_df, val_df = train_test_split(
dataframe_for_generator,
test_size=config['trainparams'].getfloat('test_fraction'),
random_state=42) # before it was 42
total_train = len(train_df)
total_val = len(val_df)
print("The number of objects in the whole training sample is: ",
total_train)
print("The number of objects in the whole validation sample is: ",
total_val)
test_fraction = float(config["trainparams"]["test_fraction"])
print("The test fraction is: ", test_fraction)
if config['trainparams']['subsample_train'] == 'total':
subsample_train = total_train
subsample_val = total_val
else:
try:
subsample_train = int(config['trainparams']['subsample_train'])
subsample_val = int(subsample_train * test_fraction /
(1. - test_fraction))
except:
raise ValueError('subsample_train should be \'total\' or int.')
print("The number of objects in the training subsample is: ",
subsample_train)
print("The number of objects in the validation subsample is: ",
subsample_val)
train_steps_per_epoch = int(subsample_train // batch_size)
val_steps_per_epoch = int(subsample_val // batch_size)
print("The number of training steps is: ", train_steps_per_epoch)
print("The number of validation steps is: ", val_steps_per_epoch)
augment_train_data = bool(int(config['trainparams']['augment_train_data']))
# Create Tiff Image Data Generator objects for train and validation
image_data_gen_train = TiffImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
zca_epsilon=1e-06,
#rotation_range=0,
#width_shift_range=0.0,
#height_shift_range=0.0,
#brightness_range=(0.99, 1.01),
#shear_range=0.0,
#zoom_range=(0.99, 1.01),
#channel_shift_range=0.0,
#fill_mode='wrap',
#cval=0.0,
horizontal_flip=True,
vertical_flip=True,
rescale=None,
preprocessing_function=None,
data_format='channels_last',
dtype='float32')
image_data_gen_val = TiffImageDataGenerator(dtype='float32')
bands = [
config['bands'].getboolean('VIS0'), config['bands'].getboolean('NIR1'),
config['bands'].getboolean('NIR2'), config['bands'].getboolean('NIR3')
]
print("The bands are: ", bands)
# Create generators for Images and Labels
ratio = float(config['trainparams']['lens_nolens_ratio'])
binary = bool(int(config['general']['binary']))
train_data_gen = image_data_gen_train.prop_image_generator_dataframe(
train_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=not (augment_train_data),
ratio=ratio,
bands=bands,
binary=binary)
val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=True,
ratio=ratio,
bands=bands,
binary=binary)
roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=subsample_val,
validation=True,
ratio=ratio,
bands=bands,
binary=binary)
# Obtain the shape of the input data (train images)
temp_data_gen = image_data_gen_train.image_generator_dataframe(
train_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=True,
bands=bands)
image, _ = next(temp_data_gen)
input_shape = image[0].shape
# Create model
if version == 2:
model = myf.resnet_v2(input_shape=input_shape,
depth=depth,
num_classes=num_classes)
else:
model = myf.resnet_v1(input_shape=input_shape,
depth=depth,
num_classes=num_classes)
# Define metrics for the model.
metrics = [
keras.metrics.TruePositives(name='tp'),
keras.metrics.FalsePositives(name='fp'),
keras.metrics.TrueNegatives(name='tn'),
keras.metrics.FalseNegatives(name='fn'),
keras.metrics.BinaryAccuracy(name='acc'),
keras.metrics.AUC(name='auc')
]
model.compile(
loss='binary_crossentropy',
optimizer=tf.keras.optimizers.Adam(learning_rate=myf.lr_schedule(0)),
metrics=metrics)
model.summary()
model.get_config()
# Prepare model model saving directory.
save_dir = os.path.join(RESULTS, 'checkpoints/resnet/')
model_name = '%s_Tr%i_Te%i_bs%i_ep%.03d_aug%i_VIS%i_NIR%i%i%i_DB%s_ratio%.01f_dropout.h5' % (
model_type, subsample_train, subsample_val, batch_size, epochs,
int(augment_train_data), config['bands'].getint('VIS0'),
config['bands'].getint('NIR1'), config['bands'].getint('NIR2'),
config['bands'].getint('NIR3'), config['trainparams']['data_bias'],
ratio)
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)
print("The model name is: ", model_name)
# Prepare callbacks for model saving and for learning rate adjustment.
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=filepath,
monitor='val_acc',
verbose=1,
save_best_only=True)
# save_weights_only=True
lr_scheduler = tf.keras.callbacks.LearningRateScheduler(myf.lr_schedule)
lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=5,
min_lr=0.5e-6)
callbacks = [checkpoint, lr_reducer, lr_scheduler]
# Define class weights for unevenly distributed (biased) dataset.
if data_bias == 'natural':
sys.stdout.write(
'Using natural data bias: 1000x more non lenses than lenses.\n')
class_coeff = natural_class_coeff
elif data_bias == 'none':
sys.stdout.write(
'Using no data bias (simulate equal proportion among classes).\n')
class_coeff = equal_class_coeff
elif data_bias == 'raw':
sys.stdout.write('Using the raw bias (no weights applied).\n')
class_coeff = [1., 1.]
else:
raise NotImplementedError('data_bias must be either natural or none.')
class_weights = {0: class_coeff[0], 1: class_coeff[1]}
sys.stdout.write('Using weights: %s\n' % class_weights)
# Train the ResNet
print('Train the ResNet using real-time data augmentation.')
history = model.fit_generator(train_data_gen,
steps_per_epoch=train_steps_per_epoch,
epochs=epochs,
validation_data=val_data_gen,
validation_steps=val_steps_per_epoch,
callbacks=callbacks,
class_weight=class_weights,
use_multiprocessing=True,
verbose=2)
# Score trained model.
scores = model.evaluate_generator(val_data_gen,
verbose=2,
steps=val_steps_per_epoch)
#images_val, labels_true = next(roc_val_data_gen)
#labels_score = model.predict(images_val, batch_size=subsample_val, verbose=2)
#fpr, tpr, thresholds = roc_curve(np.ravel(labels_true), np.ravel(labels_score))
# print(fpr)
# print(tpr)
model.save(os.path.join(RESULTS, model_name))
with open(os.path.join(RESULTS, model_name.replace('h5', 'history')),
'wb') as file_pi:
pickle.dump(history.history, file_pi)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
def main():
"""Fit the model.
If checkpoint does not exist in checkpoints or in the results directory, a new model is created and fitted
according to parameters set in the config file. If a checkpoint or end model (in results dir) is found, it is loaded
and the training is resumed according to values defined in the config file. By default, a checkpoint is preferred
over an end model since it is assumed to be more recent (in case of manual stopping of training)."""
if len(sys.argv) != 2:
config_file = 'config_lesta_df.ini'
else:
config_file = sys.argv[1]
if not os.path.isfile(config_file):
sys.exit('ERROR:\tThe config file %s was not found.' % config_file)
config = configparser.ConfigParser()
config.read(config_file)
print("\nConfiguration file:\n")
for section in config.sections():
print("Section: %s" % section)
for options in config.options(section):
print(" %s = %s" % (options, config.get(section, options)))
if not bool(config['general'].getboolean('use_gpu')):
sys.stdout.write('\nNot using GPU.\n')
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
# Paths
WORKDIR = config['general']['workdir']
sys.stdout.write('Project directory: %s\n' % WORKDIR)
SRC = os.path.join(WORKDIR, 'src')
DATA = os.path.join(WORKDIR, 'data')
RESULTS = os.path.join(WORKDIR, 'results')
TRAIN_MULTIBAND = config['general']['train_multiband']
TEST_MULTIBAND = os.path.join(DATA, 'test_multiband')
# Catalog
lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'),
index_col=0)
dataframe_for_generator = build_generator_dataframe(
lens_df, TRAIN_MULTIBAND)
# Extract data proportions for loss weighting
n_lens_clean = len(lens_df[lens_df['is_lens'] == True])
n_nolens_clean = len(lens_df[lens_df['is_lens'] == False])
equal_class_coeff = np.array([n_lens_clean / n_nolens_clean, 1])
natural_class_coeff = np.array([1000 * n_lens_clean / n_nolens_clean, 1])
# Training parameters
batch_size = config['trainparams'].getint('batch_size')
epochs = config['trainparams'].getint('epochs')
data_bias = config['trainparams']['data_bias']
test_fraction = config['trainparams'].getfloat('test_fraction')
augment_train_data = bool(int(config['trainparams']['augment_train_data']))
kernel_size_1 = int(config['trainparams']['kernel_size_1'])
kernel_size_2 = int(config['trainparams']['kernel_size_2'])
dropout_config = config['trainparams'][
'dropout_kind'] #Import dropout and check values are valid.
if dropout_config == 'dropout':
dropout_kind = Dropout
elif dropout_config == 'spatialdropout':
dropout_kind = SpatialDropout2D
else:
raise NotImplementedError(
'dropout_kind must be \'dropout\' or \'spatialdropout\'\nPlease check config file.'
)
pool_size = int(config['trainparams']['pool_size'])
bands = [
config['bands'].getboolean('VIS0'), config['bands'].getboolean('NIR1'),
config['bands'].getboolean('NIR2'), config['bands'].getboolean('NIR3')
]
print("The bands are: ", bands)
binary = bool(int(config['general']['binary']))
ratio = float(config['trainparams']['lens_nolens_ratio'])
# Split catalog in train and test (validation) sets. We used fixed state 42.
train_df, val_df = train_test_split(dataframe_for_generator,
test_size=test_fraction,
random_state=42)
total_train = len(train_df)
total_val = len(val_df)
print("The number of objects in the whole training sample is: ",
total_train)
print("The number of objects in the whole validation sample is: ",
total_val)
print("The test fraction is: ", test_fraction)
if config['trainparams'][
'subsample_train'] == 'total': #Import subsample size and check values are as expected.
subsample_train = total_train
subsample_val = total_val
else:
try:
subsample_train = int(config['trainparams']['subsample_train'])
subsample_val = int(subsample_train * test_fraction /
(1. - test_fraction))
except:
raise ValueError('subsample_train should be \'total\' or int.')
print("The number of objects in the training subsample is: ",
subsample_train)
print("The number of objects in the validation subsample is: ",
subsample_val)
train_steps_per_epoch = int(subsample_train // batch_size)
val_steps_per_epoch = int(subsample_val // batch_size)
print("The number of training steps is: ", train_steps_per_epoch)
print("The number of validation steps is: ", val_steps_per_epoch)
# Create TiffImageDataGenerator objects to inherit random transformations from Keras' class.
image_data_gen_train = TiffImageDataGenerator(featurewise_center=False,
rotation_range=0,
fill_mode='wrap',
horizontal_flip=True,
vertical_flip=True,
preprocessing_function=None,
data_format='channels_last',
dtype='float32')
image_data_gen_val = TiffImageDataGenerator(dtype='float32')
# Create Generator objects from the initialized TiffImageDataGenerators.
# To train
train_data_gen = image_data_gen_train.prop_image_generator_dataframe(
train_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=not (augment_train_data),
ratio=ratio,
bands=bands,
binary=binary)
# To validate
val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=True,
ratio=ratio,
bands=bands,
binary=binary)
# To predict/evaluate
roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=batch_size,
validation=True,
ratio=ratio,
bands=bands,
binary=binary)
# To safely obtain image size
temp_data_gen = image_data_gen_train.image_generator_dataframe(
train_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=1,
validation=True,
bands=bands,
binary=binary)
# Obtain image size
image, _ = next(temp_data_gen)
input_shape = image[0].shape
# Define correct bias to initialize (use if not forcing generator to load equal proportions of data)
output_bias = tf.keras.initializers.Constant(
np.log(n_lens_clean / n_nolens_clean))
# Path to save checkpoints
model_type = 'lastro_cnn'
save_dir = os.path.join(RESULTS, 'checkpoints/%s/' % model_type)
model_name = '%s_Tr%i_Te%i_bs%i_ep%.03d_aug%i_VIS%i_NIR%i%i%i_DB%s_ratio%.01f_ks%i%i_ps%i_%s_%s.h5' % (
model_type, subsample_train, subsample_val, batch_size, epochs,
int(augment_train_data), bands[0], bands[1], bands[2], bands[3],
data_bias, ratio, kernel_size_1, kernel_size_2, pool_size,
dropout_kind.__name__, os.path.basename(TRAIN_MULTIBAND))
# Create path of checkpoints if necessary
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
# Checkpoint file path
checkpoint_filepath = os.path.join(save_dir, model_name)
# Final model file path
end_model_name = os.path.join(RESULTS, model_name)
print("The model name is: ", model_name)
history_path = os.path.join(RESULTS, model_name.replace('h5', 'history'))
# Callbacks
# Checkpoint callback to save every epoch for better resuming training
cp_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_best_only=False,
verbose=1,
monitor='val_acc',
save_freq='epoch')
# Checkpoint best model
cp_best_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath.replace('.h5', '_BEST.h5'),
save_best_only=True,
verbose=1,
monitor='val_acc')
# Early stopping callback (currently using a very high patience to avoid it triggering)
es_callback = tf.keras.callbacks.EarlyStopping(
monitor='val_acc',
#min_delta=0.1,
patience=30,
verbose=1,
mode='auto',
baseline=None,
restore_best_weights=True)
# Learning rate reducer callback.
lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=20,
min_lr=0.5e-6,
monitor='val_acc',
verbose=1,
mode='auto')
# Callback to save log to csv. It is probably better to use this than save-resume history.
logger_callback = tf.keras.callbacks.CSVLogger(checkpoint_filepath.replace(
'.h5', '.log'),
separator=',',
append=True)
# Callback to resume history if history_path exists.
history_callback = ResumeHistory(history_path)
# Callback to use Tensorboard
log_dir = os.path.join(
RESULTS,
"logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1,
profile_batch=0)
# Define metrics for the model.
metrics = [
keras.metrics.TruePositives(name='tp'),
keras.metrics.FalsePositives(name='fp'),
keras.metrics.TrueNegatives(name='tn'),
keras.metrics.FalseNegatives(name='fn'),
keras.metrics.BinaryAccuracy(name='acc'),
keras.metrics.AUC(name='auc')
]
# If there are no checkpoints or final models saved, compile a new one.
if not os.path.isfile(checkpoint_filepath) and not os.path.isfile(
end_model_name):
model = build_lastro_model(kernel_size_1, kernel_size_2, pool_size,
input_shape, dropout_kind)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=metrics)
elif not os.path.isfile(checkpoint_filepath) and os.path.isfile(
end_model_name):
print('Loading existing model from result.')
model = tf.keras.models.load_model(end_model_name)
epochs = int(config['trainparams']['new_epochs'])
learning_rate = config['trainparams']['learning_rate']
change_learning_rate(model, learning_rate)
elif os.path.isfile(checkpoint_filepath):
print('Loading existing model from checkpoint.')
model = tf.keras.models.load_model(checkpoint_filepath)
epochs = int(config['trainparams']['new_epochs'])
learning_rate = config['trainparams']['learning_rate']
change_learning_rate(model, learning_rate)
model.summary()
# Define class weights for unevenly distributed (biased) dataset.
if data_bias == 'natural':
sys.stdout.write(
'Using natural data bias: 1000x more non lenses than lenses.\n')
class_coeff = natural_class_coeff
elif data_bias == 'none':
sys.stdout.write(
'Using no data bias (simulate equal proportion among classes).\n')
class_coeff = equal_class_coeff
elif data_bias == 'raw':
sys.stdout.write('Using the raw bias (no weights applied).\n')
class_coeff = [1, 1]
else:
raise NotImplementedError(
'data_bias must be either natural, none or raw.')
class_weights = {0: class_coeff[0], 1: class_coeff[1]}
sys.stdout.write('Using loss weights: %s\n' % class_weights)
# Fit the model and save the history callback.
# Use multiprocessing True when using > 1 workers. (Seems to cause problems)
# Expect tf update to use threadsafe_iter class.
history = model.fit_generator(
train_data_gen,
steps_per_epoch=subsample_train // batch_size,
epochs=epochs,
validation_data=val_data_gen,
validation_steps=subsample_val // batch_size,
callbacks=[
cp_callback, es_callback, lr_reducer, cp_best_callback,
history_callback, logger_callback, tensorboard_callback
],
class_weight=class_weights,
# use_multiprocessing=True,
verbose=1,
# workers=16
)
model.save(end_model_name)
# If training finishes normally (Is not stopped by user), save final model.
# Save complete history if the training was resumed.
if history_callback.use_history_file_flag:
with open(history_path, 'wb') as file_pi:
pickle.dump(history_callback.complete_history, file_pi)
else:
with open(history_path, 'wb') as file_pi:
pickle.dump(history.history, file_pi)
# Score trained model.
scores = model.evaluate_generator(val_data_gen,
verbose=2,
steps=val_steps_per_epoch)
images_val, labels_true = next(roc_val_data_gen)
labels_score = model.predict(images_val, batch_size=batch_size, verbose=2)
fpr, tpr, thresholds = roc_curve(np.ravel(labels_true),
np.ravel(labels_score))
auc = history.history['val_auc'][-1]
acc = history.history['val_acc'][-1]
# Save TPR and FPR metrics to plot ROC.
np.savetxt(os.path.join(RESULTS, model_name.replace('h5', 'FPRvsTPR.dat')),
np.array([fpr, tpr]).T,
header='auc=%.3f\nacc=%.3f' % (auc, acc))
def main():
if len(sys.argv) == 2:
config_file = 'config_lastro.ini'
model_name = sys.argv[1]
elif len(sys.argv) == 3:
config_file = sys.argv[1]
model_name = sys.argv[2]
if not os.path.isfile(config_file):
sys.exit('ERROR:\tThe config file %s was not found.' % config_file)
# Avoid using GPU to evaluate models.
sys.stdout.write('\nNot using GPU.\n')
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
# Import configuration file
config = configparser.ConfigParser()
config.read(config_file)
# Extract parameters from model name
if 'train_multiband_bin' in model_name:
datadir = 'train_multiband_bin'
elif 'train_multiband_noclip_bin' in model_name:
datadir = 'train_multiband_noclip_bin'
else:
datadir = 'train_multiband_noclip_bin'
# Extract bands from filename
bands = []
if 'VIS0' in model_name:
bands.append(False)
elif 'VIS1' in model_name:
bands.append(True)
if 'NIR000' in model_name:
[bands.append(False) for i in range(3)]
elif 'NIR111' in model_name:
[bands.append(True) for i in range(3)]
bands = list(np.array(bands).reshape(-1))
print("The bands are: ", bands)
# Extract split ratio from filename
for param in model_name.split('_'):
if 'ratio' in param:
ratio = float(param.replace('ratio', ''))
# Paths
WORKDIR = config['general']['workdir']
sys.stdout.write('Project directory: %s\n' % WORKDIR)
DATA = os.path.join(WORKDIR, 'data')
RESULTS = os.path.join(WORKDIR, 'results')
TRAIN_MULTIBAND = os.path.join(DATA, datadir)
image_catalog = pd.read_csv(os.path.join(
DATA, 'catalog/image_catalog2.0train.csv'),
comment='#',
index_col=0)
print('The shape of the image catalog: ' + str(image_catalog.shape) + "\n")
lens_df = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels_old.csv'),
index_col=0)
lens_df_new = pd.read_csv(os.path.join(RESULTS, 'lens_id_labels.csv'),
index_col=0)
dataframe_for_generator = build_generator_dataframe(
lens_df, TRAIN_MULTIBAND)
# Split the TRAIN_MULTIBAND set into train and validation sets. Set test_size below!
train_df, val_df = train_test_split(
dataframe_for_generator,
test_size=config['trainparams'].getfloat('test_fraction'),
random_state=42)
total_train = len(train_df)
total_val = len(val_df)
print(train_df)
sys.exit(0)
print("The number of objects in the whole training sample is: ",
total_train)
print("The number of objects in the whole validation sample is: ",
total_val)
test_fraction = float(config["trainparams"]["test_fraction"])
print("The test fraction is: ", test_fraction)
if config['trainparams']['subsample_train'] == 'total':
subsample_train = total_train
subsample_val = total_val
else:
try:
subsample_train = int(config['trainparams']['subsample_train'])
subsample_val = int(subsample_train * test_fraction /
(1. - test_fraction))
except:
raise ValueError('subsample_train should be \'total\' or int.')
print("The number of objects in the training subsample is: ",
subsample_train)
print("The number of objects in the validation subsample is: ",
subsample_val)
augment_train_data = bool(int(config['trainparams']['augment_train_data']))
# Create Tiff Image Data Generator objects for train and validation
image_data_gen_train = TiffImageDataGenerator(featurewise_center=False,
rotation_range=0,
fill_mode='wrap',
horizontal_flip=True,
vertical_flip=True,
preprocessing_function=None,
data_format='channels_last',
dtype='float32')
image_data_gen_val = TiffImageDataGenerator(dtype='float32')
# Create generators for Images and Labels
roc_val_data_gen = image_data_gen_val.prop_image_generator_dataframe(
val_df,
directory=TRAIN_MULTIBAND,
x_col='filenames',
y_col='labels',
batch_size=subsample_val,
validation=True,
ratio=ratio,
bands=bands,
binary=True)
# Obtain model from the saving directory
model_name_base = os.path.basename(model_name)
model = tf.keras.models.load_model(model_name)
model.summary()
history_path = model_name.replace('h5', 'history')
# Checkpoints dir
save_dir = os.path.join(RESULTS, 'checkpoints/lastro_cnn/')
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name_base)
# Plots
# History
if os.path.isfile(history_path):
with open(history_path, 'rb') as file_pi:
history = pickle.load(file_pi)
fig, ax1 = plt.subplots(1, 1, figsize=(10, 5))
ax2 = ax1.twinx()
ax1.plot(
range(len(history['loss'])),
history['val_loss'],
label='Validation loss',
# marker='o',
c='b',
lw=3)
ax1.plot(
range(len(history['loss'])),
history['loss'],
label='Training loss',
# marker='o',
c='r',
lw=3)
ax2.set_ylim([0.5, 1])
ax2.plot(
range(len(history['loss'])),
history['val_acc'],
label='Validation accuracy',
# marker='^',
c='b',
ls='--',
fillstyle='none',
lw=3)
ax2.plot(
range(len(history['loss'])),
history['acc'],
label='Training accuracy',
# marker='^',
c='r',
ls='--',
fillstyle='none',
lw=3)
ax1.set_xlabel('Epoch')
ax1.legend(loc=(-0.1, 1))
ax2.legend(loc=(0.9, 1))
ax1.set_ylabel('Loss')
ax2.set_ylabel('Accuracy')
plt.gcf()
plt.savefig(os.path.join(
RESULTS, 'plots/' +
os.path.basename(history_path).replace('.history', '.png')),
dpi=200)
# Roc curve
images_val, labels_true = next(roc_val_data_gen)
print(labels_true)
labels_score = model.predict(images_val,
batch_size=1,
verbose=2,
workers=16,
use_multiprocessing=True)
fpr, tpr, thresholds = roc_curve(np.ravel(labels_true),
np.ravel(labels_score))
scores = model.evaluate(images_val,
labels_true,
batch_size=True,
verbose=1,
workers=16,
use_multiprocessing=True)
scores_dict = {
metric: value
for metric, value in zip(model.metrics_names, scores)
}
print(scores)
print(model.metrics_names)
acc = scores_dict['acc']
auc = scores_dict['auc']
np.savetxt(os.path.join(RESULTS,
model_name_base.replace('h5', 'FPRvsTPR.dat')),
np.array([fpr, tpr]).T,
header='auc=%.3f\nacc=%.3f' % (auc, acc))
plt.figure(2)
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.plot([0, 1], [0, 1])
plt.legend()
plt.plot(fpr,
tpr,
label='Validation\nAUC=%.3f\nACC=%.3f' % (auc, acc),
lw=3)
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.plot([0, 1], [0, 1], lw=3)
plt.legend()
plt.savefig(os.path.join(
RESULTS, 'plots/ROCsklearn_' +
os.path.basename(model_name).replace('.h5', '.png')),
dpi=200)