def predict_model_path_on_validation_set(model_path, label_encoder, config):
'''
Given a weights_path for an FC-DenseNet model,
Save predictions on each scene in the IEEE competition Validation set
'''
if 'unet' in model_path.lower():
model = models.get_compiled_unet(config,
label_encoder,
predict_logits=True)
else:
model = models.get_compiled_fc_densenet(config, label_encoder)
model.load_weights(model_path)
model_name = os.path.basename(model_path).split('_weights.h5')[0]
val_season = 'ROIs0000_validation'
# get all scenes from validation set
scene_dirs = os.listdir(config['validation_dataset_dir'])
scene_dirs = [
os.path.join(config['validation_dataset_dir'], scene)
for scene in scene_dirs
]
# predict in segmentation mode
for scene_dir in scene_dirs:
scene_name = scene_dir.split('/')[-1]
save_dir = os.path.join(config['competition_predictions_dir'],
model_name, val_season, scene_name)
save_segmentation_predictions_on_scene_dir(model,
scene_dir,
save_dir,
label_encoder,
config,
competition_mode=True)
print('finished predictions using model_path: ', model_path)
print()
def train_segmentation_model_on_patch_paths(patch_paths, weights_path, config):
'''
Input: patch_paths, weights_path, config
Output: trained segmentation model (saved to disk), training history
'''
# get train-val split
train_patch_paths, val_patch_paths = get_train_val_scene_dirs(
patch_paths, config)
print('num. training images: ', len(train_patch_paths))
print('num. validation images: ', len(val_patch_paths))
# save train-val-split
train_split_filepath = weights_path.split(
'_weights.h5')[0] + '_train-val-split.json'
with open(train_split_filepath, 'w') as f:
train_split = {
'train_scene_dirs': train_patch_paths,
'val_scene_dirs': val_patch_paths,
}
json.dump(train_split, f, indent=4)
# get datagen
train_datagen_labels = config['training_params']['label_smoothing']
train_datagen = datagen.SegmentationDataGenerator(
train_patch_paths, config, labels=train_datagen_labels)
val_datagen = datagen.SegmentationDataGenerator(val_patch_paths,
config,
labels='onehot')
# get compiled model
print('getting compiled densenet model...')
label_encoder = land_cover_utils.get_label_encoder(config)
loss = 'categorical_crossentropy'
batch_size = config['fc_densenet_params']['batch_size']
model = models.get_compiled_fc_densenet(config, label_encoder, loss=loss)
# fit keras model
print("Training keras model...")
callbacks = models.get_callbacks(weights_path, config)
history = model.fit_generator(
train_datagen,
epochs=config['training_params']['max_epochs'],
validation_data=val_datagen,
callbacks=callbacks,
max_queue_size=batch_size,
use_multiprocessing=config['training_params']['use_multiprocessing'],
workers=config['training_params']['workers'])
history = land_cover_utils.make_history_json_serializable(history.history)
# save model history
history_filepath = weights_path.split('_weights.h5')[0] + '_history.json'
with open(history_filepath, 'w') as f:
json.dump(history, f, indent=4)
print("Model history saved to: ", history_filepath)
return model, history
def main(args):
'''
Main function: train new models, or test existing models on SEN12MS seasons/scenes
'''
# get config
config_json_path = args.config_path
with open(config_json_path, 'r') as f:
config = json.load(f, object_hook=land_cover_utils.json_keys_to_int)
label_encoder = land_cover_utils.get_label_encoder(config)
# configure GPU
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
GPU_ID = config['training_params'].get('gpu_id')
if GPU_ID is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = GPU_ID
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
session = tf.Session(config=tf_config)
# show summary of keras models
if args.model_summary:
fc_densenet = models.get_compiled_fc_densenet(config,
label_encoder,
predict_seasons=True)
print('---------- FC-DENSENET MODEL SUMMARY ----------')
#print(fc_densenet.summary())
print('inputs: ', fc_densenet.inputs)
print('outputs: ', fc_densenet.outputs)
print()
# train new models on all seasons/continents
if args.train:
# # train densenet models
# for continent in config['all_continents']:
# train_fc_densenet_on_continent(continent, config)
# for season in config['all_seasons']:
# train_fc_densenet_on_season(season, config)
# train_competition_fc_densenet(config)
train_competition_unet(config)
# save each model's predictions on each scene
if args.predict:
# predict_saved_models_on_each_scene(config)
competition_model_path = os.path.join(config['model_save_dir'],
'competition',
config['competition_model'])
print(
f'predicting on competition data with model {competition_model_path}'
)
print(f'label_encoder.classes_: {label_encoder.classes_}')
predict_model_path_on_validation_set(competition_model_path,
label_encoder, config)
def predict_model_path_on_all_patches(model_path, label_encoder,
image_cluster_df, config):
'''
Given a weights_path of a general (non-cluster) FC-DenseNet model,
Save predictions on each patch in dataset
'''
# load model
model = models.get_compiled_fc_densenet(config, label_encoder)
model.load_weights(model_path)
# get model name
model_name = os.path.basename(model_path).split('_weights.h5')[0]
folder = 'by_continent' if 'continent' in model_name else 'by_season'
# get all patch_paths
patch_paths = land_cover_utils.get_all_patch_paths_from_df(
image_cluster_df, config)
print('predict_model_path_on_in_cluster_patches - len(patch_paths): ',
len(patch_paths))
# get save_dir for this model
if folder == 'by_continent':
continent = model_path.split('by_continent/')[-1].split('/')[0]
save_dir = os.path.join(config['segmentation_predictions_dir'],
'by_continent', continent, model_name)
elif folder == 'by_season':
season = model_path.split('by_season/')[-1].split('/')[0]
save_dir = os.path.join(config['segmentation_predictions_dir'],
'by_season', season, model_name)
if os.path.exists(os.path.join(save_dir, 'done.txt')):
print(
f'predict_model_path_on_in_cluster_patches - {save_dir}/done.txt already exists! skipping predictions'
)
return
print(
'predict_model_path_on_in_cluster_patches - saving predictions to save_dir: ',
save_dir)
# save predictions to save_dir
save_segmentation_predictions_on_patch_paths(model, patch_paths, save_dir,
label_encoder, config)
open(os.path.join(save_dir, 'done.txt'),
'w').close() # place a 'done' marker
print('finished predictions using model_path: ', model_path)
print()
def predict_model_path_on_each_scene(model_path, label_encoder, config):
'''
Given a weights_path,
Save predictions on each scene
'''
# TODO: delete this
# load model from model_path
if 'weights' in model_path and 'resnet' in model_path:
print('WARNING - resnet models have been deprecated!')
return
elif 'weights' in model_path and 'DenseNet' in model_path:
model = models.get_compiled_fc_densenet(config, label_encoder)
model.load_weights(model_path)
elif 'weights' in model_path and 'unet' in model_path.lower():
model = models.get_compiled_unet(config,
label_encoder,
predict_logits=True)
model.load_weights(model_path)
else:
print('ERROR: unable to load weights file!')
return
model_name = os.path.basename(model_path).split('_weights.h5')[0]
folder = 'by_continent' if 'continent' in model_name else 'by_season'
# predict on each scene
for continent in config['all_continents']:
for season in config['all_seasons']:
# get all scenes from this continent-season
scene_dirs = land_cover_utils.get_scene_dirs_for_continent_season(
continent, season, config)
# predict in segmentation mode
for scene_dir in scene_dirs:
scene_name = scene_dir.split('/')[-1]
save_dir = os.path.join(config['segmentation_predictions_dir'],
folder, model_name,
'{}-{}'.format(continent,
season), scene_name)
save_segmentation_predictions_on_scene_dir(
model, scene_dir, save_dir, label_encoder, config)
print('finished predictions using model_path: ', model_path)
print()
def train_segmentation_model_on_scene_dirs(scene_dirs, weights_path, config, \
competition_mode=False, \
predict_logits=False):
'''
Input: scene_dirs, weights_path, config
save_label_counts = config['training_params']['class_weight'] == 'balanced'
Output: trained segmentation model (saved to disk), training history
'''
# get train, val scene dirs
if competition_mode:
print("Getting competition train/val split from holdout .csv file...")
train_scene_dirs, val_scene_dirs = get_competition_train_val_scene_dirs(
scene_dirs, config)
else:
print("Performing random train/val split...")
train_scene_dirs, val_scene_dirs = get_train_val_scene_dirs(
scene_dirs, config)
print("train_scene_dirs: ", train_scene_dirs)
print("val_scene_dirs: ", val_scene_dirs)
print('num. training scenes: ', len(train_scene_dirs))
print('num. validation scenes: ', len(val_scene_dirs))
# save train-val-split
train_split_filepath = weights_path.split(
'_weights.h5')[0] + '_train-val-split.json'
with open(train_split_filepath, 'w') as f:
train_split = {
'train_scene_dirs': train_scene_dirs,
'val_scene_dirs': val_scene_dirs,
}
json.dump(train_split, f, indent=4)
# get patch paths
train_patch_paths = land_cover_utils.get_segmentation_patch_paths_for_scene_dirs(
train_scene_dirs)
val_patch_paths = land_cover_utils.get_segmentation_patch_paths_for_scene_dirs(
val_scene_dirs)
# set up data generators with label smoothing
if config['training_params']['label_smoothing'] == 'kmeans':
train_datagen_labels = 'kmeans'
print('training with kmeans label smoothing...')
else:
train_datagen_labels = 'naive'
label_smoothing_factor = config['training_params'][
'label_smoothing_factor']
print(
f'training with naive label smoothing, factor={label_smoothing_factor}...'
)
train_datagen = datagen.SegmentationDataGenerator(
train_patch_paths, config, labels=train_datagen_labels)
val_datagen = datagen.SegmentationDataGenerator(val_patch_paths,
config,
labels='onehot')
# get custom loss function
label_encoder = land_cover_utils.get_label_encoder(config)
if config['training_params']['class_weight'] == 'balanced':
print('training with balanced loss...')
class_weights = train_datagen.get_class_weights_balanced()
else:
print('training with unbalanced loss...')
class_weights = None
loss = models.get_custom_loss(label_encoder,
class_weights,
config,
from_logits=predict_logits)
# get compiled keras model
if 'unet' in weights_path.lower():
print('getting compiled unet model...')
batch_size = config['unet_params']['batch_size']
model = models.get_compiled_unet(config,
label_encoder,
loss=loss,
predict_logits=predict_logits)
else:
print('getting compiled densenet model...')
batch_size = config['fc_densenet_params']['batch_size']
model = models.get_compiled_fc_densenet(config,
label_encoder,
loss=loss)
# fit keras model
print("Training keras model...")
callbacks = models.get_callbacks(weights_path, config)
history = model.fit_generator(
train_datagen,
epochs=config['training_params']['max_epochs'],
validation_data=val_datagen,
callbacks=callbacks,
max_queue_size=batch_size,
use_multiprocessing=config['training_params']['use_multiprocessing'],
workers=config['training_params']['workers'])
history = land_cover_utils.make_history_json_serializable(history.history)
# save model history
history_filepath = weights_path.split('_weights.h5')[0] + '_history.json'
with open(history_filepath, 'w') as f:
json.dump(history, f, indent=4)
print("Model history saved to: ", history_filepath)
return model, history