def _run(model_file_name, top_example_dir_name, first_time_string,
last_time_string, num_times, num_examples_per_time, output_file_name):
"""Runs permutation test for predictor importance.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param num_times: Same.
:param num_examples_per_time: Same.
:param output_file_name: Same.
"""
print 'Reading model from: "{0:s}"...'.format(model_file_name)
model_object = traditional_cnn.read_keras_model(model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=model_file_name)
print 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
print SEPARATOR_STRING
predictor_matrix, target_values = _read_examples(
top_example_dir_name=top_example_dir_name,
first_time_string=first_time_string,
last_time_string=last_time_string,
num_times=num_times,
num_examples_per_time=num_examples_per_time,
model_metadata_dict=model_metadata_dict)
print SEPARATOR_STRING
narr_predictor_names = model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY]
result_dict = permutation.run_permutation_test(
model_object=model_object,
list_of_input_matrices=[predictor_matrix],
predictor_names_by_matrix=[narr_predictor_names],
target_values=target_values,
prediction_function=_prediction_function,
cost_function=permutation.cross_entropy_function)
print SEPARATOR_STRING
print 'Writing results to: "{0:s}"...'.format(output_file_name)
permutation.write_results(result_dict=result_dict,
pickle_file_name=output_file_name)
def _run(upconvnet_file_name, top_example_dir_name, first_time_string,
last_time_string, num_baseline_examples, num_test_examples,
percent_svd_variance_to_keep, top_output_dir_name):
"""Runs novelty detection.
:param upconvnet_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param num_baseline_examples: Same.
:param num_test_examples: Same.
:param percent_svd_variance_to_keep: Same.
:param top_output_dir_name: Same.
"""
# Read upconvnet and metadata.
ucn_metafile_name = traditional_cnn.find_metafile(
model_file_name=upconvnet_file_name, raise_error_if_missing=True)
print('Reading trained upconvnet from: "{0:s}"...'.format(
upconvnet_file_name))
ucn_model_object = traditional_cnn.read_keras_model(upconvnet_file_name)
print('Reading upconvnet metadata from: "{0:s}"...'.format(
ucn_metafile_name))
ucn_metadata_dict = upconvnet.read_model_metadata(ucn_metafile_name)
# Read CNN and metadata.
cnn_file_name = ucn_metadata_dict[upconvnet.CNN_FILE_NAME_KEY]
cnn_metafile_name = traditional_cnn.find_metafile(
model_file_name=cnn_file_name, raise_error_if_missing=True)
print 'Reading trained CNN from: "{0:s}"...'.format(cnn_file_name)
cnn_model_object = traditional_cnn.read_keras_model(cnn_file_name)
print 'Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)
cnn_metadata_dict = traditional_cnn.read_model_metadata(cnn_metafile_name)
print SEPARATOR_STRING
baseline_image_matrix, test_image_matrix = _find_baseline_and_test_examples(
top_example_dir_name=top_example_dir_name,
first_time_string=first_time_string,
last_time_string=last_time_string,
num_baseline_examples=num_baseline_examples,
num_test_examples=num_test_examples,
cnn_model_object=cnn_model_object,
cnn_metadata_dict=cnn_metadata_dict)
print SEPARATOR_STRING
novelty_dict = novelty_detection.do_novelty_detection(
baseline_image_matrix=baseline_image_matrix,
test_image_matrix=test_image_matrix,
cnn_model_object=cnn_model_object,
cnn_feature_layer_name=traditional_cnn.get_flattening_layer(
cnn_model_object),
ucn_model_object=ucn_model_object,
num_novel_test_images=num_test_examples,
norm_function=None,
denorm_function=None,
percent_svd_variance_to_keep=percent_svd_variance_to_keep)
print SEPARATOR_STRING
novelty_dict[novelty_detection.UCN_FILE_NAME_KEY] = upconvnet_file_name
novelty_file_name = '{0:s}/novelty_results.p'.format(top_output_dir_name)
print 'Writing results to: "{0:s}"...\n'.format(novelty_file_name)
novelty_detection.write_results(novelty_dict=novelty_dict,
pickle_file_name=novelty_file_name)
for i in range(num_test_examples):
_plot_results(novelty_dict=novelty_dict,
narr_predictor_names=cnn_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
test_index=i,
top_output_dir_name=top_output_dir_name)
print '\n'
def _run(input_model_file_name, narr_predictor_names, pressure_level_mb,
dilation_distance_metres, num_lead_time_steps,
predictor_time_step_offsets, num_examples_per_time,
weight_loss_function, class_fractions, top_narr_directory_name,
top_frontal_grid_dir_name, narr_mask_file_name,
first_training_time_string, last_training_time_string,
first_validation_time_string, last_validation_time_string,
num_examples_per_batch, num_epochs, num_training_batches_per_epoch,
num_validation_batches_per_epoch, output_model_file_name):
"""Trains CNN from scratch.
This is effectively the main method.
:param input_model_file_name: See documentation at top of file.
:param narr_predictor_names: Same.
:param pressure_level_mb: Same.
:param dilation_distance_metres: Same.
:param num_lead_time_steps: Same.
:param predictor_time_step_offsets: Same.
:param num_examples_per_time: Same.
:param weight_loss_function: Same.
:param class_fractions: Same.
:param top_narr_directory_name: Same.
:param top_frontal_grid_dir_name: Same.
:param narr_mask_file_name: Same.
:param first_training_time_string: Same.
:param last_training_time_string: Same.
:param first_validation_time_string: Same.
:param last_validation_time_string: Same.
:param num_examples_per_batch: Same.
:param num_epochs: Same.
:param num_training_batches_per_epoch: Same.
:param num_validation_batches_per_epoch: Same.
:param output_model_file_name: Same.
:raises: ValueError: if `num_lead_time_steps > 1`.
"""
# Process input args.
first_training_time_unix_sec = time_conversion.string_to_unix_sec(
first_training_time_string, TIME_FORMAT)
last_training_time_unix_sec = time_conversion.string_to_unix_sec(
last_training_time_string, TIME_FORMAT)
first_validation_time_unix_sec = time_conversion.string_to_unix_sec(
first_validation_time_string, TIME_FORMAT)
last_validation_time_unix_sec = time_conversion.string_to_unix_sec(
last_validation_time_string, TIME_FORMAT)
if narr_mask_file_name == '':
narr_mask_file_name = None
narr_mask_matrix = None
if num_lead_time_steps <= 1:
num_lead_time_steps = None
predictor_time_step_offsets = None
else:
error_string = (
'This script cannot yet handle num_lead_time_steps > 1 '
'(specifically {0:d}).'
).format(num_lead_time_steps)
raise ValueError(error_string)
# Read architecture.
print 'Reading architecture from: "{0:s}"...'.format(input_model_file_name)
model_object = traditional_cnn.read_keras_model(input_model_file_name)
model_object = keras.models.clone_model(model_object)
# TODO(thunderhoser): This is a HACK.
model_object.compile(
loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(),
metrics=traditional_cnn.LIST_OF_METRIC_FUNCTIONS)
print SEPARATOR_STRING
model_object.summary()
print SEPARATOR_STRING
# Write metadata.
input_tensor = model_object.input
num_grid_rows = input_tensor.get_shape().as_list()[1]
num_grid_columns = input_tensor.get_shape().as_list()[2]
num_half_rows = int(numpy.round((num_grid_rows - 1) / 2))
num_half_columns = int(numpy.round((num_grid_columns - 1) / 2))
if narr_mask_file_name is not None:
print 'Reading NARR mask from: "{0:s}"...'.format(narr_mask_file_name)
narr_mask_matrix = ml_utils.read_narr_mask(narr_mask_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=output_model_file_name, raise_error_if_missing=False)
print 'Writing metadata to: "{0:s}"...'.format(model_metafile_name)
traditional_cnn.write_model_metadata(
pickle_file_name=model_metafile_name, num_epochs=num_epochs,
num_examples_per_batch=num_examples_per_batch,
num_examples_per_target_time=num_examples_per_time,
num_training_batches_per_epoch=num_training_batches_per_epoch,
num_validation_batches_per_epoch=num_validation_batches_per_epoch,
num_rows_in_half_grid=num_half_rows,
num_columns_in_half_grid=num_half_columns,
dilation_distance_metres=dilation_distance_metres,
class_fractions=class_fractions,
weight_loss_function=weight_loss_function,
narr_predictor_names=narr_predictor_names,
pressure_level_mb=pressure_level_mb,
training_start_time_unix_sec=first_training_time_unix_sec,
training_end_time_unix_sec=last_training_time_unix_sec,
validation_start_time_unix_sec=first_validation_time_unix_sec,
validation_end_time_unix_sec=last_validation_time_unix_sec,
num_lead_time_steps=num_lead_time_steps,
predictor_time_step_offsets=predictor_time_step_offsets,
narr_mask_matrix=narr_mask_matrix)
print SEPARATOR_STRING
traditional_cnn.train_with_3d_examples(
model_object=model_object, output_file_name=output_model_file_name,
num_examples_per_batch=num_examples_per_batch, num_epochs=num_epochs,
num_training_batches_per_epoch=num_training_batches_per_epoch,
num_examples_per_target_time=num_examples_per_time,
training_start_time_unix_sec=first_training_time_unix_sec,
training_end_time_unix_sec=last_training_time_unix_sec,
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=narr_predictor_names,
pressure_level_mb=pressure_level_mb,
dilation_distance_metres=dilation_distance_metres,
class_fractions=class_fractions,
num_rows_in_half_grid=num_half_rows,
num_columns_in_half_grid=num_half_columns,
weight_loss_function=weight_loss_function,
num_validation_batches_per_epoch=num_validation_batches_per_epoch,
validation_start_time_unix_sec=first_validation_time_unix_sec,
validation_end_time_unix_sec=last_validation_time_unix_sec,
narr_mask_matrix=narr_mask_matrix)
def _run(model_file_name, first_time_string, last_time_string, randomize_times,
num_target_times, use_isotonic_regression, top_narr_directory_name,
top_frontal_grid_dir_name, output_dir_name):
"""Applies traditional CNN to full grids.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param first_time_string: Same.
:param last_time_string: Same.
:param randomize_times: Same.
:param num_target_times: Same.
:param use_isotonic_regression: Same.
:param top_narr_directory_name: Same.
:param top_frontal_grid_dir_name: Same.
:param output_dir_name: Same.
"""
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, INPUT_TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, INPUT_TIME_FORMAT)
target_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=NARR_TIME_INTERVAL_SEC, include_endpoint=True)
if randomize_times:
error_checking.assert_is_leq(
num_target_times, len(target_times_unix_sec))
numpy.random.shuffle(target_times_unix_sec)
target_times_unix_sec = target_times_unix_sec[:num_target_times]
print 'Reading model from: "{0:s}"...'.format(model_file_name)
model_object = traditional_cnn.read_keras_model(model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=model_file_name, raise_error_if_missing=True)
print 'Reading model metadata from: "{0:s}"...'.format(
model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
if use_isotonic_regression:
isotonic_file_name = isotonic_regression.find_model_file(
base_model_file_name=model_file_name, raise_error_if_missing=True)
print 'Reading isotonic-regression models from: "{0:s}"...'.format(
isotonic_file_name)
isotonic_model_object_by_class = (
isotonic_regression.read_model_for_each_class(isotonic_file_name)
)
else:
isotonic_model_object_by_class = None
if model_metadata_dict[traditional_cnn.NUM_LEAD_TIME_STEPS_KEY] is None:
num_dimensions = 3
else:
num_dimensions = 4
num_classes = len(model_metadata_dict[traditional_cnn.CLASS_FRACTIONS_KEY])
num_target_times = len(target_times_unix_sec)
print SEPARATOR_STRING
for i in range(num_target_times):
if num_dimensions == 3:
(this_class_probability_matrix, this_target_matrix
) = traditional_cnn.apply_model_to_3d_example(
model_object=model_object,
target_time_unix_sec=target_times_unix_sec[i],
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
pressure_level_mb=model_metadata_dict[
traditional_cnn.PRESSURE_LEVEL_KEY],
dilation_distance_metres=model_metadata_dict[
traditional_cnn.DILATION_DISTANCE_FOR_TARGET_KEY],
num_rows_in_half_grid=model_metadata_dict[
traditional_cnn.NUM_ROWS_IN_HALF_GRID_KEY],
num_columns_in_half_grid=model_metadata_dict[
traditional_cnn.NUM_COLUMNS_IN_HALF_GRID_KEY],
num_classes=num_classes,
isotonic_model_object_by_class=isotonic_model_object_by_class,
narr_mask_matrix=model_metadata_dict[
traditional_cnn.NARR_MASK_MATRIX_KEY])
else:
(this_class_probability_matrix, this_target_matrix
) = traditional_cnn.apply_model_to_4d_example(
model_object=model_object,
target_time_unix_sec=target_times_unix_sec[i],
predictor_time_step_offsets=model_metadata_dict[
traditional_cnn.PREDICTOR_TIME_STEP_OFFSETS_KEY],
num_lead_time_steps=model_metadata_dict[
traditional_cnn.NUM_LEAD_TIME_STEPS_KEY],
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
pressure_level_mb=model_metadata_dict[
traditional_cnn.PRESSURE_LEVEL_KEY],
dilation_distance_metres=model_metadata_dict[
traditional_cnn.DILATION_DISTANCE_FOR_TARGET_KEY],
num_rows_in_half_grid=model_metadata_dict[
traditional_cnn.NUM_ROWS_IN_HALF_GRID_KEY],
num_columns_in_half_grid=model_metadata_dict[
traditional_cnn.NUM_COLUMNS_IN_HALF_GRID_KEY],
num_classes=num_classes,
isotonic_model_object_by_class=isotonic_model_object_by_class,
narr_mask_matrix=model_metadata_dict[
traditional_cnn.NARR_MASK_MATRIX_KEY])
this_target_matrix[this_target_matrix == -1] = 0
print MINOR_SEPARATOR_STRING
this_prediction_file_name = ml_utils.find_gridded_prediction_file(
directory_name=output_dir_name,
first_target_time_unix_sec=target_times_unix_sec[i],
last_target_time_unix_sec=target_times_unix_sec[i],
raise_error_if_missing=False)
print 'Writing gridded predictions to file: "{0:s}"...'.format(
this_prediction_file_name)
ml_utils.write_gridded_predictions(
pickle_file_name=this_prediction_file_name,
class_probability_matrix=this_class_probability_matrix,
target_times_unix_sec=target_times_unix_sec[[i]],
model_file_name=model_file_name,
used_isotonic_regression=use_isotonic_regression,
target_matrix=this_target_matrix)
if i != num_target_times - 1:
print SEPARATOR_STRING
def _run(model_file_name, first_eval_time_string, last_eval_time_string,
num_times, num_examples_per_time, dilation_distance_metres,
use_isotonic_regression, top_narr_directory_name,
top_frontal_grid_dir_name, output_dir_name):
"""Evaluates CNN trained by patch classification.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param first_eval_time_string: Same.
:param last_eval_time_string: Same.
:param num_times: Same.
:param num_examples_per_time: Same.
:param dilation_distance_metres: Same.
:param use_isotonic_regression: Same.
:param top_narr_directory_name: Same.
:param top_frontal_grid_dir_name: Same.
:param output_dir_name: Same.
"""
first_eval_time_unix_sec = time_conversion.string_to_unix_sec(
first_eval_time_string, INPUT_TIME_FORMAT)
last_eval_time_unix_sec = time_conversion.string_to_unix_sec(
last_eval_time_string, INPUT_TIME_FORMAT)
print 'Reading model from: "{0:s}"...'.format(model_file_name)
model_object = traditional_cnn.read_keras_model(model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=model_file_name, raise_error_if_missing=True)
print 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
if dilation_distance_metres < 0:
dilation_distance_metres = model_metadata_dict[
traditional_cnn.DILATION_DISTANCE_FOR_TARGET_KEY] + 0.
if use_isotonic_regression:
isotonic_file_name = isotonic_regression.find_model_file(
base_model_file_name=model_file_name, raise_error_if_missing=True)
print 'Reading isotonic-regression models from: "{0:s}"...'.format(
isotonic_file_name)
isotonic_model_object_by_class = (
isotonic_regression.read_model_for_each_class(isotonic_file_name))
else:
isotonic_model_object_by_class = None
num_classes = len(model_metadata_dict[traditional_cnn.CLASS_FRACTIONS_KEY])
print SEPARATOR_STRING
class_probability_matrix, observed_labels = (
eval_utils.downsized_examples_to_eval_pairs(
model_object=model_object,
first_target_time_unix_sec=first_eval_time_unix_sec,
last_target_time_unix_sec=last_eval_time_unix_sec,
num_target_times_to_sample=num_times,
num_examples_per_time=num_examples_per_time,
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
pressure_level_mb=model_metadata_dict[
traditional_cnn.PRESSURE_LEVEL_KEY],
dilation_distance_metres=dilation_distance_metres,
num_rows_in_half_grid=model_metadata_dict[
traditional_cnn.NUM_ROWS_IN_HALF_GRID_KEY],
num_columns_in_half_grid=model_metadata_dict[
traditional_cnn.NUM_COLUMNS_IN_HALF_GRID_KEY],
num_classes=num_classes,
predictor_time_step_offsets=model_metadata_dict[
traditional_cnn.PREDICTOR_TIME_STEP_OFFSETS_KEY],
num_lead_time_steps=model_metadata_dict[
traditional_cnn.NUM_LEAD_TIME_STEPS_KEY],
isotonic_model_object_by_class=isotonic_model_object_by_class,
narr_mask_matrix=model_metadata_dict[
traditional_cnn.NARR_MASK_MATRIX_KEY]))
print SEPARATOR_STRING
model_eval_helper.run_evaluation(
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels,
output_dir_name=output_dir_name)
def _run(input_cnn_file_name, use_batch_norm_for_out_layer, use_transposed_conv,
use_conv_for_out_layer, smoothing_radius_px, top_training_dir_name,
first_training_time_string, last_training_time_string,
top_validation_dir_name, first_validation_time_string,
last_validation_time_string, num_examples_per_batch, num_epochs,
num_training_batches_per_epoch, num_validation_batches_per_epoch,
output_model_file_name):
"""Trains upconvnet.
This is effectively the main method.
:param input_cnn_file_name: See documentation at top of file.
:param use_batch_norm_for_out_layer: Same.
:param use_transposed_conv: Same.
:param use_conv_for_out_layer: Same.
:param smoothing_radius_px: Same.
:param top_training_dir_name: Same.
:param first_training_time_string: Same.
:param last_training_time_string: Same.
:param top_validation_dir_name: Same.
:param first_validation_time_string: Same.
:param last_validation_time_string: Same.
:param num_examples_per_batch: Same.
:param num_epochs: Same.
:param num_training_batches_per_epoch: Same.
:param num_validation_batches_per_epoch: Same.
:param output_model_file_name: Same.
"""
first_training_time_unix_sec = time_conversion.string_to_unix_sec(
first_training_time_string, TIME_FORMAT)
last_training_time_unix_sec = time_conversion.string_to_unix_sec(
last_training_time_string, TIME_FORMAT)
first_validation_time_unix_sec = time_conversion.string_to_unix_sec(
first_validation_time_string, TIME_FORMAT)
last_validation_time_unix_sec = time_conversion.string_to_unix_sec(
last_validation_time_string, TIME_FORMAT)
if smoothing_radius_px <= 0:
smoothing_radius_px = None
print 'Reading trained CNN from: "{0:s}"...'.format(input_cnn_file_name)
cnn_model_object = traditional_cnn.read_keras_model(input_cnn_file_name)
cnn_metafile_name = traditional_cnn.find_metafile(
model_file_name=input_cnn_file_name, raise_error_if_missing=True)
print 'Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)
cnn_metadata_dict = traditional_cnn.read_model_metadata(
cnn_metafile_name)
cnn_feature_layer_name = traditional_cnn.get_flattening_layer(
cnn_model_object)
cnn_feature_layer_object = cnn_model_object.get_layer(
name=cnn_feature_layer_name)
cnn_feature_dimensions = numpy.array(
cnn_feature_layer_object.input.shape[1:], dtype=int)
num_input_features = numpy.prod(cnn_feature_dimensions)
first_num_rows = cnn_feature_dimensions[0]
first_num_columns = cnn_feature_dimensions[1]
num_output_channels = numpy.array(
cnn_model_object.input.shape[1:], dtype=int
)[-1]
ucn_metafile_name = traditional_cnn.find_metafile(
model_file_name=output_model_file_name, raise_error_if_missing=False)
print 'Writing upconvnet metadata to: "{0:s}"...'.format(ucn_metafile_name)
upconvnet.write_model_metadata(
pickle_file_name=ucn_metafile_name,
top_training_dir_name=top_training_dir_name,
first_training_time_unix_sec=first_training_time_unix_sec,
last_training_time_unix_sec=last_training_time_unix_sec,
cnn_model_file_name=input_cnn_file_name,
cnn_feature_layer_name=cnn_feature_layer_name, num_epochs=num_epochs,
num_examples_per_batch=num_examples_per_batch,
num_training_batches_per_epoch=num_training_batches_per_epoch,
num_validation_batches_per_epoch=num_validation_batches_per_epoch,
top_validation_dir_name=top_validation_dir_name,
first_validation_time_unix_sec=first_validation_time_unix_sec,
last_validation_time_unix_sec=last_validation_time_unix_sec)
print SEPARATOR_STRING
ucn_model_object = upconvnet.create_net(
num_input_features=num_input_features, first_num_rows=first_num_rows,
first_num_columns=first_num_columns,
upsampling_factors=UPSAMPLING_FACTORS,
num_output_channels=num_output_channels,
use_activation_for_out_layer=False,
use_bn_for_out_layer=use_batch_norm_for_out_layer,
use_transposed_conv=use_transposed_conv,
use_conv_for_out_layer=use_conv_for_out_layer,
smoothing_radius_px=smoothing_radius_px)
print SEPARATOR_STRING
upconvnet.train_upconvnet(
ucn_model_object=ucn_model_object,
top_training_dir_name=top_training_dir_name,
first_training_time_unix_sec=first_training_time_unix_sec,
last_training_time_unix_sec=last_training_time_unix_sec,
cnn_model_object=cnn_model_object,
cnn_feature_layer_name=cnn_feature_layer_name,
cnn_metadata_dict=cnn_metadata_dict,
num_examples_per_batch=num_examples_per_batch,
num_epochs=num_epochs,
num_training_batches_per_epoch=num_training_batches_per_epoch,
output_model_file_name=output_model_file_name,
num_validation_batches_per_epoch=num_validation_batches_per_epoch,
top_validation_dir_name=top_validation_dir_name,
first_validation_time_unix_sec=first_validation_time_unix_sec,
last_validation_time_unix_sec=last_validation_time_unix_sec)
def _run(orig_model_file_name, top_training_dir_name,
first_training_time_string, last_training_time_string,
num_training_examples, top_validn_dir_name, first_validn_time_string,
last_validn_time_string, num_validn_examples, narr_predictor_names,
num_training_examples_per_batch, num_epochs, min_loss_decrease,
min_percentage_loss_decrease, num_steps_for_loss_decrease,
output_file_name):
"""Runs sequential forward selection.
This is effectively the main method.
:param orig_model_file_name: See documentation at top of file.
:param top_training_dir_name: Same.
:param first_training_time_string: Same.
:param last_training_time_string: Same.
:param num_training_examples: Same.
:param top_validn_dir_name: Same.
:param first_validn_time_string: Same.
:param last_validn_time_string: Same.
:param num_validn_examples: Same.
:param narr_predictor_names: Same.
:param num_training_examples_per_batch: Same.
:param num_epochs: Same.
:param min_loss_decrease: Same.
:param min_percentage_loss_decrease: Same.
:param num_steps_for_loss_decrease: Same.
:param output_file_name: Same.
"""
print 'Reading original model from: "{0:s}"...'.format(orig_model_file_name)
orig_model_object = traditional_cnn.read_keras_model(orig_model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=orig_model_file_name)
print 'Reading model metadata from: "{0:s}"...'.format(
model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
print SEPARATOR_STRING
training_predictor_matrix, training_target_values = _read_examples(
top_example_dir_name=top_training_dir_name,
first_time_string=first_training_time_string,
last_time_string=last_training_time_string,
num_examples=num_training_examples,
model_metadata_dict=model_metadata_dict)
print SEPARATOR_STRING
validn_predictor_matrix, validn_target_values = _read_examples(
top_example_dir_name=top_validn_dir_name,
first_time_string=first_validn_time_string,
last_time_string=last_validn_time_string,
num_examples=num_validn_examples,
model_metadata_dict=model_metadata_dict)
print SEPARATOR_STRING
# TODO(thunderhoser): I could make the code more efficient by making
# `narr_predictor_names` an input arg to `_read_examples`.
if narr_predictor_names[0] in ['', 'None']:
narr_predictor_names = model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY]
training_function = sequential_selection.create_training_function(
num_training_examples_per_batch=num_training_examples_per_batch,
num_epochs=num_epochs)
result_dict = sequential_selection.run_sfs(
list_of_training_matrices=[training_predictor_matrix],
training_target_values=training_target_values,
list_of_validation_matrices=[validn_predictor_matrix],
validation_target_values=validn_target_values,
predictor_names_by_matrix=[narr_predictor_names],
model_builder=_create_model_builder(orig_model_object),
training_function=training_function,
min_loss_decrease=min_loss_decrease,
min_percentage_loss_decrease=min_percentage_loss_decrease,
num_steps_for_loss_decrease=num_steps_for_loss_decrease)
print SEPARATOR_STRING
result_dict.update({
ORIG_MODEL_FILE_ARG_NAME: orig_model_file_name,
TRAINING_DIR_ARG_NAME: top_training_dir_name,
FIRST_TRAINING_TIME_ARG_NAME: first_training_time_string,
LAST_TRAINING_TIME_ARG_NAME: last_training_time_string,
NUM_TRAINING_EXAMPLES_ARG_NAME: num_training_examples,
VALIDN_DIR_ARG_NAME: top_validn_dir_name,
FIRST_VALIDN_TIME_ARG_NAME: first_validn_time_string,
LAST_VALIDN_TIME_ARG_NAME: last_validn_time_string,
NUM_VALIDN_EXAMPLES_ARG_NAME: num_validn_examples
})
print 'Writing results to: "{0:s}"...'.format(output_file_name)
sequential_selection.write_results(
result_dict=result_dict, pickle_file_name=output_file_name)
def _run(model_file_name, example_file_name, num_examples, example_indices,
component_type_string, target_class, layer_name, ideal_activation,
neuron_indices, channel_index, output_file_name):
"""Creates saliency map for each example, based on the same CNN.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param example_file_name: Same.
:param num_examples: Same.
:param example_indices: Same.
:param component_type_string: Same.
:param target_class: Same.
:param layer_name: Same.
:param ideal_activation: Same.
:param neuron_indices: Same.
:param channel_index: Same.
:param output_file_name: Same.
"""
if num_examples <= 0:
num_examples = None
if num_examples is None:
error_checking.assert_is_geq_numpy_array(example_indices, 0)
else:
error_checking.assert_is_greater(num_examples, 0)
print 'Reading model from: "{0:s}"...'.format(model_file_name)
model_object = traditional_cnn.read_keras_model(model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=model_file_name)
print 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
print 'Reading normalized examples from: "{0:s}"...'.format(
example_file_name)
example_dict = trainval_io.read_downsized_3d_examples(
netcdf_file_name=example_file_name,
predictor_names_to_keep=model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
num_half_rows_to_keep=model_metadata_dict[
traditional_cnn.NUM_ROWS_IN_HALF_GRID_KEY],
num_half_columns_to_keep=model_metadata_dict[
traditional_cnn.NUM_COLUMNS_IN_HALF_GRID_KEY])
predictor_matrix = example_dict[trainval_io.PREDICTOR_MATRIX_KEY]
if num_examples is not None:
num_examples_total = predictor_matrix.shape[0]
example_indices = numpy.linspace(0,
num_examples_total - 1,
num=num_examples_total,
dtype=int)
num_examples = min([num_examples, num_examples_total])
example_indices = numpy.random.choice(example_indices,
size=num_examples,
replace=False)
predictor_matrix = predictor_matrix[example_indices, ...]
if component_type_string == CLASS_COMPONENT_TYPE_STRING:
print 'Computing saliency maps for target class {0:d}...'.format(
target_class)
saliency_matrix = (
gg_saliency_maps.get_saliency_maps_for_class_activation(
model_object=model_object,
target_class=target_class,
list_of_input_matrices=[predictor_matrix])[0])
elif component_type_string == NEURON_COMPONENT_TYPE_STRING:
print('Computing saliency maps for neuron {0:s} in layer "{1:s}"...'
).format(str(neuron_indices), layer_name)
saliency_matrix = (
gg_saliency_maps.get_saliency_maps_for_neuron_activation(
model_object=model_object,
layer_name=layer_name,
neuron_indices=neuron_indices,
list_of_input_matrices=[predictor_matrix],
ideal_activation=ideal_activation)[0])
else:
print('Computing saliency maps for channel {0:d} in layer "{1:s}"...'
).format(channel_index, layer_name)
saliency_matrix = (
gg_saliency_maps.get_saliency_maps_for_channel_activation(
model_object=model_object,
layer_name=layer_name,
channel_index=channel_index,
list_of_input_matrices=[predictor_matrix],
stat_function_for_neuron_activations=K.max,
ideal_activation=ideal_activation)[0])
print 'Writing results to: "{0:s}"...'.format(output_file_name)
ge_saliency_maps.write_file(pickle_file_name=output_file_name,
normalized_predictor_matrix=predictor_matrix,
saliency_matrix=saliency_matrix,
model_file_name=model_file_name,
component_type_string=component_type_string,
target_class=target_class,
layer_name=layer_name,
ideal_activation=ideal_activation,
neuron_indices=neuron_indices,
channel_index=channel_index)
def _run(model_file_name, example_file_name, num_examples, example_indices,
component_type_string, target_class, layer_name, ideal_activation,
neuron_indices, channel_index, num_iterations, learning_rate,
output_file_name):
"""Runs backwards optimization on a trained CNN.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param example_file_name: Same.
:param num_examples: Same.
:param example_indices: Same.
:param component_type_string: Same.
:param target_class: Same.
:param layer_name: Same.
:param ideal_activation: Same.
:param neuron_indices: Same.
:param channel_index: Same.
:param num_iterations: Same.
:param learning_rate: Same.
:param output_file_name: Same.
"""
if num_examples <= 0:
num_examples = None
print 'Reading model from: "{0:s}"...'.format(model_file_name)
model_object = traditional_cnn.read_keras_model(model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=model_file_name)
print 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
print 'Reading normalized examples from: "{0:s}"...'.format(
example_file_name)
example_dict = trainval_io.read_downsized_3d_examples(
netcdf_file_name=example_file_name,
predictor_names_to_keep=model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
num_half_rows_to_keep=model_metadata_dict[
traditional_cnn.NUM_ROWS_IN_HALF_GRID_KEY],
num_half_columns_to_keep=model_metadata_dict[
traditional_cnn.NUM_COLUMNS_IN_HALF_GRID_KEY])
predictor_matrix = example_dict[trainval_io.PREDICTOR_MATRIX_KEY]
if num_examples is None:
error_checking.assert_is_geq_numpy_array(example_indices, 0)
num_examples = len(example_indices)
else:
error_checking.assert_is_greater(num_examples, 0)
num_examples_total = predictor_matrix.shape[0]
example_indices = numpy.linspace(0,
num_examples_total - 1,
num=num_examples_total,
dtype=int)
num_examples = min([num_examples, num_examples_total])
example_indices = numpy.random.choice(example_indices,
size=num_examples,
replace=False)
predictor_matrix = predictor_matrix[example_indices, ...]
optimized_predictor_matrix = numpy.full(predictor_matrix.shape, numpy.nan)
print SEPARATOR_STRING
for i in range(num_examples):
if component_type_string == CLASS_COMPONENT_TYPE_STRING:
print(
'Optimizing {0:d}th of {1:d} images for target class {2:d}...'
).format(i + 1, num_examples, target_class)
optimized_predictor_matrix[i, ...] = (
backwards_opt.optimize_input_for_class(
model_object=model_object,
target_class=target_class,
init_function_or_matrices=[predictor_matrix[[i], ...]],
num_iterations=num_iterations,
learning_rate=learning_rate)[0])
elif component_type_string == NEURON_COMPONENT_TYPE_STRING:
print(
'Optimizing {0:d}th of {1:d} images for neuron {2:s} in layer '
'"{3:s}"...').format(i + 1, num_examples, str(neuron_indices),
layer_name)
optimized_predictor_matrix[i, ...] = (
backwards_opt.optimize_input_for_neuron(
model_object=model_object,
layer_name=layer_name,
neuron_indices=neuron_indices,
init_function_or_matrices=[predictor_matrix[[i], ...]],
num_iterations=num_iterations,
learning_rate=learning_rate,
ideal_activation=ideal_activation)[0])
else:
print(
'Optimizing {0:d}th of {1:d} images for channel {2:d} in layer '
'"{3:s}"...').format(i + 1, num_examples, channel_index,
layer_name)
optimized_predictor_matrix[i, ...] = (
backwards_opt.optimize_input_for_channel(
model_object=model_object,
layer_name=layer_name,
channel_index=channel_index,
init_function_or_matrices=[predictor_matrix[[i], ...]],
stat_function_for_neuron_activations=K.max,
num_iterations=num_iterations,
learning_rate=learning_rate,
ideal_activation=ideal_activation)[0])
print SEPARATOR_STRING
print 'Writing results to: "{0:s}"...'.format(output_file_name)
backwards_opt.write_results(
pickle_file_name=output_file_name,
list_of_optimized_input_matrices=[optimized_predictor_matrix],
model_file_name=model_file_name,
init_function_name_or_matrices=[predictor_matrix],
num_iterations=num_iterations,
learning_rate=learning_rate,
component_type_string=component_type_string,
target_class=target_class,
layer_name=layer_name,
neuron_indices=neuron_indices,
channel_index=channel_index,
ideal_activation=ideal_activation)
def _run(input_model_file_name, narr_predictor_names, top_training_dir_name,
first_training_time_string, last_training_time_string,
top_validation_dir_name, first_validation_time_string,
last_validation_time_string, num_examples_per_batch, num_epochs,
num_training_batches_per_epoch, num_validation_batches_per_epoch,
output_model_file_name):
"""Trains CNN with example files.
This is effectively the main method.
:param input_model_file_name: See documentation at top of file.
:param narr_predictor_names: Same.
:param top_training_dir_name: Same.
:param first_training_time_string: Same.
:param last_training_time_string: Same.
:param top_validation_dir_name: Same.
:param first_validation_time_string: Same.
:param last_validation_time_string: Same.
:param num_examples_per_batch: Same.
:param num_epochs: Same.
:param num_training_batches_per_epoch: Same.
:param num_validation_batches_per_epoch: Same.
:param output_model_file_name: Same.
"""
# Process input args.
first_training_time_unix_sec = time_conversion.string_to_unix_sec(
first_training_time_string, TIME_FORMAT)
last_training_time_unix_sec = time_conversion.string_to_unix_sec(
last_training_time_string, TIME_FORMAT)
first_validation_time_unix_sec = time_conversion.string_to_unix_sec(
first_validation_time_string, TIME_FORMAT)
last_validation_time_unix_sec = time_conversion.string_to_unix_sec(
last_validation_time_string, TIME_FORMAT)
# Read architecture.
print 'Reading architecture from: "{0:s}"...'.format(input_model_file_name)
model_object = traditional_cnn.read_keras_model(input_model_file_name)
model_object = keras.models.clone_model(model_object)
# TODO(thunderhoser): This is a HACK.
model_object.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(),
metrics=traditional_cnn.LIST_OF_METRIC_FUNCTIONS)
print SEPARATOR_STRING
model_object.summary()
print SEPARATOR_STRING
# Write metadata.
input_tensor = model_object.input
num_grid_rows = input_tensor.get_shape().as_list()[1]
num_grid_columns = input_tensor.get_shape().as_list()[2]
num_half_rows = int(numpy.round((num_grid_rows - 1) / 2))
num_half_columns = int(numpy.round((num_grid_columns - 1) / 2))
print 'Reading NARR mask from: "{0:s}"...'.format(NARR_MASK_FILE_NAME)
narr_mask_matrix = ml_utils.read_narr_mask(NARR_MASK_FILE_NAME)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=output_model_file_name, raise_error_if_missing=False)
print 'Writing metadata to: "{0:s}"...'.format(model_metafile_name)
traditional_cnn.write_model_metadata(
pickle_file_name=model_metafile_name,
num_epochs=num_epochs,
num_examples_per_batch=num_examples_per_batch,
num_examples_per_target_time=NUM_EXAMPLES_PER_TIME_DUMMY,
num_training_batches_per_epoch=num_training_batches_per_epoch,
num_validation_batches_per_epoch=num_validation_batches_per_epoch,
num_rows_in_half_grid=num_half_rows,
num_columns_in_half_grid=num_half_columns,
dilation_distance_metres=DILATION_DISTANCE_METRES,
class_fractions=CLASS_FRACTIONS,
weight_loss_function=WEIGHT_LOSS_FLAG,
narr_predictor_names=narr_predictor_names,
pressure_level_mb=PRESSURE_LEVEL_MB,
training_start_time_unix_sec=first_training_time_unix_sec,
training_end_time_unix_sec=last_training_time_unix_sec,
validation_start_time_unix_sec=first_validation_time_unix_sec,
validation_end_time_unix_sec=last_validation_time_unix_sec,
num_lead_time_steps=None,
predictor_time_step_offsets=None,
narr_mask_matrix=narr_mask_matrix)
print SEPARATOR_STRING
traditional_cnn.quick_train_3d(
model_object=model_object,
output_file_name=output_model_file_name,
num_examples_per_batch=num_examples_per_batch,
num_epochs=num_epochs,
num_training_batches_per_epoch=num_training_batches_per_epoch,
training_start_time_unix_sec=first_training_time_unix_sec,
training_end_time_unix_sec=last_training_time_unix_sec,
top_training_dir_name=top_training_dir_name,
top_validation_dir_name=top_validation_dir_name,
narr_predictor_names=narr_predictor_names,
num_classes=NUM_CLASSES,
num_rows_in_half_grid=num_half_rows,
num_columns_in_half_grid=num_half_columns,
num_validation_batches_per_epoch=num_validation_batches_per_epoch,
validation_start_time_unix_sec=first_validation_time_unix_sec,
validation_end_time_unix_sec=last_validation_time_unix_sec)
def _run(upconvnet_file_name, example_file_name, num_examples, example_indices,
top_output_dir_name):
"""Applies upconvnet to one or more examples.
This is effectively the main method.
:param upconvnet_file_name: See documentation at top of file.
:param example_file_name: Same.
:param num_examples: Same.
:param example_indices: Same.
:param top_output_dir_name: Same.
"""
# Check input args.
if num_examples <= 0:
num_examples = None
if num_examples is None:
error_checking.assert_is_geq_numpy_array(example_indices, 0)
else:
error_checking.assert_is_greater(num_examples, 0)
# Read upconvnet and metadata.
ucn_metafile_name = traditional_cnn.find_metafile(
model_file_name=upconvnet_file_name, raise_error_if_missing=True)
print('Reading trained upconvnet from: "{0:s}"...'.format(
upconvnet_file_name))
ucn_model_object = traditional_cnn.read_keras_model(upconvnet_file_name)
print('Reading upconvnet metadata from: "{0:s}"...'.format(
ucn_metafile_name))
ucn_metadata_dict = upconvnet.read_model_metadata(ucn_metafile_name)
# Read CNN and metadata.
cnn_file_name = ucn_metadata_dict[upconvnet.CNN_FILE_NAME_KEY]
cnn_metafile_name = traditional_cnn.find_metafile(
model_file_name=cnn_file_name, raise_error_if_missing=True)
print 'Reading trained CNN from: "{0:s}"...'.format(cnn_file_name)
cnn_model_object = traditional_cnn.read_keras_model(cnn_file_name)
print 'Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)
cnn_metadata_dict = traditional_cnn.read_model_metadata(cnn_metafile_name)
print SEPARATOR_STRING
actual_image_matrix = _read_input_examples(
example_file_name=example_file_name,
cnn_metadata_dict=cnn_metadata_dict,
num_examples=num_examples,
example_indices=example_indices)
print SEPARATOR_STRING
reconstructed_image_matrix = upconvnet.apply_upconvnet(
actual_image_matrix=actual_image_matrix,
cnn_model_object=cnn_model_object,
ucn_model_object=ucn_model_object)
print SEPARATOR_STRING
_plot_examples(actual_image_matrix=actual_image_matrix,
reconstructed_image_matrix=reconstructed_image_matrix,
narr_predictor_names=cnn_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
top_output_dir_name=top_output_dir_name)
def _run(model_file_name, example_file_name, num_examples, example_indices,
layer_names, top_output_dir_name):
"""Plots feature maps for each example and CNN layer.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param example_file_name: Same.
:param num_examples: Same.
:param example_indices: Same.
:param layer_names: Same.
:param top_output_dir_name: Same.
"""
if num_examples <= 0:
num_examples = None
if num_examples is None:
error_checking.assert_is_geq_numpy_array(example_indices, 0)
else:
error_checking.assert_is_greater(num_examples, 0)
print 'Reading model from: "{0:s}"...'.format(model_file_name)
model_object = traditional_cnn.read_keras_model(model_file_name)
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=model_file_name)
print 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
print 'Reading normalized examples from: "{0:s}"...'.format(
example_file_name)
example_dict = trainval_io.read_downsized_3d_examples(
netcdf_file_name=example_file_name,
predictor_names_to_keep=model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY],
num_half_rows_to_keep=model_metadata_dict[
traditional_cnn.NUM_ROWS_IN_HALF_GRID_KEY],
num_half_columns_to_keep=model_metadata_dict[
traditional_cnn.NUM_COLUMNS_IN_HALF_GRID_KEY])
print SEPARATOR_STRING
predictor_matrix = example_dict[trainval_io.PREDICTOR_MATRIX_KEY]
if num_examples is not None:
num_examples_total = predictor_matrix.shape[0]
example_indices = numpy.linspace(0,
num_examples_total - 1,
num=num_examples_total,
dtype=int)
num_examples = min([num_examples, num_examples_total])
example_indices = numpy.random.choice(example_indices,
size=num_examples,
replace=False)
predictor_matrix = predictor_matrix[example_indices, ...]
num_examples = predictor_matrix.shape[0]
num_layers = len(layer_names)
feature_matrix_by_layer = [None] * num_layers
for k in range(num_layers):
print 'Creating feature maps for layer "{0:s}"...'.format(
layer_names[k])
this_partial_model_object = cnn.model_to_feature_generator(
model_object=model_object, feature_layer_name=layer_names[k])
feature_matrix_by_layer[k] = this_partial_model_object.predict(
predictor_matrix, batch_size=num_examples)
print SEPARATOR_STRING
for k in range(num_layers):
this_output_dir_name = '{0:s}/{1:s}'.format(top_output_dir_name,
layer_names[k])
file_system_utils.mkdir_recursive_if_necessary(
directory_name=this_output_dir_name)
_plot_feature_maps_one_layer(feature_matrix=feature_matrix_by_layer[k],
layer_name=layer_names[k],
output_dir_name=this_output_dir_name)
print SEPARATOR_STRING