def prediction_function_2d3d_cnn(model_object, list_of_input_matrices):
"""Prediction function for hybrid 2-and-3-D CNN.
:param model_object: See doc for `prediction_function_2d_cnn`.
:param list_of_input_matrices: Same.
:return: class_probability_matrix: Same.
"""
num_input_matrices = len(list_of_input_matrices)
upsample_refl = not (
num_input_matrices > 1 and len(list_of_input_matrices[1].shape) == 4
)
if upsample_refl:
if num_input_matrices == 2:
sounding_matrix = list_of_input_matrices[-1]
else:
sounding_matrix = None
return cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=list_of_input_matrices[0],
sounding_matrix=sounding_matrix, verbose=True)
if num_input_matrices == 3:
sounding_matrix = list_of_input_matrices[-1]
else:
sounding_matrix = None
return cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=list_of_input_matrices[0],
azimuthal_shear_matrix_s01=list_of_input_matrices[1],
sounding_matrix=sounding_matrix, verbose=True)
def prediction_function_2d3d_cnn(model_object, list_of_input_matrices):
"""Prediction function for hybrid 2D/3D GewitterGefahr CNN.
E = number of examples
K = number of target classes
:param model_object: See doc for `run_permutation_test`.
:param list_of_input_matrices: Same.
:return: class_probability_matrix: E-by-K numpy array, where
class_probability_matrix[i, k] is the probability that the [i]th example
belongs to the [k]th class.
"""
num_input_matrices = len(list_of_input_matrices)
first_num_dimensions = len(list_of_input_matrices[0].shape)
upsample_refl = first_num_dimensions == 5
if upsample_refl:
if num_input_matrices == 2:
sounding_matrix = list_of_input_matrices[-1]
else:
sounding_matrix = None
return cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=list_of_input_matrices[0],
sounding_matrix=sounding_matrix)
if num_input_matrices == 3:
sounding_matrix = list_of_input_matrices[-1]
else:
sounding_matrix = None
return cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=list_of_input_matrices[0],
azimuthal_shear_matrix_s01=list_of_input_matrices[1],
sounding_matrix=sounding_matrix)
def _run(model_file_name, layer_names, top_example_dir_name,
storm_metafile_name, num_examples, top_output_dir_name):
"""Evaluates CNN (convolutional neural net) predictions.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param layer_names: Same.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if feature maps do not have 2 or 3 spatial dimensions.
"""
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0])
print(
'Reading model metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
print(
'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
full_id_strings, storm_times_unix_sec = tracking_io.read_ids_and_times(
storm_metafile_name)
print(SEPARATOR_STRING)
if 0 < num_examples < len(full_id_strings):
full_id_strings = full_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
list_of_predictor_matrices = testing_io.read_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
list_of_layer_operation_dicts=model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY])[0]
print(SEPARATOR_STRING)
include_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY]
is not None)
if include_soundings:
sounding_matrix = list_of_predictor_matrices[-1]
else:
sounding_matrix = None
num_layers = len(layer_names)
feature_matrix_by_layer = [None] * num_layers
for k in range(num_layers):
if model_metadata_dict[cnn.CONV_2D3D_KEY]:
if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
feature_matrix_by_layer[k] = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=list_of_predictor_matrices[0],
sounding_matrix=sounding_matrix,
return_features=True,
feature_layer_name=layer_names[k])
else:
feature_matrix_by_layer[k] = cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=list_of_predictor_matrices[0],
azimuthal_shear_matrix_s01=list_of_predictor_matrices[1],
sounding_matrix=sounding_matrix,
return_features=True,
feature_layer_name=layer_names[k])
else:
feature_matrix_by_layer[k] = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=list_of_predictor_matrices[0],
sounding_matrix=sounding_matrix,
return_features=True,
feature_layer_name=layer_names[k])
for k in range(num_layers):
this_output_dir_name = '{0:s}/{1:s}'.format(top_output_dir_name,
layer_names[k])
file_system_utils.mkdir_recursive_if_necessary(
directory_name=this_output_dir_name)
_plot_feature_maps_one_layer(feature_matrix=feature_matrix_by_layer[k],
full_id_strings=full_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
layer_name=layer_names[k],
output_dir_name=this_output_dir_name)
print(SEPARATOR_STRING)
def _run(model_file_name, example_file_name, first_time_string,
last_time_string, top_output_dir_name):
"""Applies CNN to one example file.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param example_file_name: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param top_output_dir_name: Same.
"""
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
model_directory_name, _ = os.path.split(model_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(model_directory_name)
print('Reading metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, INPUT_TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, INPUT_TIME_FORMAT)
training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None
training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = [example_file_name]
training_option_dict[
trainval_io.FIRST_STORM_TIME_KEY] = first_time_unix_sec
training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = last_time_unix_sec
if model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None:
generator_object = testing_io.gridrad_generator_2d_reduced(
option_dict=training_option_dict,
list_of_operation_dicts=model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY],
num_examples_total=LARGE_INTEGER)
elif model_metadata_dict[cnn.CONV_2D3D_KEY]:
generator_object = testing_io.myrorss_generator_2d3d(
option_dict=training_option_dict, num_examples_total=LARGE_INTEGER)
else:
generator_object = testing_io.generator_2d_or_3d(
option_dict=training_option_dict, num_examples_total=LARGE_INTEGER)
include_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY]
is not None)
try:
storm_object_dict = next(generator_object)
except StopIteration:
storm_object_dict = None
print(SEPARATOR_STRING)
if storm_object_dict is not None:
observed_labels = storm_object_dict[testing_io.TARGET_ARRAY_KEY]
list_of_predictor_matrices = storm_object_dict[
testing_io.INPUT_MATRICES_KEY]
if include_soundings:
sounding_matrix = list_of_predictor_matrices[-1]
else:
sounding_matrix = None
if model_metadata_dict[cnn.CONV_2D3D_KEY]:
if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
class_probability_matrix = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=list_of_predictor_matrices[0],
sounding_matrix=sounding_matrix,
verbose=True)
else:
class_probability_matrix = cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=list_of_predictor_matrices[0],
azimuthal_shear_matrix_s01=list_of_predictor_matrices[1],
sounding_matrix=sounding_matrix,
verbose=True)
else:
class_probability_matrix = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=list_of_predictor_matrices[0],
sounding_matrix=sounding_matrix,
verbose=True)
print(SEPARATOR_STRING)
num_examples = class_probability_matrix.shape[0]
for k in [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]:
print(
'{0:d}th percentile of {1:d} forecast probs = {2:.4f}'.format(
k, num_examples,
numpy.percentile(class_probability_matrix[:, 1], k)))
print('\n')
target_param_dict = target_val_utils.target_name_to_params(
training_option_dict[trainval_io.TARGET_NAME_KEY])
event_type_string = target_param_dict[target_val_utils.EVENT_TYPE_KEY]
if event_type_string == linkage.TORNADO_EVENT_STRING:
genesis_only = False
elif event_type_string == linkage.TORNADOGENESIS_EVENT_STRING:
genesis_only = True
else:
genesis_only = None
target_name = target_val_utils.target_params_to_name(
min_lead_time_sec=target_param_dict[
target_val_utils.MIN_LEAD_TIME_KEY],
max_lead_time_sec=target_param_dict[
target_val_utils.MAX_LEAD_TIME_KEY],
min_link_distance_metres=target_param_dict[
target_val_utils.MIN_LINKAGE_DISTANCE_KEY],
max_link_distance_metres=10000.,
genesis_only=genesis_only)
output_file_name = prediction_io.find_file(
top_prediction_dir_name=top_output_dir_name,
first_init_time_unix_sec=first_time_unix_sec,
last_init_time_unix_sec=last_time_unix_sec,
gridded=False,
raise_error_if_missing=False)
print('Writing "{0:s}" predictions to: "{1:s}"...'.format(
target_name, output_file_name))
if storm_object_dict is None:
num_output_neurons = (
model_object.layers[-1].output.get_shape().as_list()[-1])
num_classes = max([num_output_neurons, 2])
class_probability_matrix = numpy.full((0, num_classes), numpy.nan)
prediction_io.write_ungridded_predictions(
netcdf_file_name=output_file_name,
class_probability_matrix=class_probability_matrix,
storm_ids=[],
storm_times_unix_sec=numpy.array([], dtype=int),
target_name=target_name,
observed_labels=numpy.array([], dtype=int))
return
prediction_io.write_ungridded_predictions(
netcdf_file_name=output_file_name,
class_probability_matrix=class_probability_matrix,
storm_ids=storm_object_dict[testing_io.FULL_IDS_KEY],
storm_times_unix_sec=storm_object_dict[testing_io.STORM_TIMES_KEY],
target_name=target_name,
observed_labels=observed_labels)
def _run(model_file_name, top_example_dir_name, first_spc_date_string,
last_spc_date_string, num_examples, num_bootstrap_reps,
confidence_level, class_fraction_keys, class_fraction_values,
output_dir_name):
"""Evaluates CNN (convolutional neural net) predictions.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param num_examples: Same.
:param num_bootstrap_reps: Same.
:param confidence_level: Same.
:param class_fraction_keys: Same.
:param class_fraction_values: Same.
:param output_dir_name: Same.
:raises: ValueError: if the model does multi-class classification.
"""
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
num_output_neurons = (
model_object.layers[-1].output.get_shape().as_list()[-1])
if num_output_neurons > 2:
error_string = (
'The model has {0:d} output neurons, which suggests {0:d}-class '
'classification. This script handles only binary classification.'
).format(num_output_neurons)
raise ValueError(error_string)
soundings_only = False
if isinstance(model_object.input, list):
list_of_input_tensors = model_object.input
else:
list_of_input_tensors = [model_object.input]
if len(list_of_input_tensors) == 1:
these_spatial_dim = numpy.array(
list_of_input_tensors[0].get_shape().as_list()[1:-1], dtype=int)
soundings_only = len(these_spatial_dim) == 1
model_directory_name, _ = os.path.split(model_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(model_directory_name)
print('Reading metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
if len(class_fraction_keys) > 1:
class_to_sampling_fraction_dict = dict(
list(zip(class_fraction_keys, class_fraction_values)))
else:
class_to_sampling_fraction_dict = None
training_option_dict[
trainval_io.SAMPLING_FRACTIONS_KEY] = class_to_sampling_fraction_dict
example_file_names = input_examples.find_many_example_files(
top_directory_name=top_example_dir_name,
shuffled=False,
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string,
raise_error_if_any_missing=False)
training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names
training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = (
time_conversion.get_start_of_spc_date(first_spc_date_string))
training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = (
time_conversion.get_end_of_spc_date(last_spc_date_string))
if soundings_only:
generator_object = testing_io.sounding_generator(
option_dict=training_option_dict, num_examples_total=num_examples)
elif model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None:
generator_object = testing_io.gridrad_generator_2d_reduced(
option_dict=training_option_dict,
list_of_operation_dicts=model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY],
num_examples_total=num_examples)
elif model_metadata_dict[cnn.CONV_2D3D_KEY]:
generator_object = testing_io.myrorss_generator_2d3d(
option_dict=training_option_dict, num_examples_total=num_examples)
else:
generator_object = testing_io.generator_2d_or_3d(
option_dict=training_option_dict, num_examples_total=num_examples)
include_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY]
is not None)
forecast_probabilities = numpy.array([])
observed_labels = numpy.array([], dtype=int)
for _ in range(len(example_file_names)):
try:
this_storm_object_dict = next(generator_object)
print(SEPARATOR_STRING)
except StopIteration:
break
observed_labels = numpy.concatenate(
(observed_labels,
this_storm_object_dict[testing_io.TARGET_ARRAY_KEY]))
if soundings_only:
these_predictor_matrices = [
this_storm_object_dict[testing_io.SOUNDING_MATRIX_KEY]
]
else:
these_predictor_matrices = this_storm_object_dict[
testing_io.INPUT_MATRICES_KEY]
if include_soundings:
this_sounding_matrix = these_predictor_matrices[-1]
else:
this_sounding_matrix = None
if soundings_only:
this_probability_matrix = cnn.apply_cnn_soundings_only(
model_object=model_object,
sounding_matrix=this_sounding_matrix,
verbose=True)
elif model_metadata_dict[cnn.CONV_2D3D_KEY]:
if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
this_probability_matrix = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=these_predictor_matrices[0],
sounding_matrix=this_sounding_matrix,
verbose=True)
else:
this_probability_matrix = cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=these_predictor_matrices[0],
azimuthal_shear_matrix_s01=these_predictor_matrices[1],
sounding_matrix=this_sounding_matrix,
verbose=True)
else:
this_probability_matrix = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=these_predictor_matrices[0],
sounding_matrix=this_sounding_matrix,
verbose=True)
print(SEPARATOR_STRING)
forecast_probabilities = numpy.concatenate(
(forecast_probabilities, this_probability_matrix[:, -1]))
model_eval_helper.run_evaluation(
forecast_probabilities=forecast_probabilities,
observed_labels=observed_labels,
num_bootstrap_reps=num_bootstrap_reps,
confidence_level=confidence_level,
output_dir_name=output_dir_name)
def _run(model_file_name, top_example_dir_name, storm_metafile_name,
output_dir_name):
"""Uses trained CNN to make predictions for specific examples.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param output_dir_name: Same.
:raises: ValueError: if the model does multi-class classification.
"""
print('Reading CNN from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
num_output_neurons = (
model_object.layers[-1].output.get_shape().as_list()[-1]
)
if num_output_neurons > 2:
error_string = (
'The model has {0:d} output neurons, which suggests {0:d}-class '
'classification. This script handles only binary classification.'
).format(num_output_neurons)
raise ValueError(error_string)
soundings_only = False
if isinstance(model_object.input, list):
list_of_input_tensors = model_object.input
else:
list_of_input_tensors = [model_object.input]
if len(list_of_input_tensors) == 1:
these_spatial_dim = numpy.array(
list_of_input_tensors[0].get_shape().as_list()[1:-1], dtype=int
)
soundings_only = len(these_spatial_dim) == 1
cnn_metafile_name = cnn.find_metafile(
model_file_name=model_file_name, raise_error_if_missing=True
)
print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name))
cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name)
print('Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
desired_full_id_strings, desired_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name)
)
unique_spc_date_strings = list(set([
time_conversion.time_to_spc_date_string(t)
for t in desired_times_unix_sec
]))
example_file_names = [
input_examples.find_example_file(
top_directory_name=top_example_dir_name, shuffled=False,
spc_date_string=d, raise_error_if_missing=True
) for d in unique_spc_date_strings
]
first_spc_date_string = time_conversion.time_to_spc_date_string(
numpy.min(desired_times_unix_sec)
)
last_spc_date_string = time_conversion.time_to_spc_date_string(
numpy.max(desired_times_unix_sec)
)
training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names
training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = (
time_conversion.get_start_of_spc_date(first_spc_date_string)
)
training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = (
time_conversion.get_end_of_spc_date(last_spc_date_string)
)
training_option_dict[trainval_io.NUM_EXAMPLES_PER_BATCH_KEY] = (
NUM_EXAMPLES_PER_BATCH
)
if soundings_only:
generator_object = testing_io.sounding_generator(
option_dict=training_option_dict,
desired_full_id_strings=desired_full_id_strings,
desired_times_unix_sec=desired_times_unix_sec)
elif cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None:
generator_object = testing_io.gridrad_generator_2d_reduced(
option_dict=training_option_dict,
desired_full_id_strings=desired_full_id_strings,
desired_times_unix_sec=desired_times_unix_sec,
list_of_operation_dicts=cnn_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
)
elif cnn_metadata_dict[cnn.CONV_2D3D_KEY]:
generator_object = testing_io.myrorss_generator_2d3d(
option_dict=training_option_dict,
desired_full_id_strings=desired_full_id_strings,
desired_times_unix_sec=desired_times_unix_sec)
else:
generator_object = testing_io.generator_2d_or_3d(
option_dict=training_option_dict,
desired_full_id_strings=desired_full_id_strings,
desired_times_unix_sec=desired_times_unix_sec)
include_soundings = (
training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None
)
full_storm_id_strings = []
storm_times_unix_sec = numpy.array([], dtype=int)
observed_labels = numpy.array([], dtype=int)
class_probability_matrix = None
while True:
try:
this_storm_object_dict = next(generator_object)
print(SEPARATOR_STRING)
except StopIteration:
break
full_storm_id_strings += this_storm_object_dict[testing_io.FULL_IDS_KEY]
storm_times_unix_sec = numpy.concatenate((
storm_times_unix_sec,
this_storm_object_dict[testing_io.STORM_TIMES_KEY]
))
observed_labels = numpy.concatenate((
observed_labels, this_storm_object_dict[testing_io.TARGET_ARRAY_KEY]
))
if soundings_only:
these_predictor_matrices = [
this_storm_object_dict[testing_io.SOUNDING_MATRIX_KEY]
]
else:
these_predictor_matrices = this_storm_object_dict[
testing_io.INPUT_MATRICES_KEY]
if include_soundings:
this_sounding_matrix = these_predictor_matrices[-1]
else:
this_sounding_matrix = None
if soundings_only:
this_probability_matrix = cnn.apply_cnn_soundings_only(
model_object=model_object, sounding_matrix=this_sounding_matrix,
verbose=True)
elif cnn_metadata_dict[cnn.CONV_2D3D_KEY]:
if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
this_probability_matrix = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=these_predictor_matrices[0],
sounding_matrix=this_sounding_matrix, verbose=True)
else:
this_probability_matrix = cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=these_predictor_matrices[0],
azimuthal_shear_matrix_s01=these_predictor_matrices[1],
sounding_matrix=this_sounding_matrix, verbose=True)
else:
this_probability_matrix = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=these_predictor_matrices[0],
sounding_matrix=this_sounding_matrix, verbose=True)
print(SEPARATOR_STRING)
if class_probability_matrix is None:
class_probability_matrix = this_probability_matrix + 0.
else:
class_probability_matrix = numpy.concatenate(
(class_probability_matrix, this_probability_matrix), axis=0
)
output_file_name = prediction_io.find_ungridded_file(
directory_name=output_dir_name, raise_error_if_missing=False)
print('Writing results to: "{0:s}"...'.format(output_file_name))
prediction_io.write_ungridded_predictions(
netcdf_file_name=output_file_name,
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels, storm_ids=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
target_name=training_option_dict[trainval_io.TARGET_NAME_KEY],
model_file_name=model_file_name
)