def _run(storm_metafile_name, warning_dir_name): """Finds which storms are linked to an NWS tornado warning. This is effectively the main method. :param storm_metafile_name: See documentation at top of file. :param warning_dir_name: Same. """ print( 'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name)) full_storm_id_strings, valid_times_unix_sec = ( tracking_io.read_ids_and_times(storm_metafile_name)) secondary_id_strings = ( temporal_tracking.full_to_partial_ids(full_storm_id_strings)[-1]) these_times_unix_sec = numpy.concatenate( (valid_times_unix_sec, valid_times_unix_sec - NUM_SECONDS_PER_DAY, valid_times_unix_sec + NUM_SECONDS_PER_DAY)) spc_date_strings = [ time_conversion.time_to_spc_date_string(t) for t in these_times_unix_sec ] spc_date_strings = numpy.unique(numpy.array(spc_date_strings)) linked_secondary_id_strings = [] for this_spc_date_string in spc_date_strings: this_file_name = '{0:s}/tornado_warnings_{1:s}.p'.format( warning_dir_name, this_spc_date_string) print('Reading warnings from: "{0:s}"...'.format(this_file_name)) this_file_handle = open(this_file_name, 'rb') this_warning_table = pickle.load(this_file_handle) this_file_handle.close() this_num_warnings = len(this_warning_table.index) for k in range(this_num_warnings): linked_secondary_id_strings += ( this_warning_table[LINKED_SECONDARY_IDS_KEY].values[k]) print(SEPARATOR_STRING) storm_warned_flags = numpy.array( [s in linked_secondary_id_strings for s in secondary_id_strings], dtype=bool) print(('{0:d} of {1:d} storm objects are linked to an NWS tornado warning!' ).format(numpy.sum(storm_warned_flags), len(storm_warned_flags)))
def _run(input_activation_file_name, unique_storm_cells, num_low_activation_examples, num_high_activation_examples, num_hits, num_misses, num_false_alarms, num_correct_nulls, top_target_dir_name, output_dir_name): """Finds extreme examples (storm objects), based on model activations. This is effectively the main method. :param input_activation_file_name: See documentation at top of file. :param unique_storm_cells: Same. :param num_low_activation_examples: Same. :param num_high_activation_examples: Same. :param num_hits: Same. :param num_misses: Same. :param num_false_alarms: Same. :param num_correct_nulls: Same. :param top_target_dir_name: Same. :param output_dir_name: Same. :raises: ValueError: if the activation file contains activations for more than one model component. """ # Check input args. example_counts = numpy.array([ num_low_activation_examples, num_high_activation_examples, num_hits, num_misses, num_false_alarms, num_correct_nulls ], dtype=int) error_checking.assert_is_geq_numpy_array(example_counts, 0) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) # Read activations. print('Reading activations from: "{0:s}"...'.format( input_activation_file_name)) activation_matrix, activation_metadata_dict = model_activation.read_file( input_activation_file_name) num_model_components = activation_matrix.shape[1] if num_model_components > 1: error_string = ( 'The file should contain activations for only one model component, ' 'not {0:d}.').format(num_model_components) raise ValueError(error_string) storm_activations = activation_matrix[:, 0] full_id_strings = activation_metadata_dict[model_activation.FULL_IDS_KEY] storm_times_unix_sec = activation_metadata_dict[ model_activation.STORM_TIMES_KEY] num_storm_objects = len(full_id_strings) error_checking.assert_is_leq(numpy.sum(example_counts), num_storm_objects) # Find high- and low-activation examples. if num_low_activation_examples + num_high_activation_examples > 0: high_indices, low_indices = ( model_activation.get_hilo_activation_examples( storm_activations=storm_activations, num_low_activation_examples=num_low_activation_examples, num_high_activation_examples=num_high_activation_examples, unique_storm_cells=unique_storm_cells, full_storm_id_strings=full_id_strings)) else: high_indices = numpy.array([], dtype=int) low_indices = numpy.array([], dtype=int) # Write high-activation examples to file. if len(high_indices) > 0: high_activation_file_name = '{0:s}/high_activation_examples.p'.format( output_dir_name) print(( 'Writing IDs and times for high-activation examples to: "{0:s}"...' ).format(high_activation_file_name)) this_activation_matrix = numpy.reshape(storm_activations[high_indices], (len(high_indices), 1)) model_activation.write_file( pickle_file_name=high_activation_file_name, activation_matrix=this_activation_matrix, full_id_strings=[full_id_strings[k] for k in high_indices], storm_times_unix_sec=storm_times_unix_sec[high_indices], model_file_name=activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY], component_type_string=activation_metadata_dict[ model_activation.COMPONENT_TYPE_KEY], target_class=activation_metadata_dict[ model_activation.TARGET_CLASS_KEY], layer_name=activation_metadata_dict[ model_activation.LAYER_NAME_KEY], neuron_index_matrix=activation_metadata_dict[ model_activation.NEURON_INDICES_KEY], channel_indices=activation_metadata_dict[ model_activation.CHANNEL_INDICES_KEY]) # Write low-activation examples to file if len(low_indices) > 0: low_activation_file_name = '{0:s}/low_activation_examples.p'.format( output_dir_name) print( ('Writing IDs and times for low-activation examples to: "{0:s}"...' ).format(low_activation_file_name)) this_activation_matrix = numpy.reshape(storm_activations[low_indices], (len(low_indices), 1)) model_activation.write_file( pickle_file_name=low_activation_file_name, activation_matrix=this_activation_matrix, full_id_strings=[full_id_strings[k] for k in low_indices], storm_times_unix_sec=storm_times_unix_sec[low_indices], model_file_name=activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY], component_type_string=activation_metadata_dict[ model_activation.COMPONENT_TYPE_KEY], target_class=activation_metadata_dict[ model_activation.TARGET_CLASS_KEY], layer_name=activation_metadata_dict[ model_activation.LAYER_NAME_KEY], neuron_index_matrix=activation_metadata_dict[ model_activation.NEURON_INDICES_KEY], channel_indices=activation_metadata_dict[ model_activation.CHANNEL_INDICES_KEY]) if num_hits + num_misses + num_false_alarms + num_correct_nulls == 0: return print(SEPARATOR_STRING) target_value_dict = _read_target_values( top_target_dir_name=top_target_dir_name, storm_activations=storm_activations, activation_metadata_dict=activation_metadata_dict) print(SEPARATOR_STRING) full_id_strings = target_value_dict[FULL_IDS_KEY] storm_times_unix_sec = target_value_dict[STORM_TIMES_KEY] storm_activations = target_value_dict[STORM_ACTIVATIONS_KEY] storm_target_values = target_value_dict[TARGET_VALUES_KEY] ct_extreme_dict = model_activation.get_contingency_table_extremes( storm_activations=storm_activations, storm_target_values=storm_target_values, num_hits=num_hits, num_misses=num_misses, num_false_alarms=num_false_alarms, num_correct_nulls=num_correct_nulls, unique_storm_cells=unique_storm_cells, full_storm_id_strings=full_id_strings) hit_indices = ct_extreme_dict[model_activation.HIT_INDICES_KEY] miss_indices = ct_extreme_dict[model_activation.MISS_INDICES_KEY] false_alarm_indices = ( ct_extreme_dict[model_activation.FALSE_ALARM_INDICES_KEY]) correct_null_indices = ( ct_extreme_dict[model_activation.CORRECT_NULL_INDICES_KEY]) hit_id_strings = [full_id_strings[k] for k in hit_indices] miss_id_strings = [full_id_strings[k] for k in miss_indices] false_alarm_id_strings = [full_id_strings[k] for k in false_alarm_indices] correct_null_id_strings = [ full_id_strings[k] for k in correct_null_indices ] hit_primary_id_strings = ( temporal_tracking.full_to_partial_ids(hit_id_strings)[0]) miss_primary_id_strings = ( temporal_tracking.full_to_partial_ids(miss_id_strings)[0]) fa_primary_id_strings = ( temporal_tracking.full_to_partial_ids(false_alarm_id_strings)[0]) cn_primary_id_strings = ( temporal_tracking.full_to_partial_ids(correct_null_id_strings)[0]) these_flags = numpy.array( [i in miss_primary_id_strings for i in hit_primary_id_strings], dtype=bool) print( ('Number of primary IDs in best hits AND worst misses = {0:d}').format( numpy.sum(these_flags))) these_flags = numpy.array( [i in fa_primary_id_strings for i in hit_primary_id_strings], dtype=bool) print(('Number of primary IDs in best hits AND worst false alarms = {0:d}' ).format(numpy.sum(these_flags))) these_flags = numpy.array( [i in cn_primary_id_strings for i in hit_primary_id_strings], dtype=bool) print(('Number of primary IDs in best hits AND best correct nulls = {0:d}' ).format(numpy.sum(these_flags))) these_flags = numpy.array( [i in fa_primary_id_strings for i in miss_primary_id_strings], dtype=bool) print( ('Number of primary IDs in worst misses AND worst false alarms = {0:d}' ).format(numpy.sum(these_flags))) these_flags = numpy.array( [i in cn_primary_id_strings for i in miss_primary_id_strings], dtype=bool) print( ('Number of primary IDs in worst misses AND best correct nulls = {0:d}' ).format(numpy.sum(these_flags))) these_flags = numpy.array( [i in cn_primary_id_strings for i in fa_primary_id_strings], dtype=bool) print(( 'Number of primary IDs in worst false alarms AND best correct nulls = ' '{0:d}').format(numpy.sum(these_flags))) # Write best hits to file. if len(hit_indices) > 0: best_hit_file_name = '{0:s}/best_hits.p'.format(output_dir_name) print('Writing IDs and times for best hits to: "{0:s}"...'.format( best_hit_file_name)) this_activation_matrix = numpy.reshape(storm_activations[hit_indices], (len(hit_indices), 1)) model_activation.write_file( pickle_file_name=best_hit_file_name, activation_matrix=this_activation_matrix, full_id_strings=[full_id_strings[k] for k in hit_indices], storm_times_unix_sec=storm_times_unix_sec[hit_indices], model_file_name=activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY], component_type_string=activation_metadata_dict[ model_activation.COMPONENT_TYPE_KEY], target_class=activation_metadata_dict[ model_activation.TARGET_CLASS_KEY], layer_name=activation_metadata_dict[ model_activation.LAYER_NAME_KEY], neuron_index_matrix=activation_metadata_dict[ model_activation.NEURON_INDICES_KEY], channel_indices=activation_metadata_dict[ model_activation.CHANNEL_INDICES_KEY]) # Write worst misses to file. if len(miss_indices) > 0: worst_miss_file_name = '{0:s}/worst_misses.p'.format(output_dir_name) print('Writing IDs and times for worst misses to: "{0:s}"...'.format( worst_miss_file_name)) this_activation_matrix = numpy.reshape(storm_activations[miss_indices], (len(miss_indices), 1)) model_activation.write_file( pickle_file_name=worst_miss_file_name, activation_matrix=this_activation_matrix, full_id_strings=[full_id_strings[k] for k in miss_indices], storm_times_unix_sec=storm_times_unix_sec[miss_indices], model_file_name=activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY], component_type_string=activation_metadata_dict[ model_activation.COMPONENT_TYPE_KEY], target_class=activation_metadata_dict[ model_activation.TARGET_CLASS_KEY], layer_name=activation_metadata_dict[ model_activation.LAYER_NAME_KEY], neuron_index_matrix=activation_metadata_dict[ model_activation.NEURON_INDICES_KEY], channel_indices=activation_metadata_dict[ model_activation.CHANNEL_INDICES_KEY]) # Write worst false alarms to file. if len(false_alarm_indices) > 0: worst_fa_file_name = '{0:s}/worst_false_alarms.p'.format( output_dir_name) print(('Writing IDs and times for worst false alarms to: "{0:s}"...' ).format(worst_fa_file_name)) this_activation_matrix = numpy.reshape( storm_activations[false_alarm_indices], (len(false_alarm_indices), 1)) model_activation.write_file( pickle_file_name=worst_fa_file_name, activation_matrix=this_activation_matrix, full_id_strings=[full_id_strings[k] for k in false_alarm_indices], storm_times_unix_sec=storm_times_unix_sec[false_alarm_indices], model_file_name=activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY], component_type_string=activation_metadata_dict[ model_activation.COMPONENT_TYPE_KEY], target_class=activation_metadata_dict[ model_activation.TARGET_CLASS_KEY], layer_name=activation_metadata_dict[ model_activation.LAYER_NAME_KEY], neuron_index_matrix=activation_metadata_dict[ model_activation.NEURON_INDICES_KEY], channel_indices=activation_metadata_dict[ model_activation.CHANNEL_INDICES_KEY]) # Write best correct nulls to file. if len(correct_null_indices) > 0: best_cn_file_name = '{0:s}/best_correct_nulls.p'.format( output_dir_name) print(('Writing IDs and times for best correct nulls to: "{0:s}"...' ).format(best_cn_file_name)) this_activation_matrix = numpy.reshape( storm_activations[correct_null_indices], (len(correct_null_indices), 1)) model_activation.write_file( pickle_file_name=best_cn_file_name, activation_matrix=this_activation_matrix, full_id_strings=[full_id_strings[k] for k in correct_null_indices], storm_times_unix_sec=storm_times_unix_sec[correct_null_indices], model_file_name=activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY], component_type_string=activation_metadata_dict[ model_activation.COMPONENT_TYPE_KEY], target_class=activation_metadata_dict[ model_activation.TARGET_CLASS_KEY], layer_name=activation_metadata_dict[ model_activation.LAYER_NAME_KEY], neuron_index_matrix=activation_metadata_dict[ model_activation.NEURON_INDICES_KEY], channel_indices=activation_metadata_dict[ model_activation.CHANNEL_INDICES_KEY])
def _plot_one_example(full_id_string, storm_time_unix_sec, target_name, forecast_probability, tornado_dir_name, top_tracking_dir_name, top_myrorss_dir_name, radar_field_name, radar_height_m_asl, latitude_buffer_deg, longitude_buffer_deg, top_output_dir_name, aux_forecast_probabilities=None, aux_activation_dict=None): """Plots one example with surrounding context at several times. N = number of storm objects read from auxiliary activation file :param full_id_string: Full storm ID. :param storm_time_unix_sec: Storm time. :param target_name: Name of target variable. :param forecast_probability: Forecast tornado probability for this example. :param tornado_dir_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param top_myrorss_dir_name: Same. :param radar_field_name: Same. :param radar_height_m_asl: Same. :param latitude_buffer_deg: Same. :param longitude_buffer_deg: Same. :param top_output_dir_name: Same. :param aux_forecast_probabilities: length-N numpy array of forecast probabilities. If this is None, will not plot forecast probs in maps. :param aux_activation_dict: Dictionary returned by `model_activation.read_file` from auxiliary file. If this is None, will not plot forecast probs in maps. """ storm_time_string = time_conversion.unix_sec_to_string( storm_time_unix_sec, TIME_FORMAT) # Create output directory for this example. output_dir_name = '{0:s}/{1:s}_{2:s}'.format(top_output_dir_name, full_id_string, storm_time_string) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) # Find tracking files. tracking_file_names = _find_tracking_files_one_example( valid_time_unix_sec=storm_time_unix_sec, top_tracking_dir_name=top_tracking_dir_name, target_name=target_name) tracking_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int) tracking_time_strings = [ time_conversion.unix_sec_to_string(t, TIME_FORMAT) for t in tracking_times_unix_sec ] # Read tracking files. storm_object_table = tracking_io.read_many_files(tracking_file_names) print('\n') if aux_activation_dict is not None: these_indices = tracking_utils.find_storm_objects( all_id_strings=aux_activation_dict[model_activation.FULL_IDS_KEY], all_times_unix_sec=aux_activation_dict[ model_activation.STORM_TIMES_KEY], id_strings_to_keep=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), times_to_keep_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, allow_missing=True) storm_object_probs = numpy.array([ aux_forecast_probabilities[k] if k >= 0 else numpy.nan for k in these_indices ]) storm_object_table = storm_object_table.assign( **{FORECAST_PROBABILITY_COLUMN: storm_object_probs}) primary_id_string = temporal_tracking.full_to_partial_ids([full_id_string ])[0][0] this_storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.PRIMARY_ID_COLUMN] == primary_id_string] latitude_limits_deg, longitude_limits_deg = _get_plotting_limits( storm_object_table=this_storm_object_table, latitude_buffer_deg=latitude_buffer_deg, longitude_buffer_deg=longitude_buffer_deg) storm_min_latitudes_deg = numpy.array([ numpy.min(numpy.array(p.exterior.xy[1])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) storm_max_latitudes_deg = numpy.array([ numpy.max(numpy.array(p.exterior.xy[1])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) storm_min_longitudes_deg = numpy.array([ numpy.min(numpy.array(p.exterior.xy[0])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) storm_max_longitudes_deg = numpy.array([ numpy.max(numpy.array(p.exterior.xy[0])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) min_latitude_flags = numpy.logical_and( storm_min_latitudes_deg >= latitude_limits_deg[0], storm_min_latitudes_deg <= latitude_limits_deg[1]) max_latitude_flags = numpy.logical_and( storm_max_latitudes_deg >= latitude_limits_deg[0], storm_max_latitudes_deg <= latitude_limits_deg[1]) latitude_flags = numpy.logical_or(min_latitude_flags, max_latitude_flags) min_longitude_flags = numpy.logical_and( storm_min_longitudes_deg >= longitude_limits_deg[0], storm_min_longitudes_deg <= longitude_limits_deg[1]) max_longitude_flags = numpy.logical_and( storm_max_longitudes_deg >= longitude_limits_deg[0], storm_max_longitudes_deg <= longitude_limits_deg[1]) longitude_flags = numpy.logical_or(min_longitude_flags, max_longitude_flags) good_indices = numpy.where( numpy.logical_and(latitude_flags, longitude_flags))[0] storm_object_table = storm_object_table.iloc[good_indices] # Read tornado reports. target_param_dict = target_val_utils.target_name_to_params(target_name) min_lead_time_seconds = target_param_dict[ target_val_utils.MIN_LEAD_TIME_KEY] max_lead_time_seconds = target_param_dict[ target_val_utils.MAX_LEAD_TIME_KEY] tornado_table = linkage._read_input_tornado_reports( input_directory_name=tornado_dir_name, storm_times_unix_sec=numpy.array([storm_time_unix_sec], dtype=int), max_time_before_storm_start_sec=-1 * min_lead_time_seconds, max_time_after_storm_end_sec=max_lead_time_seconds, genesis_only=True) tornado_table = tornado_table.loc[ (tornado_table[linkage.EVENT_LATITUDE_COLUMN] >= latitude_limits_deg[0] ) & (tornado_table[linkage.EVENT_LATITUDE_COLUMN] <= latitude_limits_deg[1])] tornado_table = tornado_table.loc[ (tornado_table[linkage.EVENT_LONGITUDE_COLUMN] >= longitude_limits_deg[0]) & (tornado_table[linkage.EVENT_LONGITUDE_COLUMN] <= longitude_limits_deg[1])] for i in range(len(tracking_file_names)): this_storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.VALID_TIME_COLUMN] == tracking_times_unix_sec[i]] _plot_one_example_one_time( storm_object_table=this_storm_object_table, full_id_string=full_id_string, valid_time_unix_sec=tracking_times_unix_sec[i], tornado_table=copy.deepcopy(tornado_table), top_myrorss_dir_name=top_myrorss_dir_name, radar_field_name=radar_field_name, radar_height_m_asl=radar_height_m_asl, latitude_limits_deg=latitude_limits_deg, longitude_limits_deg=longitude_limits_deg) if aux_activation_dict is None: this_title_string = ( 'Valid time = {0:s} ... forecast prob at {1:s} = {2:.3f}' ).format(tracking_time_strings[i], storm_time_string, forecast_probability) pyplot.title(this_title_string, fontsize=TITLE_FONT_SIZE) this_file_name = '{0:s}/{1:s}.jpg'.format(output_dir_name, tracking_time_strings[i]) print('Saving figure to file: "{0:s}"...\n'.format(this_file_name)) pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close() imagemagick_utils.trim_whitespace(input_file_name=this_file_name, output_file_name=this_file_name)
def _plot_one_example_one_time(storm_object_table, full_id_string, valid_time_unix_sec, tornado_table, top_myrorss_dir_name, radar_field_name, radar_height_m_asl, latitude_limits_deg, longitude_limits_deg): """Plots one example with surrounding context at one time. :param storm_object_table: pandas DataFrame, containing only storm objects at one time with the relevant primary ID. Columns are documented in `storm_tracking_io.write_file`. :param full_id_string: Full ID of storm of interest. :param valid_time_unix_sec: Valid time. :param tornado_table: pandas DataFrame created by `linkage._read_input_tornado_reports`. :param top_myrorss_dir_name: See documentation at top of file. :param radar_field_name: Same. :param radar_height_m_asl: Same. :param latitude_limits_deg: See doc for `_get_plotting_limits`. :param longitude_limits_deg: Same. """ min_plot_latitude_deg = latitude_limits_deg[0] max_plot_latitude_deg = latitude_limits_deg[1] min_plot_longitude_deg = longitude_limits_deg[0] max_plot_longitude_deg = longitude_limits_deg[1] radar_file_name = myrorss_and_mrms_io.find_raw_file( top_directory_name=top_myrorss_dir_name, spc_date_string=time_conversion.time_to_spc_date_string( valid_time_unix_sec), unix_time_sec=valid_time_unix_sec, data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_field_name, height_m_asl=radar_height_m_asl, raise_error_if_missing=True) print('Reading data from: "{0:s}"...'.format(radar_file_name)) radar_metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file( netcdf_file_name=radar_file_name, data_source=radar_utils.MYRORSS_SOURCE_ID) sparse_grid_table = (myrorss_and_mrms_io.read_data_from_sparse_grid_file( netcdf_file_name=radar_file_name, field_name_orig=radar_metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=radar_metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN]) ) radar_matrix, grid_point_latitudes_deg, grid_point_longitudes_deg = ( radar_s2f.sparse_to_full_grid(sparse_grid_table=sparse_grid_table, metadata_dict=radar_metadata_dict)) radar_matrix = numpy.flip(radar_matrix, axis=0) grid_point_latitudes_deg = grid_point_latitudes_deg[::-1] axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')[1:]) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS) radar_plotting.plot_latlng_grid( field_matrix=radar_matrix, field_name=radar_field_name, axes_object=axes_object, min_grid_point_latitude_deg=numpy.min(grid_point_latitudes_deg), min_grid_point_longitude_deg=numpy.min(grid_point_longitudes_deg), latitude_spacing_deg=numpy.diff(grid_point_latitudes_deg[:2])[0], longitude_spacing_deg=numpy.diff(grid_point_longitudes_deg[:2])[0]) colour_map_object, colour_norm_object = ( radar_plotting.get_default_colour_scheme(radar_field_name)) plotting_utils.plot_colour_bar(axes_object_or_matrix=axes_object, data_matrix=radar_matrix, colour_map_object=colour_map_object, colour_norm_object=colour_norm_object, orientation_string='horizontal', extend_min=False, extend_max=True, fraction_of_axis_length=0.8) first_list, second_list = temporal_tracking.full_to_partial_ids( [full_id_string]) primary_id_string = first_list[0] secondary_id_string = second_list[0] # Plot outlines of unrelated storms (with different primary IDs). this_storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.PRIMARY_ID_COLUMN] != primary_id_string] storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=axes_object, basemap_object=basemap_object, line_width=2, line_colour='k', line_style='dashed') # Plot outlines of related storms (with the same primary ID). this_storm_object_table = storm_object_table.loc[ (storm_object_table[tracking_utils.PRIMARY_ID_COLUMN] == primary_id_string) & (storm_object_table[ tracking_utils.SECONDARY_ID_COLUMN] != secondary_id_string)] this_num_storm_objects = len(this_storm_object_table.index) if this_num_storm_objects > 0: storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=axes_object, basemap_object=basemap_object, line_width=2, line_colour='k', line_style='solid') for j in range(len(this_storm_object_table)): axes_object.text( this_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values[j], this_storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values[j], 'P', fontsize=FONT_SIZE, color=FONT_COLOUR, fontweight='bold', horizontalalignment='center', verticalalignment='center') # Plot outline of storm of interest (same secondary ID). this_storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.SECONDARY_ID_COLUMN] == secondary_id_string] storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=axes_object, basemap_object=basemap_object, line_width=4, line_colour='k', line_style='solid') this_num_storm_objects = len(this_storm_object_table.index) plot_forecast = (this_num_storm_objects > 0 and FORECAST_PROBABILITY_COLUMN in list(this_storm_object_table)) if plot_forecast: this_polygon_object_latlng = this_storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values[0] this_latitude_deg = numpy.min( numpy.array(this_polygon_object_latlng.exterior.xy[1])) this_longitude_deg = this_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values[0] label_string = 'Prob = {0:.3f}\nat {1:s}'.format( this_storm_object_table[FORECAST_PROBABILITY_COLUMN].values[0], time_conversion.unix_sec_to_string(valid_time_unix_sec, TORNADO_TIME_FORMAT)) bounding_box_dict = { 'facecolor': plotting_utils.colour_from_numpy_to_tuple( PROBABILITY_BACKGROUND_COLOUR), 'alpha': PROBABILITY_BACKGROUND_OPACITY, 'edgecolor': 'k', 'linewidth': 1 } axes_object.text(this_longitude_deg, this_latitude_deg, label_string, fontsize=FONT_SIZE, color=plotting_utils.colour_from_numpy_to_tuple( PROBABILITY_FONT_COLOUR), fontweight='bold', bbox=bounding_box_dict, horizontalalignment='center', verticalalignment='top', zorder=1e10) tornado_latitudes_deg = tornado_table[linkage.EVENT_LATITUDE_COLUMN].values tornado_longitudes_deg = tornado_table[ linkage.EVENT_LONGITUDE_COLUMN].values tornado_times_unix_sec = tornado_table[linkage.EVENT_TIME_COLUMN].values tornado_time_strings = [ time_conversion.unix_sec_to_string(t, TORNADO_TIME_FORMAT) for t in tornado_times_unix_sec ] axes_object.plot(tornado_longitudes_deg, tornado_latitudes_deg, linestyle='None', marker=TORNADO_MARKER_TYPE, markersize=TORNADO_MARKER_SIZE, markeredgewidth=TORNADO_MARKER_EDGE_WIDTH, markerfacecolor=plotting_utils.colour_from_numpy_to_tuple( TORNADO_MARKER_COLOUR), markeredgecolor=plotting_utils.colour_from_numpy_to_tuple( TORNADO_MARKER_COLOUR)) num_tornadoes = len(tornado_latitudes_deg) for j in range(num_tornadoes): axes_object.text(tornado_longitudes_deg[j] + 0.02, tornado_latitudes_deg[j] - 0.02, tornado_time_strings[j], fontsize=FONT_SIZE, color=FONT_COLOUR, fontweight='bold', horizontalalignment='left', verticalalignment='top')
def _run(storm_metafile_name, top_tracking_dir_name, lead_time_seconds, output_file_name): """Plots spatial distribution of examples (storm objects) in file. This is effectively the main method. :param storm_metafile_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param lead_time_seconds: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) # Read storm metadata. print( 'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name)) orig_full_id_strings, orig_times_unix_sec = ( tracking_io.read_ids_and_times(storm_metafile_name)) orig_primary_id_strings = temporal_tracking.full_to_partial_ids( orig_full_id_strings)[0] # Find relevant tracking files. spc_date_strings = [ time_conversion.time_to_spc_date_string(t) for t in orig_times_unix_sec ] spc_date_strings += [ time_conversion.time_to_spc_date_string(t + lead_time_seconds) for t in orig_times_unix_sec ] spc_date_strings = list(set(spc_date_strings)) tracking_file_names = [] for this_spc_date_string in spc_date_strings: tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] file_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int) num_orig_storm_objects = len(orig_full_id_strings) num_files = len(file_times_unix_sec) keep_file_flags = numpy.full(num_files, 0, dtype=bool) for i in range(num_orig_storm_objects): these_flags = numpy.logical_and( file_times_unix_sec >= orig_times_unix_sec[i], file_times_unix_sec <= orig_times_unix_sec[i] + lead_time_seconds) keep_file_flags = numpy.logical_or(keep_file_flags, these_flags) del file_times_unix_sec keep_file_indices = numpy.where(keep_file_flags)[0] tracking_file_names = [tracking_file_names[k] for k in keep_file_indices] # Read relevant tracking files. num_files = len(tracking_file_names) storm_object_tables = [None] * num_files print(SEPARATOR_STRING) for i in range(num_files): print('Reading data from: "{0:s}"...'.format(tracking_file_names[i])) this_table = tracking_io.read_file(tracking_file_names[i]) storm_object_tables[i] = this_table.loc[this_table[ tracking_utils.PRIMARY_ID_COLUMN].isin( numpy.array(orig_primary_id_strings))] if i == 0: continue storm_object_tables[i] = storm_object_tables[i].align( storm_object_tables[0], axis=1)[0] storm_object_table = pandas.concat(storm_object_tables, axis=0, ignore_index=True) print(SEPARATOR_STRING) # Find relevant storm objects. orig_object_rows = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, id_strings_to_keep=orig_full_id_strings, times_to_keep_unix_sec=orig_times_unix_sec) good_object_rows = numpy.array([], dtype=int) for i in range(num_orig_storm_objects): # Non-merging successors only! first_rows = temporal_tracking.find_successors( storm_object_table=storm_object_table, target_row=orig_object_rows[i], num_seconds_forward=lead_time_seconds, max_num_sec_id_changes=1, change_type_string=temporal_tracking.SPLIT_STRING, return_all_on_path=True) second_rows = temporal_tracking.find_successors( storm_object_table=storm_object_table, target_row=orig_object_rows[i], num_seconds_forward=lead_time_seconds, max_num_sec_id_changes=0, change_type_string=temporal_tracking.MERGER_STRING, return_all_on_path=True) first_rows = first_rows.tolist() second_rows = second_rows.tolist() these_rows = set(first_rows) & set(second_rows) these_rows = numpy.array(list(these_rows), dtype=int) good_object_rows = numpy.concatenate((good_object_rows, these_rows)) good_object_rows = numpy.unique(good_object_rows) storm_object_table = storm_object_table.iloc[good_object_rows] times_of_day_sec = numpy.mod( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values, NUM_SECONDS_IN_DAY) storm_object_table = storm_object_table.assign( **{tracking_utils.VALID_TIME_COLUMN: times_of_day_sec}) min_plot_latitude_deg = -LATLNG_BUFFER_DEG + numpy.min( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values) max_plot_latitude_deg = LATLNG_BUFFER_DEG + numpy.max( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values) min_plot_longitude_deg = -LATLNG_BUFFER_DEG + numpy.min( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values) max_plot_longitude_deg = LATLNG_BUFFER_DEG + numpy.max( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values) _, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR, line_width=BORDER_WIDTH * 2) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR, line_width=BORDER_WIDTH) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR, line_width=BORDER_WIDTH) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS, line_width=BORDER_WIDTH) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS, line_width=BORDER_WIDTH) # colour_bar_object = storm_plotting.plot_storm_tracks( # storm_object_table=storm_object_table, axes_object=axes_object, # basemap_object=basemap_object, colour_map_object=COLOUR_MAP_OBJECT, # colour_min_unix_sec=0, colour_max_unix_sec=NUM_SECONDS_IN_DAY - 1, # line_width=TRACK_LINE_WIDTH, # start_marker_type=None, end_marker_type=None # ) colour_bar_object = storm_plotting.plot_storm_centroids( storm_object_table=storm_object_table, axes_object=axes_object, basemap_object=basemap_object, colour_map_object=COLOUR_MAP_OBJECT, colour_min_unix_sec=0, colour_max_unix_sec=NUM_SECONDS_IN_DAY - 1) tick_times_unix_sec = numpy.linspace(0, NUM_SECONDS_IN_DAY, num=NUM_HOURS_IN_DAY + 1, dtype=int) tick_times_unix_sec = tick_times_unix_sec[:-1] tick_times_unix_sec = tick_times_unix_sec[::2] tick_time_strings = [ time_conversion.unix_sec_to_string(t, COLOUR_BAR_TIME_FORMAT) for t in tick_times_unix_sec ] colour_bar_object.set_ticks(tick_times_unix_sec) colour_bar_object.set_ticklabels(tick_time_strings) print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close()
def _plot_one_example_one_time( storm_object_table, full_id_string, valid_time_unix_sec, tornado_table, top_myrorss_dir_name, radar_field_name, radar_height_m_asl, latitude_limits_deg, longitude_limits_deg): """Plots one example with surrounding context at one time. :param storm_object_table: pandas DataFrame, containing only storm objects at one time with the relevant primary ID. Columns are documented in `storm_tracking_io.write_file`. :param full_id_string: Full ID of storm of interest. :param valid_time_unix_sec: Valid time. :param tornado_table: pandas DataFrame created by `linkage._read_input_tornado_reports`. :param top_myrorss_dir_name: See documentation at top of file. :param radar_field_name: Same. :param radar_height_m_asl: Same. :param latitude_limits_deg: See doc for `_get_plotting_limits`. :param longitude_limits_deg: Same. """ min_plot_latitude_deg = latitude_limits_deg[0] max_plot_latitude_deg = latitude_limits_deg[1] min_plot_longitude_deg = longitude_limits_deg[0] max_plot_longitude_deg = longitude_limits_deg[1] radar_file_name = myrorss_and_mrms_io.find_raw_file_inexact_time( top_directory_name=top_myrorss_dir_name, desired_time_unix_sec=valid_time_unix_sec, spc_date_string=time_conversion.time_to_spc_date_string( valid_time_unix_sec), data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_field_name, height_m_asl=radar_height_m_asl, max_time_offset_sec= myrorss_and_mrms_io.DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC, raise_error_if_missing=True) print('Reading data from: "{0:s}"...'.format(radar_file_name)) radar_metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file( netcdf_file_name=radar_file_name, data_source=radar_utils.MYRORSS_SOURCE_ID) sparse_grid_table = ( myrorss_and_mrms_io.read_data_from_sparse_grid_file( netcdf_file_name=radar_file_name, field_name_orig=radar_metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=radar_metadata_dict[ radar_utils.SENTINEL_VALUE_COLUMN] ) ) radar_matrix, grid_point_latitudes_deg, grid_point_longitudes_deg = ( radar_s2f.sparse_to_full_grid( sparse_grid_table=sparse_grid_table, metadata_dict=radar_metadata_dict) ) radar_matrix = numpy.flip(radar_matrix, axis=0) grid_point_latitudes_deg = grid_point_latitudes_deg[::-1] axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='h' )[1:] ) plotting_utils.plot_coastlines( basemap_object=basemap_object, axes_object=axes_object, line_colour=plotting_utils.DEFAULT_COUNTRY_COLOUR) plotting_utils.plot_countries( basemap_object=basemap_object, axes_object=axes_object) plotting_utils.plot_states_and_provinces( basemap_object=basemap_object, axes_object=axes_object) plotting_utils.plot_parallels( basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS, line_width=0) plotting_utils.plot_meridians( basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS, line_width=0) radar_plotting.plot_latlng_grid( field_matrix=radar_matrix, field_name=radar_field_name, axes_object=axes_object, min_grid_point_latitude_deg=numpy.min(grid_point_latitudes_deg), min_grid_point_longitude_deg=numpy.min(grid_point_longitudes_deg), latitude_spacing_deg=numpy.diff(grid_point_latitudes_deg[:2])[0], longitude_spacing_deg=numpy.diff(grid_point_longitudes_deg[:2])[0] ) colour_map_object, colour_norm_object = ( radar_plotting.get_default_colour_scheme(radar_field_name) ) plotting_utils.plot_colour_bar( axes_object_or_matrix=axes_object, data_matrix=radar_matrix, colour_map_object=colour_map_object, colour_norm_object=colour_norm_object, orientation_string='horizontal', padding=0.05, extend_min=False, extend_max=True, fraction_of_axis_length=0.8) first_list, second_list = temporal_tracking.full_to_partial_ids( [full_id_string] ) primary_id_string = first_list[0] secondary_id_string = second_list[0] # Plot outlines of unrelated storms (with different primary IDs). this_storm_object_table = storm_object_table.loc[ storm_object_table[tracking_utils.PRIMARY_ID_COLUMN] != primary_id_string ] storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=axes_object, basemap_object=basemap_object, line_width=AUXILIARY_STORM_WIDTH, line_colour='k', line_style='dashed') # Plot outlines of related storms (with the same primary ID). this_storm_object_table = storm_object_table.loc[ (storm_object_table[tracking_utils.PRIMARY_ID_COLUMN] == primary_id_string) & (storm_object_table[tracking_utils.SECONDARY_ID_COLUMN] != secondary_id_string) ] this_num_storm_objects = len(this_storm_object_table.index) if this_num_storm_objects > 0: storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=axes_object, basemap_object=basemap_object, line_width=AUXILIARY_STORM_WIDTH, line_colour='k', line_style='solid' ) for j in range(len(this_storm_object_table)): axes_object.text( this_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN ].values[j], this_storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN ].values[j], 'P', fontsize=MAIN_FONT_SIZE, color=FONT_COLOUR, fontweight='bold', horizontalalignment='center', verticalalignment='center' ) # Plot outline of storm of interest (same secondary ID). this_storm_object_table = storm_object_table.loc[ storm_object_table[tracking_utils.SECONDARY_ID_COLUMN] == secondary_id_string ] storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=axes_object, basemap_object=basemap_object, line_width=MAIN_STORM_WIDTH, line_colour='k', line_style='solid') this_num_storm_objects = len(this_storm_object_table.index) plot_forecast = ( this_num_storm_objects > 0 and FORECAST_PROBABILITY_COLUMN in list(this_storm_object_table) ) if plot_forecast: label_string = 'Prob = {0:.3f}\nat {1:s}'.format( this_storm_object_table[FORECAST_PROBABILITY_COLUMN].values[0], time_conversion.unix_sec_to_string( valid_time_unix_sec, TORNADO_TIME_FORMAT) ) axes_object.set_title( label_string.replace('\n', ' '), fontsize=TITLE_FONT_SIZE ) tornado_id_strings = tornado_table[tornado_io.TORNADO_ID_COLUMN].values for this_tornado_id_string in numpy.unique(tornado_id_strings): these_rows = numpy.where( tornado_id_strings == this_tornado_id_string )[0] this_tornado_table = tornado_table.iloc[these_rows].sort_values( linkage.EVENT_TIME_COLUMN, axis=0, ascending=True, inplace=False ) _plot_one_tornado( tornado_table=this_tornado_table, axes_object=axes_object )
def _run(top_input_dir_name, target_name_for_downsampling, first_spc_date_string, last_spc_date_string, downsampling_classes, downsampling_fractions, for_training, top_output_dir_name): """Downsamples storm objects, based on target values. This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param target_name_for_downsampling: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param downsampling_classes: Same. :param downsampling_fractions: Same. :param for_training: Same. :param top_output_dir_name: Same. """ all_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) downsampling_dict = dict( list(zip(downsampling_classes, downsampling_fractions))) target_param_dict = target_val_utils.target_name_to_params( target_name_for_downsampling) event_type_string = target_param_dict[target_val_utils.EVENT_TYPE_KEY] input_target_file_names = [] spc_date_string_by_file = [] for this_spc_date_string in all_spc_date_strings: this_file_name = target_val_utils.find_target_file( top_directory_name=top_input_dir_name, event_type_string=event_type_string, spc_date_string=this_spc_date_string, raise_error_if_missing=False) if not os.path.isfile(this_file_name): continue input_target_file_names.append(this_file_name) spc_date_string_by_file.append(this_spc_date_string) num_files = len(input_target_file_names) target_dict_by_file = [None] * num_files full_id_strings = [] storm_times_unix_sec = numpy.array([], dtype=int) storm_to_file_indices = numpy.array([], dtype=int) target_names = [] target_matrix = None for i in range(num_files): print('Reading data from: "{0:s}"...'.format( input_target_file_names[i])) target_dict_by_file[i] = target_val_utils.read_target_values( netcdf_file_name=input_target_file_names[i]) if i == 0: target_names = ( target_dict_by_file[i][target_val_utils.TARGET_NAMES_KEY]) these_full_id_strings = ( target_dict_by_file[i][target_val_utils.FULL_IDS_KEY]) full_id_strings += these_full_id_strings this_num_storm_objects = len(these_full_id_strings) storm_times_unix_sec = numpy.concatenate( (storm_times_unix_sec, target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY])) storm_to_file_indices = numpy.concatenate( (storm_to_file_indices, numpy.full(this_num_storm_objects, i, dtype=int))) this_target_matrix = ( target_dict_by_file[i][target_val_utils.TARGET_MATRIX_KEY]) if target_matrix is None: target_matrix = this_target_matrix + 0 else: target_matrix = numpy.concatenate( (target_matrix, this_target_matrix), axis=0) print(SEPARATOR_STRING) downsampling_index = target_names.index(target_name_for_downsampling) good_indices = numpy.where(target_matrix[:, downsampling_index] != target_val_utils.INVALID_STORM_INTEGER)[0] full_id_strings = [full_id_strings[k] for k in good_indices] storm_times_unix_sec = storm_times_unix_sec[good_indices] target_matrix = target_matrix[good_indices, :] storm_to_file_indices = storm_to_file_indices[good_indices] primary_id_strings = temporal_tracking.full_to_partial_ids( full_id_strings)[0] if for_training: indices_to_keep = fancy_downsampling.downsample_for_training( primary_id_strings=primary_id_strings, storm_times_unix_sec=storm_times_unix_sec, target_values=target_matrix[:, downsampling_index], target_name=target_name_for_downsampling, class_fraction_dict=downsampling_dict) else: indices_to_keep = fancy_downsampling.downsample_for_non_training( primary_id_strings=primary_id_strings, storm_times_unix_sec=storm_times_unix_sec, target_values=target_matrix[:, downsampling_index], target_name=target_name_for_downsampling, class_fraction_dict=downsampling_dict) print(SEPARATOR_STRING) for i in range(num_files): these_object_subindices = numpy.where( storm_to_file_indices[indices_to_keep] == i)[0] these_object_indices = indices_to_keep[these_object_subindices] if len(these_object_indices) == 0: continue these_indices_in_file = tracking_utils.find_storm_objects( all_id_strings=target_dict_by_file[i][ target_val_utils.FULL_IDS_KEY], all_times_unix_sec=target_dict_by_file[i][ target_val_utils.VALID_TIMES_KEY], id_strings_to_keep=[ full_id_strings[k] for k in these_object_indices ], times_to_keep_unix_sec=storm_times_unix_sec[these_object_indices], allow_missing=False) this_output_dict = { tracking_utils.FULL_ID_COLUMN: [ target_dict_by_file[i][target_val_utils.FULL_IDS_KEY][k] for k in these_indices_in_file ], tracking_utils.VALID_TIME_COLUMN: target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY] [these_indices_in_file] } for j in range(len(target_names)): this_output_dict[target_names[j]] = (target_dict_by_file[i][ target_val_utils.TARGET_MATRIX_KEY][these_indices_in_file, j]) this_output_table = pandas.DataFrame.from_dict(this_output_dict) this_new_file_name = target_val_utils.find_target_file( top_directory_name=top_output_dir_name, event_type_string=event_type_string, spc_date_string=spc_date_string_by_file[i], raise_error_if_missing=False) print(( 'Writing {0:d} downsampled storm objects (out of {1:d} total) to: ' '"{2:s}"...').format( len(this_output_table.index), len(target_dict_by_file[i][target_val_utils.FULL_IDS_KEY]), this_new_file_name)) target_val_utils.write_target_values( storm_to_events_table=this_output_table, target_names=target_names, netcdf_file_name=this_new_file_name)