def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string, top_myrorss_dir_name, radar_field_name, output_file_name): """Evaluates a set of storm tracks. This is effectively the main method. :param top_tracking_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param top_myrorss_dir_name: Same. :param radar_field_name: Same. :param output_file_name: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) list_of_storm_object_tables = [] for this_spc_date_string in spc_date_strings: these_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=echo_top_tracking. DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_file_names) == 0: continue this_storm_object_table = tracking_io.read_many_files( these_file_names)[STORM_OBJECT_COLUMNS] list_of_storm_object_tables.append(this_storm_object_table) if this_spc_date_string != spc_date_strings[-1]: print(MINOR_SEPARATOR_STRING) if len(list_of_storm_object_tables) == 1: continue list_of_storm_object_tables[-1] = list_of_storm_object_tables[ -1].align(list_of_storm_object_tables[0], axis=1)[0] print(SEPARATOR_STRING) storm_object_table = pandas.concat(list_of_storm_object_tables, axis=0, ignore_index=True) evaluation_dict = tracking_eval.evaluate_tracks( storm_object_table=storm_object_table, top_myrorss_dir_name=top_myrorss_dir_name, radar_field_name=radar_field_name) print('Writing results to: "{0:s}"...'.format(output_file_name)) tracking_eval.write_file(evaluation_dict=evaluation_dict, pickle_file_name=output_file_name)
def _run(tornado_dir_name, top_gridrad_dir_name, first_spc_date_string, last_spc_date_string, output_dir_name): """Plots histograms for GridRad dataset. This is effectively the main method. :param tornado_dir_name: See documentation at top of file. :param top_gridrad_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) all_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) spc_date_strings = [] for this_spc_date_string in all_spc_date_strings: this_gridrad_file_name = _find_gridrad_file_for_date( top_gridrad_dir_name=top_gridrad_dir_name, spc_date_string=this_spc_date_string) if this_gridrad_file_name is None: continue spc_date_strings.append(this_spc_date_string) first_year, last_year = _spc_dates_to_years(spc_date_strings) tornado_table = _read_tornado_reports(tornado_dir_name=tornado_dir_name, first_year=first_year, last_year=last_year) print(SEPARATOR_STRING) num_days = len(spc_date_strings) num_tornadoes_by_day = numpy.full(num_days, -1, dtype=int) for i in range(num_days): num_tornadoes_by_day[i] = _get_num_tornadoes_in_day( tornado_table=tornado_table, spc_date_string=spc_date_strings[i]) print('Number of tornadoes on SPC date "{0:s}" = {1:d}'.format( spc_date_strings[i], num_tornadoes_by_day[i])) print(SEPARATOR_STRING) _plot_tornado_histogram( num_tornadoes_by_day=num_tornadoes_by_day, output_file_name='{0:s}/tornado_histogram.jpg'.format(output_dir_name)) _plot_month_histogram( spc_date_strings=spc_date_strings, output_file_name='{0:s}/month_histogram.jpg'.format(output_dir_name))
def _run(echo_top_field_name, top_radar_dir_name_tarred, top_radar_dir_name, top_echo_classifn_dir_name, min_echo_top_km_asl, min_grid_cells_in_polygon, top_output_dir_name, first_spc_date_string, last_spc_date_string): """Runs echo-top-based storm-tracking. This is effectively the main method. :param echo_top_field_name: See documentation at top of file. :param top_radar_dir_name_tarred: Same. :param top_radar_dir_name: Same. :param top_echo_classifn_dir_name: Same. :param min_echo_top_km_asl: Same. :param min_grid_cells_in_polygon: Same. :param top_output_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. """ if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( top_radar_dir_name_tarred, this_spc_date_string[:4], this_spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=this_tar_file_name, field_names=[echo_top_field_name], spc_date_string=this_spc_date_string, top_target_directory_name=top_radar_dir_name) print SEPARATOR_STRING if top_echo_classifn_dir_name in ['', 'None']: top_echo_classifn_dir_name = None echo_top_tracking.run_tracking( top_radar_dir_name=top_radar_dir_name, top_output_dir_name=top_output_dir_name, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, echo_top_field_name=echo_top_field_name, top_echo_classifn_dir_name=top_echo_classifn_dir_name, min_echo_top_height_km_asl=min_echo_top_km_asl, min_grid_cells_in_polygon=min_grid_cells_in_polygon) print SEPARATOR_STRING if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: for this_spc_date_string in spc_date_strings: myrorss_io.remove_unzipped_data_1day( spc_date_string=this_spc_date_string, top_directory_name=top_radar_dir_name, field_names=[echo_top_field_name])
def test_get_spc_dates_in_range_many_dates(self): """Ensures correct output from get_spc_dates_in_range. In this case, there are many dates in the range. """ these_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=FIRST_SPC_DATE_STRING, last_spc_date_string=LAST_SPC_DATE_STRING) self.assertTrue(these_spc_date_strings == ALL_SPC_DATE_STRINGS)
def test_get_spc_dates_in_range_one_date(self): """Ensures correct output from get_spc_dates_in_range. In this case, there is only one date in the range. """ these_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=FIRST_SPC_DATE_STRING, last_spc_date_string=FIRST_SPC_DATE_STRING) self.assertTrue(these_spc_date_strings == [FIRST_SPC_DATE_STRING])
def _run(top_input_dir_name, radar_field_names, refl_heights_m_asl, first_spc_date_string, last_spc_date_string, top_output_dir_name): """Untars MYRORSS data. This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param radar_field_names: Same. :param refl_heights_m_asl: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param top_output_dir_name: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) az_shear_field_names = list( set(radar_field_names) & set(radar_utils.SHEAR_NAMES)) non_shear_field_names = list( set(radar_field_names) - set(radar_utils.SHEAR_NAMES)) for this_spc_date_string in spc_date_strings: if len(az_shear_field_names) > 0: this_tar_file_name = ( '{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar').format( top_input_dir_name, this_spc_date_string[:4], this_spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=this_tar_file_name, field_names=az_shear_field_names, spc_date_string=this_spc_date_string, top_target_directory_name=top_output_dir_name) print(SEPARATOR_STRING) if len(non_shear_field_names) > 0: this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( top_input_dir_name, this_spc_date_string[:4], this_spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=this_tar_file_name, field_names=non_shear_field_names, spc_date_string=this_spc_date_string, top_target_directory_name=top_output_dir_name, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING)
def _run(tornado_dir_name, top_tracking_dir_name, tracking_scale_metres2, genesis_only, first_spc_date_string, last_spc_date_string, top_output_dir_name): """Runs `linkage.link_tornadoes_to_storms`. This is effectively the main method. :param tornado_dir_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param tracking_scale_metres2: Same. :param genesis_only: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param top_output_dir_name: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] for this_spc_date_string in spc_date_strings: these_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_file_names) == 0: if len(tracking_file_names) > 0: _link_tornadoes_one_period( tracking_file_names=tracking_file_names, tornado_dir_name=tornado_dir_name, genesis_only=genesis_only, top_output_dir_name=top_output_dir_name) print(SEPARATOR_STRING) tracking_file_names = [] continue tracking_file_names += these_file_names _link_tornadoes_one_period(tracking_file_names=tracking_file_names, tornado_dir_name=tornado_dir_name, genesis_only=genesis_only, top_output_dir_name=top_output_dir_name)
def _write_csv_file_for_thea( first_spc_date_string, last_spc_date_string, top_tracking_dir_name, tracking_scale_metres2, output_file_name): """Converts storm tracks to a single CSV file for Thea. :param first_spc_date_string: See documentation at top of file. :param last_spc_date_string: Same. :param top_tracking_dir_name: Same. :param tracking_scale_metres2: Same. :param output_file_name: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] for this_spc_date_string in spc_date_strings: (these_tracking_file_names, _ ) = tracking_io.find_processed_files_one_spc_date( spc_date_string=this_spc_date_string, data_source=tracking_utils.SEGMOTION_SOURCE_ID, top_processed_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2) tracking_file_names += these_tracking_file_names num_files = len(tracking_file_names) list_of_storm_object_tables = [None] * num_files for i in range(num_files): print 'Reading {0:d}th of {1:d} input files: "{2:s}"...'.format( i + 1, num_files, tracking_file_names[i]) list_of_storm_object_tables[i] = tracking_io.read_processed_file( tracking_file_names[i]) if i == 0: continue list_of_storm_object_tables[i], _ = ( list_of_storm_object_tables[i].align( list_of_storm_object_tables[0], axis=1)) storm_object_table = pandas.concat( list_of_storm_object_tables, axis=0, ignore_index=True) print 'Writing all storm tracks to "{0:s}"...'.format(output_file_name) best_tracks.write_simple_output_for_thea( storm_object_table, output_file_name)
def _run(top_input_dir_name, first_spc_date_string, last_spc_date_string, resolution_factor, top_output_dir_name): """Converts examples from GridRad to MYRORSS format. This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param resolution_factor: Same. :param top_output_dir_name: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) input_file_names = [ input_examples.find_example_file( top_directory_name=top_input_dir_name, shuffled=False, spc_date_string=d, raise_error_if_missing=False ) for d in spc_date_strings ] output_file_names = [ input_examples.find_example_file( top_directory_name=top_output_dir_name, shuffled=False, spc_date_string=d, raise_error_if_missing=False ) for d in spc_date_strings ] num_spc_dates = len(spc_date_strings) for i in range(num_spc_dates): if not os.path.isfile(input_file_names[i]): continue _convert_one_file( input_file_name=input_file_names[i], resolution_factor=resolution_factor, output_file_name=output_file_names[i] ) print('\n')
def _run(top_orig_tracking_dir_name, top_new_tracking_dir_name, first_spc_date_string, last_spc_date_string, output_file_name): """Plots storms that were removed by remove_storms_outside_conus.py. This is effectively the main method. :param top_orig_tracking_dir_name: See documentation at top of file. :param top_new_tracking_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) orig_tracking_file_names = [] for d in spc_date_strings: orig_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_orig_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=d, raise_error_if_missing=False)[0] valid_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in orig_tracking_file_names], dtype=int) new_tracking_file_names = [ tracking_io.find_file( top_tracking_dir_name=top_new_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=t, spc_date_string=time_conversion.time_to_spc_date_string(t), raise_error_if_missing=True) for t in valid_times_unix_sec ] orig_storm_object_table = tracking_io.read_many_files( orig_tracking_file_names) print(SEPARATOR_STRING) new_storm_object_table = tracking_io.read_many_files( new_tracking_file_names) print(SEPARATOR_STRING) orig_storm_id_strings = ( orig_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) orig_storm_times_unix_sec = ( orig_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) new_storm_id_strings = ( new_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) new_storm_times_unix_sec = ( new_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) num_orig_storm_objects = len(orig_storm_object_table.index) orig_kept_flags = numpy.full(num_orig_storm_objects, 0, dtype=bool) these_indices = tracking_utils.find_storm_objects( all_id_strings=orig_storm_id_strings, all_times_unix_sec=orig_storm_times_unix_sec, id_strings_to_keep=new_storm_id_strings, times_to_keep_unix_sec=new_storm_times_unix_sec, allow_missing=False) orig_kept_flags[these_indices] = True orig_removed_indices = numpy.where(numpy.invert(orig_kept_flags))[0] print('{0:d} of {1:d} storm objects were outside CONUS.'.format( len(orig_removed_indices), num_orig_storm_objects)) removed_storm_object_table = orig_storm_object_table.iloc[ orig_removed_indices] removed_latitudes_deg = removed_storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values removed_longitudes_deg = removed_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values figure_object, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=numpy.min(removed_latitudes_deg) - 1., max_latitude_deg=numpy.max(removed_latitudes_deg) + 1., min_longitude_deg=numpy.min(removed_longitudes_deg) - 1., max_longitude_deg=numpy.max(removed_longitudes_deg) + 1., resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS) conus_latitudes_deg, conus_longitudes_deg = ( conus_boundary.read_from_netcdf()) conus_latitudes_deg, conus_longitudes_deg = conus_boundary.erode_boundary( latitudes_deg=conus_latitudes_deg, longitudes_deg=conus_longitudes_deg, erosion_distance_metres=EROSION_DISTANCE_METRES) axes_object.plot(conus_longitudes_deg, conus_latitudes_deg, color=LINE_COLOUR, linestyle='solid', linewidth=LINE_WIDTH) axes_object.plot(removed_longitudes_deg, removed_latitudes_deg, linestyle='None', marker=MARKER_TYPE, markersize=MARKER_SIZE, markeredgewidth=0, markerfacecolor=MARKER_COLOUR, markeredgecolor=MARKER_COLOUR) print('Saving figure to: "{0:s}"...'.format(output_file_name)) figure_object.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object)
def _run(top_gridrad_dir_name, first_spc_date_string, last_spc_date_string): """Makes LaTeX table with GridRad days and domains. This is effectively the main method. :param top_gridrad_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. """ all_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) nice_date_strings = [] latitude_strings = [] longitude_strings = [] for this_spc_date_string in all_spc_date_strings: these_limits_deg = _find_domain_for_date( top_gridrad_dir_name=top_gridrad_dir_name, spc_date_string=this_spc_date_string ) if these_limits_deg is None: continue print(SEPARATOR_STRING) this_time_unix_sec = time_conversion.spc_date_string_to_unix_sec( this_spc_date_string ) this_nice_date_string = time_conversion.unix_sec_to_string( this_time_unix_sec, NICE_TIME_FORMAT ) nice_date_strings.append(this_nice_date_string) latitude_strings.append('{0:.1f}-{1:.1f}'.format( these_limits_deg[0], these_limits_deg[1] )) longitude_strings.append('{0:.1f}-{1:.1f}'.format( these_limits_deg[3], these_limits_deg[2] )) table_string = '' for i in range(len(nice_date_strings)): if i != 0: if numpy.mod(i, 2) == 0: table_string += ' \\\\\n\t\t\t' else: table_string += ' & ' table_string += '{0:s}, {1:s}'.format( nice_date_strings[i], latitude_strings[i] ) # table_string += ' $^{\\circ}$N' table_string += ', {0:s}'.format(longitude_strings[i]) # table_string += ' $^{\\circ}$W' print(table_string)
def _run(top_input_dir_name, target_name, first_spc_date_string, last_spc_date_string, class_fraction_keys, class_fraction_values, for_training, top_output_dir_name): """Downsamples storm objects, based on target values. This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param target_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param class_fraction_keys: Same. :param class_fraction_values: Same. :param for_training: Same. :param top_output_dir_name: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) class_fraction_dict = dict(zip(class_fraction_keys, class_fraction_values)) target_param_dict = target_val_utils.target_name_to_params(target_name) input_target_file_names = [] spc_date_string_by_file = [] for this_spc_date_string in spc_date_strings: this_file_name = target_val_utils.find_target_file( top_directory_name=top_input_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=this_spc_date_string, raise_error_if_missing=False) if not os.path.isfile(this_file_name): continue input_target_file_names.append(this_file_name) spc_date_string_by_file.append(this_spc_date_string) num_files = len(input_target_file_names) spc_date_strings = spc_date_string_by_file target_dict_by_file = [None] * num_files storm_ids = [] storm_times_unix_sec = numpy.array([], dtype=int) target_values = numpy.array([], dtype=int) for i in range(num_files): print 'Reading "{0:s}" from: "{1:s}"...'.format( target_name, input_target_file_names[i]) target_dict_by_file[i] = target_val_utils.read_target_values( netcdf_file_name=input_target_file_names[i], target_name=target_name) storm_ids += target_dict_by_file[i][target_val_utils.STORM_IDS_KEY] storm_times_unix_sec = numpy.concatenate( (storm_times_unix_sec, target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY])) target_values = numpy.concatenate( (target_values, target_dict_by_file[i][target_val_utils.TARGET_VALUES_KEY])) print SEPARATOR_STRING good_indices = numpy.where( target_values != target_val_utils.INVALID_STORM_INTEGER)[0] storm_ids = [storm_ids[k] for k in good_indices] storm_times_unix_sec = storm_times_unix_sec[good_indices] target_values = target_values[good_indices] if for_training: storm_ids, storm_times_unix_sec, target_values = ( fancy_downsampling.downsample_for_training( storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec, target_values=target_values, target_name=target_name, class_fraction_dict=class_fraction_dict)) else: storm_ids, storm_times_unix_sec, target_values = ( fancy_downsampling.downsample_for_non_training( storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec, target_values=target_values, target_name=target_name, class_fraction_dict=class_fraction_dict)) print SEPARATOR_STRING for i in range(num_files): these_indices = tracking_utils.find_storm_objects( all_storm_ids=target_dict_by_file[i][ target_val_utils.STORM_IDS_KEY], all_times_unix_sec=target_dict_by_file[i][ target_val_utils.VALID_TIMES_KEY], storm_ids_to_keep=storm_ids, times_to_keep_unix_sec=storm_times_unix_sec, allow_missing=True) these_indices = these_indices[these_indices >= 0] if len(these_indices) == 0: continue this_output_dict = { tracking_utils.STORM_ID_COLUMN: [ target_dict_by_file[i][target_val_utils.STORM_IDS_KEY][k] for k in these_indices ], tracking_utils.TIME_COLUMN: target_dict_by_file[i][ target_val_utils.VALID_TIMES_KEY][these_indices], target_name: target_dict_by_file[i][target_val_utils.TARGET_VALUES_KEY] [these_indices] } this_output_table = pandas.DataFrame.from_dict(this_output_dict) this_new_file_name = target_val_utils.find_target_file( top_directory_name=top_output_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=spc_date_strings[i], raise_error_if_missing=False) print( 'Writing {0:d} downsampled storm objects (out of {1:d} total) to: ' '"{2:s}"...').format( len(this_output_table.index), len(target_dict_by_file[i][target_val_utils.STORM_IDS_KEY]), this_new_file_name) target_val_utils.write_target_values( storm_to_events_table=this_output_table, target_names=[target_name], netcdf_file_name=this_new_file_name)
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string, colour_map_name, min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg, max_plot_longitude_deg, output_file_name): """Plots storm tracks for a continuous time period. This is effectively the main method. :param top_tracking_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param colour_map_name: Same. :param min_plot_latitude_deg: Same. :param max_plot_latitude_deg: Same. :param min_plot_longitude_deg: Same. :param max_plot_longitude_deg: Same. :param output_file_name: Same. """ if colour_map_name in ['', 'None']: colour_map_object = 'random' else: colour_map_object = pyplot.cm.get_cmap(colour_map_name) if min_plot_latitude_deg <= SENTINEL_VALUE: min_plot_latitude_deg = None if max_plot_latitude_deg <= SENTINEL_VALUE: max_plot_latitude_deg = None if min_plot_longitude_deg <= SENTINEL_VALUE: min_plot_longitude_deg = None if max_plot_longitude_deg <= SENTINEL_VALUE: max_plot_longitude_deg = None file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) list_of_storm_object_tables = [] for this_spc_date_string in spc_date_strings: these_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=echo_top_tracking. DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_file_names) == 0: continue this_storm_object_table = tracking_io.read_many_files( these_file_names)[REQUIRED_COLUMNS] list_of_storm_object_tables.append(this_storm_object_table) if this_spc_date_string != spc_date_strings[-1]: print(MINOR_SEPARATOR_STRING) if len(list_of_storm_object_tables) == 1: continue list_of_storm_object_tables[-1] = list_of_storm_object_tables[ -1].align(list_of_storm_object_tables[0], axis=1)[0] print(SEPARATOR_STRING) storm_object_table = pandas.concat(list_of_storm_object_tables, axis=0, ignore_index=True) if min_plot_latitude_deg is None: min_plot_latitude_deg = numpy.min( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values ) - LATLNG_BUFFER_DEG if max_plot_latitude_deg is None: max_plot_latitude_deg = numpy.max( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values ) + LATLNG_BUFFER_DEG if min_plot_longitude_deg is None: min_plot_longitude_deg = numpy.min( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values ) - LATLNG_BUFFER_DEG if max_plot_longitude_deg is None: max_plot_longitude_deg = numpy.max( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values ) + LATLNG_BUFFER_DEG _, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS) storm_plotting.plot_storm_tracks(storm_object_table=storm_object_table, axes_object=axes_object, basemap_object=basemap_object, colour_map_object=colour_map_object) print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close()
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string, colour_map_name, min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg, max_plot_longitude_deg, output_file_name): """Plots storm tracks for a continuous time period. This is effectively the main method. :param top_tracking_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param colour_map_name: Same. :param min_plot_latitude_deg: Same. :param max_plot_latitude_deg: Same. :param min_plot_longitude_deg: Same. :param max_plot_longitude_deg: Same. :param output_file_name: Same. """ if colour_map_name in ['', 'None']: colour_map_object = 'random' else: colour_map_object = pyplot.cm.get_cmap(colour_map_name) if min_plot_latitude_deg <= SENTINEL_VALUE: min_plot_latitude_deg = None if max_plot_latitude_deg <= SENTINEL_VALUE: max_plot_latitude_deg = None if min_plot_longitude_deg <= SENTINEL_VALUE: min_plot_longitude_deg = None if max_plot_longitude_deg <= SENTINEL_VALUE: max_plot_longitude_deg = None file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) list_of_storm_object_tables = [] for this_spc_date_string in spc_date_strings: these_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=echo_top_tracking. DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_file_names) == 0: continue this_storm_object_table = tracking_io.read_many_files( these_file_names)[REQUIRED_COLUMNS] list_of_storm_object_tables.append(this_storm_object_table) if this_spc_date_string != spc_date_strings[-1]: print(MINOR_SEPARATOR_STRING) if len(list_of_storm_object_tables) == 1: continue list_of_storm_object_tables[-1] = list_of_storm_object_tables[ -1].align(list_of_storm_object_tables[0], axis=1)[0] print(SEPARATOR_STRING) storm_object_table = pandas.concat(list_of_storm_object_tables, axis=0, ignore_index=True) # TODO(thunderhoser): HACK first_time_unix_sec = time_conversion.string_to_unix_sec( '2011-04-27-20', '%Y-%m-%d-%H') storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.VALID_TIME_COLUMN] >= first_time_unix_sec] if min_plot_latitude_deg is None: min_plot_latitude_deg = numpy.min( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values ) - LATLNG_BUFFER_DEG if max_plot_latitude_deg is None: max_plot_latitude_deg = numpy.max( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values ) + LATLNG_BUFFER_DEG if min_plot_longitude_deg is None: min_plot_longitude_deg = numpy.min( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values ) - LATLNG_BUFFER_DEG if max_plot_longitude_deg is None: max_plot_longitude_deg = numpy.max( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values ) + LATLNG_BUFFER_DEG _, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')) # plotting_utils.plot_coastlines( # basemap_object=basemap_object, axes_object=axes_object, # line_colour=BORDER_COLOUR # ) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS, line_colour=numpy.full(3, 1.)) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS, line_colour=numpy.full(3, 1.)) colour_bar_object = storm_plotting.plot_storm_tracks( storm_object_table=storm_object_table, axes_object=axes_object, basemap_object=basemap_object, colour_map_object=colour_map_object) valid_times_unix_sec = ( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) # TODO(thunderhoser): HACK tick_times_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=numpy.min(valid_times_unix_sec), end_time_unix_sec=numpy.max(valid_times_unix_sec), time_interval_sec=1800, include_endpoint=True) tick_time_strings = [ time_conversion.unix_sec_to_string(t, COLOUR_BAR_TIME_FORMAT) for t in tick_times_unix_sec ] colour_bar_object.set_ticks(tick_times_unix_sec) colour_bar_object.set_ticklabels(tick_time_strings) print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close()
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string, storm_colour, storm_opacity, include_secondary_ids, min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg, max_plot_longitude_deg, top_myrorss_dir_name, radar_field_name, radar_height_m_asl, output_dir_name): """Plots storm outlines (along with IDs) at each time step. This is effectively the main method. :param top_tracking_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param storm_colour: Same. :param storm_opacity: Same. :param include_secondary_ids: Same. :param min_plot_latitude_deg: Same. :param max_plot_latitude_deg: Same. :param min_plot_longitude_deg: Same. :param max_plot_longitude_deg: Same. :param top_myrorss_dir_name: Same. :param radar_field_name: Same. :param radar_height_m_asl: Same. :param output_dir_name: Same. """ if top_myrorss_dir_name in ['', 'None']: top_myrorss_dir_name = None if radar_field_name != radar_utils.REFL_NAME: radar_height_m_asl = None file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] for this_spc_date_string in spc_date_strings: tracking_file_names += (tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=DUMMY_SOURCE_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0]) storm_object_table = tracking_io.read_many_files(tracking_file_names) print(SEPARATOR_STRING) latitude_limits_deg, longitude_limits_deg = _get_plotting_limits( min_plot_latitude_deg=min_plot_latitude_deg, max_plot_latitude_deg=max_plot_latitude_deg, min_plot_longitude_deg=min_plot_longitude_deg, max_plot_longitude_deg=max_plot_longitude_deg, storm_object_table=storm_object_table) min_plot_latitude_deg = latitude_limits_deg[0] max_plot_latitude_deg = latitude_limits_deg[1] min_plot_longitude_deg = longitude_limits_deg[0] max_plot_longitude_deg = longitude_limits_deg[1] valid_times_unix_sec = numpy.unique( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) num_times = len(valid_times_unix_sec) for i in range(num_times): these_current_rows = numpy.where( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values == valid_times_unix_sec[i])[0] these_current_subrows = _filter_storm_objects_latlng( storm_object_table=storm_object_table.iloc[these_current_rows], min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg) if len(these_current_subrows) == 0: continue these_current_rows = these_current_rows[these_current_subrows] this_storm_object_table = _find_relevant_storm_objects( storm_object_table=storm_object_table, current_rows=these_current_rows) these_latlng_rows = _filter_storm_objects_latlng( storm_object_table=this_storm_object_table, min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg) if top_myrorss_dir_name is None: this_radar_matrix = None these_radar_latitudes_deg = None these_radar_longitudes_deg = None else: this_myrorss_file_name = myrorss_and_mrms_io.find_raw_file( top_directory_name=top_myrorss_dir_name, unix_time_sec=valid_times_unix_sec[i], spc_date_string=time_conversion.time_to_spc_date_string( valid_times_unix_sec[i]), field_name=radar_field_name, data_source=radar_utils.MYRORSS_SOURCE_ID, height_m_asl=radar_height_m_asl, raise_error_if_missing=True) print( 'Reading data from: "{0:s}"...'.format(this_myrorss_file_name)) this_metadata_dict = ( myrorss_and_mrms_io.read_metadata_from_raw_file( netcdf_file_name=this_myrorss_file_name, data_source=radar_utils.MYRORSS_SOURCE_ID)) this_sparse_grid_table = ( myrorss_and_mrms_io.read_data_from_sparse_grid_file( netcdf_file_name=this_myrorss_file_name, field_name_orig=this_metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=this_metadata_dict[ radar_utils.SENTINEL_VALUE_COLUMN])) (this_radar_matrix, these_radar_latitudes_deg, these_radar_longitudes_deg) = radar_s2f.sparse_to_full_grid( sparse_grid_table=this_sparse_grid_table, metadata_dict=this_metadata_dict) this_radar_matrix = numpy.flipud(this_radar_matrix) these_radar_latitudes_deg = these_radar_latitudes_deg[::-1] _, this_axes_object, this_basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')) _plot_storm_outlines_one_time( storm_object_table=this_storm_object_table.iloc[these_latlng_rows], valid_time_unix_sec=valid_times_unix_sec[i], axes_object=this_axes_object, basemap_object=this_basemap_object, storm_colour=storm_colour, storm_opacity=storm_opacity, include_secondary_ids=include_secondary_ids, output_dir_name=output_dir_name, radar_matrix=this_radar_matrix, radar_field_name=radar_field_name, radar_latitudes_deg=these_radar_latitudes_deg, radar_longitudes_deg=these_radar_longitudes_deg)
def _run(top_radar_dir_name, top_radar_dir_name_tarred, top_echo_classifn_dir_name, first_spc_date_string, last_spc_date_string, echo_top_field_name, min_echo_top_km, min_size_pixels, min_intermax_distance_metres, max_velocity_diff_m_s01, max_link_distance_m_s01, top_output_dir_name): """Tracks storms based on echo top. This is effectively the main method. :param top_radar_dir_name: See documentation at top of file. :param top_radar_dir_name_tarred: Same. :param top_echo_classifn_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param echo_top_field_name: Same. :param min_echo_top_km: Same. :param min_size_pixels: Same. :param min_intermax_distance_metres: Same. :param max_velocity_diff_m_s01: Same. :param max_link_distance_m_s01: Same. :param top_output_dir_name: Same. """ if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( top_radar_dir_name_tarred, this_spc_date_string[:4], this_spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=this_tar_file_name, field_names=[echo_top_field_name], spc_date_string=this_spc_date_string, top_target_directory_name=top_radar_dir_name) print(SEPARATOR_STRING) if top_echo_classifn_dir_name in ['', 'None']: top_echo_classifn_dir_name = None echo_top_tracking.run_tracking( top_radar_dir_name=top_radar_dir_name, top_output_dir_name=top_output_dir_name, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, echo_top_field_name=echo_top_field_name, top_echo_classifn_dir_name=top_echo_classifn_dir_name, min_echo_top_km=min_echo_top_km, min_intermax_distance_metres=min_intermax_distance_metres, min_polygon_size_pixels=min_size_pixels, max_velocity_diff_m_s01=max_velocity_diff_m_s01, max_link_distance_m_s01=max_link_distance_m_s01, min_track_duration_seconds=0) print(SEPARATOR_STRING) if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: for this_spc_date_string in spc_date_strings: myrorss_io.remove_unzipped_data_1day( spc_date_string=this_spc_date_string, top_directory_name=top_radar_dir_name, field_names=[echo_top_field_name] )
def _run(top_input_dir_name, tracking_scale_metres2, first_spc_date_string, last_spc_date_string, min_distances_metres, max_distances_metres, top_output_dir_name): """Creates one or more distance buffers around each storm object (polygon). This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param tracking_scale_metres2: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param min_distances_metres: Same. :param max_distances_metres: Same. :param top_output_dir_name: Same. """ min_distances_metres[min_distances_metres < 0] = numpy.nan spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: these_input_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_input_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_input_file_names) == 0: continue for this_input_file_name in these_input_file_names: print('Reading input tracks from: "{0:s}"...'.format( this_input_file_name)) this_storm_object_table = tracking_io.read_file( this_input_file_name) this_storm_object_table = (tracking_utils.create_distance_buffers( storm_object_table=this_storm_object_table, min_distances_metres=min_distances_metres, max_distances_metres=max_distances_metres)) this_output_file_name = tracking_io.find_file( top_tracking_dir_name=top_output_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=tracking_io.file_name_to_time( this_input_file_name), spc_date_string=this_spc_date_string, raise_error_if_missing=False) print('Writing input tracks + buffers to: "{0:s}"...\n'.format( this_output_file_name)) tracking_io.write_file(storm_object_table=this_storm_object_table, pickle_file_name=this_output_file_name) print(SEPARATOR_STRING)
def _run(top_input_dir_name, target_name_for_downsampling, first_spc_date_string, last_spc_date_string, downsampling_classes, downsampling_fractions, for_training, top_output_dir_name): """Downsamples storm objects, based on target values. This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param target_name_for_downsampling: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param downsampling_classes: Same. :param downsampling_fractions: Same. :param for_training: Same. :param top_output_dir_name: Same. """ all_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) downsampling_dict = dict( list(zip(downsampling_classes, downsampling_fractions))) target_param_dict = target_val_utils.target_name_to_params( target_name_for_downsampling) event_type_string = target_param_dict[target_val_utils.EVENT_TYPE_KEY] input_target_file_names = [] spc_date_string_by_file = [] for this_spc_date_string in all_spc_date_strings: this_file_name = target_val_utils.find_target_file( top_directory_name=top_input_dir_name, event_type_string=event_type_string, spc_date_string=this_spc_date_string, raise_error_if_missing=False) if not os.path.isfile(this_file_name): continue input_target_file_names.append(this_file_name) spc_date_string_by_file.append(this_spc_date_string) num_files = len(input_target_file_names) target_dict_by_file = [None] * num_files full_id_strings = [] storm_times_unix_sec = numpy.array([], dtype=int) storm_to_file_indices = numpy.array([], dtype=int) target_names = [] target_matrix = None for i in range(num_files): print('Reading data from: "{0:s}"...'.format( input_target_file_names[i])) target_dict_by_file[i] = target_val_utils.read_target_values( netcdf_file_name=input_target_file_names[i]) if i == 0: target_names = ( target_dict_by_file[i][target_val_utils.TARGET_NAMES_KEY]) these_full_id_strings = ( target_dict_by_file[i][target_val_utils.FULL_IDS_KEY]) full_id_strings += these_full_id_strings this_num_storm_objects = len(these_full_id_strings) storm_times_unix_sec = numpy.concatenate( (storm_times_unix_sec, target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY])) storm_to_file_indices = numpy.concatenate( (storm_to_file_indices, numpy.full(this_num_storm_objects, i, dtype=int))) this_target_matrix = ( target_dict_by_file[i][target_val_utils.TARGET_MATRIX_KEY]) if target_matrix is None: target_matrix = this_target_matrix + 0 else: target_matrix = numpy.concatenate( (target_matrix, this_target_matrix), axis=0) print(SEPARATOR_STRING) downsampling_index = target_names.index(target_name_for_downsampling) good_indices = numpy.where(target_matrix[:, downsampling_index] != target_val_utils.INVALID_STORM_INTEGER)[0] full_id_strings = [full_id_strings[k] for k in good_indices] storm_times_unix_sec = storm_times_unix_sec[good_indices] target_matrix = target_matrix[good_indices, :] storm_to_file_indices = storm_to_file_indices[good_indices] primary_id_strings = temporal_tracking.full_to_partial_ids( full_id_strings)[0] if for_training: indices_to_keep = fancy_downsampling.downsample_for_training( primary_id_strings=primary_id_strings, storm_times_unix_sec=storm_times_unix_sec, target_values=target_matrix[:, downsampling_index], target_name=target_name_for_downsampling, class_fraction_dict=downsampling_dict) else: indices_to_keep = fancy_downsampling.downsample_for_non_training( primary_id_strings=primary_id_strings, storm_times_unix_sec=storm_times_unix_sec, target_values=target_matrix[:, downsampling_index], target_name=target_name_for_downsampling, class_fraction_dict=downsampling_dict) print(SEPARATOR_STRING) for i in range(num_files): these_object_subindices = numpy.where( storm_to_file_indices[indices_to_keep] == i)[0] these_object_indices = indices_to_keep[these_object_subindices] if len(these_object_indices) == 0: continue these_indices_in_file = tracking_utils.find_storm_objects( all_id_strings=target_dict_by_file[i][ target_val_utils.FULL_IDS_KEY], all_times_unix_sec=target_dict_by_file[i][ target_val_utils.VALID_TIMES_KEY], id_strings_to_keep=[ full_id_strings[k] for k in these_object_indices ], times_to_keep_unix_sec=storm_times_unix_sec[these_object_indices], allow_missing=False) this_output_dict = { tracking_utils.FULL_ID_COLUMN: [ target_dict_by_file[i][target_val_utils.FULL_IDS_KEY][k] for k in these_indices_in_file ], tracking_utils.VALID_TIME_COLUMN: target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY] [these_indices_in_file] } for j in range(len(target_names)): this_output_dict[target_names[j]] = (target_dict_by_file[i][ target_val_utils.TARGET_MATRIX_KEY][these_indices_in_file, j]) this_output_table = pandas.DataFrame.from_dict(this_output_dict) this_new_file_name = target_val_utils.find_target_file( top_directory_name=top_output_dir_name, event_type_string=event_type_string, spc_date_string=spc_date_string_by_file[i], raise_error_if_missing=False) print(( 'Writing {0:d} downsampled storm objects (out of {1:d} total) to: ' '"{2:s}"...').format( len(this_output_table.index), len(target_dict_by_file[i][target_val_utils.FULL_IDS_KEY]), this_new_file_name)) target_val_utils.write_target_values( storm_to_events_table=this_output_table, target_names=target_names, netcdf_file_name=this_new_file_name)
def _find_tracking_files_one_example(top_tracking_dir_name, valid_time_unix_sec, target_name): """Finds tracking files needed to make plots for one example. :param top_tracking_dir_name: See documentation at top of file. :param valid_time_unix_sec: Valid time for example. :param target_name: Name of target variable. :return: tracking_file_names: 1-D list of paths to tracking files. :raises: ValueError: if no tracking files are found. """ target_param_dict = target_val_utils.target_name_to_params(target_name) min_lead_time_seconds = target_param_dict[ target_val_utils.MIN_LEAD_TIME_KEY] max_lead_time_seconds = target_param_dict[ target_val_utils.MAX_LEAD_TIME_KEY] first_time_unix_sec = valid_time_unix_sec + min_lead_time_seconds last_time_unix_sec = valid_time_unix_sec + max_lead_time_seconds first_spc_date_string = time_conversion.time_to_spc_date_string( first_time_unix_sec - TIME_INTERVAL_SECONDS) last_spc_date_string = time_conversion.time_to_spc_date_string( last_time_unix_sec + TIME_INTERVAL_SECONDS) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] for this_spc_date_string in spc_date_strings: these_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=echo_top_tracking. DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] tracking_file_names += these_file_names if len(tracking_file_names) == 0: error_string = ( 'Cannot find any tracking files for SPC dates "{0:s}" to "{1:s}".' ).format(first_spc_date_string, last_spc_date_string) raise ValueError(error_string) tracking_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int) sort_indices = numpy.argsort(tracking_times_unix_sec) tracking_times_unix_sec = tracking_times_unix_sec[sort_indices] tracking_file_names = [tracking_file_names[k] for k in sort_indices] these_indices = numpy.where( tracking_times_unix_sec <= first_time_unix_sec)[0] if len(these_indices) == 0: first_index = 0 else: first_index = these_indices[-1] these_indices = numpy.where( tracking_times_unix_sec >= last_time_unix_sec)[0] if len(these_indices) == 0: last_index = len(tracking_file_names) - 1 else: last_index = these_indices[0] return tracking_file_names[first_index:(last_index + 1)]
def _run(top_linkage_dir_name, first_spc_date_string, last_spc_date_string, min_lead_times_sec, max_lead_times_sec, min_link_distances_metres, max_link_distances_metres, event_type_string, wind_speed_percentile_level, wind_speed_cutoffs_kt, top_output_dir_name): """Computes target value for ea storm object, lead-time window, and buffer. This is effectively the main method. :param top_linkage_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param min_lead_times_sec: Same. :param max_lead_times_sec: Same. :param min_link_distances_metres: Same. :param max_link_distances_metres: Same. :param event_type_string: Same. :param wind_speed_percentile_level: Same. :param wind_speed_cutoffs_kt: Same. :param top_output_dir_name: Same. """ num_lead_time_windows = len(min_lead_times_sec) these_expected_dim = numpy.array([num_lead_time_windows], dtype=int) error_checking.assert_is_numpy_array(max_lead_times_sec, exact_dimensions=these_expected_dim) num_distance_buffers = len(min_link_distances_metres) these_expected_dim = numpy.array([num_distance_buffers], dtype=int) error_checking.assert_is_numpy_array(max_link_distances_metres, exact_dimensions=these_expected_dim) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: this_linkage_file_name = linkage.find_linkage_file( top_directory_name=top_linkage_dir_name, event_type_string=event_type_string, spc_date_string=this_spc_date_string, raise_error_if_missing=False) if not os.path.isfile(this_linkage_file_name): continue print('Reading data from: "{0:s}"...'.format(this_linkage_file_name)) this_storm_to_events_table = linkage.read_linkage_file( this_linkage_file_name)[0] _compute_targets_one_day( storm_to_events_table=this_storm_to_events_table, spc_date_string=this_spc_date_string, min_lead_times_sec=min_lead_times_sec, max_lead_times_sec=max_lead_times_sec, min_link_distances_metres=min_link_distances_metres, max_link_distances_metres=max_link_distances_metres, event_type_string=event_type_string, wind_speed_percentile_level=wind_speed_percentile_level, wind_speed_cutoffs_kt=wind_speed_cutoffs_kt, top_output_dir_name=top_output_dir_name)
FIRST_SPC_DATE_STRING = '20110601' LAST_SPC_DATE_STRING = '20110831' TOP_PROCESSED_DIR_NAME = '/condo/swatwork/ralager/myrorss_40dbz_echo_tops/final_tracks/reanalyzed/' TRACKING_SCALE_METRES2 = 314159265 HOUR_FORMAT = '%H' NUM_HOURS_PER_DAY = 24 BIRTH_CLIMATOLOGY_TYPE = 'birth' DEATH_CLIMATOLOGY_TYPE = 'death' PASSAGE_CLIMATOLOGY_TYPE = 'passage' if __name__ == '__main__': spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=FIRST_SPC_DATE_STRING, last_spc_date_string=LAST_SPC_DATE_STRING) num_spc_dates = len(spc_date_strings) storm_object_table_by_spc_date = [None] * num_spc_dates num_storm_objects_by_hour = numpy.full(NUM_HOURS_PER_DAY, 0, dtype=int) speeds = [] for working_date_index in range(num_spc_dates): date_in_memory_indices = utils._get_dates_needed( working_date_index=working_date_index, num_dates=num_spc_dates, climatology_type=PASSAGE_CLIMATOLOGY_TYPE) for i in range(num_spc_dates): if i in date_in_memory_indices:
def _find_tracking_gaps(first_spc_date_string, last_spc_date_string, top_tracking_dir_name, tracking_scale_metres2, source_name, min_time_diff_seconds): """Finds gaps (temporal discontinuities) between storm-tracking files. :param first_spc_date_string: See documentation at top of file. :param last_spc_date_string: Same. :param top_tracking_dir_name: Same. :param tracking_scale_metres2: Same. :param source_name: Same. :param min_time_diff_seconds: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] unix_times_sec = numpy.array([], dtype=int) num_spc_dates = len(spc_date_strings) for i in range(num_spc_dates): print('Finding tracking files for SPC date "{0:s}"...'.format( spc_date_strings[i])) these_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_strings[i], source_name=source_name, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2, raise_error_if_missing=False)[0] print(len(these_file_names)) if not len(these_file_names): continue these_file_sizes_bytes = numpy.array( [os.path.getsize(f) for f in these_file_names], dtype=int) these_valid_indices = numpy.where( these_file_sizes_bytes > FILE_SIZE_WITHOUT_STORMS_BYTES)[0] these_file_names = [these_file_names[k] for k in these_valid_indices] these_unix_times_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in these_file_names], dtype=int) these_sort_indices = numpy.argsort(these_unix_times_sec) these_unix_times_sec = these_unix_times_sec[these_sort_indices] these_file_names = [these_file_names[k] for k in these_sort_indices] tracking_file_names += these_file_names unix_times_sec = numpy.concatenate( (unix_times_sec, these_unix_times_sec)) time_diffs_seconds = numpy.diff(unix_times_sec) time_gap_indices = numpy.where( time_diffs_seconds >= min_time_diff_seconds)[0] num_time_gaps = len(time_gap_indices) print(( '\nThere are {0:d} time gaps (successive files >= {1:d} seconds apart),' ' listed below:\n').format(num_time_gaps, min_time_diff_seconds)) for i in time_gap_indices: this_start_time_string = time_conversion.unix_sec_to_string( unix_times_sec[i], TIME_FORMAT) this_end_time_string = time_conversion.unix_sec_to_string( unix_times_sec[i + 1], TIME_FORMAT) print('Gap between {0:s} and {1:s} = {2:d} seconds'.format( this_start_time_string, this_end_time_string, time_diffs_seconds[i]))
def _run(top_linkage_dir_name, genesis_only, max_link_distance_metres, first_spc_date_string, last_spc_date_string, num_colours, min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg, max_plot_longitude_deg, output_file_name): """Plots tornado reports, storm tracks, and linkages. This is effectively the main method. :param top_linkage_dir_name: See documentation at top of file. :param genesis_only: Same. :param max_link_distance_metres: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param num_colours: Same. :param min_plot_latitude_deg: Same. :param max_plot_latitude_deg: Same. :param min_plot_longitude_deg: Same. :param max_plot_longitude_deg: Same. :param output_file_name: Same. """ if max_link_distance_metres < 0: max_link_distance_metres = None colour_map_object = _truncate_colour_map( orig_colour_map_object=pyplot.cm.get_cmap('YlOrRd'), num_colours=num_colours) event_type_string = (linkage.TORNADOGENESIS_EVENT_STRING if genesis_only else linkage.TORNADO_EVENT_STRING) if min_plot_latitude_deg <= SENTINEL_VALUE: min_plot_latitude_deg = None if max_plot_latitude_deg <= SENTINEL_VALUE: max_plot_latitude_deg = None if min_plot_longitude_deg <= SENTINEL_VALUE: min_plot_longitude_deg = None if max_plot_longitude_deg <= SENTINEL_VALUE: max_plot_longitude_deg = None file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) list_of_linkage_tables = [] list_of_tornado_tables = [] linkage_metadata_dict = None for this_spc_date_string in spc_date_strings: this_file_name = linkage.find_linkage_file( top_directory_name=top_linkage_dir_name, event_type_string=event_type_string, spc_date_string=this_spc_date_string, raise_error_if_missing=False) if not os.path.isfile(this_file_name): continue print('Reading data from: "{0:s}"...'.format(this_file_name)) this_linkage_table, linkage_metadata_dict, this_tornado_table = ( linkage.read_linkage_file(this_file_name)) list_of_linkage_tables.append(this_linkage_table) list_of_tornado_tables.append(this_tornado_table) if len(list_of_linkage_tables) == 1: continue list_of_linkage_tables[-1] = list_of_linkage_tables[-1].align( list_of_linkage_tables[0], axis=1)[0] list_of_tornado_tables[-1] = list_of_tornado_tables[-1].align( list_of_tornado_tables[0], axis=1)[0] print(SEPARATOR_STRING) storm_to_tornadoes_table = pandas.concat(list_of_linkage_tables, axis=0, ignore_index=True) tornado_table = pandas.concat(list_of_tornado_tables, axis=0, ignore_index=True) column_dict_old_to_new = { linkage.EVENT_TIME_COLUMN: tornado_io.TIME_COLUMN, linkage.EVENT_LATITUDE_COLUMN: tornado_io.LATITUDE_COLUMN, linkage.EVENT_LONGITUDE_COLUMN: tornado_io.LONGITUDE_COLUMN } tornado_table.rename(columns=column_dict_old_to_new, inplace=True) tornado_table = tornado_io.segments_to_tornadoes(tornado_table) tornado_table = tornado_table.assign( **{ SHORT_TORNADO_ID_COLUMN: _long_to_short_tornado_ids(tornado_table[ tornado_io.TORNADO_ID_COLUMN].values) }) if min_plot_latitude_deg is None: min_plot_latitude_deg = numpy.min( storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN]. values) - LATLNG_BUFFER_DEG if max_plot_latitude_deg is None: max_plot_latitude_deg = numpy.max( storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN]. values) + LATLNG_BUFFER_DEG if min_plot_longitude_deg is None: min_plot_longitude_deg = numpy.min( storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN]. values) - LATLNG_BUFFER_DEG if max_plot_longitude_deg is None: max_plot_longitude_deg = numpy.max( storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN]. values) + LATLNG_BUFFER_DEG # TODO(thunderhoser): Should maybe restrict this to an inner domain. storm_to_tornadoes_table = storm_to_tornadoes_table.loc[ (storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN] >= min_plot_latitude_deg) & (storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN] <= max_plot_latitude_deg)] storm_to_tornadoes_table = storm_to_tornadoes_table.loc[ (storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN] >= min_plot_longitude_deg) & (storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN] <= max_plot_longitude_deg)] tornado_io.subset_tornadoes(tornado_table=tornado_table, min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg) # TODO(thunderhoser): Make this subsetting optional. storm_to_tornadoes_table = _subset_storms_by_time( storm_to_tornadoes_table=storm_to_tornadoes_table, tornado_table=tornado_table, linkage_metadata_dict=linkage_metadata_dict, genesis_only=genesis_only) print(SEPARATOR_STRING) _, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS) print('Plotting storm tracks...') storm_plotting.plot_storm_tracks( storm_object_table=storm_to_tornadoes_table, axes_object=axes_object, basemap_object=basemap_object, colour_map_object=colour_map_object, start_marker_type=None, end_marker_type=None) num_tornadoes = len(tornado_table.index) if num_tornadoes == 0: print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close() return colour_norm_object = pyplot.Normalize( numpy.min( storm_to_tornadoes_table[tracking_utils.VALID_TIME_COLUMN].values), numpy.max( storm_to_tornadoes_table[tracking_utils.VALID_TIME_COLUMN].values)) print('Plotting tornado markers...') _plot_tornadoes(tornado_table=tornado_table, colour_map_object=colour_map_object, colour_norm_object=colour_norm_object, genesis_only=genesis_only, axes_object=axes_object, basemap_object=basemap_object) print('Plotting tornado IDs with storm objects...') num_storm_objects = len(storm_to_tornadoes_table.index) for i in range(num_storm_objects): _plot_linkages_one_storm_object( storm_to_tornadoes_table=storm_to_tornadoes_table, storm_object_index=i, tornado_table=tornado_table, colour_map_object=colour_map_object, colour_norm_object=colour_norm_object, axes_object=axes_object, basemap_object=basemap_object, max_link_distance_metres=max_link_distance_metres) print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close()
def _run(myrorss_tracking_dir_name, gridrad_tracking_dir_name, max_distance_metres, source_dataset_name, first_spc_date_string, last_spc_date_string, output_dir_name): """Matches storm objects between MYRORSS and GridRad datasets. This is effectively the main method. :param myrorss_tracking_dir_name: See documentation at end of file. :param gridrad_tracking_dir_name: Same. :param max_distance_metres: Same. :param source_dataset_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_dir_name: Same. :raises: ValueError: if `source_dataset_name not in VALID_DATASET_NAMES`. """ if source_dataset_name not in VALID_DATASET_NAMES: error_string = ( '\n{0:s}\nValid datasets (listed above) do not include "{1:s}".' ).format(str(VALID_DATASET_NAMES), source_dataset_name) raise ValueError(error_string) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) if source_dataset_name == radar_utils.MYRORSS_SOURCE_ID: source_tracking_dir_name = myrorss_tracking_dir_name target_tracking_dir_name = gridrad_tracking_dir_name target_dataset_name = radar_utils.GRIDRAD_SOURCE_ID else: source_tracking_dir_name = gridrad_tracking_dir_name target_tracking_dir_name = myrorss_tracking_dir_name target_dataset_name = radar_utils.MYRORSS_SOURCE_ID source_tracking_file_names = [] target_tracking_file_names = [] for this_spc_date_string in spc_date_strings: source_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=source_tracking_dir_name, tracking_scale_metres2=TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=True)[0] target_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=target_tracking_dir_name, tracking_scale_metres2=TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=True)[0] source_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in source_tracking_file_names], dtype=int) target_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in target_tracking_file_names], dtype=int) source_to_target_indices = _match_all_times( source_times_unix_sec=source_times_unix_sec, target_times_unix_sec=target_times_unix_sec, max_diff_seconds=MAX_TIME_DIFF_SECONDS) print(SEPARATOR_STRING) del target_times_unix_sec target_tracking_file_names = [ target_tracking_file_names[k] for k in source_to_target_indices ] num_source_times = len(source_times_unix_sec) for i in range(num_source_times): print('Reading data from: "{0:s}"...'.format( source_tracking_file_names[i])) this_source_object_table = tracking_io.read_file( source_tracking_file_names[i]) print('Reading data from: "{0:s}"...'.format( target_tracking_file_names[i])) this_target_object_table = tracking_io.read_file( target_tracking_file_names[i]) this_source_to_target_dict = _match_locations_one_time( source_object_table=this_source_object_table, target_object_table=this_target_object_table, max_distance_metres=max_distance_metres) this_match_file_name = tracking_io.find_match_file( top_directory_name=output_dir_name, valid_time_unix_sec=source_times_unix_sec[i], raise_error_if_missing=False) print('Writing results to: "{0:s}"...\n'.format(this_match_file_name)) tracking_io.write_matches( pickle_file_name=this_match_file_name, source_to_target_dict=this_source_to_target_dict, max_time_diff_seconds=MAX_TIME_DIFF_SECONDS, max_distance_metres=max_distance_metres, source_dataset_name=source_dataset_name, source_tracking_dir_name=source_tracking_dir_name, target_dataset_name=target_dataset_name, target_tracking_dir_name=target_tracking_dir_name)