def test_file_name_to_time_segmotion(self): """Ensures correct output from file_name_to_time. In this case, data source is segmotion. """ this_time_unix_sec = tracking_io.file_name_to_time(SEGMOTION_FILE_NAME) self.assertTrue(this_time_unix_sec == VALID_TIME_UNIX_SEC)
def test_file_name_to_time_probsevere(self): """Ensures correct output from file_name_to_time. In this case, data source is probSevere. """ this_time_unix_sec = tracking_io.file_name_to_time( PROBSEVERE_FILE_NAME) self.assertTrue(this_time_unix_sec == VALID_TIME_UNIX_SEC)
def _run(top_orig_tracking_dir_name, top_new_tracking_dir_name, first_spc_date_string, last_spc_date_string, output_file_name): """Plots storms that were removed by remove_storms_outside_conus.py. This is effectively the main method. :param top_orig_tracking_dir_name: See documentation at top of file. :param top_new_tracking_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) orig_tracking_file_names = [] for d in spc_date_strings: orig_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_orig_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=d, raise_error_if_missing=False)[0] valid_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in orig_tracking_file_names], dtype=int) new_tracking_file_names = [ tracking_io.find_file( top_tracking_dir_name=top_new_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=t, spc_date_string=time_conversion.time_to_spc_date_string(t), raise_error_if_missing=True) for t in valid_times_unix_sec ] orig_storm_object_table = tracking_io.read_many_files( orig_tracking_file_names) print(SEPARATOR_STRING) new_storm_object_table = tracking_io.read_many_files( new_tracking_file_names) print(SEPARATOR_STRING) orig_storm_id_strings = ( orig_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) orig_storm_times_unix_sec = ( orig_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) new_storm_id_strings = ( new_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) new_storm_times_unix_sec = ( new_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) num_orig_storm_objects = len(orig_storm_object_table.index) orig_kept_flags = numpy.full(num_orig_storm_objects, 0, dtype=bool) these_indices = tracking_utils.find_storm_objects( all_id_strings=orig_storm_id_strings, all_times_unix_sec=orig_storm_times_unix_sec, id_strings_to_keep=new_storm_id_strings, times_to_keep_unix_sec=new_storm_times_unix_sec, allow_missing=False) orig_kept_flags[these_indices] = True orig_removed_indices = numpy.where(numpy.invert(orig_kept_flags))[0] print('{0:d} of {1:d} storm objects were outside CONUS.'.format( len(orig_removed_indices), num_orig_storm_objects)) removed_storm_object_table = orig_storm_object_table.iloc[ orig_removed_indices] removed_latitudes_deg = removed_storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values removed_longitudes_deg = removed_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values figure_object, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=numpy.min(removed_latitudes_deg) - 1., max_latitude_deg=numpy.max(removed_latitudes_deg) + 1., min_longitude_deg=numpy.min(removed_longitudes_deg) - 1., max_longitude_deg=numpy.max(removed_longitudes_deg) + 1., resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS) conus_latitudes_deg, conus_longitudes_deg = ( conus_boundary.read_from_netcdf()) conus_latitudes_deg, conus_longitudes_deg = conus_boundary.erode_boundary( latitudes_deg=conus_latitudes_deg, longitudes_deg=conus_longitudes_deg, erosion_distance_metres=EROSION_DISTANCE_METRES) axes_object.plot(conus_longitudes_deg, conus_latitudes_deg, color=LINE_COLOUR, linestyle='solid', linewidth=LINE_WIDTH) axes_object.plot(removed_longitudes_deg, removed_latitudes_deg, linestyle='None', marker=MARKER_TYPE, markersize=MARKER_SIZE, markeredgewidth=0, markerfacecolor=MARKER_COLOUR, markeredgecolor=MARKER_COLOUR) print('Saving figure to: "{0:s}"...'.format(output_file_name)) figure_object.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object)
def _find_tracking_gaps(first_spc_date_string, last_spc_date_string, top_tracking_dir_name, tracking_scale_metres2, source_name, min_time_diff_seconds): """Finds gaps (temporal discontinuities) between storm-tracking files. :param first_spc_date_string: See documentation at top of file. :param last_spc_date_string: Same. :param top_tracking_dir_name: Same. :param tracking_scale_metres2: Same. :param source_name: Same. :param min_time_diff_seconds: Same. """ spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] unix_times_sec = numpy.array([], dtype=int) num_spc_dates = len(spc_date_strings) for i in range(num_spc_dates): print('Finding tracking files for SPC date "{0:s}"...'.format( spc_date_strings[i])) these_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_strings[i], source_name=source_name, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2, raise_error_if_missing=False)[0] print(len(these_file_names)) if not len(these_file_names): continue these_file_sizes_bytes = numpy.array( [os.path.getsize(f) for f in these_file_names], dtype=int) these_valid_indices = numpy.where( these_file_sizes_bytes > FILE_SIZE_WITHOUT_STORMS_BYTES)[0] these_file_names = [these_file_names[k] for k in these_valid_indices] these_unix_times_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in these_file_names], dtype=int) these_sort_indices = numpy.argsort(these_unix_times_sec) these_unix_times_sec = these_unix_times_sec[these_sort_indices] these_file_names = [these_file_names[k] for k in these_sort_indices] tracking_file_names += these_file_names unix_times_sec = numpy.concatenate( (unix_times_sec, these_unix_times_sec)) time_diffs_seconds = numpy.diff(unix_times_sec) time_gap_indices = numpy.where( time_diffs_seconds >= min_time_diff_seconds)[0] num_time_gaps = len(time_gap_indices) print(( '\nThere are {0:d} time gaps (successive files >= {1:d} seconds apart),' ' listed below:\n').format(num_time_gaps, min_time_diff_seconds)) for i in time_gap_indices: this_start_time_string = time_conversion.unix_sec_to_string( unix_times_sec[i], TIME_FORMAT) this_end_time_string = time_conversion.unix_sec_to_string( unix_times_sec[i + 1], TIME_FORMAT) print('Gap between {0:s} and {1:s} = {2:d} seconds'.format( this_start_time_string, this_end_time_string, time_diffs_seconds[i]))
def _run(top_input_dir_name, tracking_scale_metres2, first_spc_date_string, last_spc_date_string, min_distances_metres, max_distances_metres, top_output_dir_name): """Creates one or more distance buffers around each storm object (polygon). This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param tracking_scale_metres2: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param min_distances_metres: Same. :param max_distances_metres: Same. :param top_output_dir_name: Same. """ min_distances_metres[min_distances_metres < 0] = numpy.nan spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: these_input_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_input_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_input_file_names) == 0: continue for this_input_file_name in these_input_file_names: print('Reading input tracks from: "{0:s}"...'.format( this_input_file_name)) this_storm_object_table = tracking_io.read_file( this_input_file_name) this_storm_object_table = (tracking_utils.create_distance_buffers( storm_object_table=this_storm_object_table, min_distances_metres=min_distances_metres, max_distances_metres=max_distances_metres)) this_output_file_name = tracking_io.find_file( top_tracking_dir_name=top_output_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=tracking_io.file_name_to_time( this_input_file_name), spc_date_string=this_spc_date_string, raise_error_if_missing=False) print('Writing input tracks + buffers to: "{0:s}"...\n'.format( this_output_file_name)) tracking_io.write_file(storm_object_table=this_storm_object_table, pickle_file_name=this_output_file_name) print(SEPARATOR_STRING)
def _plot_one_example(full_id_string, storm_time_unix_sec, target_name, forecast_probability, tornado_dir_name, top_tracking_dir_name, top_myrorss_dir_name, radar_field_name, radar_height_m_asl, latitude_buffer_deg, longitude_buffer_deg, top_output_dir_name, aux_forecast_probabilities=None, aux_activation_dict=None): """Plots one example with surrounding context at several times. N = number of storm objects read from auxiliary activation file :param full_id_string: Full storm ID. :param storm_time_unix_sec: Storm time. :param target_name: Name of target variable. :param forecast_probability: Forecast tornado probability for this example. :param tornado_dir_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param top_myrorss_dir_name: Same. :param radar_field_name: Same. :param radar_height_m_asl: Same. :param latitude_buffer_deg: Same. :param longitude_buffer_deg: Same. :param top_output_dir_name: Same. :param aux_forecast_probabilities: length-N numpy array of forecast probabilities. If this is None, will not plot forecast probs in maps. :param aux_activation_dict: Dictionary returned by `model_activation.read_file` from auxiliary file. If this is None, will not plot forecast probs in maps. """ storm_time_string = time_conversion.unix_sec_to_string( storm_time_unix_sec, TIME_FORMAT) # Create output directory for this example. output_dir_name = '{0:s}/{1:s}_{2:s}'.format(top_output_dir_name, full_id_string, storm_time_string) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) # Find tracking files. tracking_file_names = _find_tracking_files_one_example( valid_time_unix_sec=storm_time_unix_sec, top_tracking_dir_name=top_tracking_dir_name, target_name=target_name) tracking_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int) tracking_time_strings = [ time_conversion.unix_sec_to_string(t, TIME_FORMAT) for t in tracking_times_unix_sec ] # Read tracking files. storm_object_table = tracking_io.read_many_files(tracking_file_names) print('\n') if aux_activation_dict is not None: these_indices = tracking_utils.find_storm_objects( all_id_strings=aux_activation_dict[model_activation.FULL_IDS_KEY], all_times_unix_sec=aux_activation_dict[ model_activation.STORM_TIMES_KEY], id_strings_to_keep=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), times_to_keep_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, allow_missing=True) storm_object_probs = numpy.array([ aux_forecast_probabilities[k] if k >= 0 else numpy.nan for k in these_indices ]) storm_object_table = storm_object_table.assign( **{FORECAST_PROBABILITY_COLUMN: storm_object_probs}) primary_id_string = temporal_tracking.full_to_partial_ids([full_id_string ])[0][0] this_storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.PRIMARY_ID_COLUMN] == primary_id_string] latitude_limits_deg, longitude_limits_deg = _get_plotting_limits( storm_object_table=this_storm_object_table, latitude_buffer_deg=latitude_buffer_deg, longitude_buffer_deg=longitude_buffer_deg) storm_min_latitudes_deg = numpy.array([ numpy.min(numpy.array(p.exterior.xy[1])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) storm_max_latitudes_deg = numpy.array([ numpy.max(numpy.array(p.exterior.xy[1])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) storm_min_longitudes_deg = numpy.array([ numpy.min(numpy.array(p.exterior.xy[0])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) storm_max_longitudes_deg = numpy.array([ numpy.max(numpy.array(p.exterior.xy[0])) for p in storm_object_table[ tracking_utils.LATLNG_POLYGON_COLUMN].values ]) min_latitude_flags = numpy.logical_and( storm_min_latitudes_deg >= latitude_limits_deg[0], storm_min_latitudes_deg <= latitude_limits_deg[1]) max_latitude_flags = numpy.logical_and( storm_max_latitudes_deg >= latitude_limits_deg[0], storm_max_latitudes_deg <= latitude_limits_deg[1]) latitude_flags = numpy.logical_or(min_latitude_flags, max_latitude_flags) min_longitude_flags = numpy.logical_and( storm_min_longitudes_deg >= longitude_limits_deg[0], storm_min_longitudes_deg <= longitude_limits_deg[1]) max_longitude_flags = numpy.logical_and( storm_max_longitudes_deg >= longitude_limits_deg[0], storm_max_longitudes_deg <= longitude_limits_deg[1]) longitude_flags = numpy.logical_or(min_longitude_flags, max_longitude_flags) good_indices = numpy.where( numpy.logical_and(latitude_flags, longitude_flags))[0] storm_object_table = storm_object_table.iloc[good_indices] # Read tornado reports. target_param_dict = target_val_utils.target_name_to_params(target_name) min_lead_time_seconds = target_param_dict[ target_val_utils.MIN_LEAD_TIME_KEY] max_lead_time_seconds = target_param_dict[ target_val_utils.MAX_LEAD_TIME_KEY] tornado_table = linkage._read_input_tornado_reports( input_directory_name=tornado_dir_name, storm_times_unix_sec=numpy.array([storm_time_unix_sec], dtype=int), max_time_before_storm_start_sec=-1 * min_lead_time_seconds, max_time_after_storm_end_sec=max_lead_time_seconds, genesis_only=True) tornado_table = tornado_table.loc[ (tornado_table[linkage.EVENT_LATITUDE_COLUMN] >= latitude_limits_deg[0] ) & (tornado_table[linkage.EVENT_LATITUDE_COLUMN] <= latitude_limits_deg[1])] tornado_table = tornado_table.loc[ (tornado_table[linkage.EVENT_LONGITUDE_COLUMN] >= longitude_limits_deg[0]) & (tornado_table[linkage.EVENT_LONGITUDE_COLUMN] <= longitude_limits_deg[1])] for i in range(len(tracking_file_names)): this_storm_object_table = storm_object_table.loc[storm_object_table[ tracking_utils.VALID_TIME_COLUMN] == tracking_times_unix_sec[i]] _plot_one_example_one_time( storm_object_table=this_storm_object_table, full_id_string=full_id_string, valid_time_unix_sec=tracking_times_unix_sec[i], tornado_table=copy.deepcopy(tornado_table), top_myrorss_dir_name=top_myrorss_dir_name, radar_field_name=radar_field_name, radar_height_m_asl=radar_height_m_asl, latitude_limits_deg=latitude_limits_deg, longitude_limits_deg=longitude_limits_deg) if aux_activation_dict is None: this_title_string = ( 'Valid time = {0:s} ... forecast prob at {1:s} = {2:.3f}' ).format(tracking_time_strings[i], storm_time_string, forecast_probability) pyplot.title(this_title_string, fontsize=TITLE_FONT_SIZE) this_file_name = '{0:s}/{1:s}.jpg'.format(output_dir_name, tracking_time_strings[i]) print('Saving figure to file: "{0:s}"...\n'.format(this_file_name)) pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close() imagemagick_utils.trim_whitespace(input_file_name=this_file_name, output_file_name=this_file_name)
def _find_tracking_files_one_example(top_tracking_dir_name, valid_time_unix_sec, target_name): """Finds tracking files needed to make plots for one example. :param top_tracking_dir_name: See documentation at top of file. :param valid_time_unix_sec: Valid time for example. :param target_name: Name of target variable. :return: tracking_file_names: 1-D list of paths to tracking files. :raises: ValueError: if no tracking files are found. """ target_param_dict = target_val_utils.target_name_to_params(target_name) min_lead_time_seconds = target_param_dict[ target_val_utils.MIN_LEAD_TIME_KEY] max_lead_time_seconds = target_param_dict[ target_val_utils.MAX_LEAD_TIME_KEY] first_time_unix_sec = valid_time_unix_sec + min_lead_time_seconds last_time_unix_sec = valid_time_unix_sec + max_lead_time_seconds first_spc_date_string = time_conversion.time_to_spc_date_string( first_time_unix_sec - TIME_INTERVAL_SECONDS) last_spc_date_string = time_conversion.time_to_spc_date_string( last_time_unix_sec + TIME_INTERVAL_SECONDS) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) tracking_file_names = [] for this_spc_date_string in spc_date_strings: these_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=echo_top_tracking. DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] tracking_file_names += these_file_names if len(tracking_file_names) == 0: error_string = ( 'Cannot find any tracking files for SPC dates "{0:s}" to "{1:s}".' ).format(first_spc_date_string, last_spc_date_string) raise ValueError(error_string) tracking_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int) sort_indices = numpy.argsort(tracking_times_unix_sec) tracking_times_unix_sec = tracking_times_unix_sec[sort_indices] tracking_file_names = [tracking_file_names[k] for k in sort_indices] these_indices = numpy.where( tracking_times_unix_sec <= first_time_unix_sec)[0] if len(these_indices) == 0: first_index = 0 else: first_index = these_indices[-1] these_indices = numpy.where( tracking_times_unix_sec >= last_time_unix_sec)[0] if len(these_indices) == 0: last_index = len(tracking_file_names) - 1 else: last_index = these_indices[0] return tracking_file_names[first_index:(last_index + 1)]
def _extract_storm_images( num_image_rows, num_image_columns, rotate_grids, rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl, spc_date_string, first_time_string, last_time_string, tarred_myrorss_dir_name, untarred_myrorss_dir_name, top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2, target_name, top_target_dir_name, top_output_dir_name): """Extracts storm-centered img for each field/height pair and storm object. :param num_image_rows: See documentation at top of file. :param num_image_columns: Same. :param rotate_grids: Same. :param rotated_grid_spacing_metres: Same. :param radar_field_names: Same. :param refl_heights_m_agl: Same. :param spc_date_string: Same. :param first_time_string: Same. :param last_time_string: Same. :param tarred_myrorss_dir_name: Same. :param untarred_myrorss_dir_name: Same. :param top_tracking_dir_name: Same. :param elevation_dir_name: Same. :param tracking_scale_metres2: Same. :param target_name: Same. :param top_target_dir_name: Same. :param top_output_dir_name: Same. :raises: ValueError: if `first_time_string` and `last_time_string` have different SPC dates. """ if elevation_dir_name in ['', 'None']: elevation_dir_name = None if elevation_dir_name is None: host_name = socket.gethostname() if 'casper' in host_name: elevation_dir_name = '/glade/work/ryanlage/elevation' else: elevation_dir_name = '/condo/swatwork/ralager/elevation' if spc_date_string in ['', 'None']: first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, TIME_FORMAT) first_spc_date_string = time_conversion.time_to_spc_date_string( first_time_unix_sec) last_spc_date_string = time_conversion.time_to_spc_date_string( last_time_unix_sec) if first_spc_date_string != last_spc_date_string: error_string = ( 'First ({0:s}) and last ({1:s}) times have different SPC dates.' ' This script can handle only one SPC date.' ).format(first_time_string, last_time_string) raise ValueError(error_string) spc_date_string = first_spc_date_string else: first_time_unix_sec = 0 last_time_unix_sec = int(1e12) if tarred_myrorss_dir_name in ['', 'None']: tarred_myrorss_dir_name = None if target_name in ['', 'None']: target_name = None if target_name is not None: target_param_dict = target_val_utils.target_name_to_params(target_name) target_file_name = target_val_utils.find_target_file( top_directory_name=top_target_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=spc_date_string) print('Reading data from: "{0:s}"...'.format(target_file_name)) target_dict = target_val_utils.read_target_values( netcdf_file_name=target_file_name, target_names=[target_name] ) print('\n') refl_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) # Untar files. if tarred_myrorss_dir_name is not None: az_shear_field_names = list( set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES) ) if len(az_shear_field_names) > 0: az_shear_tar_file_name = ( '{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar' ).format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string ) myrorss_io.unzip_1day_tar_file( tar_file_name=az_shear_tar_file_name, field_names=az_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name) print(SEPARATOR_STRING) non_shear_field_names = list( set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES) ) if len(non_shear_field_names) > 0: non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string ) myrorss_io.unzip_1day_tar_file( tar_file_name=non_shear_tar_file_name, field_names=non_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING) # Read storm tracks for the given SPC date. tracking_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_string, source_name=tracking_utils.SEGMOTION_NAME, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2 )[0] file_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int ) good_indices = numpy.where(numpy.logical_and( file_times_unix_sec >= first_time_unix_sec, file_times_unix_sec <= last_time_unix_sec ))[0] tracking_file_names = [tracking_file_names[k] for k in good_indices] storm_object_table = tracking_io.read_many_files( tracking_file_names )[storm_images.STORM_COLUMNS_NEEDED] print(SEPARATOR_STRING) if target_name is not None: print(( 'Removing storm objects without target values (variable = ' '"{0:s}")...' ).format(target_name)) these_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values.astype(int), id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY], times_to_keep_unix_sec=target_dict[ target_val_utils.VALID_TIMES_KEY], allow_missing=False) num_storm_objects_orig = len(storm_object_table.index) storm_object_table = storm_object_table.iloc[these_indices] num_storm_objects = len(storm_object_table.index) print('Removed {0:d} of {1:d} storm objects!\n'.format( num_storm_objects_orig - num_storm_objects, num_storm_objects_orig )) # Extract storm-centered radar images. storm_images.extract_storm_images_myrorss_or_mrms( storm_object_table=storm_object_table, radar_source=radar_utils.MYRORSS_SOURCE_ID, top_radar_dir_name=untarred_myrorss_dir_name, top_output_dir_name=top_output_dir_name, elevation_dir_name=elevation_dir_name, num_storm_image_rows=num_image_rows, num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids, rotated_grid_spacing_metres=rotated_grid_spacing_metres, radar_field_names=radar_field_names, reflectivity_heights_m_agl=refl_heights_m_agl) print(SEPARATOR_STRING) # Remove untarred MYRORSS files. if tarred_myrorss_dir_name is not None: myrorss_io.remove_unzipped_data_1day( spc_date_string=spc_date_string, top_directory_name=untarred_myrorss_dir_name, field_names=radar_field_names, refl_heights_m_asl=refl_heights_m_asl)
def _run(myrorss_tracking_dir_name, gridrad_tracking_dir_name, max_distance_metres, source_dataset_name, first_spc_date_string, last_spc_date_string, output_dir_name): """Matches storm objects between MYRORSS and GridRad datasets. This is effectively the main method. :param myrorss_tracking_dir_name: See documentation at end of file. :param gridrad_tracking_dir_name: Same. :param max_distance_metres: Same. :param source_dataset_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_dir_name: Same. :raises: ValueError: if `source_dataset_name not in VALID_DATASET_NAMES`. """ if source_dataset_name not in VALID_DATASET_NAMES: error_string = ( '\n{0:s}\nValid datasets (listed above) do not include "{1:s}".' ).format(str(VALID_DATASET_NAMES), source_dataset_name) raise ValueError(error_string) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) if source_dataset_name == radar_utils.MYRORSS_SOURCE_ID: source_tracking_dir_name = myrorss_tracking_dir_name target_tracking_dir_name = gridrad_tracking_dir_name target_dataset_name = radar_utils.GRIDRAD_SOURCE_ID else: source_tracking_dir_name = gridrad_tracking_dir_name target_tracking_dir_name = myrorss_tracking_dir_name target_dataset_name = radar_utils.MYRORSS_SOURCE_ID source_tracking_file_names = [] target_tracking_file_names = [] for this_spc_date_string in spc_date_strings: source_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=source_tracking_dir_name, tracking_scale_metres2=TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=True)[0] target_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=target_tracking_dir_name, tracking_scale_metres2=TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=True)[0] source_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in source_tracking_file_names], dtype=int) target_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in target_tracking_file_names], dtype=int) source_to_target_indices = _match_all_times( source_times_unix_sec=source_times_unix_sec, target_times_unix_sec=target_times_unix_sec, max_diff_seconds=MAX_TIME_DIFF_SECONDS) print(SEPARATOR_STRING) del target_times_unix_sec target_tracking_file_names = [ target_tracking_file_names[k] for k in source_to_target_indices ] num_source_times = len(source_times_unix_sec) for i in range(num_source_times): print('Reading data from: "{0:s}"...'.format( source_tracking_file_names[i])) this_source_object_table = tracking_io.read_file( source_tracking_file_names[i]) print('Reading data from: "{0:s}"...'.format( target_tracking_file_names[i])) this_target_object_table = tracking_io.read_file( target_tracking_file_names[i]) this_source_to_target_dict = _match_locations_one_time( source_object_table=this_source_object_table, target_object_table=this_target_object_table, max_distance_metres=max_distance_metres) this_match_file_name = tracking_io.find_match_file( top_directory_name=output_dir_name, valid_time_unix_sec=source_times_unix_sec[i], raise_error_if_missing=False) print('Writing results to: "{0:s}"...\n'.format(this_match_file_name)) tracking_io.write_matches( pickle_file_name=this_match_file_name, source_to_target_dict=this_source_to_target_dict, max_time_diff_seconds=MAX_TIME_DIFF_SECONDS, max_distance_metres=max_distance_metres, source_dataset_name=source_dataset_name, source_tracking_dir_name=source_tracking_dir_name, target_dataset_name=target_dataset_name, target_tracking_dir_name=target_tracking_dir_name)
def _run(storm_metafile_name, top_tracking_dir_name, lead_time_seconds, output_file_name): """Plots spatial distribution of examples (storm objects) in file. This is effectively the main method. :param storm_metafile_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param lead_time_seconds: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) # Read storm metadata. print( 'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name)) orig_full_id_strings, orig_times_unix_sec = ( tracking_io.read_ids_and_times(storm_metafile_name)) orig_primary_id_strings = temporal_tracking.full_to_partial_ids( orig_full_id_strings)[0] # Find relevant tracking files. spc_date_strings = [ time_conversion.time_to_spc_date_string(t) for t in orig_times_unix_sec ] spc_date_strings += [ time_conversion.time_to_spc_date_string(t + lead_time_seconds) for t in orig_times_unix_sec ] spc_date_strings = list(set(spc_date_strings)) tracking_file_names = [] for this_spc_date_string in spc_date_strings: tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] file_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int) num_orig_storm_objects = len(orig_full_id_strings) num_files = len(file_times_unix_sec) keep_file_flags = numpy.full(num_files, 0, dtype=bool) for i in range(num_orig_storm_objects): these_flags = numpy.logical_and( file_times_unix_sec >= orig_times_unix_sec[i], file_times_unix_sec <= orig_times_unix_sec[i] + lead_time_seconds) keep_file_flags = numpy.logical_or(keep_file_flags, these_flags) del file_times_unix_sec keep_file_indices = numpy.where(keep_file_flags)[0] tracking_file_names = [tracking_file_names[k] for k in keep_file_indices] # Read relevant tracking files. num_files = len(tracking_file_names) storm_object_tables = [None] * num_files print(SEPARATOR_STRING) for i in range(num_files): print('Reading data from: "{0:s}"...'.format(tracking_file_names[i])) this_table = tracking_io.read_file(tracking_file_names[i]) storm_object_tables[i] = this_table.loc[this_table[ tracking_utils.PRIMARY_ID_COLUMN].isin( numpy.array(orig_primary_id_strings))] if i == 0: continue storm_object_tables[i] = storm_object_tables[i].align( storm_object_tables[0], axis=1)[0] storm_object_table = pandas.concat(storm_object_tables, axis=0, ignore_index=True) print(SEPARATOR_STRING) # Find relevant storm objects. orig_object_rows = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, id_strings_to_keep=orig_full_id_strings, times_to_keep_unix_sec=orig_times_unix_sec) good_object_rows = numpy.array([], dtype=int) for i in range(num_orig_storm_objects): # Non-merging successors only! first_rows = temporal_tracking.find_successors( storm_object_table=storm_object_table, target_row=orig_object_rows[i], num_seconds_forward=lead_time_seconds, max_num_sec_id_changes=1, change_type_string=temporal_tracking.SPLIT_STRING, return_all_on_path=True) second_rows = temporal_tracking.find_successors( storm_object_table=storm_object_table, target_row=orig_object_rows[i], num_seconds_forward=lead_time_seconds, max_num_sec_id_changes=0, change_type_string=temporal_tracking.MERGER_STRING, return_all_on_path=True) first_rows = first_rows.tolist() second_rows = second_rows.tolist() these_rows = set(first_rows) & set(second_rows) these_rows = numpy.array(list(these_rows), dtype=int) good_object_rows = numpy.concatenate((good_object_rows, these_rows)) good_object_rows = numpy.unique(good_object_rows) storm_object_table = storm_object_table.iloc[good_object_rows] times_of_day_sec = numpy.mod( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values, NUM_SECONDS_IN_DAY) storm_object_table = storm_object_table.assign( **{tracking_utils.VALID_TIME_COLUMN: times_of_day_sec}) min_plot_latitude_deg = -LATLNG_BUFFER_DEG + numpy.min( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values) max_plot_latitude_deg = LATLNG_BUFFER_DEG + numpy.max( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values) min_plot_longitude_deg = -LATLNG_BUFFER_DEG + numpy.min( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values) max_plot_longitude_deg = LATLNG_BUFFER_DEG + numpy.max( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values) _, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR, line_width=BORDER_WIDTH * 2) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR, line_width=BORDER_WIDTH) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR, line_width=BORDER_WIDTH) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS, line_width=BORDER_WIDTH) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS, line_width=BORDER_WIDTH) # colour_bar_object = storm_plotting.plot_storm_tracks( # storm_object_table=storm_object_table, axes_object=axes_object, # basemap_object=basemap_object, colour_map_object=COLOUR_MAP_OBJECT, # colour_min_unix_sec=0, colour_max_unix_sec=NUM_SECONDS_IN_DAY - 1, # line_width=TRACK_LINE_WIDTH, # start_marker_type=None, end_marker_type=None # ) colour_bar_object = storm_plotting.plot_storm_centroids( storm_object_table=storm_object_table, axes_object=axes_object, basemap_object=basemap_object, colour_map_object=COLOUR_MAP_OBJECT, colour_min_unix_sec=0, colour_max_unix_sec=NUM_SECONDS_IN_DAY - 1) tick_times_unix_sec = numpy.linspace(0, NUM_SECONDS_IN_DAY, num=NUM_HOURS_IN_DAY + 1, dtype=int) tick_times_unix_sec = tick_times_unix_sec[:-1] tick_times_unix_sec = tick_times_unix_sec[::2] tick_time_strings = [ time_conversion.unix_sec_to_string(t, COLOUR_BAR_TIME_FORMAT) for t in tick_times_unix_sec ] colour_bar_object.set_ticks(tick_times_unix_sec) colour_bar_object.set_ticklabels(tick_time_strings) print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close()
def _find_io_files_for_renaming(top_input_dir_name, first_date_unix_sec, last_date_unix_sec, top_output_dir_name): """Finds input and output files for renaming storms. N = number of dates :param top_input_dir_name: See documentation for `rename_storms.` :param first_date_unix_sec: Same. :param last_date_unix_sec: Same. :param top_output_dir_name: Same. :return: input_file_names_by_date: length-N list, where the [i]th item is a numpy array of paths to input files for the [i]th date. :return: output_file_names_by_date: Same as above, but for output files. :return: valid_times_by_date_unix_sec: Same as above, but for valid times. All 3 arrays for the [i]th date have the same length. """ dates_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=first_date_unix_sec, end_time_unix_sec=last_date_unix_sec, time_interval_sec=DAYS_TO_SECONDS, include_endpoint=True) date_strings = [ time_conversion.unix_sec_to_string(t, DATE_FORMAT) for t in dates_unix_sec ] num_dates = len(date_strings) input_file_names_by_date = [numpy.array([], dtype=object)] * num_dates output_file_names_by_date = [numpy.array([], dtype=object)] * num_dates valid_times_by_date_unix_sec = [numpy.array([], dtype=int)] * num_dates for i in range(num_dates): print('Finding input files for date {0:s}...'.format(date_strings[i])) these_input_file_names = tracking_io.find_files_one_spc_date( spc_date_string=date_strings[i], source_name=tracking_utils.PROBSEVERE_NAME, top_tracking_dir_name=top_input_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, raise_error_if_missing=True)[0] these_input_file_names.sort() these_valid_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in these_input_file_names], dtype=int) these_output_file_names = [] for t in these_valid_times_unix_sec: these_output_file_names.append( tracking_io.find_file( valid_time_unix_sec=t, source_name=tracking_utils.PROBSEVERE_NAME, top_tracking_dir_name=top_output_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, raise_error_if_missing=False)) input_file_names_by_date[i] = numpy.array(these_input_file_names, dtype=object) output_file_names_by_date[i] = numpy.array(these_output_file_names, dtype=object) valid_times_by_date_unix_sec[i] = these_valid_times_unix_sec print(SEPARATOR_STRING) return (input_file_names_by_date, output_file_names_by_date, valid_times_by_date_unix_sec)