def test_find_file_probsevere(self): """Ensures correct output from find_file. In this case, data source is probSevere. """ this_file_name = tracking_io.find_file( top_tracking_dir_name=TOP_PROBSEVERE_DIR_NAME, tracking_scale_metres2=TRACKING_SCALE_METRES2, source_name=tracking_utils.PROBSEVERE_NAME, valid_time_unix_sec=VALID_TIME_UNIX_SEC, raise_error_if_missing=False) self.assertTrue(this_file_name == PROBSEVERE_FILE_NAME)
def test_find_file_segmotion(self): """Ensures correct output from find_file. In this case, data source is segmotion. """ this_file_name = tracking_io.find_file( top_tracking_dir_name=TOP_SEGMOTION_DIR_NAME, tracking_scale_metres2=TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=VALID_TIME_UNIX_SEC, spc_date_string=VALID_SPC_DATE_STRING, raise_error_if_missing=False) self.assertTrue(this_file_name == SEGMOTION_FILE_NAME)
def _read_storm_locations_one_time( top_tracking_dir_name, valid_time_unix_sec, desired_full_id_strings): """Reads storm locations at one time. K = number of storm objects desired :param top_tracking_dir_name: See documentation at top of file. :param valid_time_unix_sec: Valid time. :param desired_full_id_strings: length-K list of full storm IDs. Locations will be read for these storms only. :return: desired_latitudes_deg: length-K numpy array of latitudes (deg N). :return: desired_longitudes_deg: length-K numpy array of longitudes (deg E). """ spc_date_string = time_conversion.time_to_spc_date_string( valid_time_unix_sec) desired_times_unix_sec = numpy.full( len(desired_full_id_strings), valid_time_unix_sec, dtype=int ) tracking_file_name = tracking_io.find_file( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=valid_time_unix_sec, spc_date_string=spc_date_string, raise_error_if_missing=True) print('Reading storm locations from: "{0:s}"...'.format(tracking_file_name)) storm_object_table = tracking_io.read_file(tracking_file_name) desired_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, id_strings_to_keep=desired_full_id_strings, times_to_keep_unix_sec=desired_times_unix_sec, allow_missing=False) desired_latitudes_deg = storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values[desired_indices] desired_longitudes_deg = storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values[desired_indices] return desired_latitudes_deg, desired_longitudes_deg
def _convert_files(top_probsevere_dir_name, date_string, top_output_dir_name): """Converts probSevere tracking files for one day. :param top_probsevere_dir_name: See documentation at top of file. :param date_string: Same. :param top_output_dir_name: Same. """ date_unix_sec = time_conversion.string_to_unix_sec(date_string, DATE_FORMAT) raw_file_names = probsevere_io.find_raw_files_one_day( top_directory_name=top_probsevere_dir_name, unix_time_sec=date_unix_sec, file_extension=probsevere_io.ASCII_FILE_EXTENSION, raise_error_if_all_missing=True) for this_raw_file_name in raw_file_names: print('Reading data from "{0:s}"...'.format(this_raw_file_name)) this_storm_object_table = probsevere_io.read_raw_file( this_raw_file_name) this_storm_object_table = ( temporal_tracking.finish_segmotion_or_probsevere_ids( this_storm_object_table)) this_new_file_name = tracking_io.find_file( valid_time_unix_sec=probsevere_io.raw_file_name_to_time( this_raw_file_name), source_name=tracking_utils.PROBSEVERE_NAME, top_tracking_dir_name=top_output_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, raise_error_if_missing=False) print('Writing data to "{0:s}"...'.format(this_new_file_name)) tracking_io.write_file(storm_object_table=this_storm_object_table, pickle_file_name=this_new_file_name)
def _run(top_orig_tracking_dir_name, top_new_tracking_dir_name, first_spc_date_string, last_spc_date_string, output_file_name): """Plots storms that were removed by remove_storms_outside_conus.py. This is effectively the main method. :param top_orig_tracking_dir_name: See documentation at top of file. :param top_new_tracking_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) orig_tracking_file_names = [] for d in spc_date_strings: orig_tracking_file_names += tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_orig_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=d, raise_error_if_missing=False)[0] valid_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in orig_tracking_file_names], dtype=int) new_tracking_file_names = [ tracking_io.find_file( top_tracking_dir_name=top_new_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=t, spc_date_string=time_conversion.time_to_spc_date_string(t), raise_error_if_missing=True) for t in valid_times_unix_sec ] orig_storm_object_table = tracking_io.read_many_files( orig_tracking_file_names) print(SEPARATOR_STRING) new_storm_object_table = tracking_io.read_many_files( new_tracking_file_names) print(SEPARATOR_STRING) orig_storm_id_strings = ( orig_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) orig_storm_times_unix_sec = ( orig_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) new_storm_id_strings = ( new_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) new_storm_times_unix_sec = ( new_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) num_orig_storm_objects = len(orig_storm_object_table.index) orig_kept_flags = numpy.full(num_orig_storm_objects, 0, dtype=bool) these_indices = tracking_utils.find_storm_objects( all_id_strings=orig_storm_id_strings, all_times_unix_sec=orig_storm_times_unix_sec, id_strings_to_keep=new_storm_id_strings, times_to_keep_unix_sec=new_storm_times_unix_sec, allow_missing=False) orig_kept_flags[these_indices] = True orig_removed_indices = numpy.where(numpy.invert(orig_kept_flags))[0] print('{0:d} of {1:d} storm objects were outside CONUS.'.format( len(orig_removed_indices), num_orig_storm_objects)) removed_storm_object_table = orig_storm_object_table.iloc[ orig_removed_indices] removed_latitudes_deg = removed_storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values removed_longitudes_deg = removed_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values figure_object, axes_object, basemap_object = ( plotting_utils.create_equidist_cylindrical_map( min_latitude_deg=numpy.min(removed_latitudes_deg) - 1., max_latitude_deg=numpy.max(removed_latitudes_deg) + 1., min_longitude_deg=numpy.min(removed_longitudes_deg) - 1., max_longitude_deg=numpy.max(removed_longitudes_deg) + 1., resolution_string='i')) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS) conus_latitudes_deg, conus_longitudes_deg = ( conus_boundary.read_from_netcdf()) conus_latitudes_deg, conus_longitudes_deg = conus_boundary.erode_boundary( latitudes_deg=conus_latitudes_deg, longitudes_deg=conus_longitudes_deg, erosion_distance_metres=EROSION_DISTANCE_METRES) axes_object.plot(conus_longitudes_deg, conus_latitudes_deg, color=LINE_COLOUR, linestyle='solid', linewidth=LINE_WIDTH) axes_object.plot(removed_longitudes_deg, removed_latitudes_deg, linestyle='None', marker=MARKER_TYPE, markersize=MARKER_SIZE, markeredgewidth=0, markerfacecolor=MARKER_COLOUR, markeredgecolor=MARKER_COLOUR) print('Saving figure to: "{0:s}"...'.format(output_file_name)) figure_object.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object)
def _run(top_input_dir_name, tracking_scale_metres2, first_spc_date_string, last_spc_date_string, min_distances_metres, max_distances_metres, top_output_dir_name): """Creates one or more distance buffers around each storm object (polygon). This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param tracking_scale_metres2: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param min_distances_metres: Same. :param max_distances_metres: Same. :param top_output_dir_name: Same. """ min_distances_metres[min_distances_metres < 0] = numpy.nan spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: these_input_file_names = tracking_io.find_files_one_spc_date( top_tracking_dir_name=top_input_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, spc_date_string=this_spc_date_string, raise_error_if_missing=False)[0] if len(these_input_file_names) == 0: continue for this_input_file_name in these_input_file_names: print('Reading input tracks from: "{0:s}"...'.format( this_input_file_name)) this_storm_object_table = tracking_io.read_file( this_input_file_name) this_storm_object_table = (tracking_utils.create_distance_buffers( storm_object_table=this_storm_object_table, min_distances_metres=min_distances_metres, max_distances_metres=max_distances_metres)) this_output_file_name = tracking_io.find_file( top_tracking_dir_name=top_output_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=tracking_io.file_name_to_time( this_input_file_name), spc_date_string=this_spc_date_string, raise_error_if_missing=False) print('Writing input tracks + buffers to: "{0:s}"...\n'.format( this_output_file_name)) tracking_io.write_file(storm_object_table=this_storm_object_table, pickle_file_name=this_output_file_name) print(SEPARATOR_STRING)
def _plot_rapruc_one_example( full_storm_id_string, storm_time_unix_sec, top_tracking_dir_name, latitude_buffer_deg, longitude_buffer_deg, lead_time_seconds, field_name_grib1, output_dir_name, rap_file_name=None, ruc_file_name=None): """Plots RAP or RUC field for one example. :param full_storm_id_string: Full storm ID. :param storm_time_unix_sec: Valid time. :param top_tracking_dir_name: See documentation at top of file. :param latitude_buffer_deg: Same. :param longitude_buffer_deg: Same. :param lead_time_seconds: Same. :param field_name_grib1: Same. :param output_dir_name: Same. :param rap_file_name: Path to file with RAP analysis. :param ruc_file_name: [used only if `rap_file_name is None`] Path to file with RUC analysis. """ tracking_file_name = tracking_io.find_file( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=storm_time_unix_sec, spc_date_string= time_conversion.time_to_spc_date_string(storm_time_unix_sec), raise_error_if_missing=True ) print('Reading data from: "{0:s}"...'.format(tracking_file_name)) storm_object_table = tracking_io.read_file(tracking_file_name) storm_object_table = storm_object_table.loc[ storm_object_table[tracking_utils.FULL_ID_COLUMN] == full_storm_id_string ] extrap_times_sec = numpy.array([0, lead_time_seconds], dtype=int) storm_object_table = soundings._create_target_points_for_interp( storm_object_table=storm_object_table, lead_times_seconds=extrap_times_sec ) orig_latitude_deg = ( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values[0] ) orig_longitude_deg = ( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values[0] ) extrap_latitude_deg = ( storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values[1] ) extrap_longitude_deg = ( storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values[1] ) if rap_file_name is None: grib_file_name = ruc_file_name model_name = nwp_model_utils.RUC_MODEL_NAME else: grib_file_name = rap_file_name model_name = nwp_model_utils.RAP_MODEL_NAME pathless_grib_file_name = os.path.split(grib_file_name)[-1] grid_name = pathless_grib_file_name.split('_')[1] host_name = socket.gethostname() if 'casper' in host_name: wgrib_exe_name = '/glade/work/ryanlage/wgrib/wgrib' wgrib2_exe_name = '/glade/work/ryanlage/wgrib2/wgrib2/wgrib2' else: wgrib_exe_name = '/condo/swatwork/ralager/wgrib/wgrib' wgrib2_exe_name = '/condo/swatwork/ralager/grib2/wgrib2/wgrib2' print('Reading field "{0:s}" from: "{1:s}"...'.format( field_name_grib1, grib_file_name )) main_field_matrix = nwp_model_io.read_field_from_grib_file( grib_file_name=grib_file_name, field_name_grib1=field_name_grib1, model_name=model_name, grid_id=grid_name, wgrib_exe_name=wgrib_exe_name, wgrib2_exe_name=wgrib2_exe_name ) u_wind_name_grib1 = 'UGRD:{0:s}'.format( field_name_grib1.split(':')[-1] ) u_wind_name_grib1 = u_wind_name_grib1.replace('2 m', '10 m') print('Reading field "{0:s}" from: "{1:s}"...'.format( u_wind_name_grib1, grib_file_name )) u_wind_matrix_m_s01 = nwp_model_io.read_field_from_grib_file( grib_file_name=grib_file_name, field_name_grib1=u_wind_name_grib1, model_name=model_name, grid_id=grid_name, wgrib_exe_name=wgrib_exe_name, wgrib2_exe_name=wgrib2_exe_name ) v_wind_name_grib1 = 'VGRD:{0:s}'.format( u_wind_name_grib1.split(':')[-1] ) print('Reading field "{0:s}" from: "{1:s}"...'.format( v_wind_name_grib1, grib_file_name )) v_wind_matrix_m_s01 = nwp_model_io.read_field_from_grib_file( grib_file_name=grib_file_name, field_name_grib1=v_wind_name_grib1, model_name=model_name, grid_id=grid_name, wgrib_exe_name=wgrib_exe_name, wgrib2_exe_name=wgrib2_exe_name ) latitude_matrix_deg, longitude_matrix_deg = ( nwp_model_utils.get_latlng_grid_point_matrices( model_name=model_name, grid_name=grid_name) ) cosine_matrix, sine_matrix = nwp_model_utils.get_wind_rotation_angles( latitudes_deg=latitude_matrix_deg, longitudes_deg=longitude_matrix_deg, model_name=model_name ) u_wind_matrix_m_s01, v_wind_matrix_m_s01 = ( nwp_model_utils.rotate_winds_to_earth_relative( u_winds_grid_relative_m_s01=u_wind_matrix_m_s01, v_winds_grid_relative_m_s01=v_wind_matrix_m_s01, rotation_angle_cosines=cosine_matrix, rotation_angle_sines=sine_matrix) ) min_plot_latitude_deg = ( min([orig_latitude_deg, extrap_latitude_deg]) - latitude_buffer_deg ) max_plot_latitude_deg = ( max([orig_latitude_deg, extrap_latitude_deg]) + latitude_buffer_deg ) min_plot_longitude_deg = ( min([orig_longitude_deg, extrap_longitude_deg]) - longitude_buffer_deg ) max_plot_longitude_deg = ( max([orig_longitude_deg, extrap_longitude_deg]) + longitude_buffer_deg ) row_limits, column_limits = nwp_plotting.latlng_limits_to_rowcol_limits( min_latitude_deg=min_plot_latitude_deg, max_latitude_deg=max_plot_latitude_deg, min_longitude_deg=min_plot_longitude_deg, max_longitude_deg=max_plot_longitude_deg, model_name=model_name, grid_id=grid_name ) main_field_matrix = main_field_matrix[ row_limits[0]:(row_limits[1] + 1), column_limits[0]:(column_limits[1] + 1) ] u_wind_matrix_m_s01 = u_wind_matrix_m_s01[ row_limits[0]:(row_limits[1] + 1), column_limits[0]:(column_limits[1] + 1) ] v_wind_matrix_m_s01 = v_wind_matrix_m_s01[ row_limits[0]:(row_limits[1] + 1), column_limits[0]:(column_limits[1] + 1) ] _, axes_object, basemap_object = nwp_plotting.init_basemap( model_name=model_name, grid_id=grid_name, first_row_in_full_grid=row_limits[0], last_row_in_full_grid=row_limits[1], first_column_in_full_grid=column_limits[0], last_column_in_full_grid=column_limits[1] ) plotting_utils.plot_coastlines( basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR ) plotting_utils.plot_countries( basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR ) plotting_utils.plot_states_and_provinces( basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR ) plotting_utils.plot_parallels( basemap_object=basemap_object, axes_object=axes_object, num_parallels=NUM_PARALLELS ) plotting_utils.plot_meridians( basemap_object=basemap_object, axes_object=axes_object, num_meridians=NUM_MERIDIANS ) min_colour_value = numpy.nanpercentile( main_field_matrix, 100. - MAX_COLOUR_PERCENTILE ) max_colour_value = numpy.nanpercentile( main_field_matrix, MAX_COLOUR_PERCENTILE ) nwp_plotting.plot_subgrid( field_matrix=main_field_matrix, model_name=model_name, grid_id=grid_name, axes_object=axes_object, basemap_object=basemap_object, colour_map_object=COLOUR_MAP_OBJECT, min_colour_value=min_colour_value, max_colour_value=max_colour_value, first_row_in_full_grid=row_limits[0], first_column_in_full_grid=column_limits[0] ) nwp_plotting.plot_wind_barbs_on_subgrid( u_wind_matrix_m_s01=u_wind_matrix_m_s01, v_wind_matrix_m_s01=v_wind_matrix_m_s01, model_name=model_name, grid_id=grid_name, axes_object=axes_object, basemap_object=basemap_object, first_row_in_full_grid=row_limits[0], first_column_in_full_grid=column_limits[0], plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB, plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB, barb_length=WIND_BARB_LENGTH, empty_barb_radius=EMPTY_WIND_BARB_RADIUS, fill_empty_barb=True, colour_map=WIND_COLOUR_MAP_OBJECT, colour_minimum_kt=MIN_WIND_SPEED_KT, colour_maximum_kt=MAX_WIND_SPEED_KT ) orig_x_metres, orig_y_metres = basemap_object( orig_longitude_deg, orig_latitude_deg ) axes_object.plot( orig_x_metres, orig_y_metres, linestyle='None', marker=ORIGIN_MARKER_TYPE, markersize=ORIGIN_MARKER_SIZE, markeredgewidth=ORIGIN_MARKER_EDGE_WIDTH, markerfacecolor=MARKER_COLOUR, markeredgecolor=MARKER_COLOUR ) extrap_x_metres, extrap_y_metres = basemap_object( extrap_longitude_deg, extrap_latitude_deg ) axes_object.plot( extrap_x_metres, extrap_y_metres, linestyle='None', marker=EXTRAP_MARKER_TYPE, markersize=EXTRAP_MARKER_SIZE, markeredgewidth=EXTRAP_MARKER_EDGE_WIDTH, markerfacecolor=MARKER_COLOUR, markeredgecolor=MARKER_COLOUR ) plotting_utils.plot_linear_colour_bar( axes_object_or_matrix=axes_object, data_matrix=main_field_matrix, colour_map_object=COLOUR_MAP_OBJECT, min_value=min_colour_value, max_value=max_colour_value, orientation_string='vertical' ) output_file_name = '{0:s}/{1:s}_{2:s}.jpg'.format( output_dir_name, full_storm_id_string.replace('_', '-'), time_conversion.unix_sec_to_string( storm_time_unix_sec, FILE_NAME_TIME_FORMAT ) ) print('Saving figure to: "{0:s}"...'.format(output_file_name)) pyplot.savefig( output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight' ) pyplot.close()
def _run(input_prediction_file_name, top_tracking_dir_name, tracking_scale_metres2, x_spacing_metres, y_spacing_metres, effective_radius_metres, smoothing_method_name, smoothing_cutoff_radius_metres, smoothing_efold_radius_metres, top_output_dir_name): """Projects CNN forecasts onto the RAP grid. This is effectively the same method. :param input_prediction_file_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param tracking_scale_metres2: Same. :param x_spacing_metres: Same. :param y_spacing_metres: Same. :param effective_radius_metres: Same. :param smoothing_method_name: Same. :param smoothing_cutoff_radius_metres: Same. :param smoothing_efold_radius_metres: Same. :param top_output_dir_name: Same. """ print('Reading data from: "{0:s}"...'.format(input_prediction_file_name)) ungridded_forecast_dict = prediction_io.read_ungridded_predictions( input_prediction_file_name) target_param_dict = target_val_utils.target_name_to_params( ungridded_forecast_dict[prediction_io.TARGET_NAME_KEY]) min_buffer_dist_metres = target_param_dict[ target_val_utils.MIN_LINKAGE_DISTANCE_KEY] # TODO(thunderhoser): This is HACKY. if min_buffer_dist_metres == 0: min_buffer_dist_metres = numpy.nan max_buffer_dist_metres = target_param_dict[ target_val_utils.MAX_LINKAGE_DISTANCE_KEY] min_lead_time_seconds = target_param_dict[ target_val_utils.MIN_LEAD_TIME_KEY] max_lead_time_seconds = target_param_dict[ target_val_utils.MAX_LEAD_TIME_KEY] forecast_column_name = gridded_forecasts._buffer_to_column_name( min_buffer_dist_metres=min_buffer_dist_metres, max_buffer_dist_metres=max_buffer_dist_metres, column_type=gridded_forecasts.FORECAST_COLUMN_TYPE) init_times_unix_sec = numpy.unique( ungridded_forecast_dict[prediction_io.STORM_TIMES_KEY]) tracking_file_names = [] for this_time_unix_sec in init_times_unix_sec: this_tracking_file_name = tracking_io.find_file( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=this_time_unix_sec, spc_date_string=time_conversion.time_to_spc_date_string( this_time_unix_sec), raise_error_if_missing=True) tracking_file_names.append(this_tracking_file_name) storm_object_table = tracking_io.read_many_files(tracking_file_names) print(SEPARATOR_STRING) tracking_utils.find_storm_objects( all_id_strings=ungridded_forecast_dict[prediction_io.STORM_IDS_KEY], all_times_unix_sec=ungridded_forecast_dict[ prediction_io.STORM_TIMES_KEY], id_strings_to_keep=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), times_to_keep_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, allow_missing=False) sort_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values, id_strings_to_keep=ungridded_forecast_dict[ prediction_io.STORM_IDS_KEY], times_to_keep_unix_sec=ungridded_forecast_dict[ prediction_io.STORM_TIMES_KEY], allow_missing=False) forecast_probabilities = ungridded_forecast_dict[ prediction_io.PROBABILITY_MATRIX_KEY][sort_indices, 1] storm_object_table = storm_object_table.assign( **{forecast_column_name: forecast_probabilities}) gridded_forecast_dict = gridded_forecasts.create_forecast_grids( storm_object_table=storm_object_table, min_lead_time_sec=min_lead_time_seconds, max_lead_time_sec=max_lead_time_seconds, lead_time_resolution_sec=gridded_forecasts. DEFAULT_LEAD_TIME_RES_SECONDS, grid_spacing_x_metres=x_spacing_metres, grid_spacing_y_metres=y_spacing_metres, interp_to_latlng_grid=False, prob_radius_for_grid_metres=effective_radius_metres, smoothing_method=smoothing_method_name, smoothing_e_folding_radius_metres=smoothing_efold_radius_metres, smoothing_cutoff_radius_metres=smoothing_cutoff_radius_metres) print(SEPARATOR_STRING) output_file_name = prediction_io.find_file( top_prediction_dir_name=top_output_dir_name, first_init_time_unix_sec=numpy.min( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values), last_init_time_unix_sec=numpy.max( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values), gridded=True, raise_error_if_missing=False) print(('Writing results (forecast grids for {0:d} initial times) to: ' '"{1:s}"...').format( len(gridded_forecast_dict[prediction_io.INIT_TIMES_KEY]), output_file_name)) prediction_io.write_gridded_predictions( gridded_forecast_dict=gridded_forecast_dict, pickle_file_name=output_file_name)
def _find_io_files_for_renaming(top_input_dir_name, first_date_unix_sec, last_date_unix_sec, top_output_dir_name): """Finds input and output files for renaming storms. N = number of dates :param top_input_dir_name: See documentation for `rename_storms.` :param first_date_unix_sec: Same. :param last_date_unix_sec: Same. :param top_output_dir_name: Same. :return: input_file_names_by_date: length-N list, where the [i]th item is a numpy array of paths to input files for the [i]th date. :return: output_file_names_by_date: Same as above, but for output files. :return: valid_times_by_date_unix_sec: Same as above, but for valid times. All 3 arrays for the [i]th date have the same length. """ dates_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=first_date_unix_sec, end_time_unix_sec=last_date_unix_sec, time_interval_sec=DAYS_TO_SECONDS, include_endpoint=True) date_strings = [ time_conversion.unix_sec_to_string(t, DATE_FORMAT) for t in dates_unix_sec ] num_dates = len(date_strings) input_file_names_by_date = [numpy.array([], dtype=object)] * num_dates output_file_names_by_date = [numpy.array([], dtype=object)] * num_dates valid_times_by_date_unix_sec = [numpy.array([], dtype=int)] * num_dates for i in range(num_dates): print('Finding input files for date {0:s}...'.format(date_strings[i])) these_input_file_names = tracking_io.find_files_one_spc_date( spc_date_string=date_strings[i], source_name=tracking_utils.PROBSEVERE_NAME, top_tracking_dir_name=top_input_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, raise_error_if_missing=True)[0] these_input_file_names.sort() these_valid_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in these_input_file_names], dtype=int) these_output_file_names = [] for t in these_valid_times_unix_sec: these_output_file_names.append( tracking_io.find_file( valid_time_unix_sec=t, source_name=tracking_utils.PROBSEVERE_NAME, top_tracking_dir_name=top_output_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, raise_error_if_missing=False)) input_file_names_by_date[i] = numpy.array(these_input_file_names, dtype=object) output_file_names_by_date[i] = numpy.array(these_output_file_names, dtype=object) valid_times_by_date_unix_sec[i] = these_valid_times_unix_sec print(SEPARATOR_STRING) return (input_file_names_by_date, output_file_names_by_date, valid_times_by_date_unix_sec)
def _run(prediction_file_name, top_tracking_dir_name, prob_threshold, grid_spacing_metres, output_dir_name): """Plots spatial distribution of false alarms. This is effectively the main method. :param prediction_file_name: See documentation at top of file. :param top_tracking_dir_name: Same. :param prob_threshold: Same. :param grid_spacing_metres: Same. :param output_dir_name: Same. """ # Process input args. file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) error_checking.assert_is_greater(prob_threshold, 0.) error_checking.assert_is_less_than(prob_threshold, 1.) grid_metadata_dict = grids.create_equidistant_grid( min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG, min_longitude_deg=MIN_LONGITUDE_DEG, max_longitude_deg=MAX_LONGITUDE_DEG, x_spacing_metres=grid_spacing_metres, y_spacing_metres=grid_spacing_metres, azimuthal=False) # Read predictions and find positive forecasts and false alarms. print('Reading predictions from: "{0:s}"...'.format(prediction_file_name)) prediction_dict = prediction_io.read_ungridded_predictions( prediction_file_name) observed_labels = prediction_dict[prediction_io.OBSERVED_LABELS_KEY] forecast_labels = ( prediction_dict[prediction_io.PROBABILITY_MATRIX_KEY][:, -1] >= prob_threshold).astype(int) pos_forecast_indices = numpy.where(forecast_labels == 1)[0] false_alarm_indices = numpy.where( numpy.logical_and(observed_labels == 0, forecast_labels == 1))[0] num_examples = len(observed_labels) num_positive_forecasts = len(pos_forecast_indices) num_false_alarms = len(false_alarm_indices) print(('Probability threshold = {0:.3f} ... number of examples, positive ' 'forecasts, false alarms = {1:d}, {2:d}, {3:d}').format( prob_threshold, num_examples, num_positive_forecasts, num_false_alarms)) # Find and read tracking files. pos_forecast_id_strings = [ prediction_dict[prediction_io.STORM_IDS_KEY][k] for k in pos_forecast_indices ] pos_forecast_times_unix_sec = ( prediction_dict[prediction_io.STORM_TIMES_KEY][pos_forecast_indices]) file_times_unix_sec = numpy.unique(pos_forecast_times_unix_sec) num_files = len(file_times_unix_sec) storm_object_tables = [None] * num_files print(SEPARATOR_STRING) for i in range(num_files): this_tracking_file_name = tracking_io.find_file( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=file_times_unix_sec[i], spc_date_string=time_conversion.time_to_spc_date_string( file_times_unix_sec[i]), raise_error_if_missing=True) print('Reading data from: "{0:s}"...'.format(this_tracking_file_name)) this_table = tracking_io.read_file(this_tracking_file_name) storm_object_tables[i] = this_table.loc[this_table[ tracking_utils.FULL_ID_COLUMN].isin(pos_forecast_id_strings)] if i == 0: continue storm_object_tables[i] = storm_object_tables[i].align( storm_object_tables[0], axis=1)[0] storm_object_table = pandas.concat(storm_object_tables, axis=0, ignore_index=True) print(SEPARATOR_STRING) # Find latitudes and longitudes of false alarms. all_id_strings = ( storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()) all_times_unix_sec = ( storm_object_table[tracking_utils.VALID_TIME_COLUMN].values) good_indices = tracking_utils.find_storm_objects( all_id_strings=all_id_strings, all_times_unix_sec=all_times_unix_sec, id_strings_to_keep=pos_forecast_id_strings, times_to_keep_unix_sec=pos_forecast_times_unix_sec, allow_missing=False) pos_forecast_latitudes_deg = storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values[good_indices] pos_forecast_longitudes_deg = storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values[good_indices] false_alarm_id_strings = [ prediction_dict[prediction_io.STORM_IDS_KEY][k] for k in false_alarm_indices ] false_alarm_times_unix_sec = ( prediction_dict[prediction_io.STORM_TIMES_KEY][false_alarm_indices]) good_indices = tracking_utils.find_storm_objects( all_id_strings=all_id_strings, all_times_unix_sec=all_times_unix_sec, id_strings_to_keep=false_alarm_id_strings, times_to_keep_unix_sec=false_alarm_times_unix_sec, allow_missing=False) false_alarm_latitudes_deg = storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values[good_indices] false_alarm_longitudes_deg = storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values[good_indices] pos_forecast_x_coords_metres, pos_forecast_y_coords_metres = ( projections.project_latlng_to_xy( latitudes_deg=pos_forecast_latitudes_deg, longitudes_deg=pos_forecast_longitudes_deg, projection_object=grid_metadata_dict[grids.PROJECTION_KEY])) num_pos_forecasts_matrix = grids.count_events_on_equidistant_grid( event_x_coords_metres=pos_forecast_x_coords_metres, event_y_coords_metres=pos_forecast_y_coords_metres, grid_point_x_coords_metres=grid_metadata_dict[grids.X_COORDS_KEY], grid_point_y_coords_metres=grid_metadata_dict[grids.Y_COORDS_KEY])[0] print(SEPARATOR_STRING) false_alarm_x_coords_metres, false_alarm_y_coords_metres = ( projections.project_latlng_to_xy( latitudes_deg=false_alarm_latitudes_deg, longitudes_deg=false_alarm_longitudes_deg, projection_object=grid_metadata_dict[grids.PROJECTION_KEY])) num_false_alarms_matrix = grids.count_events_on_equidistant_grid( event_x_coords_metres=false_alarm_x_coords_metres, event_y_coords_metres=false_alarm_y_coords_metres, grid_point_x_coords_metres=grid_metadata_dict[grids.X_COORDS_KEY], grid_point_y_coords_metres=grid_metadata_dict[grids.Y_COORDS_KEY])[0] print(SEPARATOR_STRING) num_pos_forecasts_matrix = num_pos_forecasts_matrix.astype(float) num_pos_forecasts_matrix[num_pos_forecasts_matrix == 0] = numpy.nan num_false_alarms_matrix = num_false_alarms_matrix.astype(float) num_false_alarms_matrix[num_false_alarms_matrix == 0] = numpy.nan far_matrix = num_false_alarms_matrix / num_pos_forecasts_matrix this_max_value = numpy.nanpercentile(num_false_alarms_matrix, MAX_COUNT_PERCENTILE_TO_PLOT) if this_max_value < 10: this_max_value = numpy.nanmax(num_false_alarms_matrix) figure_object = plotter._plot_one_value( data_matrix=num_false_alarms_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=CMAP_OBJECT_FOR_COUNTS, min_colour_value=0, max_colour_value=this_max_value, plot_cbar_min_arrow=False, plot_cbar_max_arrow=True)[0] num_false_alarms_file_name = '{0:s}/num_false_alarms.jpg'.format( output_dir_name) print('Saving figure to: "{0:s}"...'.format(num_false_alarms_file_name)) figure_object.savefig(num_false_alarms_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) this_max_value = numpy.nanpercentile(num_pos_forecasts_matrix, MAX_COUNT_PERCENTILE_TO_PLOT) if this_max_value < 10: this_max_value = numpy.nanmax(num_pos_forecasts_matrix) figure_object = plotter._plot_one_value( data_matrix=num_pos_forecasts_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=CMAP_OBJECT_FOR_COUNTS, min_colour_value=0, max_colour_value=this_max_value, plot_cbar_min_arrow=False, plot_cbar_max_arrow=True)[0] num_pos_forecasts_file_name = '{0:s}/num_positive_forecasts.jpg'.format( output_dir_name) print('Saving figure to: "{0:s}"...'.format(num_pos_forecasts_file_name)) figure_object.savefig(num_pos_forecasts_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) this_max_value = numpy.nanpercentile(far_matrix, MAX_FAR_PERCENTILE_TO_PLOT) this_min_value = numpy.nanpercentile(far_matrix, 100. - MAX_FAR_PERCENTILE_TO_PLOT) figure_object = plotter._plot_one_value( data_matrix=far_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=CMAP_OBJECT_FOR_FAR, min_colour_value=this_min_value, max_colour_value=this_max_value, plot_cbar_min_arrow=this_min_value > 0., plot_cbar_max_arrow=this_max_value < 1.)[0] far_file_name = '{0:s}/false_alarm_ratio.jpg'.format(output_dir_name) print('Saving figure to: "{0:s}"...'.format(far_file_name)) figure_object.savefig(far_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object)
def _run(top_radar_dir_name, top_echo_classifn_dir_name, valid_time_string, min_latitude_deg, max_latitude_deg, min_longitude_deg, max_longitude_deg, output_dir_name): """Makes figure to explain storm detection. This is effectively the main method. :param top_radar_dir_name: See documentation at top of file. :param top_echo_classifn_dir_name: Same. :param valid_time_string: Same. :param min_latitude_deg: Same. :param max_latitude_deg: Same. :param min_longitude_deg: Same. :param max_longitude_deg: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name ) valid_time_unix_sec = time_conversion.string_to_unix_sec( valid_time_string, TIME_FORMAT ) spc_date_string = time_conversion.time_to_spc_date_string( valid_time_unix_sec ) num_trials = len(MIN_POLYGON_SIZES_PX) tracking_dir_names = [None] * num_trials for k in range(num_trials): tracking_dir_names[k] = ( '{0:s}/tracking/min-polygon-size-px={1:d}_recompute-centroids={2:d}' ).format( output_dir_name, MIN_POLYGON_SIZES_PX[k], int(RECOMPUTE_CENTROID_FLAGS[k]) ) echo_top_tracking.run_tracking( top_radar_dir_name=top_radar_dir_name, top_output_dir_name=tracking_dir_names[k], first_spc_date_string=spc_date_string, last_spc_date_string=spc_date_string, first_time_unix_sec=valid_time_unix_sec, last_time_unix_sec=valid_time_unix_sec + 1, top_echo_classifn_dir_name=top_echo_classifn_dir_name, min_polygon_size_pixels=MIN_POLYGON_SIZES_PX[k], recompute_centroids=RECOMPUTE_CENTROID_FLAGS[k] ) print(SEPARATOR_STRING) echo_top_file_name = myrorss_and_mrms_io.find_raw_file( top_directory_name=top_radar_dir_name, unix_time_sec=valid_time_unix_sec, spc_date_string=spc_date_string, field_name=radar_utils.ECHO_TOP_40DBZ_NAME, data_source=radar_utils.MYRORSS_SOURCE_ID, raise_error_if_missing=True ) print('Reading data from: "{0:s}"...'.format(echo_top_file_name)) metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file( netcdf_file_name=echo_top_file_name, data_source=radar_utils.MYRORSS_SOURCE_ID ) sparse_grid_table = myrorss_and_mrms_io.read_data_from_sparse_grid_file( netcdf_file_name=echo_top_file_name, field_name_orig=metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN] ) echo_top_matrix_km_asl, radar_latitudes_deg, radar_longitudes_deg = ( radar_s2f.sparse_to_full_grid( sparse_grid_table=sparse_grid_table, metadata_dict=metadata_dict) ) echo_top_matrix_km_asl = numpy.flip(echo_top_matrix_km_asl, axis=0) radar_latitudes_deg = radar_latitudes_deg[::-1] echo_classifn_file_name = echo_classifn.find_classification_file( top_directory_name=top_echo_classifn_dir_name, valid_time_unix_sec=valid_time_unix_sec, desire_zipped=True, allow_zipped_or_unzipped=True, raise_error_if_missing=True ) print('Reading data from: "{0:s}"...'.format(echo_classifn_file_name)) convective_flag_matrix = echo_classifn.read_classifications( echo_classifn_file_name )[0] good_indices = numpy.where(numpy.logical_and( radar_latitudes_deg >= min_latitude_deg, radar_latitudes_deg <= max_latitude_deg ))[0] echo_top_matrix_km_asl = echo_top_matrix_km_asl[good_indices, ...] convective_flag_matrix = convective_flag_matrix[good_indices, ...] radar_latitudes_deg = radar_latitudes_deg[good_indices] good_indices = numpy.where(numpy.logical_and( radar_longitudes_deg >= min_longitude_deg, radar_longitudes_deg <= max_longitude_deg ))[0] echo_top_matrix_km_asl = echo_top_matrix_km_asl[..., good_indices] convective_flag_matrix = convective_flag_matrix[..., good_indices] radar_longitudes_deg = radar_longitudes_deg[good_indices] this_figure_object, this_axes_object = _plot_echo_tops( echo_top_matrix_km_asl=echo_top_matrix_km_asl, latitudes_deg=radar_latitudes_deg, longitudes_deg=radar_longitudes_deg, plot_colour_bar=False, convective_flag_matrix=None )[:2] this_axes_object.set_title('All echoes') plotting_utils.label_axes(axes_object=this_axes_object, label_string='(a)') panel_file_names = [ '{0:s}/before_echo_classification.jpg'.format(output_dir_name) ] print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) this_figure_object.savefig( panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight' ) pyplot.close(this_figure_object) this_figure_object, this_axes_object = _plot_echo_tops( echo_top_matrix_km_asl=echo_top_matrix_km_asl, latitudes_deg=radar_latitudes_deg, longitudes_deg=radar_longitudes_deg, plot_colour_bar=False, convective_flag_matrix=convective_flag_matrix )[:2] this_axes_object.set_title('Convective echoes only') plotting_utils.label_axes(axes_object=this_axes_object, label_string='(b)') panel_file_names.append( '{0:s}/after_echo_classification.jpg'.format(output_dir_name) ) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) this_figure_object.savefig( panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight' ) pyplot.close(this_figure_object) letter_label = 'b' for k in range(num_trials): this_tracking_file_name = tracking_io.find_file( top_tracking_dir_name=tracking_dir_names[k], tracking_scale_metres2= echo_top_tracking.DUMMY_TRACKING_SCALE_METRES2, source_name=tracking_utils.SEGMOTION_NAME, valid_time_unix_sec=valid_time_unix_sec, spc_date_string=spc_date_string, raise_error_if_missing=True ) print('Reading data from: "{0:s}"...'.format(this_tracking_file_name)) this_storm_object_table = tracking_io.read_file(this_tracking_file_name) this_figure_object, this_axes_object, this_basemap_object = ( _plot_echo_tops( echo_top_matrix_km_asl=echo_top_matrix_km_asl, latitudes_deg=radar_latitudes_deg, longitudes_deg=radar_longitudes_deg, plot_colour_bar=k > 0, convective_flag_matrix=convective_flag_matrix) ) storm_plotting.plot_storm_outlines( storm_object_table=this_storm_object_table, axes_object=this_axes_object, basemap_object=this_basemap_object, line_width=POLYGON_WIDTH, line_colour=POLYGON_COLOUR ) these_x_metres, these_y_metres = this_basemap_object( this_storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values, this_storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values ) this_axes_object.plot( these_x_metres, these_y_metres, linestyle='None', marker=MARKER_TYPE, markersize=MARKER_SIZE, markerfacecolor=MARKER_COLOUR, markeredgecolor=MARKER_COLOUR, markeredgewidth=MARKER_EDGE_WIDTH ) this_title_string = ( 'Minimum size = {0:d} GP, {1:s} storm centers' ).format( MIN_POLYGON_SIZES_PX[k], 'recomputed' if RECOMPUTE_CENTROID_FLAGS[k] else 'original' ) this_axes_object.set_title(this_title_string) letter_label = chr(ord(letter_label) + 1) plotting_utils.label_axes( axes_object=this_axes_object, label_string='({0:s})'.format(letter_label) ) panel_file_names.append( '{0:s}/detection{1:d}.jpg'.format(output_dir_name, k) ) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) this_figure_object.savefig( panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight' ) pyplot.close(this_figure_object) concat_file_name = '{0:s}/storm_detection.jpg'.format(output_dir_name) print('Concatenating panels to: "{0:s}"...'.format(concat_file_name)) imagemagick_utils.concatenate_images( input_file_names=panel_file_names, output_file_name=concat_file_name, num_panel_rows=3, num_panel_columns=2 )