def test_get_valid_heights_storm_id(self): """Ensures correct output from get_valid_heights. In this case, field is storm ID, which is not defined at certain heights. """ with self.assertRaises(ValueError): radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.STORM_ID_NAME)
def test_get_valid_heights_gridrad(self): """Ensures correct output from get_valid_heights. In this case, data source is GridRad. """ these_valid_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.GRIDRAD_SOURCE_ID) self.assertTrue(len(these_valid_heights_m_asl) > 1)
def test_get_valid_heights_reflectivity(self): """Ensures correct output from get_valid_heights. In this case, field is reflectivity (defined at many height levels). """ these_valid_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MRMS_SOURCE_ID, field_name=radar_utils.REFL_NAME) self.assertTrue(len(these_valid_heights_m_asl) > 1)
def test_get_valid_heights_mrms_shear(self): """Ensures correct output from get_valid_heights. In this case, data source and field are azimuthal shear in MRMS. """ these_valid_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MRMS_SOURCE_ID, field_name=radar_utils.MID_LEVEL_SHEAR_NAME) self.assertTrue( numpy.array_equal(these_valid_heights_m_asl, SHEAR_HEIGHTS_M_ASL))
def test_get_valid_heights_mrms_non_shear(self): """Ensures correct output from get_valid_heights. In this case, data source is MRMS and field is *not* azimuthal shear. """ these_valid_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MRMS_SOURCE_ID, field_name=radar_utils.REFL_M10CELSIUS_NAME) self.assertTrue(numpy.array_equal( these_valid_heights_m_asl, NON_SHEAR_HEIGHTS_MRMS_M_ASL))
def fields_and_refl_heights_to_pairs(field_names, data_source, refl_heights_m_asl=None): """Converts unique arrays (field names and refl heights) to non-unique ones. F = number of fields N = number of field-height pairs :param field_names: length-F list with names of radar fields in GewitterGefahr format. :param data_source: Data source (string). :param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres above sea level). :return: field_name_by_pair: length-N list of field names. :return: height_by_pair_m_asl: length-N numpy array of corresponding heights (metres above sea level). """ check_data_source(data_source) error_checking.assert_is_string_list(field_names) error_checking.assert_is_numpy_array(numpy.array(field_names), num_dimensions=1) field_name_by_pair = [] height_by_pair_m_asl = numpy.array([]) for this_field_name in field_names: if this_field_name == radar_utils.REFL_NAME: radar_utils.check_heights(data_source=data_source, heights_m_asl=refl_heights_m_asl, field_name=this_field_name) these_heights_m_asl = copy.deepcopy(refl_heights_m_asl) else: these_heights_m_asl = radar_utils.get_valid_heights( data_source=data_source, field_name=this_field_name) field_name_by_pair += [this_field_name] * len(these_heights_m_asl) height_by_pair_m_asl = numpy.concatenate( (height_by_pair_m_asl, these_heights_m_asl)) return field_name_by_pair, height_by_pair_m_asl
def get_relative_dir_for_raw_files(field_name, data_source, height_m_asl=None): """Generates relative path for raw files. :param field_name: Name of radar field in GewitterGefahr format. :param data_source: Data source (string). :param height_m_asl: Radar height (metres above sea level). :return: relative_directory_name: Relative path for raw files. """ if field_name == radar_utils.REFL_NAME: radar_utils.check_heights(data_source=data_source, heights_m_asl=numpy.array([height_m_asl]), field_name=radar_utils.REFL_NAME) else: height_m_asl = radar_utils.get_valid_heights(data_source=data_source, field_name=field_name)[0] return '{0:s}/{1:05.2f}'.format( radar_utils.field_name_new_to_orig(field_name=field_name, data_source_name=data_source), float(height_m_asl) * METRES_TO_KM)
def fields_and_refl_heights_to_dict(field_names, data_source, refl_heights_m_asl=None): """Converts two arrays (field names and reflectivity heights) to dictionary. :param field_names: 1-D list with names of radar fields in GewitterGefahr format. :param data_source: Data source (string). :param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres above sea level). :return: field_to_heights_dict_m_asl: Dictionary, where each key is a field name and each value is a 1-D numpy array of heights (metres above sea level). """ check_data_source(data_source) error_checking.assert_is_string_list(field_names) error_checking.assert_is_numpy_array(numpy.array(field_names), num_dimensions=1) field_to_heights_dict_m_asl = {} for this_field_name in field_names: if this_field_name == radar_utils.REFL_NAME: radar_utils.check_heights(data_source=data_source, heights_m_asl=refl_heights_m_asl, field_name=this_field_name) field_to_heights_dict_m_asl.update( {this_field_name: refl_heights_m_asl}) else: field_to_heights_dict_m_asl.update({ this_field_name: radar_utils.get_valid_heights(data_source=data_source, field_name=this_field_name) }) return field_to_heights_dict_m_asl
def get_echo_tops( unix_time_sec, spc_date_string, top_directory_name, critical_reflectivity_dbz, top_height_to_consider_m_asl=DEFAULT_TOP_INPUT_HEIGHT_FOR_ECHO_TOPS_M_ASL, lowest_refl_to_consider_dbz=None): """Finds echo top at each horizontal location. "Echo top" is max height with reflectivity >= critical reflectivity. M = number of rows (unique grid-point latitudes) N = number of columns (unique grid-point longitudes) :param unix_time_sec: Valid time. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_directory_name: Name of top-level directory with MYRORSS files. :param critical_reflectivity_dbz: Critical reflectivity (used to define echo top). :param top_height_to_consider_m_asl: Top height level to consider (metres above sea level). :param lowest_refl_to_consider_dbz: Lowest reflectivity to consider in echo top calculations. If None, will consider all reflectivities. :return: echo_top_matrix_m_asl: M-by-N matrix of echo tops (metres above sea level). Latitude increases down each column, and longitude increases to the right along each row. :return: grid_point_latitudes_deg: length-M numpy array with latitudes (deg N) of grid points, sorted in ascending order. :return: grid_point_longitudes_deg: length-N numpy array with longitudes (deg E) of grid points, sorted in ascending order. :return: metadata_dict: Dictionary created by `myrorss_and_mrms_io.read_metadata_from_raw_file` for column-max reflectivity. """ error_checking.assert_is_greater(critical_reflectivity_dbz, 0.) error_checking.assert_is_greater(top_height_to_consider_m_asl, 0) top_height_to_consider_m_asl = int( numpy.round(top_height_to_consider_m_asl)) if lowest_refl_to_consider_dbz is None: lowest_refl_to_consider_dbz = 0. error_checking.assert_is_less_than(lowest_refl_to_consider_dbz, critical_reflectivity_dbz) grid_point_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) grid_point_heights_m_asl = grid_point_heights_m_asl[ grid_point_heights_m_asl <= top_height_to_consider_m_asl] column_max_refl_file_name = myrorss_and_mrms_io.find_raw_file( unix_time_sec=unix_time_sec, spc_date_string=spc_date_string, field_name=radar_utils.REFL_COLUMN_MAX_NAME, data_source=radar_utils.MYRORSS_SOURCE_ID, top_directory_name=top_directory_name) num_grid_heights = len(grid_point_heights_m_asl) single_height_refl_file_names = [''] * num_grid_heights for k in range(num_grid_heights): single_height_refl_file_names[k] = myrorss_and_mrms_io.find_raw_file( unix_time_sec=unix_time_sec, spc_date_string=spc_date_string, field_name=radar_utils.REFL_NAME, data_source=radar_utils.MYRORSS_SOURCE_ID, top_directory_name=top_directory_name, height_m_asl=grid_point_heights_m_asl[k]) print 'Reading "{0:s}" for echo-top calculation...'.format( column_max_refl_file_name) metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file( column_max_refl_file_name, data_source=radar_utils.MYRORSS_SOURCE_ID) this_sparse_grid_table = ( myrorss_and_mrms_io.read_data_from_sparse_grid_file( column_max_refl_file_name, field_name_orig=metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN])) (column_max_refl_matrix_dbz, grid_point_latitudes_deg, grid_point_longitudes_deg) = radar_s2f.sparse_to_full_grid( this_sparse_grid_table, metadata_dict) num_grid_rows = len(grid_point_latitudes_deg) num_grid_columns = len(grid_point_longitudes_deg) linear_indices_to_consider = numpy.where( numpy.reshape(column_max_refl_matrix_dbz, num_grid_rows * num_grid_columns) >= critical_reflectivity_dbz)[0] print( 'Echo-top calculation is needed at only {0:d}/{1:d} horizontal grid ' 'points!').format(len(linear_indices_to_consider), num_grid_rows * num_grid_columns) echo_top_matrix_m_asl = numpy.full((num_grid_rows, num_grid_columns), numpy.nan) num_horiz_points_to_consider = len(linear_indices_to_consider) if num_horiz_points_to_consider == 0: return echo_top_matrix_m_asl grid_rows_to_consider, grid_columns_to_consider = numpy.unravel_index( linear_indices_to_consider, (num_grid_rows, num_grid_columns)) reflectivity_matrix_dbz = numpy.full( (num_grid_heights, num_horiz_points_to_consider), numpy.nan) for k in range(num_grid_heights): print 'Reading "{0:s}" for echo-top calculation...'.format( single_height_refl_file_names[k]) this_metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file( single_height_refl_file_names[k], data_source=radar_utils.MYRORSS_SOURCE_ID) this_sparse_grid_table = ( myrorss_and_mrms_io.read_data_from_sparse_grid_file( single_height_refl_file_names[k], field_name_orig=this_metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=this_metadata_dict[ radar_utils.SENTINEL_VALUE_COLUMN])) this_reflectivity_matrix_dbz, _, _ = radar_s2f.sparse_to_full_grid( this_sparse_grid_table, this_metadata_dict, ignore_if_below=lowest_refl_to_consider_dbz) reflectivity_matrix_dbz[k, :] = this_reflectivity_matrix_dbz[ grid_rows_to_consider, grid_columns_to_consider] print 'Computing echo tops at the {0:d} horizontal grid points...'.format( num_horiz_points_to_consider) for i in range(num_horiz_points_to_consider): echo_top_matrix_m_asl[ grid_rows_to_consider[i], grid_columns_to_consider[i]] = ( radar_utils.get_echo_top_single_column( reflectivities_dbz=reflectivity_matrix_dbz[:, i], heights_m_asl=grid_point_heights_m_asl, critical_reflectivity_dbz=critical_reflectivity_dbz)) return (numpy.flipud(echo_top_matrix_m_asl), grid_point_latitudes_deg[::-1], grid_point_longitudes_deg, metadata_dict)
AZIMUTHAL_RADAR_FIELD_NAMES = [ radar_utils.LOW_LEVEL_SHEAR_NAME, radar_utils.MID_LEVEL_SHEAR_NAME ] DEFAULT_FIELDS_TO_REMOVE = [ radar_utils.ECHO_TOP_18DBZ_NAME, radar_utils.ECHO_TOP_50DBZ_NAME, radar_utils.LOW_LEVEL_SHEAR_NAME, radar_utils.MID_LEVEL_SHEAR_NAME, radar_utils.REFL_NAME, radar_utils.REFL_COLUMN_MAX_NAME, radar_utils.MESH_NAME, radar_utils.REFL_0CELSIUS_NAME, radar_utils.REFL_M10CELSIUS_NAME, radar_utils.REFL_M20CELSIUS_NAME, radar_utils.REFL_LOWEST_ALTITUDE_NAME, radar_utils.SHI_NAME, radar_utils.VIL_NAME ] DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_ASL = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) def unzip_1day_tar_file( tar_file_name, field_names, spc_date_string, top_target_directory_name, refl_heights_m_asl=None): """Unzips 1-day tar file (containing raw MYRORSS data for one SPC date). :param tar_file_name: Path to input file. :param field_names: 1-D list with names of radar fields. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_target_directory_name: Name of top-level directory for unzipped MYRORSS files. This method will create a subdirectory therein for the SPC date. :param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres above sea level).
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids, rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl, spc_date_string, tarred_myrorss_dir_name, untarred_myrorss_dir_name, top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2, target_name, top_target_dir_name, top_output_dir_name): """Extracts storm-centered img for each field/height pair and storm object. :param num_image_rows: See documentation at top of file. :param num_image_columns: Same. :param rotate_grids: Same. :param rotated_grid_spacing_metres: Same. :param radar_field_names: Same. :param refl_heights_m_agl: Same. :param spc_date_string: Same. :param tarred_myrorss_dir_name: Same. :param untarred_myrorss_dir_name: Same. :param top_tracking_dir_name: Same. :param elevation_dir_name: Same. :param tracking_scale_metres2: Same. :param target_name: Same. :param top_target_dir_name: Same. :param top_output_dir_name: Same. """ if target_name in ['', 'None']: target_name = None if target_name is not None: target_param_dict = target_val_utils.target_name_to_params(target_name) target_file_name = target_val_utils.find_target_file( top_directory_name=top_target_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=spc_date_string) print('Reading data from: "{0:s}"...'.format(target_file_name)) target_dict = target_val_utils.read_target_values( netcdf_file_name=target_file_name, target_names=[target_name]) print('\n') refl_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) # Untar files with azimuthal shear. az_shear_field_names = list( set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES)) if len(az_shear_field_names): az_shear_tar_file_name = ( '{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar').format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=az_shear_tar_file_name, field_names=az_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name) print(SEPARATOR_STRING) # Untar files with other radar fields. non_shear_field_names = list( set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES)) if len(non_shear_field_names): non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=non_shear_tar_file_name, field_names=non_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING) # Read storm tracks for the given SPC date. tracking_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_string, source_name=tracking_utils.SEGMOTION_NAME, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2)[0] storm_object_table = tracking_io.read_many_files(tracking_file_names)[ storm_images.STORM_COLUMNS_NEEDED] print(SEPARATOR_STRING) if target_name is not None: print(('Removing storm objects without target values (variable = ' '"{0:s}")...').format(target_name)) these_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values.astype(int), id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY], times_to_keep_unix_sec=target_dict[ target_val_utils.VALID_TIMES_KEY], allow_missing=False) num_storm_objects_orig = len(storm_object_table.index) storm_object_table = storm_object_table.iloc[these_indices] num_storm_objects = len(storm_object_table.index) print('Removed {0:d} of {1:d} storm objects!\n'.format( num_storm_objects_orig - num_storm_objects, num_storm_objects_orig)) # Extract storm-centered radar images. storm_images.extract_storm_images_myrorss_or_mrms( storm_object_table=storm_object_table, radar_source=radar_utils.MYRORSS_SOURCE_ID, top_radar_dir_name=untarred_myrorss_dir_name, top_output_dir_name=top_output_dir_name, elevation_dir_name=elevation_dir_name, num_storm_image_rows=num_image_rows, num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids, rotated_grid_spacing_metres=rotated_grid_spacing_metres, radar_field_names=radar_field_names, reflectivity_heights_m_agl=refl_heights_m_agl) print(SEPARATOR_STRING) # Remove untarred MYRORSS files. myrorss_io.remove_unzipped_data_1day( spc_date_string=spc_date_string, top_directory_name=untarred_myrorss_dir_name, field_names=radar_field_names, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING)
def _extract_storm_images( num_image_rows, num_image_columns, rotate_grids, rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl, spc_date_string, first_time_string, last_time_string, tarred_myrorss_dir_name, untarred_myrorss_dir_name, top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2, target_name, top_target_dir_name, top_output_dir_name): """Extracts storm-centered img for each field/height pair and storm object. :param num_image_rows: See documentation at top of file. :param num_image_columns: Same. :param rotate_grids: Same. :param rotated_grid_spacing_metres: Same. :param radar_field_names: Same. :param refl_heights_m_agl: Same. :param spc_date_string: Same. :param first_time_string: Same. :param last_time_string: Same. :param tarred_myrorss_dir_name: Same. :param untarred_myrorss_dir_name: Same. :param top_tracking_dir_name: Same. :param elevation_dir_name: Same. :param tracking_scale_metres2: Same. :param target_name: Same. :param top_target_dir_name: Same. :param top_output_dir_name: Same. :raises: ValueError: if `first_time_string` and `last_time_string` have different SPC dates. """ if elevation_dir_name in ['', 'None']: elevation_dir_name = None if elevation_dir_name is None: host_name = socket.gethostname() if 'casper' in host_name: elevation_dir_name = '/glade/work/ryanlage/elevation' else: elevation_dir_name = '/condo/swatwork/ralager/elevation' if spc_date_string in ['', 'None']: first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, TIME_FORMAT) first_spc_date_string = time_conversion.time_to_spc_date_string( first_time_unix_sec) last_spc_date_string = time_conversion.time_to_spc_date_string( last_time_unix_sec) if first_spc_date_string != last_spc_date_string: error_string = ( 'First ({0:s}) and last ({1:s}) times have different SPC dates.' ' This script can handle only one SPC date.' ).format(first_time_string, last_time_string) raise ValueError(error_string) spc_date_string = first_spc_date_string else: first_time_unix_sec = 0 last_time_unix_sec = int(1e12) if tarred_myrorss_dir_name in ['', 'None']: tarred_myrorss_dir_name = None if target_name in ['', 'None']: target_name = None if target_name is not None: target_param_dict = target_val_utils.target_name_to_params(target_name) target_file_name = target_val_utils.find_target_file( top_directory_name=top_target_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=spc_date_string) print('Reading data from: "{0:s}"...'.format(target_file_name)) target_dict = target_val_utils.read_target_values( netcdf_file_name=target_file_name, target_names=[target_name] ) print('\n') refl_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) # Untar files. if tarred_myrorss_dir_name is not None: az_shear_field_names = list( set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES) ) if len(az_shear_field_names) > 0: az_shear_tar_file_name = ( '{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar' ).format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string ) myrorss_io.unzip_1day_tar_file( tar_file_name=az_shear_tar_file_name, field_names=az_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name) print(SEPARATOR_STRING) non_shear_field_names = list( set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES) ) if len(non_shear_field_names) > 0: non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string ) myrorss_io.unzip_1day_tar_file( tar_file_name=non_shear_tar_file_name, field_names=non_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING) # Read storm tracks for the given SPC date. tracking_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_string, source_name=tracking_utils.SEGMOTION_NAME, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2 )[0] file_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int ) good_indices = numpy.where(numpy.logical_and( file_times_unix_sec >= first_time_unix_sec, file_times_unix_sec <= last_time_unix_sec ))[0] tracking_file_names = [tracking_file_names[k] for k in good_indices] storm_object_table = tracking_io.read_many_files( tracking_file_names )[storm_images.STORM_COLUMNS_NEEDED] print(SEPARATOR_STRING) if target_name is not None: print(( 'Removing storm objects without target values (variable = ' '"{0:s}")...' ).format(target_name)) these_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values.astype(int), id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY], times_to_keep_unix_sec=target_dict[ target_val_utils.VALID_TIMES_KEY], allow_missing=False) num_storm_objects_orig = len(storm_object_table.index) storm_object_table = storm_object_table.iloc[these_indices] num_storm_objects = len(storm_object_table.index) print('Removed {0:d} of {1:d} storm objects!\n'.format( num_storm_objects_orig - num_storm_objects, num_storm_objects_orig )) # Extract storm-centered radar images. storm_images.extract_storm_images_myrorss_or_mrms( storm_object_table=storm_object_table, radar_source=radar_utils.MYRORSS_SOURCE_ID, top_radar_dir_name=untarred_myrorss_dir_name, top_output_dir_name=top_output_dir_name, elevation_dir_name=elevation_dir_name, num_storm_image_rows=num_image_rows, num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids, rotated_grid_spacing_metres=rotated_grid_spacing_metres, radar_field_names=radar_field_names, reflectivity_heights_m_agl=refl_heights_m_agl) print(SEPARATOR_STRING) # Remove untarred MYRORSS files. if tarred_myrorss_dir_name is not None: myrorss_io.remove_unzipped_data_1day( spc_date_string=spc_date_string, top_directory_name=untarred_myrorss_dir_name, field_names=radar_field_names, refl_heights_m_asl=refl_heights_m_asl)
from gewittergefahr.gg_utils import grids from gewittergefahr.gg_utils import radar_utils from gewittergefahr.gg_utils import echo_classification as echo_classifn TOLERANCE = 1e-6 # The following constants are used to test _estimate_melting_levels. MELTING_LEVEL_LATITUDES_DEG = numpy.linspace(-90., 90., num=19) MELTING_LEVEL_TIME_UNIX_SEC = 1541823287 # 041447 UTC 10 Nov 2018 MELTING_LEVELS_M_ASL = (echo_classifn.MELT_LEVEL_INTERCEPT_BY_MONTH_M_ASL[10] + echo_classifn.MELT_LEVEL_SLOPE_BY_MONTH_M_DEG01[10] * numpy.absolute(MELTING_LEVEL_LATITUDES_DEG)) # The following constants are used to test _neigh_metres_to_rowcol. LARGE_GRID_HEIGHTS_M_ASL = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) LARGE_GRID_METADATA_DICT = { echo_classifn.MIN_LATITUDE_KEY: 20., echo_classifn.LATITUDE_SPACING_KEY: 0.01, echo_classifn.MIN_LONGITUDE_KEY: 230., echo_classifn.LONGITUDE_SPACING_KEY: 0.01, echo_classifn.HEIGHTS_KEY: LARGE_GRID_HEIGHTS_M_ASL } THESE_LATITUDES_DEG, THESE_LONGITUDES_DEG = grids.get_latlng_grid_points( min_latitude_deg=LARGE_GRID_METADATA_DICT[echo_classifn.MIN_LATITUDE_KEY], min_longitude_deg=LARGE_GRID_METADATA_DICT[ echo_classifn.MIN_LONGITUDE_KEY], lat_spacing_deg=LARGE_GRID_METADATA_DICT[
def evaluate_tracks(storm_object_table, top_myrorss_dir_name, radar_field_name): """Evaluates a set of storm tracks, following Lakshmanan and Smith (2010). T = number of storm tracks :param storm_object_table: pandas DataFrame with storm objects. Should contain columns listed in `storm_tracking_io.write_file`. :param top_myrorss_dir_name: Name of top-level directory with MYRORSS data. Files therein will be found by `myrorss_and_mrms_io.find_raw_file` and read by `myrorss_and_mrms_io.read_data_from_sparse_grid_file`. :param radar_field_name: Name of radar field to use in computing mismatch error. Must be accepted by `radar_utils.check_field_name`. :return: evaluation_dict: Dictionary with the following keys. evaluation_dict['track_durations_sec']: length-T numpy array of storm durations. evaluation_dict['track_linearity_errors_metres']: length-T numpy array of linearity errors. The "linearity error" for one track is the RMSE (root mean square error) of Theil-Sen estimates over all time steps. evaluation_dict['track_mismatch_errors']: length-T numpy array of mismatch errors. The "mismatch error" is the standard deviation of X over all time steps, where X = median value of `radar_field_name` inside the storm. evaluation_dict['mean_linearity_error_metres']: Mean linearity error. This is the mean of trackwise linearity errors for tracks with duration >= median duration. evaluation_dict['mean_mismatch_error']: Mean mismatch error. This is the mean of trackwise mismatch errors for tracks with duration >= median duration. evaluation_dict['radar_field_name']: Same as input (metadata). """ # Add x-y coordinates. projection_object = projections.init_azimuthal_equidistant_projection( central_latitude_deg=echo_top_tracking.CENTRAL_PROJ_LATITUDE_DEG, central_longitude_deg=echo_top_tracking.CENTRAL_PROJ_LONGITUDE_DEG) x_coords_metres, y_coords_metres = projections.project_latlng_to_xy( latitudes_deg=storm_object_table[ tracking_utils.CENTROID_LATITUDE_COLUMN].values, longitudes_deg=storm_object_table[ tracking_utils.CENTROID_LONGITUDE_COLUMN].values, projection_object=projection_object, false_easting_metres=0., false_northing_metres=0.) storm_object_table = storm_object_table.assign( **{ tracking_utils.CENTROID_X_COLUMN: x_coords_metres, tracking_utils.CENTROID_Y_COLUMN: y_coords_metres }) # Convert list of storm objects to list of tracks. storm_track_table = tracking_utils.storm_objects_to_tracks( storm_object_table) # Fit Theil-Sen model to each track. print(SEPARATOR_STRING) storm_track_table = _fit_theil_sen_many_tracks(storm_track_table) print(SEPARATOR_STRING) # Compute storm durations. num_tracks = len(storm_track_table.index) track_durations_sec = numpy.full(num_tracks, -1, dtype=int) for i in range(num_tracks): this_start_time_unix_sec = numpy.min( storm_track_table[tracking_utils.TRACK_TIMES_COLUMN].values[i]) this_end_time_unix_sec = numpy.max( storm_track_table[tracking_utils.TRACK_TIMES_COLUMN].values[i]) track_durations_sec[i] = (this_end_time_unix_sec - this_start_time_unix_sec) for this_percentile_level in DURATION_PERCENTILE_LEVELS: this_percentile_seconds = numpy.percentile(track_durations_sec, this_percentile_level) print('{0:d}th percentile of track durations = {1:.1f} seconds'.format( int(numpy.round(this_percentile_level)), this_percentile_seconds)) print('\n') for this_percentile_level in DURATION_PERCENTILE_LEVELS: this_percentile_seconds = numpy.percentile( track_durations_sec[track_durations_sec != 0], this_percentile_level) print( ('{0:d}th percentile of non-zero track durations = {1:.1f} seconds' ).format(int(numpy.round(this_percentile_level)), this_percentile_seconds)) print(SEPARATOR_STRING) median_duration_sec = numpy.median( track_durations_sec[track_durations_sec != 0]) long_track_flags = track_durations_sec >= median_duration_sec long_track_indices = numpy.where(long_track_flags)[0] # Compute linearity error for each track. track_linearity_errors_metres = numpy.full(num_tracks, numpy.nan) for i in range(num_tracks): if numpy.mod(i, 50) == 0: print(('Have computed linearity error for {0:d} of {1:d} tracks...' ).format(i, num_tracks)) track_linearity_errors_metres[i] = _get_mean_ts_error_one_track( storm_track_table=storm_track_table, row_index=i) print('Have computed linearity error for all {0:d} tracks!'.format( num_tracks)) good_indices = numpy.where( numpy.logical_and( long_track_flags, track_linearity_errors_metres <= MAX_LINEARITY_ERROR_METRES))[0] mean_linearity_error_metres = numpy.mean( track_linearity_errors_metres[good_indices]) print('Mean linearity error = {0:.1f} metres'.format( mean_linearity_error_metres)) print(SEPARATOR_STRING) # Compute mismatch error for each track. radar_statistic_table = ( radar_stats.get_storm_based_radar_stats_myrorss_or_mrms( storm_object_table=storm_object_table, top_radar_dir_name=top_myrorss_dir_name, radar_metadata_dict_for_tracking=None, statistic_names=[], percentile_levels=numpy.array([50.]), radar_field_names=[radar_field_name], radar_source=radar_utils.MYRORSS_SOURCE_ID)) print(SEPARATOR_STRING) radar_height_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_field_name)[0] median_column_name = radar_stats.radar_field_and_percentile_to_column_name( radar_field_name=radar_field_name, radar_height_m_asl=radar_height_m_asl, percentile_level=50.) median_by_storm_object = radar_statistic_table[median_column_name] track_mismatch_errors = numpy.full(num_tracks, numpy.nan) for i in range(num_tracks): these_object_indices = storm_track_table[ tracking_utils.OBJECT_INDICES_COLUMN].values[i] if len(these_object_indices) < 2: continue track_mismatch_errors[i] = numpy.std( median_by_storm_object[these_object_indices], ddof=1) mean_mismatch_error = numpy.nanmean( track_mismatch_errors[long_track_indices]) print('Mean mismatch error for "{0:s}" = {1:.4e}'.format( radar_field_name, mean_mismatch_error)) return { DURATIONS_KEY: track_durations_sec, LINEARITY_ERRORS_KEY: track_linearity_errors_metres, MISMATCH_ERRORS_KEY: track_mismatch_errors, MEAN_LINEARITY_ERROR_KEY: mean_linearity_error_metres, MEAN_MISMATCH_ERROR_KEY: mean_mismatch_error, RADAR_FIELD_KEY: radar_field_name }