def test_field_name_new_to_orig_gridrad_invalid(self): """Ensures correct output from field_name_new_to_orig. In this case, trying to convert field name to GridRad format, but field does not exist in GridRad. """ with self.assertRaises(ValueError): radar_utils.field_name_new_to_orig( LL_SHEAR_NAME_NEW, data_source=radar_utils.GRIDRAD_SOURCE_ID)
def test_field_name_new_to_orig_mrms_invalid(self): """Ensures correct output from field_name_new_to_orig. In this case, trying to convert field name to MRMS format, but field does not exist in MRMS. """ with self.assertRaises(ValueError): radar_utils.field_name_new_to_orig( radar_utils.DIFFERENTIAL_REFL_NAME, data_source=radar_utils.MRMS_SOURCE_ID)
def read_field_from_full_grid_file(netcdf_file_name, field_name=None, metadata_dict=None, raise_error_if_fails=True): """Reads one radar field from full-grid (not sparse-grid) file. This file should contain all radar variables for one time step. M = number of rows (unique grid-point latitudes) N = number of columns (unique grid-point longitudes) H = number of height levels (unique grid-point heights) :param netcdf_file_name: Path to input file. :param field_name: Name of radar field. :param metadata_dict: Dictionary created by read_metadata_from_full_grid_file. :param raise_error_if_fails: Boolean flag. If True and file cannot be opened, this method will raise an error. If False and file cannot be opened, will return None for all output variables. :return: field_matrix: H-by-M-by-N numpy array with values of radar field. :return: grid_point_heights_m_asl: length-H numpy array of height levels (integer metres above sea level). If array is increasing (decreasing), height increases (decreases) with the first index of field_matrix. :return: grid_point_latitudes_deg: length-M numpy array of grid-point latitudes (deg N). If array is increasing (decreasing), latitude increases (decreases) with the second index of field_matrix. :return: grid_point_longitudes_deg: length-N numpy array of grid-point longitudes (deg N). If array is increasing (decreasing), latitude increases (decreases) with the third index of field_matrix. """ error_checking.assert_file_exists(netcdf_file_name) netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name, raise_error_if_fails) if netcdf_dataset is None: return None, None, None, None field_name_orig = radar_utils.field_name_new_to_orig( field_name=field_name, data_source_name=radar_utils.GRIDRAD_SOURCE_ID) field_matrix = numpy.array( netcdf_dataset.variables[field_name_orig][0, :, :, :]) grid_point_latitudes_deg = numpy.array( netcdf_dataset.variables[LATITUDE_NAME_ORIG]) grid_point_longitudes_deg = lng_conversion.convert_lng_positive_in_west( numpy.array(netcdf_dataset.variables[LONGITUDE_NAME_ORIG])) _check_grid_points(grid_point_latitudes_deg=grid_point_latitudes_deg, grid_point_longitudes_deg=grid_point_longitudes_deg, metadata_dict=metadata_dict) grid_point_heights_m_asl = KM_TO_METRES * numpy.array( netcdf_dataset.variables[HEIGHT_NAME_ORIG]) grid_point_heights_m_asl = numpy.round(grid_point_heights_m_asl).astype( int) netcdf_dataset.close() return (field_matrix, grid_point_heights_m_asl, grid_point_latitudes_deg, grid_point_longitudes_deg)
def test_field_name_new_to_orig_mrms_valid(self): """Ensures correct output from field_name_new_to_orig. In this case, field name is being converted to MRMS format. """ this_field_name_mrms = radar_utils.field_name_new_to_orig( LL_SHEAR_NAME_NEW, data_source=radar_utils.MRMS_SOURCE_ID) self.assertTrue(this_field_name_mrms == LL_SHEAR_NAME_MRMS)
def test_field_name_new_to_orig_myrorss_valid(self): """Ensures correct output from field_name_new_to_orig. In this case, field name is being converted to MYRORSS format. """ this_field_name_myrorss = radar_utils.field_name_new_to_orig( field_name=LL_SHEAR_NAME, data_source_name=radar_utils.MYRORSS_SOURCE_ID) self.assertTrue(this_field_name_myrorss == LL_SHEAR_NAME_MYRORSS)
def test_field_name_new_to_orig_gridrad_valid(self): """Ensures correct output from field_name_new_to_orig. In this case, field name is being converted to GridRad format. """ this_field_name_gridrad = radar_utils.field_name_new_to_orig( radar_utils.DIFFERENTIAL_REFL_NAME, data_source=radar_utils.GRIDRAD_SOURCE_ID) self.assertTrue(this_field_name_gridrad == radar_utils.DIFFERENTIAL_REFL_NAME_GRIDRAD)
def fields_and_refl_heights_to_pairs(field_names, heights_m_asl): """Converts unique arrays (field names and heights) to non-unique ones. F = number of fields H = number of heights N = F * H = number of field/height pairs :param field_names: length-F list with names of radar fields in GewitterGefahr format. :param heights_m_asl: length-H numpy array of heights (metres above sea level). :return: field_name_by_pair: length-N list of field names. :return: height_by_pair_m_asl: length-N numpy array of corresponding heights (metres above sea level). """ error_checking.assert_is_string_list(field_names) error_checking.assert_is_numpy_array(numpy.array(field_names), num_dimensions=1) radar_utils.check_heights(data_source=radar_utils.GRIDRAD_SOURCE_ID, heights_m_asl=heights_m_asl) field_name_by_pair = [] height_by_pair_m_asl = numpy.array([], dtype=int) for this_field_name in field_names: radar_utils.field_name_new_to_orig( field_name=this_field_name, data_source_name=radar_utils.GRIDRAD_SOURCE_ID) field_name_by_pair += [this_field_name] * len(heights_m_asl) height_by_pair_m_asl = numpy.concatenate( (height_by_pair_m_asl, heights_m_asl)) return field_name_by_pair, height_by_pair_m_asl
def get_relative_dir_for_raw_files(field_name, data_source, height_m_asl=None): """Generates relative path for raw files. :param field_name: Name of radar field in GewitterGefahr format. :param data_source: Data source (string). :param height_m_asl: Radar height (metres above sea level). :return: relative_directory_name: Relative path for raw files. """ if field_name == radar_utils.REFL_NAME: radar_utils.check_heights(data_source=data_source, heights_m_asl=numpy.array([height_m_asl]), field_name=radar_utils.REFL_NAME) else: height_m_asl = radar_utils.get_valid_heights(data_source=data_source, field_name=field_name)[0] return '{0:s}/{1:05.2f}'.format( radar_utils.field_name_new_to_orig(field_name=field_name, data_source_name=data_source), float(height_m_asl) * METRES_TO_KM)
def write_field_to_myrorss_file(field_matrix, netcdf_file_name, field_name, metadata_dict, height_m_asl=None): """Writes field to MYRORSS-formatted file. M = number of rows (unique grid-point latitudes) N = number of columns (unique grid-point longitudes) :param field_matrix: M-by-N numpy array with one radar variable at one time. Latitude should increase down each column, and longitude should increase to the right along each row. :param netcdf_file_name: Path to output file. :param field_name: Name of radar field in GewitterGefahr format. :param metadata_dict: Dictionary created by either `gridrad_io.read_metadata_from_full_grid_file` or `read_metadata_from_raw_file`. :param height_m_asl: Height of radar field (metres above sea level). """ if field_name == radar_utils.REFL_NAME: field_to_heights_dict_m_asl = ( myrorss_and_mrms_utils.fields_and_refl_heights_to_dict( field_names=[field_name], data_source=radar_utils.MYRORSS_SOURCE_ID, refl_heights_m_asl=numpy.array([height_m_asl]))) else: field_to_heights_dict_m_asl = ( myrorss_and_mrms_utils.fields_and_refl_heights_to_dict( field_names=[field_name], data_source=radar_utils.MYRORSS_SOURCE_ID)) field_name = list(field_to_heights_dict_m_asl.keys())[0] radar_height_m_asl = field_to_heights_dict_m_asl[field_name][0] if field_name in radar_utils.ECHO_TOP_NAMES: field_matrix = METRES_TO_KM * field_matrix field_name_myrorss = radar_utils.field_name_new_to_orig( field_name=field_name, data_source_name=radar_utils.MYRORSS_SOURCE_ID) file_system_utils.mkdir_recursive_if_necessary(file_name=netcdf_file_name) netcdf_dataset = Dataset(netcdf_file_name, 'w', format='NETCDF3_64BIT_OFFSET') netcdf_dataset.setncattr(FIELD_NAME_COLUMN_ORIG, field_name_myrorss) netcdf_dataset.setncattr('DataType', 'SparseLatLonGrid') netcdf_dataset.setncattr( NW_GRID_POINT_LAT_COLUMN_ORIG, rounder.round_to_nearest( metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN], LATLNG_MULTIPLE_DEG)) netcdf_dataset.setncattr( NW_GRID_POINT_LNG_COLUMN_ORIG, rounder.round_to_nearest( metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN], LATLNG_MULTIPLE_DEG)) netcdf_dataset.setncattr(HEIGHT_COLUMN_ORIG, METRES_TO_KM * numpy.float(radar_height_m_asl)) netcdf_dataset.setncattr( UNIX_TIME_COLUMN_ORIG, numpy.int32(metadata_dict[radar_utils.UNIX_TIME_COLUMN])) netcdf_dataset.setncattr('FractionalTime', 0.) netcdf_dataset.setncattr('attributes', ' ColorMap SubType Unit') netcdf_dataset.setncattr('ColorMap-unit', 'dimensionless') netcdf_dataset.setncattr('ColorMap-value', '') netcdf_dataset.setncattr('SubType-unit', 'dimensionless') netcdf_dataset.setncattr('SubType-value', numpy.float(radar_height_m_asl)) netcdf_dataset.setncattr('Unit-unit', 'dimensionless') netcdf_dataset.setncattr('Unit-value', 'dimensionless') netcdf_dataset.setncattr( LAT_SPACING_COLUMN_ORIG, rounder.round_to_nearest(metadata_dict[radar_utils.LAT_SPACING_COLUMN], LATLNG_MULTIPLE_DEG)) netcdf_dataset.setncattr( LNG_SPACING_COLUMN_ORIG, rounder.round_to_nearest(metadata_dict[radar_utils.LNG_SPACING_COLUMN], LATLNG_MULTIPLE_DEG)) netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[0], numpy.double(-99000.)) netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[1], numpy.double(-99001.)) min_latitude_deg = metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] - ( metadata_dict[radar_utils.LAT_SPACING_COLUMN] * (metadata_dict[radar_utils.NUM_LAT_COLUMN] - 1)) unique_grid_point_lats_deg, unique_grid_point_lngs_deg = ( grids.get_latlng_grid_points( min_latitude_deg=min_latitude_deg, min_longitude_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LNG_COLUMN], lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN], lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN], num_rows=metadata_dict[radar_utils.NUM_LAT_COLUMN], num_columns=metadata_dict[radar_utils.NUM_LNG_COLUMN])) num_grid_rows = len(unique_grid_point_lats_deg) num_grid_columns = len(unique_grid_point_lngs_deg) field_vector = numpy.reshape(field_matrix, num_grid_rows * num_grid_columns) grid_point_lat_matrix, grid_point_lng_matrix = ( grids.latlng_vectors_to_matrices(unique_grid_point_lats_deg, unique_grid_point_lngs_deg)) grid_point_lat_vector = numpy.reshape(grid_point_lat_matrix, num_grid_rows * num_grid_columns) grid_point_lng_vector = numpy.reshape(grid_point_lng_matrix, num_grid_rows * num_grid_columns) real_value_indices = numpy.where(numpy.invert( numpy.isnan(field_vector)))[0] netcdf_dataset.createDimension(NUM_LAT_COLUMN_ORIG, num_grid_rows - 1) netcdf_dataset.createDimension(NUM_LNG_COLUMN_ORIG, num_grid_columns - 1) netcdf_dataset.createDimension(NUM_PIXELS_COLUMN_ORIG, len(real_value_indices)) row_index_vector, column_index_vector = radar_utils.latlng_to_rowcol( grid_point_lat_vector, grid_point_lng_vector, nw_grid_point_lat_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LAT_COLUMN], nw_grid_point_lng_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LNG_COLUMN], lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN], lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN]) netcdf_dataset.createVariable(field_name_myrorss, numpy.single, (NUM_PIXELS_COLUMN_ORIG, )) netcdf_dataset.createVariable(GRID_ROW_COLUMN_ORIG, numpy.int16, (NUM_PIXELS_COLUMN_ORIG, )) netcdf_dataset.createVariable(GRID_COLUMN_COLUMN_ORIG, numpy.int16, (NUM_PIXELS_COLUMN_ORIG, )) netcdf_dataset.createVariable(NUM_GRID_CELL_COLUMN_ORIG, numpy.int32, (NUM_PIXELS_COLUMN_ORIG, )) netcdf_dataset.variables[field_name_myrorss].setncattr( 'BackgroundValue', numpy.int32(-99900)) netcdf_dataset.variables[field_name_myrorss].setncattr( 'units', 'dimensionless') netcdf_dataset.variables[field_name_myrorss].setncattr( 'NumValidRuns', numpy.int32(len(real_value_indices))) netcdf_dataset.variables[field_name_myrorss][:] = field_vector[ real_value_indices] netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:] = ( row_index_vector[real_value_indices]) netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:] = ( column_index_vector[real_value_indices]) netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:] = (numpy.full( len(real_value_indices), 1, dtype=int)) netcdf_dataset.close()
"""Unit tests for myrorss_and_mrms_io.py.""" import unittest import numpy import pandas from gewittergefahr.gg_io import myrorss_and_mrms_io from gewittergefahr.gg_utils import radar_utils TOLERANCE = 1e-6 LL_SHEAR_NAME = radar_utils.LOW_LEVEL_SHEAR_NAME LL_SHEAR_NAME_MYRORSS = radar_utils.field_name_new_to_orig( field_name=radar_utils.LOW_LEVEL_SHEAR_NAME, data_source_name=radar_utils.MYRORSS_SOURCE_ID) LL_SHEAR_NAME_MRMS = radar_utils.field_name_new_to_orig( field_name=radar_utils.LOW_LEVEL_SHEAR_NAME, data_source_name=radar_utils.MRMS_SOURCE_ID) # The following constants are used to test _get_pathless_raw_file_pattern and # _get_pathless_raw_file_name. FILE_TIME_UNIX_SEC = 1507234802 FILE_SPC_DATE_STRING = '20171005' PATHLESS_ZIPPED_FILE_NAME = '20171005-202002.netcdf.gz' PATHLESS_UNZIPPED_FILE_NAME = '20171005-202002.netcdf' PATHLESS_FILE_PATTERN = '20171005-2020*.netcdf*' # The following constants are used to test _remove_sentinels_from_sparse_grid. THESE_GRID_ROWS = numpy.linspace(0, 10, num=11, dtype=int) THESE_GRID_COLUMNS = numpy.linspace(0, 10, num=11, dtype=int) THESE_NUM_GRID_CELLS = numpy.linspace(0, 10, num=11, dtype=int)