def find_prediction_file(top_directory_name, spc_date_string, raise_error_if_missing=False): """Finds file with upconvnet predictions (reconstructed radar images). :param top_directory_name: Name of top-level directory with upconvnet predictions. :param spc_date_string: SPC date (format "yyyymmdd"). :param raise_error_if_missing: Boolean flag. If file is missing and `raise_error_if_missing = True`, this method will error out. :return: prediction_file_name: Path to prediction file. If file is missing and `raise_error_if_missing = False`, this will be the expected path. :raises: ValueError: if file is missing and `raise_error_if_missing = True`. """ error_checking.assert_is_string(top_directory_name) error_checking.assert_is_boolean(raise_error_if_missing) time_conversion.spc_date_string_to_unix_sec(spc_date_string) prediction_file_name = ( '{0:s}/{1:s}/{2:s}_{3:s}.p' ).format( top_directory_name, spc_date_string[:4], PATHLESS_FILE_NAME_PREFIX, spc_date_string ) if raise_error_if_missing and not os.path.isfile(prediction_file_name): error_string = 'Cannot find file. Expected at: "{0:s}"'.format( prediction_file_name) raise ValueError(error_string) return prediction_file_name
def find_target_file(top_directory_name, event_type_string, spc_date_string, raise_error_if_missing=True, unix_time_sec=None): """Locates file with target values for either one time or one SPC date. :param top_directory_name: Name of top-level directory with target files. :param event_type_string: Event type (must be accepted by `linkage.check_event_type`). :param spc_date_string: SPC date (format "yyyymmdd"). :param raise_error_if_missing: Boolean flag. If file is missing and `raise_error_if_missing = True`, this method will error out. :param unix_time_sec: Valid time. :return: target_file_name: Path to linkage file. If file is missing and `raise_error_if_missing = False`, this will be the *expected* path. :raises: ValueError: if file is missing and `raise_error_if_missing = True`. """ error_checking.assert_is_string(top_directory_name) linkage.check_event_type(event_type_string) error_checking.assert_is_boolean(raise_error_if_missing) if unix_time_sec is None: time_conversion.spc_date_string_to_unix_sec(spc_date_string) if event_type_string == linkage.WIND_EVENT_STRING: target_file_name = '{0:s}/{1:s}/wind_labels_{2:s}.nc'.format( top_directory_name, spc_date_string[:4], spc_date_string) else: target_file_name = '{0:s}/{1:s}/tornado_labels_{2:s}.nc'.format( top_directory_name, spc_date_string[:4], spc_date_string) else: spc_date_string = time_conversion.time_to_spc_date_string( unix_time_sec) valid_time_string = time_conversion.unix_sec_to_string( unix_time_sec, TIME_FORMAT) if event_type_string == linkage.WIND_EVENT_STRING: target_file_name = '{0:s}/{1:s}/{2:s}/wind_labels_{3:s}.nc'.format( top_directory_name, spc_date_string[:4], spc_date_string, valid_time_string) else: target_file_name = ( '{0:s}/{1:s}/{2:s}/tornado_labels_{3:s}.nc').format( top_directory_name, spc_date_string[:4], spc_date_string, valid_time_string) if raise_error_if_missing and not os.path.isfile(target_file_name): error_string = 'Cannot find file. Expected at: "{0:s}"'.format( target_file_name) raise ValueError(error_string) return target_file_name
def find_files_one_spc_date( top_tracking_dir_name, tracking_scale_metres2, source_name, spc_date_string, raise_error_if_missing=True): """Finds tracking files for the given SPC date. :param top_tracking_dir_name: See doc for `_check_file_finding_args`. :param tracking_scale_metres2: Same. :param source_name: Same. :param spc_date_string: SPC date (format "yyyymmdd"). :param raise_error_if_missing: Boolean flag. If no files are found and `raise_error_if_missing = True`, this method will error out. If no files are found and `raise_error_if_missing = False`, will return empty list. :return: tracking_file_names: 1-D list of paths to tracking files. :return: glob_pattern: glob pattern used to find tracking files. :raises: ValueError: if no files are found are `raise_error_if_missing = True`. """ tracking_scale_metres2 = _check_file_finding_args( top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2, source_name=source_name, raise_error_if_missing=raise_error_if_missing) time_conversion.spc_date_string_to_unix_sec(spc_date_string) directory_name = '{0:s}/{1:s}/{2:s}/scale_{3:d}m2'.format( top_tracking_dir_name, spc_date_string[:4], spc_date_string, tracking_scale_metres2 ) glob_pattern = ( '{0:s}/{1:s}_{2:s}_{3:s}{4:s}' ).format( directory_name, FILE_NAME_PREFIX, source_name, FILE_NAME_TIME_REGEX, FILE_EXTENSION ) tracking_file_names = glob.glob(glob_pattern) if raise_error_if_missing and not len(tracking_file_names): error_string = 'Could not find any files with pattern: "{0:s}"'.format( glob_pattern) raise ValueError(error_string) return tracking_file_names, glob_pattern
def remove_unzipped_data_1day( spc_date_string, top_directory_name, field_names=DEFAULT_FIELDS_TO_REMOVE, refl_heights_m_agl=DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_AGL): """Removes unzipped MYRORSS data for one SPC date. Basically, this method cleans up after unzip_1day_tar_file. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_directory_name: Name of top-level directory with unzipped MYRORSS files. This method will find the subdirectory in `top_directory_name` for the given SPC date. :param field_names: 1-D list with names of radar fields. Only these will be deleted. :param refl_heights_m_agl: 1-D integer numpy array of reflectivity heights (metres above ground level). """ spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( spc_date_string) field_to_heights_dict_m_agl = radar_io.field_and_height_arrays_to_dict( field_names, refl_heights_m_agl=refl_heights_m_agl, data_source=radar_io.MYRORSS_SOURCE_ID) for this_field_name in field_to_heights_dict_m_agl.keys(): these_heights_m_agl = field_to_heights_dict_m_agl[this_field_name] for this_height_m_agl in these_heights_m_agl: example_file_name = radar_io.find_raw_file( unix_time_sec=spc_date_unix_sec, spc_date_unix_sec=spc_date_unix_sec, field_name=this_field_name, height_m_agl=this_height_m_agl, data_source=radar_io.MYRORSS_SOURCE_ID, top_directory_name=top_directory_name, raise_error_if_missing=False) example_directory_name, _ = os.path.split(example_file_name) directory_name_parts = example_directory_name.split('/') remove_all_heights = False if this_field_name == radar_io.REFL_NAME: if (set(these_heights_m_agl) == set(DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_AGL)): remove_all_heights = True dir_name_to_remove = '/'.join(directory_name_parts[:-1]) else: dir_name_to_remove = '/'.join(directory_name_parts) else: dir_name_to_remove = '/'.join(directory_name_parts[:-1]) if os.path.isdir(dir_name_to_remove): print 'Removing directory "{0:s}"...'.format(dir_name_to_remove) shutil.rmtree(dir_name_to_remove, ignore_errors=True) if remove_all_heights: break
def find_raw_file(unix_time_sec, spc_date_string, field_name, data_source, top_directory_name, height_m_asl=None, raise_error_if_missing=True): """Finds raw file. File should contain one field at one time step (e.g., MESH at 123502 UTC, reflectivity at 500 m above sea level and 123502 UTC). :param unix_time_sec: Valid time. :param spc_date_string: SPC date (format "yyyymmdd"). :param field_name: Name of radar field in GewitterGefahr format. :param data_source: Data source (string). :param top_directory_name: Name of top-level directory with raw files. :param height_m_asl: Radar height (metres above sea level). :param raise_error_if_missing: Boolean flag. If True and file is missing, this method will raise an error. If False and file is missing, will return *expected* path to raw file. :return: raw_file_name: Path to raw file. :raises: ValueError: if raise_error_if_missing = True and file is missing. """ # Error-checking. _ = time_conversion.spc_date_string_to_unix_sec(spc_date_string) error_checking.assert_is_string(top_directory_name) error_checking.assert_is_boolean(raise_error_if_missing) relative_directory_name = get_relative_dir_for_raw_files( field_name=field_name, height_m_asl=height_m_asl, data_source=data_source) directory_name = '{0:s}/{1:s}/{2:s}/{3:s}'.format(top_directory_name, spc_date_string[:4], spc_date_string, relative_directory_name) pathless_file_name = _get_pathless_raw_file_name(unix_time_sec, zipped=True) raw_file_name = '{0:s}/{1:s}'.format(directory_name, pathless_file_name) if raise_error_if_missing and not os.path.isfile(raw_file_name): pathless_file_name = _get_pathless_raw_file_name(unix_time_sec, zipped=False) raw_file_name = '{0:s}/{1:s}'.format(directory_name, pathless_file_name) if raise_error_if_missing and not os.path.isfile(raw_file_name): raise ValueError('Cannot find raw file. Expected at: "{0:s}"'.format( raw_file_name)) return raw_file_name
def read_stats_from_xml(xml_file_name, spc_date_string): """Reads storm statistics from XML file. :param xml_file_name: Path to input file. :param spc_date_string: SPC date (format "yyyymmdd"). :return: stats_table: pandas DataFrame with the following columns. stats_table.storm_id: String ID for storm cell. stats_table.east_velocity_m_s01: Eastward velocity (m/s). stats_table.north_velocity_m_s01: Northward velocity (m/s). stats_table.age_sec: Age of storm cell (seconds). """ # Verification. _ = time_conversion.spc_date_string_to_unix_sec(spc_date_string) error_checking.assert_file_exists(xml_file_name) xml_tree = _open_xml_file(xml_file_name) storm_dict = {} this_column_name = None this_column_name_orig = None this_column_values = None for this_element in xml_tree.iter(): if this_element.tag == 'datacolumn': if this_column_name_orig in XML_COLUMN_NAMES_ORIG: storm_dict.update({this_column_name: this_column_values}) this_column_name_orig = this_element.attrib['name'] if this_column_name_orig in XML_COLUMN_NAMES_ORIG: this_column_name = _xml_column_name_orig_to_new( this_column_name_orig) this_column_values = [] continue if this_column_name_orig not in XML_COLUMN_NAMES_ORIG: continue if this_column_name == tracking_utils.STORM_ID_COLUMN: this_column_values.append(this_element.attrib['value']) elif this_column_name == tracking_utils.NORTH_VELOCITY_COLUMN: this_column_values.append(-1 * float(this_element.attrib['value'])) elif this_column_name == tracking_utils.EAST_VELOCITY_COLUMN: this_column_values.append(float(this_element.attrib['value'])) elif this_column_name == tracking_utils.AGE_COLUMN: this_column_values.append( int(numpy.round(float(this_element.attrib['value'])))) stats_table = pandas.DataFrame.from_dict(storm_dict) storm_ids = _append_spc_date_to_storm_ids( stats_table[tracking_utils.STORM_ID_COLUMN].values, spc_date_string) stats_table = stats_table.assign( **{tracking_utils.STORM_ID_COLUMN: storm_ids}) return tracking_utils.remove_nan_rows_from_dataframe(stats_table)
def find_processed_files_one_spc_date(top_processed_dir_name, tracking_scale_metres2, data_source, spc_date_string, raise_error_if_missing=True): """Finds processed files with valid time in one SPC date. :param top_processed_dir_name: See doc for `_check_input_args_for_file_finding`. :param tracking_scale_metres2: Same. :param data_source: Same. :param spc_date_string: SPC date (format "yyyymmdd"). :param raise_error_if_missing: See doc for `find_processed_files_at_times`. :return: processed_file_names: 1-D list of paths to processed tracking files. :return: glob_pattern: glob pattern used to find files. :raises: ValueError: if no files are found and `raise_error_if_missing = True`. """ tracking_scale_metres2 = _check_input_args_for_file_finding( top_processed_dir_name=top_processed_dir_name, tracking_scale_metres2=tracking_scale_metres2, data_source=data_source, raise_error_if_missing=raise_error_if_missing) time_conversion.spc_date_string_to_unix_sec(spc_date_string) glob_pattern = ( '{0:s}/{1:s}/{2:s}/scale_{3:d}m2/{4:s}_{5:s}_{6:s}{7:s}').format( top_processed_dir_name, spc_date_string[:4], spc_date_string, tracking_scale_metres2, PREFIX_FOR_PATHLESS_FILE_NAMES, data_source, TIME_FORMAT_IN_FILE_NAMES_AS_REGEX, FILE_EXTENSION) processed_file_names = glob.glob(glob_pattern) if raise_error_if_missing and not len(processed_file_names): error_string = 'Could not find any files with pattern: "{0:s}"'.format( glob_pattern) raise ValueError(error_string) return processed_file_names, glob_pattern
def find_raw_files_one_spc_date(spc_date_string, field_name, data_source, top_directory_name, height_m_asl=None, raise_error_if_missing=True): """Finds raw files for one field and one SPC date. :param spc_date_string: SPC date (format "yyyymmdd"). :param field_name: Name of radar field in GewitterGefahr format. :param data_source: Data source (string). :param top_directory_name: Name of top-level directory with raw files. :param height_m_asl: Radar height (metres above sea level). :param raise_error_if_missing: Boolean flag. If True and no files are found, will raise error. :return: raw_file_names: 1-D list of paths to raw files. :raises: ValueError: if raise_error_if_missing = True and no files are found. """ error_checking.assert_is_boolean(raise_error_if_missing) example_time_unix_sec = time_conversion.spc_date_string_to_unix_sec( spc_date_string) example_file_name = find_raw_file(unix_time_sec=example_time_unix_sec, spc_date_string=spc_date_string, field_name=field_name, data_source=data_source, top_directory_name=top_directory_name, height_m_asl=height_m_asl, raise_error_if_missing=False) example_directory_name, example_pathless_file_name = os.path.split( example_file_name) example_time_string = time_conversion.unix_sec_to_string( example_time_unix_sec, TIME_FORMAT_SECONDS) pathless_file_pattern = example_pathless_file_name.replace( example_time_string, TIME_FORMAT_SECONDS_REGEX) pathless_file_pattern = pathless_file_pattern.replace( ZIPPED_FILE_EXTENSION, '*') raw_file_pattern = '{0:s}/{1:s}'.format(example_directory_name, pathless_file_pattern) raw_file_names = glob.glob(raw_file_pattern) if raise_error_if_missing and not raw_file_names: error_string = ( 'Could not find any files with the following pattern: {0:s}' ).format(raw_file_pattern) raise ValueError(error_string) return raw_file_names
def unzip_1day_tar_file(tar_file_name, spc_date_string, top_target_dir_name, scales_to_extract_metres2): """Unzips tar file with segmotion output for one SPC date. :param tar_file_name: Path to input file. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_target_dir_name: Name of top-level output directory. :param scales_to_extract_metres2: 1-D numpy array of tracking scales to extract. :return: target_directory_name: Path to output directory. This will be "<top_target_directory_name>/<yyyymmdd>", where <yyyymmdd> is the SPC date. """ # Verification. _ = time_conversion.spc_date_string_to_unix_sec(spc_date_string) error_checking.assert_file_exists(tar_file_name) error_checking.assert_is_greater_numpy_array(scales_to_extract_metres2, 0) error_checking.assert_is_numpy_array(scales_to_extract_metres2, num_dimensions=1) scales_to_extract_metres2 = numpy.round(scales_to_extract_metres2).astype( int) num_scales_to_extract = len(scales_to_extract_metres2) directory_names_to_unzip = [] for j in range(num_scales_to_extract): this_relative_stats_dir_name = '{0:s}/{1:s}'.format( spc_date_string, _get_relative_stats_dir_physical_scale( scales_to_extract_metres2[j])) this_relative_polygon_dir_name = '{0:s}/{1:s}'.format( spc_date_string, _get_relative_polygon_dir_physical_scale( scales_to_extract_metres2[j])) directory_names_to_unzip.append( this_relative_stats_dir_name.replace(spc_date_string + '/', '')) directory_names_to_unzip.append( this_relative_polygon_dir_name.replace(spc_date_string + '/', '')) target_directory_name = '{0:s}/{1:s}/{2:s}'.format(top_target_dir_name, spc_date_string[:4], spc_date_string) unzipping.unzip_tar(tar_file_name, target_directory_name=target_directory_name, file_and_dir_names_to_unzip=directory_names_to_unzip) return target_directory_name
def find_local_polygon_file(unix_time_sec, spc_date_string, top_raw_directory_name, tracking_scale_metres2, raise_error_if_missing=True): """Finds polygon file on local machine. This file should contain storm outlines (polygons) for one time step and one tracking scale. :param unix_time_sec: Valid time. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_raw_directory_name: Name of top-level directory with raw segmotion files. :param tracking_scale_metres2: Tracking scale. :param raise_error_if_missing: Boolean flag. If True and file is missing, this method will raise an error. :return: polygon_file_name: Path to polygon file. If raise_error_if_missing = False and file is missing, this will be the *expected* path. :raises: ValueError: if raise_error_if_missing = True and file is missing. """ # Verification. _ = time_conversion.spc_date_string_to_unix_sec(spc_date_string) error_checking.assert_is_string(top_raw_directory_name) error_checking.assert_is_greater(tracking_scale_metres2, 0.) error_checking.assert_is_boolean(raise_error_if_missing) directory_name = '{0:s}/{1:s}/{2:s}/{3:s}'.format( top_raw_directory_name, spc_date_string[:4], spc_date_string, _get_relative_polygon_dir_physical_scale(tracking_scale_metres2)) pathless_file_name = _get_pathless_polygon_file_name(unix_time_sec, zipped=True) polygon_file_name = '{0:s}/{1:s}'.format(directory_name, pathless_file_name) if raise_error_if_missing and not os.path.isfile(polygon_file_name): pathless_file_name = _get_pathless_polygon_file_name(unix_time_sec, zipped=False) polygon_file_name = '{0:s}/{1:s}'.format(directory_name, pathless_file_name) if raise_error_if_missing and not os.path.isfile(polygon_file_name): raise ValueError('Cannot find polygon file. Expected at location: ' + polygon_file_name) return polygon_file_name
def find_files_for_smart_io(start_time_unix_sec=None, start_spc_date_string=None, end_time_unix_sec=None, end_spc_date_string=None, data_source=None, tracking_scale_metres2=None, top_input_dir_name=None, top_output_dir_name=None): """Finds input, output, and temporary working files for smart IO. N = number of SPC dates in period T_i = number of time steps in the [i]th SPC date :param start_time_unix_sec: Beginning of time period. :param start_spc_date_string: SPC date at beginning of time period (format "yyyymmdd"). :param end_time_unix_sec: End of time period. :param end_spc_date_string: SPC date at end of time period (format "yyyymmdd"). :param data_source: Source for input data (examples: "segmotion", "probSevere"). :param tracking_scale_metres2: Tracking scale. :param top_input_dir_name: Name of top-level directory for input files. :param top_output_dir_name: Name of top-level directory for output files. :return: file_dict: Dictionary with the following keys. file_dict.spc_dates_unix_sec: length-N numpy array of SPC dates. file_dict.temp_file_names: 1-D list of paths to temp files (will be used for intermediate IO). file_dict.input_file_names_by_spc_date: length-N list, where the [i]th element is a 1-D list (length T_i) of paths to input files. file_dict.output_file_names_by_spc_date: Same but for output files. :raises: ValueError: if start_time_unix_sec is not part of the first SPC date (determined by start_spc_date_unix_sec). :raises: ValueError: if end_time_unix_sec is not part of the last SPC date (determined by end_spc_date_unix_sec). """ if not time_conversion.is_time_in_spc_date(start_time_unix_sec, start_spc_date_string): start_time_string = time_conversion.unix_sec_to_string( start_time_unix_sec, TIME_FORMAT_FOR_MESSAGES) raise ValueError('Start time (' + start_time_string + ') is not in first SPC date (' + start_spc_date_string + ').') if not time_conversion.is_time_in_spc_date(end_time_unix_sec, end_spc_date_string): end_time_string = time_conversion.unix_sec_to_string( end_time_unix_sec, TIME_FORMAT_FOR_MESSAGES) raise ValueError('End time (' + end_time_string + ') is not in last SPC date (' + end_spc_date_string + ').') error_checking.assert_is_greater(end_time_unix_sec, start_time_unix_sec) start_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( start_spc_date_string) end_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( end_spc_date_string) num_spc_dates = int(1 + (end_spc_date_unix_sec - start_spc_date_unix_sec) / DAYS_TO_SECONDS) spc_dates_unix_sec = numpy.linspace(start_spc_date_unix_sec, end_spc_date_unix_sec, num=num_spc_dates, dtype=int) temp_file_names = [''] * num_spc_dates input_file_names_by_spc_date = [['']] * num_spc_dates output_file_names_by_spc_date = [['']] * num_spc_dates for i in range(num_spc_dates): spc_dates_unix_sec[i] = time_conversion.time_to_spc_date_unix_sec( spc_dates_unix_sec[i]) temp_file_names[i] = tempfile.NamedTemporaryFile(delete=False).name input_file_names_by_spc_date[i] = ( tracking_io.find_processed_files_one_spc_date( spc_dates_unix_sec[i], data_source=data_source, top_processed_dir_name=top_input_dir_name, tracking_scale_metres2=tracking_scale_metres2, raise_error_if_missing=True)) this_num_files = len(input_file_names_by_spc_date[i]) these_times_unix_sec = numpy.full(this_num_files, -1, dtype=int) output_file_names_by_spc_date[i] = [''] * this_num_files for j in range(this_num_files): these_times_unix_sec[j] = tracking_io.processed_file_name_to_time( input_file_names_by_spc_date[i][j]) output_file_names_by_spc_date[i][j] = ( tracking_io.find_processed_file( unix_time_sec=these_times_unix_sec[j], data_source=data_source, spc_date_unix_sec=spc_dates_unix_sec[i], top_processed_dir_name=top_output_dir_name, tracking_scale_metres2=tracking_scale_metres2, raise_error_if_missing=False)) if i == 0: keep_time_indices = numpy.where( these_times_unix_sec >= start_time_unix_sec)[0] these_times_unix_sec = these_times_unix_sec[keep_time_indices] input_file_names_by_spc_date[i] = [ input_file_names_by_spc_date[i][j] for j in keep_time_indices ] output_file_names_by_spc_date[i] = [ output_file_names_by_spc_date[i][j] for j in keep_time_indices ] if i == num_spc_dates - 1: keep_time_indices = numpy.where( these_times_unix_sec <= end_time_unix_sec)[0] input_file_names_by_spc_date[i] = [ input_file_names_by_spc_date[i][j] for j in keep_time_indices ] output_file_names_by_spc_date[i] = [ output_file_names_by_spc_date[i][j] for j in keep_time_indices ] return { SPC_DATES_KEY: spc_dates_unix_sec, TEMP_FILE_NAMES_KEY: temp_file_names, INPUT_FILE_NAMES_KEY: input_file_names_by_spc_date, OUTPUT_FILE_NAMES_KEY: output_file_names_by_spc_date }
def unzip_1day_tar_file( tar_file_name, field_names, spc_date_string, top_target_directory_name, refl_heights_m_asl=None): """Unzips 1-day tar file (containing raw MYRORSS data for one SPC date). :param tar_file_name: Path to input file. :param field_names: 1-D list with names of radar fields. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_target_directory_name: Name of top-level directory for unzipped MYRORSS files. This method will create a subdirectory therein for the SPC date. :param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres above sea level). :return: target_directory_name: Path to output directory. """ # Verification. _ = time_conversion.spc_date_string_to_unix_sec(spc_date_string) error_checking.assert_is_string_list(field_names) error_checking.assert_is_numpy_array( numpy.asarray(field_names), num_dimensions=1) error_checking.assert_is_string(top_target_directory_name) # Put azimuthal-shear fields (which are allowed to be missing) at the end. # This way, if the tar command errors out due to missing data, it will do so # after unzipping all the non-missing data. field_names_removed = [] for this_field_name in AZIMUTHAL_RADAR_FIELD_NAMES: if this_field_name in field_names: field_names.remove(this_field_name) field_names_removed.append(this_field_name) for this_field_name in field_names_removed: field_names.append(this_field_name) field_to_heights_dict_m_asl = ( myrorss_and_mrms_utils.fields_and_refl_heights_to_dict( field_names=field_names, data_source=radar_utils.MYRORSS_SOURCE_ID, refl_heights_m_asl=refl_heights_m_asl)) target_directory_name = '{0:s}/{1:s}/{2:s}'.format( top_target_directory_name, spc_date_string[:4], spc_date_string ) field_names = list(field_to_heights_dict_m_asl.keys()) directory_names_to_unzip = [] for this_field_name in field_names: these_heights_m_asl = field_to_heights_dict_m_asl[this_field_name] for this_height_m_asl in these_heights_m_asl: directory_names_to_unzip.append( myrorss_and_mrms_io.get_relative_dir_for_raw_files( field_name=this_field_name, data_source=radar_utils.MYRORSS_SOURCE_ID, height_m_asl=this_height_m_asl)) unzipping.unzip_tar( tar_file_name, target_directory_name=target_directory_name, file_and_dir_names_to_unzip=directory_names_to_unzip) return target_directory_name
def get_echo_tops( unix_time_sec, spc_date_string, top_directory_name, critical_reflectivity_dbz, top_height_to_consider_m_asl=DEFAULT_TOP_INPUT_HEIGHT_FOR_ECHO_TOPS_M_ASL, lowest_refl_to_consider_dbz=None): """Finds echo top at each horizontal location. "Echo top" is max height with reflectivity >= critical reflectivity. M = number of rows (unique grid-point latitudes) N = number of columns (unique grid-point longitudes) :param unix_time_sec: Valid time. :param spc_date_string: SPC date (format "yyyymmdd"). :param top_directory_name: Name of top-level directory with MYRORSS files. :param critical_reflectivity_dbz: Critical reflectivity (used to define echo top). :param top_height_to_consider_m_asl: Top height level to consider (metres above sea level). :param lowest_refl_to_consider_dbz: Lowest reflectivity to consider in echo top calculations. If None, will consider all reflectivities. :return: echo_top_matrix_m_asl: M-by-N matrix of echo tops (metres above sea level). Latitude increases down each column, and longitude increases to the right along each row. :return: grid_point_latitudes_deg: length-M numpy array with latitudes (deg N) of grid points, sorted in ascending order. :return: grid_point_longitudes_deg: length-N numpy array with longitudes (deg E) of grid points, sorted in ascending order. :return: metadata_dict: Dictionary created by `radar_io.read_metadata_from_raw_file` for column-max reflectivity. """ error_checking.assert_is_greater(critical_reflectivity_dbz, 0.) error_checking.assert_is_integer(top_height_to_consider_m_asl) error_checking.assert_is_greater(top_height_to_consider_m_asl, 0) if lowest_refl_to_consider_dbz is None: lowest_refl_to_consider_dbz = 0. error_checking.assert_is_less_than(lowest_refl_to_consider_dbz, critical_reflectivity_dbz) grid_point_heights_m_asl = radar_io.get_valid_heights_for_field( radar_io.REFL_NAME, data_source=radar_io.MYRORSS_SOURCE_ID) grid_point_heights_m_asl = grid_point_heights_m_asl[ grid_point_heights_m_asl <= top_height_to_consider_m_asl] spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( spc_date_string) column_max_refl_file_name = radar_io.find_raw_file( unix_time_sec=unix_time_sec, spc_date_unix_sec=spc_date_unix_sec, field_name=radar_io.REFL_COLUMN_MAX_NAME, data_source=radar_io.MYRORSS_SOURCE_ID, top_directory_name=top_directory_name) num_grid_heights = len(grid_point_heights_m_asl) single_height_refl_file_names = [''] * num_grid_heights for k in range(num_grid_heights): single_height_refl_file_names[k] = radar_io.find_raw_file( unix_time_sec=unix_time_sec, spc_date_unix_sec=spc_date_unix_sec, field_name=radar_io.REFL_NAME, height_m_agl=grid_point_heights_m_asl[k], data_source=radar_io.MYRORSS_SOURCE_ID, top_directory_name=top_directory_name) print 'Reading "{0:s}" for echo-top calculation...'.format( column_max_refl_file_name) metadata_dict = radar_io.read_metadata_from_raw_file( column_max_refl_file_name, data_source=radar_io.MYRORSS_SOURCE_ID) this_sparse_grid_table = radar_io.read_data_from_sparse_grid_file( column_max_refl_file_name, field_name_orig=metadata_dict[radar_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_io.MYRORSS_SOURCE_ID, sentinel_values=metadata_dict[radar_io.SENTINEL_VALUE_COLUMN]) (column_max_refl_matrix_dbz, grid_point_latitudes_deg, grid_point_longitudes_deg) = radar_s2f.sparse_to_full_grid( this_sparse_grid_table, metadata_dict) num_grid_rows = len(grid_point_latitudes_deg) num_grid_columns = len(grid_point_longitudes_deg) linear_indices_to_consider = numpy.where( numpy.reshape(column_max_refl_matrix_dbz, num_grid_rows * num_grid_columns) >= critical_reflectivity_dbz)[0] print( 'Echo-top calculation is needed at only {0:d}/{1:d} horizontal grid ' 'points!').format(len(linear_indices_to_consider), num_grid_rows * num_grid_columns) echo_top_matrix_m_asl = numpy.full((num_grid_rows, num_grid_columns), numpy.nan) num_horiz_points_to_consider = len(linear_indices_to_consider) if num_horiz_points_to_consider == 0: return echo_top_matrix_m_asl grid_rows_to_consider, grid_columns_to_consider = numpy.unravel_index( linear_indices_to_consider, (num_grid_rows, num_grid_columns)) reflectivity_matrix_dbz = numpy.full( (num_grid_heights, num_horiz_points_to_consider), numpy.nan) for k in range(num_grid_heights): print 'Reading "{0:s}" for echo-top calculation...'.format( single_height_refl_file_names[k]) this_metadata_dict = radar_io.read_metadata_from_raw_file( single_height_refl_file_names[k], data_source=radar_io.MYRORSS_SOURCE_ID) this_sparse_grid_table = radar_io.read_data_from_sparse_grid_file( single_height_refl_file_names[k], field_name_orig=this_metadata_dict[ radar_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_io.MYRORSS_SOURCE_ID, sentinel_values=this_metadata_dict[radar_io.SENTINEL_VALUE_COLUMN]) this_reflectivity_matrix_dbz, _, _ = radar_s2f.sparse_to_full_grid( this_sparse_grid_table, this_metadata_dict, ignore_if_below=lowest_refl_to_consider_dbz) reflectivity_matrix_dbz[k, :] = this_reflectivity_matrix_dbz[ grid_rows_to_consider, grid_columns_to_consider] print 'Computing echo tops at the {0:d} horizontal grid points...'.format( num_horiz_points_to_consider) for i in range(num_horiz_points_to_consider): echo_top_matrix_m_asl[ grid_rows_to_consider[i], grid_columns_to_consider[i]] = ( radar_utils.get_echo_top_single_column( reflectivities_dbz=reflectivity_matrix_dbz[:, i], heights_m_asl=grid_point_heights_m_asl, critical_reflectivity_dbz=critical_reflectivity_dbz)) return (numpy.flipud(echo_top_matrix_m_asl), grid_point_latitudes_deg[::-1], grid_point_longitudes_deg, metadata_dict)
"""Joins tables with storm statistics and polygons. :param stats_table: pandas DataFrame created by read_stats_from_xml. :param polygon_table: pandas DataFrame created by read_polygons_from_netcdf or `tracking_io.make_buffers_around_polygons`. :return: storm_table: pandas DataFrame with columns from both stats_table and polygon_table. """ return polygon_table.merge(stats_table, on=tracking_io.STORM_ID_COLUMN, how='inner') if __name__ == '__main__': SPC_DATE_UNIX_SEC = time_conversion.spc_date_string_to_unix_sec( SPC_DATE_STRING) STATS_TABLE = read_stats_from_xml(XML_FILE_NAME, spc_date_unix_sec=SPC_DATE_UNIX_SEC) print STATS_TABLE METADATA_DICT = radar_io.read_metadata_from_raw_file( NETCDF_FILE_NAME, data_source=radar_io.MYRORSS_SOURCE_ID) POLYGON_TABLE = read_polygons_from_netcdf( NETCDF_FILE_NAME, metadata_dict=METADATA_DICT, spc_date_unix_sec=SPC_DATE_UNIX_SEC, tracking_start_time_unix_sec=TRACKING_START_TIME_UNIX_SEC, tracking_end_time_unix_sec=TRACKING_END_TIME_UNIX_SEC) print POLYGON_TABLE
def find_many_raw_files( desired_times_unix_sec, spc_date_strings, data_source, field_names, top_directory_name, reflectivity_heights_m_asl=None, max_time_offset_for_az_shear_sec=DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC, max_time_offset_for_non_shear_sec=DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC ): """Finds raw file for each field/height pair and time step. N = number of input times T = number of unique input times F = number of field/height pairs :param desired_times_unix_sec: length-N numpy array with desired valid times. :param spc_date_strings: length-N list of corresponding SPC dates (format "yyyymmdd"). :param data_source: Data source ("myrorss" or "mrms"). :param field_names: 1-D list of field names. :param top_directory_name: Name of top-level directory with radar data from the given source. :param reflectivity_heights_m_asl: 1-D numpy array of heights (metres above sea level) for the field "reflectivity_dbz". If "reflectivity_dbz" is not in `field_names`, leave this as None. :param max_time_offset_for_az_shear_sec: Max time offset (between desired and actual valid time) for azimuthal-shear fields. :param max_time_offset_for_non_shear_sec: Max time offset (between desired and actual valid time) for non-azimuthal-shear fields. :return: file_dictionary: Dictionary with the following keys. file_dictionary['radar_file_name_matrix']: T-by-F numpy array of paths to raw files. file_dictionary['unique_times_unix_sec']: length-T numpy array of unique valid times. file_dictionary['spc_date_strings_for_unique_times']: length-T numpy array of corresponding SPC dates. file_dictionary['field_name_by_pair']: length-F list of field names. file_dictionary['height_by_pair_m_asl']: length-F numpy array of heights (metres above sea level). """ field_name_by_pair, height_by_pair_m_asl = ( myrorss_and_mrms_utils.fields_and_refl_heights_to_pairs( field_names=field_names, data_source=data_source, refl_heights_m_asl=reflectivity_heights_m_asl)) num_fields = len(field_name_by_pair) error_checking.assert_is_integer_numpy_array(desired_times_unix_sec) error_checking.assert_is_numpy_array(desired_times_unix_sec, num_dimensions=1) num_times = len(desired_times_unix_sec) error_checking.assert_is_string_list(spc_date_strings) error_checking.assert_is_numpy_array(numpy.array(spc_date_strings), exact_dimensions=numpy.array( [num_times])) spc_dates_unix_sec = numpy.array([ time_conversion.spc_date_string_to_unix_sec(s) for s in spc_date_strings ]) time_matrix = numpy.hstack( (numpy.reshape(desired_times_unix_sec, (num_times, 1)), numpy.reshape(spc_dates_unix_sec, (num_times, 1)))) unique_time_matrix = numpy.vstack( {tuple(this_row) for this_row in time_matrix}).astype(int) unique_times_unix_sec = unique_time_matrix[:, 0] spc_dates_at_unique_times_unix_sec = unique_time_matrix[:, 1] sort_indices = numpy.argsort(unique_times_unix_sec) unique_times_unix_sec = unique_times_unix_sec[sort_indices] spc_dates_at_unique_times_unix_sec = spc_dates_at_unique_times_unix_sec[ sort_indices] num_unique_times = len(unique_times_unix_sec) radar_file_name_matrix = numpy.full((num_unique_times, num_fields), '', dtype=object) for i in range(num_unique_times): this_spc_date_string = time_conversion.time_to_spc_date_string( spc_dates_at_unique_times_unix_sec[i]) for j in range(num_fields): if field_name_by_pair[j] in AZIMUTHAL_SHEAR_FIELD_NAMES: this_max_time_offset_sec = max_time_offset_for_az_shear_sec this_raise_error_flag = False else: this_max_time_offset_sec = max_time_offset_for_non_shear_sec this_raise_error_flag = True if this_max_time_offset_sec == 0: radar_file_name_matrix[i, j] = find_raw_file( unix_time_sec=unique_times_unix_sec[i], spc_date_string=this_spc_date_string, field_name=field_name_by_pair[j], data_source=data_source, top_directory_name=top_directory_name, height_m_asl=height_by_pair_m_asl[j], raise_error_if_missing=this_raise_error_flag) else: radar_file_name_matrix[i, j] = find_raw_file_inexact_time( desired_time_unix_sec=unique_times_unix_sec[i], spc_date_string=this_spc_date_string, field_name=field_name_by_pair[j], data_source=data_source, top_directory_name=top_directory_name, height_m_asl=height_by_pair_m_asl[j], max_time_offset_sec=this_max_time_offset_sec, raise_error_if_missing=this_raise_error_flag) if radar_file_name_matrix[i, j] is None: this_time_string = time_conversion.unix_sec_to_string( unique_times_unix_sec[i], TIME_FORMAT_FOR_LOG_MESSAGES) warning_string = ( 'Cannot find file for "{0:s}" at {1:d} metres ASL and ' '{2:s}.').format(field_name_by_pair[j], int(height_by_pair_m_asl[j]), this_time_string) warnings.warn(warning_string) return { RADAR_FILE_NAMES_KEY: radar_file_name_matrix, UNIQUE_TIMES_KEY: unique_times_unix_sec, SPC_DATES_AT_UNIQUE_TIMES_KEY: spc_dates_at_unique_times_unix_sec, FIELD_NAME_BY_PAIR_KEY: field_name_by_pair, HEIGHT_BY_PAIR_KEY: numpy.round(height_by_pair_m_asl).astype(int) }
def test_spc_date_string_to_unix_sec(self): """Ensures correct output from spc_date_string_to_unix_sec.""" this_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( SPC_DATE_STRING) self.assertTrue(this_spc_date_unix_sec == SPC_DATE_UNIX_SEC)
def find_raw_file_inexact_time(desired_time_unix_sec, spc_date_string, field_name, data_source, top_directory_name, height_m_asl=None, max_time_offset_sec=None, raise_error_if_missing=False): """Finds raw file at inexact time. If you know the exact valid time, use `find_raw_file`. :param desired_time_unix_sec: Desired valid time. :param spc_date_string: SPC date (format "yyyymmdd"). :param field_name: Field name in GewitterGefahr format. :param data_source: Data source (string). :param top_directory_name: Name of top-level directory with raw files. :param height_m_asl: Radar height (metres above sea level). :param max_time_offset_sec: Maximum offset between actual and desired valid time. For example, if `desired_time_unix_sec` is 162933 UTC 5 Jan 2018 and `max_time_offset_sec` = 60, this method will look for az-shear at valid times from 162833...163033 UTC 5 Jan 2018. If None, this defaults to `DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC` for azimuthal-shear fields and `DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC` for all other fields. :param raise_error_if_missing: Boolean flag. If no file is found and raise_error_if_missing = True, this method will error out. If no file is found and raise_error_if_missing = False, will return None. :return: raw_file_name: Path to raw file. :raises: ValueError: if no file is found and raise_error_if_missing = True. """ # Error-checking. error_checking.assert_is_integer(desired_time_unix_sec) _ = time_conversion.spc_date_string_to_unix_sec(spc_date_string) error_checking.assert_is_boolean(raise_error_if_missing) radar_utils.check_field_name(field_name) if max_time_offset_sec is None: if field_name in AZIMUTHAL_SHEAR_FIELD_NAMES: max_time_offset_sec = DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC else: max_time_offset_sec = DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC error_checking.assert_is_integer(max_time_offset_sec) error_checking.assert_is_greater(max_time_offset_sec, 0) first_allowed_minute_unix_sec = numpy.round( int( rounder.floor_to_nearest( float(desired_time_unix_sec - max_time_offset_sec), MINUTES_TO_SECONDS))) last_allowed_minute_unix_sec = numpy.round( int( rounder.floor_to_nearest( float(desired_time_unix_sec + max_time_offset_sec), MINUTES_TO_SECONDS))) allowed_minutes_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=first_allowed_minute_unix_sec, end_time_unix_sec=last_allowed_minute_unix_sec, time_interval_sec=MINUTES_TO_SECONDS, include_endpoint=True).astype(int) relative_directory_name = get_relative_dir_for_raw_files( field_name=field_name, data_source=data_source, height_m_asl=height_m_asl) raw_file_names = [] for this_time_unix_sec in allowed_minutes_unix_sec: this_pathless_file_pattern = _get_pathless_raw_file_pattern( this_time_unix_sec) this_file_pattern = '{0:s}/{1:s}/{2:s}/{3:s}/{4:s}'.format( top_directory_name, spc_date_string[:4], spc_date_string, relative_directory_name, this_pathless_file_pattern) raw_file_names += glob.glob(this_file_pattern) file_times_unix_sec = [] for this_raw_file_name in raw_file_names: file_times_unix_sec.append(raw_file_name_to_time(this_raw_file_name)) if len(file_times_unix_sec): file_times_unix_sec = numpy.array(file_times_unix_sec) time_differences_sec = numpy.absolute(file_times_unix_sec - desired_time_unix_sec) nearest_index = numpy.argmin(time_differences_sec) min_time_diff_sec = time_differences_sec[nearest_index] else: min_time_diff_sec = numpy.inf if min_time_diff_sec > max_time_offset_sec: if raise_error_if_missing: desired_time_string = time_conversion.unix_sec_to_string( desired_time_unix_sec, TIME_FORMAT_FOR_LOG_MESSAGES) error_string = ( 'Could not find "{0:s}" file within {1:d} seconds of {2:s}.' ).format(field_name, max_time_offset_sec, desired_time_string) raise ValueError(error_string) return None return raw_file_names[nearest_index]
def read_polygons_from_netcdf(netcdf_file_name, metadata_dict, spc_date_string, tracking_start_time_unix_sec, tracking_end_time_unix_sec, raise_error_if_fails=True): """Reads storm polygons (outlines of storm cells) from NetCDF file. P = number of grid points in storm cell (different for each storm cell) V = number of vertices in storm polygon (different for each storm cell) If file cannot be opened, returns None. :param netcdf_file_name: Path to input file. :param metadata_dict: Dictionary with metadata for NetCDF file, created by `myrorss_and_mrms_io.read_metadata_from_raw_file`. :param spc_date_string: SPC date (format "yyyymmdd"). :param tracking_start_time_unix_sec: Start time for tracking period. This can be found by `get_start_end_times_for_spc_date`. :param tracking_end_time_unix_sec: End time for tracking period. This can be found by `get_start_end_times_for_spc_date`. :param raise_error_if_fails: Boolean flag. If True and file cannot be opened, this method will raise an error. :return: polygon_table: If file cannot be opened and raise_error_if_fails = False, this is None. Otherwise, it is a pandas DataFrame with the following columns. polygon_table.storm_id: String ID for storm cell. polygon_table.unix_time_sec: Time in Unix format. polygon_table.spc_date_unix_sec: SPC date in Unix format. polygon_table.tracking_start_time_unix_sec: Start time for tracking period. polygon_table.tracking_end_time_unix_sec: End time for tracking period. polygon_table.centroid_lat_deg: Latitude at centroid of storm cell (deg N). polygon_table.centroid_lng_deg: Longitude at centroid of storm cell (deg E). polygon_table.grid_point_latitudes_deg: length-P numpy array with latitudes (deg N) of grid points in storm cell. polygon_table.grid_point_longitudes_deg: length-P numpy array with longitudes (deg E) of grid points in storm cell. polygon_table.grid_point_rows: length-P numpy array with row indices (all integers) of grid points in storm cell. polygon_table.grid_point_columns: length-P numpy array with column indices (all integers) of grid points in storm cell. polygon_table.polygon_object_latlng: Instance of `shapely.geometry.Polygon` with vertices in lat-long coordinates. polygon_table.polygon_object_rowcol: Instance of `shapely.geometry.Polygon` with vertices in row-column coordinates. """ error_checking.assert_file_exists(netcdf_file_name) error_checking.assert_is_integer(tracking_start_time_unix_sec) error_checking.assert_is_not_nan(tracking_start_time_unix_sec) error_checking.assert_is_integer(tracking_end_time_unix_sec) error_checking.assert_is_not_nan(tracking_end_time_unix_sec) netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name, raise_error_if_fails) if netcdf_dataset is None: return None storm_id_var_name = metadata_dict[radar_utils.FIELD_NAME_COLUMN] storm_id_var_name_orig = metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG] num_values = len( netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG]) if num_values == 0: sparse_grid_dict = { myrorss_and_mrms_io.GRID_ROW_COLUMN: numpy.array([], dtype=int), myrorss_and_mrms_io.GRID_COLUMN_COLUMN: numpy.array([], dtype=int), myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN: numpy.array([], dtype=int), storm_id_var_name: numpy.array([], dtype=int) } else: sparse_grid_dict = { myrorss_and_mrms_io.GRID_ROW_COLUMN: netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG] [:], myrorss_and_mrms_io.GRID_COLUMN_COLUMN: netcdf_dataset.variables[ myrorss_and_mrms_io.GRID_COLUMN_COLUMN_ORIG][:], myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN: netcdf_dataset.variables[ myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN_ORIG][:], storm_id_var_name: netcdf_dataset.variables[storm_id_var_name_orig][:] } netcdf_dataset.close() sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict) numeric_storm_id_matrix, _, _ = (radar_s2f.sparse_to_full_grid( sparse_grid_table, metadata_dict)) polygon_table = _storm_id_matrix_to_coord_lists(numeric_storm_id_matrix) num_storms = len(polygon_table.index) unix_times_sec = numpy.full(num_storms, metadata_dict[radar_utils.UNIX_TIME_COLUMN], dtype=int) spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec( spc_date_string) spc_dates_unix_sec = numpy.full(num_storms, spc_date_unix_sec, dtype=int) tracking_start_times_unix_sec = numpy.full(num_storms, tracking_start_time_unix_sec, dtype=int) tracking_end_times_unix_sec = numpy.full(num_storms, tracking_end_time_unix_sec, dtype=int) storm_ids = _append_spc_date_to_storm_ids( polygon_table[tracking_utils.STORM_ID_COLUMN].values, spc_date_string) simple_array = numpy.full(num_storms, numpy.nan) object_array = numpy.full(num_storms, numpy.nan, dtype=object) nested_array = polygon_table[[ tracking_utils.STORM_ID_COLUMN, tracking_utils.STORM_ID_COLUMN ]].values.tolist() argument_dict = { tracking_utils.STORM_ID_COLUMN: storm_ids, tracking_utils.TIME_COLUMN: unix_times_sec, tracking_utils.SPC_DATE_COLUMN: spc_dates_unix_sec, tracking_utils.TRACKING_START_TIME_COLUMN: tracking_start_times_unix_sec, tracking_utils.TRACKING_END_TIME_COLUMN: tracking_end_times_unix_sec, tracking_utils.CENTROID_LAT_COLUMN: simple_array, tracking_utils.CENTROID_LNG_COLUMN: simple_array, tracking_utils.GRID_POINT_LAT_COLUMN: nested_array, tracking_utils.GRID_POINT_LNG_COLUMN: nested_array, tracking_utils.POLYGON_OBJECT_LATLNG_COLUMN: object_array, tracking_utils.POLYGON_OBJECT_ROWCOL_COLUMN: object_array } polygon_table = polygon_table.assign(**argument_dict) for i in range(num_storms): these_vertex_rows, these_vertex_columns = ( polygons.grid_points_in_poly_to_vertices( polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i], polygon_table[ tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i])) # these_vertex_rows, these_vertex_columns = ( # polygons.grid_points_in_poly_to_vertices( # metadata_dict[radar_utils.NUM_LAT_COLUMN] - # polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i], # polygon_table[ # tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i])) # # these_vertex_rows = ( # metadata_dict[radar_utils.NUM_LAT_COLUMN] - these_vertex_rows) (polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i], polygon_table[tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i]) = ( polygons.simple_polygon_to_grid_points(these_vertex_rows, these_vertex_columns)) (polygon_table[tracking_utils.GRID_POINT_LAT_COLUMN].values[i], polygon_table[tracking_utils.GRID_POINT_LNG_COLUMN].values[i] ) = (radar_utils.rowcol_to_latlng( polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i], polygon_table[tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i], nw_grid_point_lat_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LAT_COLUMN], nw_grid_point_lng_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LNG_COLUMN], lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN], lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])) these_vertex_lat_deg, these_vertex_lng_deg = ( radar_utils.rowcol_to_latlng( these_vertex_rows, these_vertex_columns, nw_grid_point_lat_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LAT_COLUMN], nw_grid_point_lng_deg=metadata_dict[ radar_utils.NW_GRID_POINT_LNG_COLUMN], lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN], lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])) (polygon_table[tracking_utils.CENTROID_LAT_COLUMN].values[i], polygon_table[tracking_utils.CENTROID_LNG_COLUMN].values[i] ) = geodetic_utils.get_latlng_centroid( latitudes_deg=these_vertex_lat_deg, longitudes_deg=these_vertex_lng_deg) polygon_table[ tracking_utils.POLYGON_OBJECT_ROWCOL_COLUMN].values[i] = ( polygons.vertex_arrays_to_polygon_object( these_vertex_columns, these_vertex_rows)) polygon_table[ tracking_utils.POLYGON_OBJECT_LATLNG_COLUMN].values[i] = ( polygons.vertex_arrays_to_polygon_object( these_vertex_lng_deg, these_vertex_lat_deg)) return polygon_table
def _run(top_gridrad_dir_name, first_spc_date_string, last_spc_date_string): """Makes LaTeX table with GridRad days and domains. This is effectively the main method. :param top_gridrad_dir_name: See documentation at top of file. :param first_spc_date_string: Same. :param last_spc_date_string: Same. """ all_spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) nice_date_strings = [] latitude_strings = [] longitude_strings = [] for this_spc_date_string in all_spc_date_strings: these_limits_deg = _find_domain_for_date( top_gridrad_dir_name=top_gridrad_dir_name, spc_date_string=this_spc_date_string ) if these_limits_deg is None: continue print(SEPARATOR_STRING) this_time_unix_sec = time_conversion.spc_date_string_to_unix_sec( this_spc_date_string ) this_nice_date_string = time_conversion.unix_sec_to_string( this_time_unix_sec, NICE_TIME_FORMAT ) nice_date_strings.append(this_nice_date_string) latitude_strings.append('{0:.1f}-{1:.1f}'.format( these_limits_deg[0], these_limits_deg[1] )) longitude_strings.append('{0:.1f}-{1:.1f}'.format( these_limits_deg[3], these_limits_deg[2] )) table_string = '' for i in range(len(nice_date_strings)): if i != 0: if numpy.mod(i, 2) == 0: table_string += ' \\\\\n\t\t\t' else: table_string += ' & ' table_string += '{0:s}, {1:s}'.format( nice_date_strings[i], latitude_strings[i] ) # table_string += ' $^{\\circ}$N' table_string += ', {0:s}'.format(longitude_strings[i]) # table_string += ' $^{\\circ}$W' print(table_string)