def _run(echo_top_field_name, top_radar_dir_name_tarred, top_radar_dir_name, top_echo_classifn_dir_name, min_echo_top_km_asl, min_grid_cells_in_polygon, top_output_dir_name, first_spc_date_string, last_spc_date_string): """Runs echo-top-based storm-tracking. This is effectively the main method. :param echo_top_field_name: See documentation at top of file. :param top_radar_dir_name_tarred: Same. :param top_radar_dir_name: Same. :param top_echo_classifn_dir_name: Same. :param min_echo_top_km_asl: Same. :param min_grid_cells_in_polygon: Same. :param top_output_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. """ if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( top_radar_dir_name_tarred, this_spc_date_string[:4], this_spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=this_tar_file_name, field_names=[echo_top_field_name], spc_date_string=this_spc_date_string, top_target_directory_name=top_radar_dir_name) print SEPARATOR_STRING if top_echo_classifn_dir_name in ['', 'None']: top_echo_classifn_dir_name = None echo_top_tracking.run_tracking( top_radar_dir_name=top_radar_dir_name, top_output_dir_name=top_output_dir_name, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, echo_top_field_name=echo_top_field_name, top_echo_classifn_dir_name=top_echo_classifn_dir_name, min_echo_top_height_km_asl=min_echo_top_km_asl, min_grid_cells_in_polygon=min_grid_cells_in_polygon) print SEPARATOR_STRING if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: for this_spc_date_string in spc_date_strings: myrorss_io.remove_unzipped_data_1day( spc_date_string=this_spc_date_string, top_directory_name=top_radar_dir_name, field_names=[echo_top_field_name])
def _run_for_myrorss(spc_date_string, top_radar_dir_name_tarred, top_radar_dir_name_untarred, top_output_dir_name, option_dict): """Runs echo classification for MYRORSS data. :param spc_date_string: See documentation at top of file. :param top_radar_dir_name_tarred: Same. :param top_radar_dir_name_untarred: Same. :param top_output_dir_name: Same. :param option_dict: See doc for `echo_classification.find_convective_pixels`. """ if top_radar_dir_name_tarred in ['', 'None']: top_radar_dir_name_tarred = None if top_radar_dir_name_tarred is not None: tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( top_radar_dir_name_tarred, spc_date_string[:4], spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=tar_file_name, field_names=[radar_utils.REFL_NAME], spc_date_string=spc_date_string, top_target_directory_name=top_radar_dir_name_untarred, refl_heights_m_asl=RADAR_HEIGHTS_M_ASL) print SEPARATOR_STRING these_file_names = myrorss_and_mrms_io.find_raw_files_one_spc_date( spc_date_string=spc_date_string, field_name=radar_utils.REFL_NAME, data_source=radar_utils.MYRORSS_SOURCE_ID, top_directory_name=top_radar_dir_name_untarred, height_m_asl=RADAR_HEIGHTS_M_ASL[0], raise_error_if_missing=True) valid_times_unix_sec = numpy.array([ myrorss_and_mrms_io.raw_file_name_to_time(f) for f in these_file_names ], dtype=int) valid_times_unix_sec = numpy.sort(valid_times_unix_sec) start_time_unix_sec = time_conversion.get_start_of_spc_date( spc_date_string) end_time_unix_sec = time_conversion.get_end_of_spc_date(spc_date_string) good_indices = numpy.where( numpy.logical_and(valid_times_unix_sec >= start_time_unix_sec, valid_times_unix_sec <= end_time_unix_sec))[0] valid_times_unix_sec = valid_times_unix_sec[good_indices] radar_file_dict = myrorss_and_mrms_io.find_many_raw_files( desired_times_unix_sec=valid_times_unix_sec, spc_date_strings=[spc_date_string] * len(valid_times_unix_sec), data_source=radar_utils.MYRORSS_SOURCE_ID, field_names=[radar_utils.REFL_NAME], top_directory_name=top_radar_dir_name_untarred, reflectivity_heights_m_asl=RADAR_HEIGHTS_M_ASL) radar_file_name_matrix = radar_file_dict[ myrorss_and_mrms_io.RADAR_FILE_NAMES_KEY] valid_times_unix_sec = radar_file_dict[ myrorss_and_mrms_io.UNIQUE_TIMES_KEY] num_times = len(valid_times_unix_sec) num_heights = len(RADAR_HEIGHTS_M_ASL) for i in range(num_times): reflectivity_matrix_dbz = None fine_grid_point_latitudes_deg = None fine_grid_point_longitudes_deg = None found_corrupt_file = False for j in range(num_heights): print 'Reading data from: "{0:s}"...'.format( radar_file_name_matrix[i, j]) this_metadata_dict = ( myrorss_and_mrms_io.read_metadata_from_raw_file( netcdf_file_name=radar_file_name_matrix[i, j], data_source=radar_utils.MYRORSS_SOURCE_ID, raise_error_if_fails=False)) if this_metadata_dict is None: found_corrupt_file = True break this_sparse_grid_table = ( myrorss_and_mrms_io.read_data_from_sparse_grid_file( netcdf_file_name=radar_file_name_matrix[i, j], field_name_orig=this_metadata_dict[ myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG], data_source=radar_utils.MYRORSS_SOURCE_ID, sentinel_values=this_metadata_dict[ radar_utils.SENTINEL_VALUE_COLUMN])) (this_refl_matrix_dbz, fine_grid_point_latitudes_deg, fine_grid_point_longitudes_deg) = radar_s2f.sparse_to_full_grid( sparse_grid_table=this_sparse_grid_table, metadata_dict=this_metadata_dict) this_refl_matrix_dbz = numpy.expand_dims( this_refl_matrix_dbz[::2, ::2], axis=-1) if reflectivity_matrix_dbz is None: reflectivity_matrix_dbz = this_refl_matrix_dbz + 0. else: reflectivity_matrix_dbz = numpy.concatenate( (reflectivity_matrix_dbz, this_refl_matrix_dbz), axis=-1) print '\n' if found_corrupt_file: continue reflectivity_matrix_dbz = numpy.flip(reflectivity_matrix_dbz, axis=0) fine_grid_point_latitudes_deg = fine_grid_point_latitudes_deg[::-1] coarse_grid_point_latitudes_deg = fine_grid_point_latitudes_deg[::2] coarse_grid_point_longitudes_deg = fine_grid_point_longitudes_deg[::2] coarse_grid_metadata_dict = { echo_classifn.MIN_LATITUDE_KEY: numpy.min(coarse_grid_point_latitudes_deg), echo_classifn.LATITUDE_SPACING_KEY: (coarse_grid_point_latitudes_deg[1] - coarse_grid_point_latitudes_deg[0]), echo_classifn.MIN_LONGITUDE_KEY: numpy.min(coarse_grid_point_longitudes_deg), echo_classifn.LONGITUDE_SPACING_KEY: (coarse_grid_point_longitudes_deg[1] - coarse_grid_point_longitudes_deg[0]), echo_classifn.HEIGHTS_KEY: RADAR_HEIGHTS_M_ASL } fine_grid_metadata_dict = { echo_classifn.MIN_LATITUDE_KEY: numpy.min(fine_grid_point_latitudes_deg), echo_classifn.LATITUDE_SPACING_KEY: (fine_grid_point_latitudes_deg[1] - fine_grid_point_latitudes_deg[0]), echo_classifn.MIN_LONGITUDE_KEY: numpy.min(fine_grid_point_longitudes_deg), echo_classifn.LONGITUDE_SPACING_KEY: (fine_grid_point_longitudes_deg[1] - fine_grid_point_longitudes_deg[0]), echo_classifn.HEIGHTS_KEY: RADAR_HEIGHTS_M_ASL } convective_flag_matrix = echo_classifn.find_convective_pixels( reflectivity_matrix_dbz=reflectivity_matrix_dbz, grid_metadata_dict=coarse_grid_metadata_dict, valid_time_unix_sec=valid_times_unix_sec[i], option_dict=option_dict) print 'Number of convective pixels = {0:d}\n'.format( numpy.sum(convective_flag_matrix)) convective_flag_matrix = echo_classifn._double_class_resolution( coarse_convective_flag_matrix=convective_flag_matrix, coarse_grid_point_latitudes_deg=coarse_grid_point_latitudes_deg, coarse_grid_point_longitudes_deg=coarse_grid_point_longitudes_deg, fine_grid_point_latitudes_deg=fine_grid_point_latitudes_deg, fine_grid_point_longitudes_deg=fine_grid_point_longitudes_deg) this_output_file_name = echo_classifn.find_classification_file( top_directory_name=top_output_dir_name, valid_time_unix_sec=valid_times_unix_sec[i], desire_zipped=False, allow_zipped_or_unzipped=False, raise_error_if_missing=False) print 'Writing echo classifications to: "{0:s}"...'.format( this_output_file_name) echo_classifn.write_classifications( convective_flag_matrix=convective_flag_matrix, grid_metadata_dict=fine_grid_metadata_dict, valid_time_unix_sec=valid_times_unix_sec[i], option_dict=option_dict, netcdf_file_name=this_output_file_name) unzipping.gzip_file(input_file_name=this_output_file_name, delete_input_file=True) print SEPARATOR_STRING if top_radar_dir_name_tarred is None: return myrorss_io.remove_unzipped_data_1day( spc_date_string=spc_date_string, top_directory_name=top_radar_dir_name_untarred, field_names=[radar_utils.REFL_NAME], refl_heights_m_asl=RADAR_HEIGHTS_M_ASL) print SEPARATOR_STRING
def _extract_storm_images( num_image_rows, num_image_columns, rotate_grids, rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl, spc_date_string, first_time_string, last_time_string, tarred_myrorss_dir_name, untarred_myrorss_dir_name, top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2, target_name, top_target_dir_name, top_output_dir_name): """Extracts storm-centered img for each field/height pair and storm object. :param num_image_rows: See documentation at top of file. :param num_image_columns: Same. :param rotate_grids: Same. :param rotated_grid_spacing_metres: Same. :param radar_field_names: Same. :param refl_heights_m_agl: Same. :param spc_date_string: Same. :param first_time_string: Same. :param last_time_string: Same. :param tarred_myrorss_dir_name: Same. :param untarred_myrorss_dir_name: Same. :param top_tracking_dir_name: Same. :param elevation_dir_name: Same. :param tracking_scale_metres2: Same. :param target_name: Same. :param top_target_dir_name: Same. :param top_output_dir_name: Same. :raises: ValueError: if `first_time_string` and `last_time_string` have different SPC dates. """ if elevation_dir_name in ['', 'None']: elevation_dir_name = None if elevation_dir_name is None: host_name = socket.gethostname() if 'casper' in host_name: elevation_dir_name = '/glade/work/ryanlage/elevation' else: elevation_dir_name = '/condo/swatwork/ralager/elevation' if spc_date_string in ['', 'None']: first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, TIME_FORMAT) first_spc_date_string = time_conversion.time_to_spc_date_string( first_time_unix_sec) last_spc_date_string = time_conversion.time_to_spc_date_string( last_time_unix_sec) if first_spc_date_string != last_spc_date_string: error_string = ( 'First ({0:s}) and last ({1:s}) times have different SPC dates.' ' This script can handle only one SPC date.' ).format(first_time_string, last_time_string) raise ValueError(error_string) spc_date_string = first_spc_date_string else: first_time_unix_sec = 0 last_time_unix_sec = int(1e12) if tarred_myrorss_dir_name in ['', 'None']: tarred_myrorss_dir_name = None if target_name in ['', 'None']: target_name = None if target_name is not None: target_param_dict = target_val_utils.target_name_to_params(target_name) target_file_name = target_val_utils.find_target_file( top_directory_name=top_target_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=spc_date_string) print('Reading data from: "{0:s}"...'.format(target_file_name)) target_dict = target_val_utils.read_target_values( netcdf_file_name=target_file_name, target_names=[target_name] ) print('\n') refl_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) # Untar files. if tarred_myrorss_dir_name is not None: az_shear_field_names = list( set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES) ) if len(az_shear_field_names) > 0: az_shear_tar_file_name = ( '{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar' ).format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string ) myrorss_io.unzip_1day_tar_file( tar_file_name=az_shear_tar_file_name, field_names=az_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name) print(SEPARATOR_STRING) non_shear_field_names = list( set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES) ) if len(non_shear_field_names) > 0: non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string ) myrorss_io.unzip_1day_tar_file( tar_file_name=non_shear_tar_file_name, field_names=non_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING) # Read storm tracks for the given SPC date. tracking_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_string, source_name=tracking_utils.SEGMOTION_NAME, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2 )[0] file_times_unix_sec = numpy.array( [tracking_io.file_name_to_time(f) for f in tracking_file_names], dtype=int ) good_indices = numpy.where(numpy.logical_and( file_times_unix_sec >= first_time_unix_sec, file_times_unix_sec <= last_time_unix_sec ))[0] tracking_file_names = [tracking_file_names[k] for k in good_indices] storm_object_table = tracking_io.read_many_files( tracking_file_names )[storm_images.STORM_COLUMNS_NEEDED] print(SEPARATOR_STRING) if target_name is not None: print(( 'Removing storm objects without target values (variable = ' '"{0:s}")...' ).format(target_name)) these_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values.astype(int), id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY], times_to_keep_unix_sec=target_dict[ target_val_utils.VALID_TIMES_KEY], allow_missing=False) num_storm_objects_orig = len(storm_object_table.index) storm_object_table = storm_object_table.iloc[these_indices] num_storm_objects = len(storm_object_table.index) print('Removed {0:d} of {1:d} storm objects!\n'.format( num_storm_objects_orig - num_storm_objects, num_storm_objects_orig )) # Extract storm-centered radar images. storm_images.extract_storm_images_myrorss_or_mrms( storm_object_table=storm_object_table, radar_source=radar_utils.MYRORSS_SOURCE_ID, top_radar_dir_name=untarred_myrorss_dir_name, top_output_dir_name=top_output_dir_name, elevation_dir_name=elevation_dir_name, num_storm_image_rows=num_image_rows, num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids, rotated_grid_spacing_metres=rotated_grid_spacing_metres, radar_field_names=radar_field_names, reflectivity_heights_m_agl=refl_heights_m_agl) print(SEPARATOR_STRING) # Remove untarred MYRORSS files. if tarred_myrorss_dir_name is not None: myrorss_io.remove_unzipped_data_1day( spc_date_string=spc_date_string, top_directory_name=untarred_myrorss_dir_name, field_names=radar_field_names, refl_heights_m_asl=refl_heights_m_asl)
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids, rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl, spc_date_string, tarred_myrorss_dir_name, untarred_myrorss_dir_name, top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2, target_name, top_target_dir_name, top_output_dir_name): """Extracts storm-centered img for each field/height pair and storm object. :param num_image_rows: See documentation at top of file. :param num_image_columns: Same. :param rotate_grids: Same. :param rotated_grid_spacing_metres: Same. :param radar_field_names: Same. :param refl_heights_m_agl: Same. :param spc_date_string: Same. :param tarred_myrorss_dir_name: Same. :param untarred_myrorss_dir_name: Same. :param top_tracking_dir_name: Same. :param elevation_dir_name: Same. :param tracking_scale_metres2: Same. :param target_name: Same. :param top_target_dir_name: Same. :param top_output_dir_name: Same. """ if target_name in ['', 'None']: target_name = None if target_name is not None: target_param_dict = target_val_utils.target_name_to_params(target_name) target_file_name = target_val_utils.find_target_file( top_directory_name=top_target_dir_name, event_type_string=target_param_dict[ target_val_utils.EVENT_TYPE_KEY], spc_date_string=spc_date_string) print('Reading data from: "{0:s}"...'.format(target_file_name)) target_dict = target_val_utils.read_target_values( netcdf_file_name=target_file_name, target_names=[target_name]) print('\n') refl_heights_m_asl = radar_utils.get_valid_heights( data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME) # Untar files with azimuthal shear. az_shear_field_names = list( set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES)) if len(az_shear_field_names): az_shear_tar_file_name = ( '{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar').format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=az_shear_tar_file_name, field_names=az_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name) print(SEPARATOR_STRING) # Untar files with other radar fields. non_shear_field_names = list( set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES)) if len(non_shear_field_names): non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=non_shear_tar_file_name, field_names=non_shear_field_names, spc_date_string=spc_date_string, top_target_directory_name=untarred_myrorss_dir_name, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING) # Read storm tracks for the given SPC date. tracking_file_names = tracking_io.find_files_one_spc_date( spc_date_string=spc_date_string, source_name=tracking_utils.SEGMOTION_NAME, top_tracking_dir_name=top_tracking_dir_name, tracking_scale_metres2=tracking_scale_metres2)[0] storm_object_table = tracking_io.read_many_files(tracking_file_names)[ storm_images.STORM_COLUMNS_NEEDED] print(SEPARATOR_STRING) if target_name is not None: print(('Removing storm objects without target values (variable = ' '"{0:s}")...').format(target_name)) these_indices = tracking_utils.find_storm_objects( all_id_strings=storm_object_table[ tracking_utils.FULL_ID_COLUMN].values.tolist(), all_times_unix_sec=storm_object_table[ tracking_utils.VALID_TIME_COLUMN].values.astype(int), id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY], times_to_keep_unix_sec=target_dict[ target_val_utils.VALID_TIMES_KEY], allow_missing=False) num_storm_objects_orig = len(storm_object_table.index) storm_object_table = storm_object_table.iloc[these_indices] num_storm_objects = len(storm_object_table.index) print('Removed {0:d} of {1:d} storm objects!\n'.format( num_storm_objects_orig - num_storm_objects, num_storm_objects_orig)) # Extract storm-centered radar images. storm_images.extract_storm_images_myrorss_or_mrms( storm_object_table=storm_object_table, radar_source=radar_utils.MYRORSS_SOURCE_ID, top_radar_dir_name=untarred_myrorss_dir_name, top_output_dir_name=top_output_dir_name, elevation_dir_name=elevation_dir_name, num_storm_image_rows=num_image_rows, num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids, rotated_grid_spacing_metres=rotated_grid_spacing_metres, radar_field_names=radar_field_names, reflectivity_heights_m_agl=refl_heights_m_agl) print(SEPARATOR_STRING) # Remove untarred MYRORSS files. myrorss_io.remove_unzipped_data_1day( spc_date_string=spc_date_string, top_directory_name=untarred_myrorss_dir_name, field_names=radar_field_names, refl_heights_m_asl=refl_heights_m_asl) print(SEPARATOR_STRING)
def _run(top_radar_dir_name, top_radar_dir_name_tarred, top_echo_classifn_dir_name, first_spc_date_string, last_spc_date_string, echo_top_field_name, min_echo_top_km, min_size_pixels, min_intermax_distance_metres, max_velocity_diff_m_s01, max_link_distance_m_s01, top_output_dir_name): """Tracks storms based on echo top. This is effectively the main method. :param top_radar_dir_name: See documentation at top of file. :param top_radar_dir_name_tarred: Same. :param top_echo_classifn_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param echo_top_field_name: Same. :param min_echo_top_km: Same. :param min_size_pixels: Same. :param min_intermax_distance_metres: Same. :param max_velocity_diff_m_s01: Same. :param max_link_distance_m_s01: Same. :param top_output_dir_name: Same. """ if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: spc_date_strings = time_conversion.get_spc_dates_in_range( first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string) for this_spc_date_string in spc_date_strings: this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format( top_radar_dir_name_tarred, this_spc_date_string[:4], this_spc_date_string) myrorss_io.unzip_1day_tar_file( tar_file_name=this_tar_file_name, field_names=[echo_top_field_name], spc_date_string=this_spc_date_string, top_target_directory_name=top_radar_dir_name) print(SEPARATOR_STRING) if top_echo_classifn_dir_name in ['', 'None']: top_echo_classifn_dir_name = None echo_top_tracking.run_tracking( top_radar_dir_name=top_radar_dir_name, top_output_dir_name=top_output_dir_name, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, echo_top_field_name=echo_top_field_name, top_echo_classifn_dir_name=top_echo_classifn_dir_name, min_echo_top_km=min_echo_top_km, min_intermax_distance_metres=min_intermax_distance_metres, min_polygon_size_pixels=min_size_pixels, max_velocity_diff_m_s01=max_velocity_diff_m_s01, max_link_distance_m_s01=max_link_distance_m_s01, min_track_duration_seconds=0) print(SEPARATOR_STRING) if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES: for this_spc_date_string in spc_date_strings: myrorss_io.remove_unzipped_data_1day( spc_date_string=this_spc_date_string, top_directory_name=top_radar_dir_name, field_names=[echo_top_field_name] )