def _run(top_narr_dir_name, top_front_line_dir_name, top_wpc_bulletin_dir_name, first_time_string, last_time_string, pressure_level_mb, thermal_field_name, thermal_colour_map_name, max_thermal_prctile_for_colours, first_letter_label, letter_interval, output_dir_name): """Plots predictors on full NARR grid. This is effectively the main method. :param top_narr_dir_name: See documentation at top of file. :param top_front_line_dir_name: Same. :param top_wpc_bulletin_dir_name: Same. :param first_time_string: Same. :param last_time_string: Same. :param pressure_level_mb: Same. :param thermal_field_name: Same. :param thermal_colour_map_name: Same. :param max_thermal_prctile_for_colours: Same. :param first_letter_label: Same. :param letter_interval: Same. :param output_dir_name: Same. :raises: ValueError: if `thermal_field_name not in VALID_THERMAL_FIELD_NAMES`. """ # Check input args. if top_wpc_bulletin_dir_name in ['', 'None']: top_wpc_bulletin_dir_name = None if first_letter_label in ['', 'None']: first_letter_label = None if thermal_field_name not in VALID_THERMAL_FIELD_NAMES: error_string = ( '\n{0:s}\nValid thermal fields (listed above) do not include ' '"{1:s}".' ).format(str(VALID_THERMAL_FIELD_NAMES), thermal_field_name) raise ValueError(error_string) thermal_colour_map_object = pyplot.cm.get_cmap(thermal_colour_map_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, DEFAULT_TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, DEFAULT_TIME_FORMAT) valid_times_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=first_time_unix_sec, end_time_unix_sec=last_time_unix_sec, time_interval_sec=NARR_TIME_INTERVAL_SEC, include_endpoint=True) # Read metadata for NARR grid. narr_latitude_matrix_deg, narr_longitude_matrix_deg = ( nwp_model_utils.get_latlng_grid_point_matrices( model_name=nwp_model_utils.NARR_MODEL_NAME) ) narr_rotation_cos_matrix, narr_rotation_sin_matrix = ( nwp_model_utils.get_wind_rotation_angles( latitudes_deg=narr_latitude_matrix_deg, longitudes_deg=narr_longitude_matrix_deg, model_name=nwp_model_utils.NARR_MODEL_NAME) ) narr_row_limits, narr_column_limits = ( nwp_plotting.latlng_limits_to_rowcol_limits( min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG, min_longitude_deg=MIN_LONGITUDE_DEG, max_longitude_deg=MAX_LONGITUDE_DEG, model_name=nwp_model_utils.NARR_MODEL_NAME) ) narr_rotation_cos_matrix = narr_rotation_cos_matrix[ narr_row_limits[0]:(narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1) ] narr_rotation_sin_matrix = narr_rotation_sin_matrix[ narr_row_limits[0]:(narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1) ] # Do plotting. narr_field_names = [ processed_narr_io.U_WIND_GRID_RELATIVE_NAME, processed_narr_io.V_WIND_GRID_RELATIVE_NAME, thermal_field_name ] this_letter_label = None for this_time_unix_sec in valid_times_unix_sec: this_file_name = fronts_io.find_file_for_one_time( top_directory_name=top_front_line_dir_name, file_type=fronts_io.POLYLINE_FILE_TYPE, valid_time_unix_sec=this_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(this_file_name) this_polyline_table = fronts_io.read_polylines_from_file(this_file_name) if top_wpc_bulletin_dir_name is None: this_high_low_table = None else: this_file_name = wpc_bulletin_io.find_file( top_directory_name=top_wpc_bulletin_dir_name, valid_time_unix_sec=this_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(this_file_name) this_high_low_table = wpc_bulletin_io.read_highs_and_lows( this_file_name) this_predictor_matrix = None for this_field_name in narr_field_names: this_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_dir_name, field_name=this_field_name, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=this_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(this_file_name) this_field_matrix = processed_narr_io.read_fields_from_file( this_file_name )[0][0, ...] this_field_matrix = utils.fill_nans(this_field_matrix) this_field_matrix = this_field_matrix[ narr_row_limits[0]:(narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1) ] if this_field_name in [processed_narr_io.TEMPERATURE_NAME, processed_narr_io.WET_BULB_THETA_NAME]: this_field_matrix -= ZERO_CELSIUS_IN_KELVINS if this_field_name == processed_narr_io.SPECIFIC_HUMIDITY_NAME: this_field_matrix = this_field_matrix * KG_TO_GRAMS this_field_matrix = numpy.expand_dims(this_field_matrix, axis=-1) if this_predictor_matrix is None: this_predictor_matrix = this_field_matrix + 0. else: this_predictor_matrix = numpy.concatenate( (this_predictor_matrix, this_field_matrix), axis=-1) u_wind_index = narr_field_names.index( processed_narr_io.U_WIND_GRID_RELATIVE_NAME) v_wind_index = narr_field_names.index( processed_narr_io.V_WIND_GRID_RELATIVE_NAME) (this_predictor_matrix[..., u_wind_index], this_predictor_matrix[..., v_wind_index] ) = nwp_model_utils.rotate_winds_to_earth_relative( u_winds_grid_relative_m_s01=this_predictor_matrix[ ..., u_wind_index], v_winds_grid_relative_m_s01=this_predictor_matrix[ ..., v_wind_index], rotation_angle_cosines=narr_rotation_cos_matrix, rotation_angle_sines=narr_rotation_sin_matrix) this_title_string = time_conversion.unix_sec_to_string( this_time_unix_sec, NICE_TIME_FORMAT) if pressure_level_mb == 1013: this_title_string += ' at surface' else: this_title_string += ' at {0:d} mb'.format(pressure_level_mb) this_default_time_string = time_conversion.unix_sec_to_string( this_time_unix_sec, DEFAULT_TIME_FORMAT) this_output_file_name = '{0:s}/predictors_{1:s}.jpg'.format( output_dir_name, this_default_time_string) if first_letter_label is not None: if this_letter_label is None: this_letter_label = first_letter_label else: this_letter_label = chr( ord(this_letter_label) + letter_interval ) _plot_one_time( predictor_matrix=this_predictor_matrix, predictor_names=narr_field_names, front_polyline_table=this_polyline_table, high_low_table=this_high_low_table, thermal_colour_map_object=thermal_colour_map_object, max_thermal_prctile_for_colours=max_thermal_prctile_for_colours, narr_row_limits=narr_row_limits, narr_column_limits=narr_column_limits, title_string=this_title_string, letter_label=this_letter_label, output_file_name=this_output_file_name) print '\n'
def _run(valid_time_string, smoothing_radius_pixels, front_percentile, num_closing_iters, pressure_level_mb, top_narr_directory_name, narr_mask_file_name, output_dir_name): """Plots NFA (numerical frontal analysis) procedure. This is effectively the main method. :param valid_time_string: See documentation at top of file. :param smoothing_radius_pixels: Same. :param front_percentile: Same. :param num_closing_iters: Same. :param pressure_level_mb: Same. :param top_narr_directory_name: Same. :param narr_mask_file_name: Same. :param output_dir_name: Same. """ cutoff_radius_pixels = 4 * smoothing_radius_pixels valid_time_unix_sec = time_conversion.string_to_unix_sec( valid_time_string, INPUT_TIME_FORMAT) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) if narr_mask_file_name == '': num_grid_rows, num_grid_columns = nwp_model_utils.get_grid_dimensions( model_name=nwp_model_utils.NARR_MODEL_NAME) narr_mask_matrix = numpy.full( (num_grid_rows, num_grid_columns), 1, dtype=int) else: print 'Reading mask from: "{0:s}"...\n'.format(narr_mask_file_name) narr_mask_matrix = ml_utils.read_narr_mask(narr_mask_file_name) wet_bulb_theta_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_directory_name, field_name=processed_narr_io.WET_BULB_THETA_NAME, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(wet_bulb_theta_file_name) wet_bulb_theta_matrix_kelvins = processed_narr_io.read_fields_from_file( wet_bulb_theta_file_name)[0][0, ...] wet_bulb_theta_matrix_kelvins = general_utils.fill_nans( wet_bulb_theta_matrix_kelvins) u_wind_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_directory_name, field_name=processed_narr_io.U_WIND_GRID_RELATIVE_NAME, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(u_wind_file_name) u_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file( u_wind_file_name)[0][0, ...] u_wind_matrix_m_s01 = general_utils.fill_nans(u_wind_matrix_m_s01) v_wind_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_directory_name, field_name=processed_narr_io.V_WIND_GRID_RELATIVE_NAME, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(v_wind_file_name) v_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file( v_wind_file_name)[0][0, ...] v_wind_matrix_m_s01 = general_utils.fill_nans(v_wind_matrix_m_s01) unsmoothed_narr_file_name = '{0:s}/unsmoothed_narr_fields.jpg'.format( output_dir_name) _plot_narr_fields( wet_bulb_theta_matrix_kelvins=wet_bulb_theta_matrix_kelvins, u_wind_matrix_m_s01=u_wind_matrix_m_s01, v_wind_matrix_m_s01=v_wind_matrix_m_s01, title_string='Predictors before smoothing', annotation_string='(a)', output_file_name=unsmoothed_narr_file_name) wet_bulb_theta_matrix_kelvins = nfa.gaussian_smooth_2d_field( field_matrix=wet_bulb_theta_matrix_kelvins, standard_deviation_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels) u_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field( field_matrix=u_wind_matrix_m_s01, standard_deviation_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels) v_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field( field_matrix=v_wind_matrix_m_s01, standard_deviation_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels) smoothed_narr_file_name = '{0:s}/smoothed_narr_fields.jpg'.format( output_dir_name) _plot_narr_fields( wet_bulb_theta_matrix_kelvins=wet_bulb_theta_matrix_kelvins, u_wind_matrix_m_s01=u_wind_matrix_m_s01, v_wind_matrix_m_s01=v_wind_matrix_m_s01, title_string='Predictors after smoothing', annotation_string='(b)', output_file_name=smoothed_narr_file_name) x_spacing_metres, y_spacing_metres = nwp_model_utils.get_xy_grid_spacing( model_name=nwp_model_utils.NARR_MODEL_NAME) tfp_matrix_kelvins_m02 = nfa.get_thermal_front_param( thermal_field_matrix_kelvins=wet_bulb_theta_matrix_kelvins, x_spacing_metres=x_spacing_metres, y_spacing_metres=y_spacing_metres) tfp_matrix_kelvins_m02[narr_mask_matrix == 0] = 0. tfp_file_name = '{0:s}/tfp.jpg'.format(output_dir_name) tfp_title_string = ( r'Thermal front parameter ($\times$ 10$^{-10}$ K m$^{-2}$)') _plot_tfp(tfp_matrix_kelvins_m02=tfp_matrix_kelvins_m02, title_string=tfp_title_string, annotation_string='(c)', output_file_name=tfp_file_name) proj_velocity_matrix_m_s01 = nfa.project_wind_to_thermal_gradient( u_matrix_grid_relative_m_s01=u_wind_matrix_m_s01, v_matrix_grid_relative_m_s01=v_wind_matrix_m_s01, thermal_field_matrix_kelvins=wet_bulb_theta_matrix_kelvins, x_spacing_metres=x_spacing_metres, y_spacing_metres=y_spacing_metres) locating_var_matrix_m01_s01 = nfa.get_locating_variable( tfp_matrix_kelvins_m02=tfp_matrix_kelvins_m02, projected_velocity_matrix_m_s01=proj_velocity_matrix_m_s01) locating_var_file_name = '{0:s}/locating_variable.jpg'.format( output_dir_name) locating_var_title_string = ( r'Locating variable ($\times$ 10$^{-9}$ K m$^{-1}$ s$^{-1}$)') _plot_locating_variable( locating_var_matrix_m01_s01=locating_var_matrix_m01_s01, title_string=locating_var_title_string, annotation_string='(d)', output_file_name=locating_var_file_name) predicted_label_matrix = nfa.get_front_types( locating_var_matrix_m01_s01=locating_var_matrix_m01_s01, warm_front_percentile=front_percentile, cold_front_percentile=front_percentile) unclosed_fronts_file_name = '{0:s}/unclosed_fronts.jpg'.format( output_dir_name) _plot_front_types( predicted_label_matrix=predicted_label_matrix, title_string='Frontal regions before closing', annotation_string='(e)', output_file_name=unclosed_fronts_file_name) predicted_label_matrix = front_utils.close_frontal_image( ternary_image_matrix=predicted_label_matrix, num_iterations=num_closing_iters) closed_fronts_file_name = '{0:s}/closed_fronts.jpg'.format(output_dir_name) _plot_front_types( predicted_label_matrix=predicted_label_matrix, title_string='Frontal regions after closing', annotation_string='(f)', output_file_name=closed_fronts_file_name) concat_file_name = '{0:s}/nfa_procedure.jpg'.format(output_dir_name) print 'Concatenating figures to: "{0:s}"...'.format(concat_file_name) panel_file_names = [ unsmoothed_narr_file_name, smoothed_narr_file_name, tfp_file_name, locating_var_file_name, unclosed_fronts_file_name, closed_fronts_file_name ] imagemagick_utils.concatenate_images( input_file_names=panel_file_names, output_file_name=concat_file_name, num_panel_rows=3, num_panel_columns=2) imagemagick_utils.resize_image( input_file_name=concat_file_name, output_file_name=concat_file_name, output_size_pixels=CONCAT_SIZE_PIXELS)
def _run(): """Plots input example. This is effectively the main method. :return: figure_file_name: Path to output file (where the figure was saved). """ valid_time_unix_sec = time_conversion.string_to_unix_sec( VALID_TIME_STRING, TIME_FORMAT) front_file_name = fronts_io.find_file_for_one_time( top_directory_name=TOP_FRONT_DIR_NAME, file_type=fronts_io.POLYLINE_FILE_TYPE, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(front_file_name) front_line_table = fronts_io.read_polylines_from_file(front_file_name) num_narr_fields = len(NARR_FIELD_NAMES) narr_matrix_by_field = [numpy.array([])] * num_narr_fields for j in range(num_narr_fields): if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES: this_directory_name = '{0:s}/earth_relative_wind'.format( TOP_NARR_DIRECTORY_NAME) else: this_directory_name = TOP_NARR_DIRECTORY_NAME + '' this_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=this_directory_name, field_name=NARR_FIELD_NAMES[j], pressure_level_mb=PRESSURE_LEVEL_MB, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(this_file_name) narr_matrix_by_field[j] = processed_narr_io.read_fields_from_file( this_file_name)[0][0, ...] narr_matrix_by_field[j] = utils.fill_nans(narr_matrix_by_field[j]) if NARR_FIELD_NAMES[j] == processed_narr_io.WET_BULB_THETA_NAME: narr_matrix_by_field[j] = (narr_matrix_by_field[j] - ZERO_CELSIUS_IN_KELVINS) # (_, front_centroid_latitude_deg, front_centroid_longitude_deg # ) = _find_nearest_front( # front_line_table=front_line_table, # query_latitude_deg=APPROX_FRONT_LATITUDE_DEG, # query_longitude_deg=APPROX_FRONT_LONGITUDE_DEG) front_centroid_latitude_deg = APPROX_FRONT_LATITUDE_DEG + 0. front_centroid_longitude_deg = APPROX_FRONT_LONGITUDE_DEG + 0. projection_object = nwp_model_utils.init_model_projection( nwp_model_utils.NARR_MODEL_NAME) these_x_metres, these_y_metres = nwp_model_utils.project_latlng_to_xy( latitudes_deg=numpy.array([front_centroid_latitude_deg]), longitudes_deg=numpy.array([front_centroid_longitude_deg]), projection_object=projection_object, model_name=nwp_model_utils.NARR_MODEL_NAME) front_centroid_x_metres = these_x_metres[0] front_centroid_y_metres = these_y_metres[0] grid_spacing_metres, _ = nwp_model_utils.get_xy_grid_spacing( model_name=nwp_model_utils.NARR_MODEL_NAME) center_narr_row_index = int( numpy.round(front_centroid_y_metres / grid_spacing_metres)) center_narr_column_index = int( numpy.round(front_centroid_x_metres / grid_spacing_metres)) first_narr_row_index = center_narr_row_index - NUM_ROWS_IN_HALF_GRID last_narr_row_index = center_narr_row_index + NUM_ROWS_IN_HALF_GRID first_narr_column_index = (center_narr_column_index - NUM_COLUMNS_IN_HALF_GRID) last_narr_column_index = center_narr_column_index + NUM_COLUMNS_IN_HALF_GRID for j in range(num_narr_fields): narr_matrix_by_field[j] = narr_matrix_by_field[j][ first_narr_row_index:(last_narr_row_index + 1), first_narr_column_index:(last_narr_column_index + 1)] _, axes_object, basemap_object = nwp_plotting.init_basemap( model_name=nwp_model_utils.NARR_MODEL_NAME, first_row_in_full_grid=first_narr_row_index, last_row_in_full_grid=last_narr_row_index, first_column_in_full_grid=first_narr_column_index, last_column_in_full_grid=last_narr_column_index, resolution_string='i') plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, bottom_left_lat_deg=-90., upper_right_lat_deg=90., parallel_spacing_deg=PARALLEL_SPACING_DEG) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, bottom_left_lng_deg=0., upper_right_lng_deg=360., meridian_spacing_deg=MERIDIAN_SPACING_DEG) for j in range(num_narr_fields): if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES: continue min_colour_value = numpy.percentile(narr_matrix_by_field[j], MIN_COLOUR_PERCENTILE) max_colour_value = numpy.percentile(narr_matrix_by_field[j], MAX_COLOUR_PERCENTILE) nwp_plotting.plot_subgrid( field_matrix=narr_matrix_by_field[j], model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object, basemap_object=basemap_object, colour_map=THERMAL_COLOUR_MAP_OBJECT, min_value_in_colour_map=min_colour_value, max_value_in_colour_map=max_colour_value, first_row_in_full_grid=first_narr_row_index, first_column_in_full_grid=first_narr_column_index) plotting_utils.add_linear_colour_bar( axes_object_or_list=axes_object, values_to_colour=narr_matrix_by_field[j], colour_map=THERMAL_COLOUR_MAP_OBJECT, colour_min=min_colour_value, colour_max=max_colour_value, orientation='horizontal', extend_min=True, extend_max=True) u_wind_index = NARR_FIELD_NAMES.index( processed_narr_io.U_WIND_EARTH_RELATIVE_NAME) v_wind_index = NARR_FIELD_NAMES.index( processed_narr_io.V_WIND_EARTH_RELATIVE_NAME) nwp_plotting.plot_wind_barbs_on_subgrid( u_wind_matrix_m_s01=narr_matrix_by_field[u_wind_index], v_wind_matrix_m_s01=narr_matrix_by_field[v_wind_index], model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object, basemap_object=basemap_object, first_row_in_full_grid=first_narr_row_index, first_column_in_full_grid=first_narr_column_index, plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB, plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB, barb_length=WIND_BARB_LENGTH, empty_barb_radius=EMPTY_WIND_BARB_RADIUS, colour_map=WIND_COLOUR_MAP_OBJECT, colour_minimum_kt=MIN_COLOUR_WIND_SPEED_KT, colour_maximum_kt=MAX_COLOUR_WIND_SPEED_KT) num_fronts = len(front_line_table.index) for i in range(num_fronts): this_front_type_string = front_line_table[ front_utils.FRONT_TYPE_COLUMN].values[i] if this_front_type_string == front_utils.WARM_FRONT_STRING_ID: this_colour = WARM_FRONT_COLOUR else: this_colour = COLD_FRONT_COLOUR # front_plotting.plot_polyline( # latitudes_deg=front_line_table[ # front_utils.LATITUDES_COLUMN].values[i], # longitudes_deg=front_line_table[ # front_utils.LONGITUDES_COLUMN].values[i], # basemap_object=basemap_object, axes_object=axes_object, # front_type=front_line_table[ # front_utils.FRONT_TYPE_COLUMN].values[i], # line_width=FRONT_LINE_WIDTH, line_colour=this_colour) print 'Saving figure to: "{0:s}"...'.format(OUTPUT_FILE_NAME) file_system_utils.mkdir_recursive_if_necessary(file_name=OUTPUT_FILE_NAME) pyplot.savefig(OUTPUT_FILE_NAME, dpi=OUTPUT_RESOLUTION_DPI) pyplot.close() imagemagick_utils.trim_whitespace(input_file_name=OUTPUT_FILE_NAME, output_file_name=OUTPUT_FILE_NAME)
def _plot_observations_one_time( valid_time_string, title_string, annotation_string, output_file_name): """Plots observations (NARR predictors and WPC fronts) for one valid time. :param valid_time_string: Valid time (format "yyyy-mm-dd-HH"). :param title_string: Title (will be placed above figure). :param annotation_string: Text annotation (will be placed in top left of figure). :param output_file_name: Path to output file (figure will be saved here). """ (narr_row_limits, narr_column_limits ) = nwp_plotting.latlng_limits_to_rowcol_limits( min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG, min_longitude_deg=MIN_LONGITUDE_DEG, max_longitude_deg=MAX_LONGITUDE_DEG, model_name=nwp_model_utils.NARR_MODEL_NAME) valid_time_unix_sec = time_conversion.string_to_unix_sec( valid_time_string, INPUT_TIME_FORMAT) front_file_name = fronts_io.find_file_for_one_time( top_directory_name=TOP_FRONT_DIR_NAME, file_type=fronts_io.POLYLINE_FILE_TYPE, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(front_file_name) front_line_table = fronts_io.read_polylines_from_file(front_file_name) num_narr_fields = len(NARR_FIELD_NAMES) narr_matrix_by_field = [numpy.array([])] * num_narr_fields for j in range(num_narr_fields): if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES: this_directory_name = TOP_NARR_WIND_DIR_NAME + '' else: this_directory_name = TOP_NARR_DIR_NAME + '' this_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=this_directory_name, field_name=NARR_FIELD_NAMES[j], pressure_level_mb=PRESSURE_LEVEL_MB, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(this_file_name) narr_matrix_by_field[j] = processed_narr_io.read_fields_from_file( this_file_name)[0][0, ...] narr_matrix_by_field[j] = utils.fill_nans(narr_matrix_by_field[j]) narr_matrix_by_field[j] = narr_matrix_by_field[j][ narr_row_limits[0]:(narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1) ] if NARR_FIELD_NAMES[j] == processed_narr_io.WET_BULB_THETA_NAME: narr_matrix_by_field[j] = ( narr_matrix_by_field[j] - ZERO_CELSIUS_IN_KELVINS ) _, axes_object, basemap_object = nwp_plotting.init_basemap( model_name=nwp_model_utils.NARR_MODEL_NAME, first_row_in_full_grid=narr_row_limits[0], last_row_in_full_grid=narr_row_limits[1], first_column_in_full_grid=narr_column_limits[0], last_column_in_full_grid=narr_column_limits[1]) plotting_utils.plot_coastlines( basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries( basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces( basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels( basemap_object=basemap_object, axes_object=axes_object, bottom_left_lat_deg=-90., upper_right_lat_deg=90., parallel_spacing_deg=PARALLEL_SPACING_DEG) plotting_utils.plot_meridians( basemap_object=basemap_object, axes_object=axes_object, bottom_left_lng_deg=0., upper_right_lng_deg=360., meridian_spacing_deg=MERIDIAN_SPACING_DEG) for j in range(num_narr_fields): if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES: continue min_colour_value = numpy.percentile( narr_matrix_by_field[j], MIN_COLOUR_PERCENTILE) max_colour_value = numpy.percentile( narr_matrix_by_field[j], MAX_COLOUR_PERCENTILE) nwp_plotting.plot_subgrid( field_matrix=narr_matrix_by_field[j], model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object, basemap_object=basemap_object, colour_map=THERMAL_COLOUR_MAP_OBJECT, min_value_in_colour_map=min_colour_value, max_value_in_colour_map=max_colour_value, first_row_in_full_grid=narr_row_limits[0], first_column_in_full_grid=narr_column_limits[0]) plotting_utils.add_linear_colour_bar( axes_object_or_list=axes_object, values_to_colour=narr_matrix_by_field[j], colour_map=THERMAL_COLOUR_MAP_OBJECT, colour_min=min_colour_value, colour_max=max_colour_value, orientation='vertical', extend_min=True, extend_max=True, fraction_of_axis_length=LENGTH_FRACTION_FOR_THETA_COLOUR_BAR) u_wind_index = NARR_FIELD_NAMES.index( processed_narr_io.U_WIND_EARTH_RELATIVE_NAME) v_wind_index = NARR_FIELD_NAMES.index( processed_narr_io.V_WIND_EARTH_RELATIVE_NAME) nwp_plotting.plot_wind_barbs_on_subgrid( u_wind_matrix_m_s01=narr_matrix_by_field[u_wind_index], v_wind_matrix_m_s01=narr_matrix_by_field[v_wind_index], model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object, basemap_object=basemap_object, first_row_in_full_grid=narr_row_limits[0], first_column_in_full_grid=narr_column_limits[0], plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB, plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB, barb_length=WIND_BARB_LENGTH, empty_barb_radius=EMPTY_WIND_BARB_RADIUS, colour_map=WIND_COLOUR_MAP_OBJECT, colour_minimum_kt=MIN_COLOUR_WIND_SPEED_KT, colour_maximum_kt=MAX_COLOUR_WIND_SPEED_KT) num_fronts = len(front_line_table.index) for i in range(num_fronts): this_front_type_string = front_line_table[ front_utils.FRONT_TYPE_COLUMN].values[i] if this_front_type_string == front_utils.WARM_FRONT_STRING_ID: this_colour = WARM_FRONT_COLOUR else: this_colour = COLD_FRONT_COLOUR front_plotting.plot_polyline( latitudes_deg=front_line_table[ front_utils.LATITUDES_COLUMN].values[i], longitudes_deg=front_line_table[ front_utils.LONGITUDES_COLUMN].values[i], basemap_object=basemap_object, axes_object=axes_object, front_type=front_line_table[ front_utils.FRONT_TYPE_COLUMN].values[i], line_width=FRONT_LINE_WIDTH, line_colour=this_colour) pyplot.title(title_string) plotting_utils.annotate_axes( axes_object=axes_object, annotation_string=annotation_string) print 'Saving figure to: "{0:s}"...'.format(output_file_name) file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close() imagemagick_utils.trim_whitespace(input_file_name=output_file_name, output_file_name=output_file_name)
def _plot_one_time(valid_time_string, pressure_level_mb, title_string, annotation_string, narr_rotation_cos_matrix, narr_rotation_sin_matrix): """Plots WPC fronts and NARR fields at one time. M = number of grid rows in the full NARR N = number of grid columns in the full NARR :param valid_time_string: Valid time (format "yyyy-mm-dd-HH"). :param pressure_level_mb: Pressure level (millibars). :param title_string: Title (will be placed above figure). :param annotation_string: Annotation (will be placed above and left of figure). :param narr_rotation_cos_matrix: M-by-N numpy array of cosines for wind- rotation angles. :param narr_rotation_sin_matrix: M-by-N numpy array of sines for wind- rotation angles. """ narr_row_limits, narr_column_limits = ( nwp_plotting.latlng_limits_to_rowcol_limits( min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG, min_longitude_deg=MIN_LONGITUDE_DEG, max_longitude_deg=MAX_LONGITUDE_DEG, model_name=nwp_model_utils.NARR_MODEL_NAME)) valid_time_unix_sec = time_conversion.string_to_unix_sec( valid_time_string, DEFAULT_TIME_FORMAT) front_file_name = fronts_io.find_file_for_one_time( top_directory_name=TOP_FRONT_DIR_NAME, file_type=fronts_io.POLYLINE_FILE_TYPE, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(front_file_name) front_line_table = fronts_io.read_polylines_from_file(front_file_name) num_narr_fields = len(NARR_FIELD_NAMES) narr_matrix_by_field = [numpy.array([])] * num_narr_fields for j in range(num_narr_fields): this_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=TOP_NARR_DIRECTORY_NAME, field_name=NARR_FIELD_NAMES[j], pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_time_unix_sec) print 'Reading data from: "{0:s}"...'.format(this_file_name) narr_matrix_by_field[j] = processed_narr_io.read_fields_from_file( this_file_name)[0][0, ...] narr_matrix_by_field[j] = utils.fill_nans(narr_matrix_by_field[j]) narr_matrix_by_field[j] = narr_matrix_by_field[j][narr_row_limits[0]:( narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1)] if NARR_FIELD_NAMES[j] == processed_narr_io.WET_BULB_THETA_NAME: narr_matrix_by_field[j] = (narr_matrix_by_field[j] - ZERO_CELSIUS_IN_KELVINS) _, axes_object, basemap_object = nwp_plotting.init_basemap( model_name=nwp_model_utils.NARR_MODEL_NAME, first_row_in_full_grid=narr_row_limits[0], last_row_in_full_grid=narr_row_limits[1], first_column_in_full_grid=narr_column_limits[0], last_column_in_full_grid=narr_column_limits[1]) plotting_utils.plot_coastlines(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_countries(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_states_and_provinces(basemap_object=basemap_object, axes_object=axes_object, line_colour=BORDER_COLOUR) plotting_utils.plot_parallels(basemap_object=basemap_object, axes_object=axes_object, bottom_left_lat_deg=-90., upper_right_lat_deg=90., parallel_spacing_deg=PARALLEL_SPACING_DEG) plotting_utils.plot_meridians(basemap_object=basemap_object, axes_object=axes_object, bottom_left_lng_deg=0., upper_right_lng_deg=360., meridian_spacing_deg=MERIDIAN_SPACING_DEG) for j in range(num_narr_fields): if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES: continue min_colour_value = numpy.percentile(narr_matrix_by_field[j], MIN_COLOUR_PERCENTILE) max_colour_value = numpy.percentile(narr_matrix_by_field[j], MAX_COLOUR_PERCENTILE) nwp_plotting.plot_subgrid( field_matrix=narr_matrix_by_field[j], model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object, basemap_object=basemap_object, colour_map=THERMAL_COLOUR_MAP_OBJECT, min_value_in_colour_map=min_colour_value, max_value_in_colour_map=max_colour_value, first_row_in_full_grid=narr_row_limits[0], first_column_in_full_grid=narr_column_limits[0]) plotting_utils.add_linear_colour_bar( axes_object_or_list=axes_object, values_to_colour=narr_matrix_by_field[j], colour_map=THERMAL_COLOUR_MAP_OBJECT, colour_min=min_colour_value, colour_max=max_colour_value, orientation='horizontal', extend_min=True, extend_max=True, fraction_of_axis_length=0.9) this_cos_matrix = narr_rotation_cos_matrix[narr_row_limits[0]:( narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1)] this_sin_matrix = narr_rotation_sin_matrix[narr_row_limits[0]:( narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] + 1)] u_wind_index = NARR_FIELD_NAMES.index( processed_narr_io.U_WIND_GRID_RELATIVE_NAME) v_wind_index = NARR_FIELD_NAMES.index( processed_narr_io.V_WIND_GRID_RELATIVE_NAME) narr_matrix_by_field[u_wind_index], narr_matrix_by_field[v_wind_index] = ( nwp_model_utils.rotate_winds_to_earth_relative( u_winds_grid_relative_m_s01=narr_matrix_by_field[u_wind_index], v_winds_grid_relative_m_s01=narr_matrix_by_field[v_wind_index], rotation_angle_cosines=this_cos_matrix, rotation_angle_sines=this_sin_matrix)) nwp_plotting.plot_wind_barbs_on_subgrid( u_wind_matrix_m_s01=narr_matrix_by_field[u_wind_index], v_wind_matrix_m_s01=narr_matrix_by_field[v_wind_index], model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object, basemap_object=basemap_object, first_row_in_full_grid=narr_row_limits[0], first_column_in_full_grid=narr_column_limits[0], plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB, plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB, barb_length=WIND_BARB_LENGTH, empty_barb_radius=EMPTY_WIND_BARB_RADIUS, fill_empty_barb=False, colour_map=WIND_COLOUR_MAP_OBJECT, colour_minimum_kt=MIN_COLOUR_WIND_SPEED_KT, colour_maximum_kt=MAX_COLOUR_WIND_SPEED_KT) num_fronts = len(front_line_table.index) for i in range(num_fronts): this_front_type_string = front_line_table[ front_utils.FRONT_TYPE_COLUMN].values[i] if this_front_type_string == front_utils.WARM_FRONT_STRING_ID: this_colour = WARM_FRONT_COLOUR else: this_colour = COLD_FRONT_COLOUR front_plotting.plot_front_with_markers( line_latitudes_deg=front_line_table[ front_utils.LATITUDES_COLUMN].values[i], line_longitudes_deg=front_line_table[ front_utils.LONGITUDES_COLUMN].values[i], axes_object=axes_object, basemap_object=basemap_object, front_type_string=front_line_table[ front_utils.FRONT_TYPE_COLUMN].values[i], marker_colour=this_colour) pyplot.title(title_string) plotting_utils.annotate_axes(axes_object=axes_object, annotation_string=annotation_string) file_system_utils.mkdir_recursive_if_necessary( directory_name=OUTPUT_DIR_NAME) figure_file_name = '{0:s}/fronts_{1:04d}mb_{2:s}.jpg'.format( OUTPUT_DIR_NAME, pressure_level_mb, valid_time_string) print 'Saving figure to: "{0:s}"...'.format(figure_file_name) pyplot.savefig(figure_file_name, dpi=FIGURE_RESOLUTION_DPI) pyplot.close() imagemagick_utils.trim_whitespace(input_file_name=figure_file_name, output_file_name=figure_file_name) return figure_file_name
def _run(first_time_string, last_time_string, randomize_times, num_times, thermal_field_name, smoothing_radius_pixels, warm_front_percentile, cold_front_percentile, num_closing_iters, pressure_level_mb, top_narr_directory_name, narr_mask_file_name, output_dir_name): """Uses NFA (numerical frontal analysis) to predict front type at each px. This is effectively the main method. :param first_time_string: See documentation at top of file. :param last_time_string: Same. :param randomize_times: Same. :param num_times: Same. :param thermal_field_name: Same. :param smoothing_radius_pixels: Same. :param warm_front_percentile: Same. :param cold_front_percentile: Same. :param num_closing_iters: Same. :param pressure_level_mb: Same. :param top_narr_directory_name: Same. :param narr_mask_file_name: Same. :param output_dir_name: Same. :raises: ValueError: if `thermal_field_name not in VALID_THERMAL_FIELD_NAMES`. """ if thermal_field_name not in VALID_THERMAL_FIELD_NAMES: error_string = ( '\n{0:s}\nValid thermal fields (listed above) do not include ' '"{1:s}".').format(str(VALID_THERMAL_FIELD_NAMES), thermal_field_name) raise ValueError(error_string) cutoff_radius_pixels = 4 * smoothing_radius_pixels first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, INPUT_TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, INPUT_TIME_FORMAT) valid_times_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=first_time_unix_sec, end_time_unix_sec=last_time_unix_sec, time_interval_sec=NARR_TIME_INTERVAL_SEC, include_endpoint=True) if randomize_times: error_checking.assert_is_leq(num_times, len(valid_times_unix_sec)) numpy.random.shuffle(valid_times_unix_sec) valid_times_unix_sec = valid_times_unix_sec[:num_times] if narr_mask_file_name == '': num_grid_rows, num_grid_columns = nwp_model_utils.get_grid_dimensions( model_name=nwp_model_utils.NARR_MODEL_NAME) narr_mask_matrix = numpy.full((num_grid_rows, num_grid_columns), 1, dtype=int) else: print 'Reading mask from: "{0:s}"...\n'.format(narr_mask_file_name) narr_mask_matrix = ml_utils.read_narr_mask(narr_mask_file_name) x_spacing_metres, y_spacing_metres = nwp_model_utils.get_xy_grid_spacing( model_name=nwp_model_utils.NARR_MODEL_NAME) num_times = len(valid_times_unix_sec) for i in range(num_times): this_thermal_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_directory_name, field_name=thermal_field_name, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_times_unix_sec[i]) print 'Reading data from: "{0:s}"...'.format(this_thermal_file_name) this_thermal_matrix_kelvins = processed_narr_io.read_fields_from_file( this_thermal_file_name)[0][0, ...] this_thermal_matrix_kelvins = general_utils.fill_nans( this_thermal_matrix_kelvins) this_thermal_matrix_kelvins = nfa.gaussian_smooth_2d_field( field_matrix=this_thermal_matrix_kelvins, standard_deviation_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels) this_u_wind_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_directory_name, field_name=processed_narr_io.U_WIND_GRID_RELATIVE_NAME, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_times_unix_sec[i]) print 'Reading data from: "{0:s}"...'.format(this_u_wind_file_name) this_u_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file( this_u_wind_file_name)[0][0, ...] this_u_wind_matrix_m_s01 = general_utils.fill_nans( this_u_wind_matrix_m_s01) this_u_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field( field_matrix=this_u_wind_matrix_m_s01, standard_deviation_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels) this_v_wind_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_narr_directory_name, field_name=processed_narr_io.V_WIND_GRID_RELATIVE_NAME, pressure_level_mb=pressure_level_mb, valid_time_unix_sec=valid_times_unix_sec[i]) print 'Reading data from: "{0:s}"...'.format(this_v_wind_file_name) this_v_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file( this_v_wind_file_name)[0][0, ...] this_v_wind_matrix_m_s01 = general_utils.fill_nans( this_v_wind_matrix_m_s01) this_v_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field( field_matrix=this_v_wind_matrix_m_s01, standard_deviation_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels) this_tfp_matrix_kelvins_m02 = nfa.get_thermal_front_param( thermal_field_matrix_kelvins=this_thermal_matrix_kelvins, x_spacing_metres=x_spacing_metres, y_spacing_metres=y_spacing_metres) this_tfp_matrix_kelvins_m02[narr_mask_matrix == 0] = 0. this_proj_velocity_matrix_m_s01 = nfa.project_wind_to_thermal_gradient( u_matrix_grid_relative_m_s01=this_u_wind_matrix_m_s01, v_matrix_grid_relative_m_s01=this_v_wind_matrix_m_s01, thermal_field_matrix_kelvins=this_thermal_matrix_kelvins, x_spacing_metres=x_spacing_metres, y_spacing_metres=y_spacing_metres) this_locating_var_matrix_m01_s01 = nfa.get_locating_variable( tfp_matrix_kelvins_m02=this_tfp_matrix_kelvins_m02, projected_velocity_matrix_m_s01=this_proj_velocity_matrix_m_s01) this_predicted_label_matrix = nfa.get_front_types( locating_var_matrix_m01_s01=this_locating_var_matrix_m01_s01, warm_front_percentile=warm_front_percentile, cold_front_percentile=cold_front_percentile) this_predicted_label_matrix = front_utils.close_frontal_image( ternary_image_matrix=this_predicted_label_matrix, num_iterations=num_closing_iters) this_prediction_file_name = nfa.find_prediction_file( directory_name=output_dir_name, first_valid_time_unix_sec=valid_times_unix_sec[i], last_valid_time_unix_sec=valid_times_unix_sec[i], ensembled=False, raise_error_if_missing=False) print 'Writing gridded predictions to file: "{0:s}"...\n'.format( this_prediction_file_name) nfa.write_gridded_predictions( pickle_file_name=this_prediction_file_name, predicted_label_matrix=numpy.expand_dims( this_predicted_label_matrix, axis=0), valid_times_unix_sec=valid_times_unix_sec[[i]], narr_mask_matrix=narr_mask_matrix, pressure_level_mb=pressure_level_mb, smoothing_radius_pixels=smoothing_radius_pixels, cutoff_radius_pixels=cutoff_radius_pixels, warm_front_percentile=warm_front_percentile, cold_front_percentile=cold_front_percentile, num_closing_iters=num_closing_iters)
def _run(top_input_dir_name, first_time_string, last_time_string, pressure_level_mb, top_output_dir_name): """Computes theta_w (wet-bulb potential temperature) for NARR data. This is effectively the main method. :param top_input_dir_name: See documentation at top of file. :param first_time_string: Same. :param last_time_string: Same. :param pressure_level_mb: Same. :param top_output_dir_name: Same. """ if pressure_level_mb <= 0: pressure_level_mb = None if pressure_level_mb is None: pressure_in_file_name_mb = DUMMY_PRESSURE_LEVEL_MB + 0 else: pressure_in_file_name_mb = pressure_level_mb + 0 first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, INPUT_TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, INPUT_TIME_FORMAT) valid_times_unix_sec = time_periods.range_and_interval_to_list( start_time_unix_sec=first_time_unix_sec, end_time_unix_sec=last_time_unix_sec, time_interval_sec=TIME_INTERVAL_SECONDS) num_times = len(valid_times_unix_sec) this_pressure_matrix_pascals = None for i in range(num_times): this_temperature_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_input_dir_name, field_name=processed_narr_io.TEMPERATURE_NAME, pressure_level_mb=pressure_in_file_name_mb, valid_time_unix_sec=valid_times_unix_sec[i]) print 'Reading data from: "{0:s}"...'.format( this_temperature_file_name) this_temperature_matrix_kelvins = ( processed_narr_io.read_fields_from_file( this_temperature_file_name)[0]) this_humidity_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_input_dir_name, field_name=processed_narr_io.SPECIFIC_HUMIDITY_NAME, pressure_level_mb=pressure_in_file_name_mb, valid_time_unix_sec=valid_times_unix_sec[i]) print 'Reading data from: "{0:s}"...'.format(this_humidity_file_name) this_humidity_matrix_kg_kg01 = ( processed_narr_io.read_fields_from_file(this_humidity_file_name)[0] ) if pressure_level_mb is None: this_pressure_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_input_dir_name, field_name=processed_narr_io.HEIGHT_NAME, pressure_level_mb=pressure_in_file_name_mb, valid_time_unix_sec=valid_times_unix_sec[i]) print 'Reading data from: "{0:s}"...'.format( this_pressure_file_name) this_pressure_matrix_pascals = ( processed_narr_io.read_fields_from_file( this_pressure_file_name)[0]) print this_pressure_matrix_pascals[:5, :5] else: if this_pressure_matrix_pascals is None: this_pressure_matrix_pascals = numpy.full( this_humidity_matrix_kg_kg01.shape, pressure_level_mb * MB_TO_PASCALS) this_dewpoint_matrix_kelvins = ( moisture_conversions.specific_humidity_to_dewpoint( specific_humidities_kg_kg01=this_humidity_matrix_kg_kg01, total_pressures_pascals=this_pressure_matrix_pascals)) this_wb_temp_matrix_kelvins = ( ge_conversions.dewpoint_to_wet_bulb_temperature( dewpoints_kelvins=this_dewpoint_matrix_kelvins, temperatures_kelvins=this_temperature_matrix_kelvins, total_pressures_pascals=this_pressure_matrix_pascals)) this_theta_w_matrix_kelvins = ( temperature_conversions.temperatures_to_potential_temperatures( temperatures_kelvins=this_wb_temp_matrix_kelvins, total_pressures_pascals=this_pressure_matrix_pascals)) this_theta_w_file_name = processed_narr_io.find_file_for_one_time( top_directory_name=top_output_dir_name, field_name=processed_narr_io.WET_BULB_THETA_NAME, pressure_level_mb=pressure_in_file_name_mb, valid_time_unix_sec=valid_times_unix_sec[i], raise_error_if_missing=False) print 'Writing theta_w field to: "{0:s}"...\n'.format( this_theta_w_file_name) processed_narr_io.write_fields_to_file( pickle_file_name=this_theta_w_file_name, field_matrix=this_theta_w_matrix_kelvins, field_name=processed_narr_io.WET_BULB_THETA_NAME, pressure_level_pascals=pressure_in_file_name_mb * MB_TO_PASCALS, valid_times_unix_sec=valid_times_unix_sec[[i]])
def create_full_size_4d_example(target_time_unix_sec, num_lead_time_steps, predictor_time_step_offsets, top_narr_directory_name, top_frontal_grid_dir_name, narr_predictor_names, pressure_level_mb, dilation_distance_metres, num_classes): """Creates full-size 4-D examples from raw files. :param target_time_unix_sec: See doc for `create_downsized_3d_examples`. :param num_lead_time_steps: See doc for `create_downsized_4d_examples`. :param predictor_time_step_offsets: Same. :param top_narr_directory_name: See doc for `create_downsized_3d_examples`. :param top_frontal_grid_dir_name: Same. :param narr_predictor_names: Same. :param pressure_level_mb: Same. :param dilation_distance_metres: Same. :param num_classes: Same. :return: predictor_matrix: 1-by-M-by-N-by-T-by-C numpy array of predictor values. :return: target_matrix: 1-by-M-by-N numpy array of target values. Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`. """ error_checking.assert_is_integer(num_classes) error_checking.assert_is_geq(num_classes, 2) error_checking.assert_is_leq(num_classes, 3) (narr_file_name_matrix, frontal_grid_file_names) = trainval_io.find_input_files_for_4d_examples( first_target_time_unix_sec=target_time_unix_sec, last_target_time_unix_sec=target_time_unix_sec, predictor_time_step_offsets=predictor_time_step_offsets, num_lead_time_steps=num_lead_time_steps, top_narr_directory_name=top_narr_directory_name, top_frontal_grid_dir_name=top_frontal_grid_dir_name, narr_predictor_names=narr_predictor_names, pressure_level_mb=pressure_level_mb) narr_file_name_matrix = narr_file_name_matrix[0, ...] frontal_grid_file_name = frontal_grid_file_names[0] num_predictor_times_per_example = len(predictor_time_step_offsets) num_predictors = len(narr_predictor_names) tuple_of_4d_predictor_matrices = () for i in range(num_predictor_times_per_example): tuple_of_3d_predictor_matrices = () for j in range(num_predictors): print 'Reading data from: "{0:s}"...'.format( narr_file_name_matrix[i, j]) this_field_predictor_matrix = ( processed_narr_io.read_fields_from_file( narr_file_name_matrix[i, j]))[0] this_field_predictor_matrix = ( ml_utils.fill_nans_in_predictor_images( this_field_predictor_matrix)) tuple_of_3d_predictor_matrices += (this_field_predictor_matrix, ) tuple_of_4d_predictor_matrices += (ml_utils.stack_predictor_variables( tuple_of_3d_predictor_matrices), ) predictor_matrix = ml_utils.stack_time_steps( tuple_of_4d_predictor_matrices) print 'Reading data from: "{0:s}"...'.format(frontal_grid_file_name) frontal_grid_table = fronts_io.read_narr_grids_from_file( frontal_grid_file_name) print 'Processing full-size 4-D machine-learning example...' predictor_matrix, _ = ml_utils.normalize_predictors( predictor_matrix=predictor_matrix) target_matrix = ml_utils.front_table_to_images( frontal_grid_table=frontal_grid_table, num_rows_per_image=predictor_matrix.shape[1], num_columns_per_image=predictor_matrix.shape[2]) if num_classes == 2: target_matrix = ml_utils.binarize_front_images(target_matrix) predictor_matrix = ml_utils.subset_narr_grid_for_fcn_input( predictor_matrix) target_matrix = ml_utils.subset_narr_grid_for_fcn_input(target_matrix) if num_classes == 2: target_matrix = ml_utils.dilate_binary_target_images( target_matrix=target_matrix, dilation_distance_metres=dilation_distance_metres, verbose=False) else: target_matrix = ml_utils.dilate_ternary_target_images( target_matrix=target_matrix, dilation_distance_metres=dilation_distance_metres, verbose=False) predictor_matrix = predictor_matrix.astype('float32') print 'Fraction of pixels with a front = {0:.4f}'.format( numpy.mean(target_matrix > 0)) target_matrix = numpy.expand_dims(target_matrix, axis=-1) return predictor_matrix, target_matrix
def create_downsized_3d_examples(center_row_indices, center_column_indices, num_rows_in_half_grid, num_columns_in_half_grid, full_predictor_matrix=None, full_target_matrix=None, target_time_unix_sec=None, top_narr_directory_name=None, top_frontal_grid_dir_name=None, narr_predictor_names=None, pressure_level_mb=None, dilation_distance_metres=None, num_classes=None): """Creates downsized 3-D examples from raw files. This method creates one example for each center pixel, where center pixels are specified by `center_row_indices` and `center_column_indices`. If `full_predictor_matrix` and `full_target_matrix` are specified, all input args thereafter will be ignored. E = number of examples (center pixels) P = number of rows in full NARR grid Q = number of columns in full NARR grid :param center_row_indices: length-E numpy array with row for each center pixel. :param center_column_indices: length-E numpy array with column for each center pixel. :param num_rows_in_half_grid: Number of rows in half-grid for each example. Actual number of rows will be 2 * `num_rows_in_half_grid` + 1. :param num_columns_in_half_grid: Same but for columns. :param full_predictor_matrix: 1-by-P-by-Q-by-C numpy array of predictor values. :param full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values. Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`. :param target_time_unix_sec: Target time. :param top_narr_directory_name: Name of top-level directory with NARR data. Files therein will be found by `processed_narr_io.find_file_for_one_time` and read by `processed_narr_io.read_fields_from_file`. :param top_frontal_grid_dir_name: Name of top-level directory with target values (grids of front labels). Files therein will be found by `fronts_io.find_file_for_one_time` and read by `fronts_io.read_narr_grids_from_file`. :param narr_predictor_names: length-C list with names of predictor variables. Each must be accepted by `processed_narr_io.check_field_name`. :param pressure_level_mb: Pressure level (millibars) for predictors. :param dilation_distance_metres: Dilation distance. Will be used to dilate WF and CF labels, which effectively creates a distance buffer around each front, thus accounting for spatial uncertainty in front placement. :param num_classes: Number of target classes (either 2 or 3). :return: downsized_predictor_matrix: E-by-M-by-N-by-C numpy array of predictor values. :return: target_values: length-E numpy array of target values (integers from the list `front_utils.VALID_INTEGER_IDS`). :return: full_predictor_matrix: 1-by-P-by-Q-by-C numpy array of predictor values. :return: full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values. Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`. """ if full_predictor_matrix is None or full_target_matrix is None: error_checking.assert_is_integer(num_classes) error_checking.assert_is_geq(num_classes, 2) error_checking.assert_is_leq(num_classes, 3) (narr_file_name_matrix, frontal_grid_file_names ) = trainval_io.find_input_files_for_3d_examples( first_target_time_unix_sec=target_time_unix_sec, last_target_time_unix_sec=target_time_unix_sec, top_narr_directory_name=top_narr_directory_name, top_frontal_grid_dir_name=top_frontal_grid_dir_name, narr_predictor_names=narr_predictor_names, pressure_level_mb=pressure_level_mb) narr_file_names = narr_file_name_matrix[0, :] frontal_grid_file_name = frontal_grid_file_names[0] num_predictors = len(narr_predictor_names) tuple_of_full_predictor_matrices = () for j in range(num_predictors): print 'Reading data from: "{0:s}"...'.format(narr_file_names[j]) this_field_predictor_matrix = ( processed_narr_io.read_fields_from_file(narr_file_names[j]))[0] this_field_predictor_matrix = ( ml_utils.fill_nans_in_predictor_images( this_field_predictor_matrix)) tuple_of_full_predictor_matrices += (this_field_predictor_matrix, ) print 'Reading data from: "{0:s}"...'.format(frontal_grid_file_name) frontal_grid_table = fronts_io.read_narr_grids_from_file( frontal_grid_file_name) full_predictor_matrix = ml_utils.stack_predictor_variables( tuple_of_full_predictor_matrices) full_predictor_matrix, _ = ml_utils.normalize_predictors( predictor_matrix=full_predictor_matrix) full_target_matrix = ml_utils.front_table_to_images( frontal_grid_table=frontal_grid_table, num_rows_per_image=full_predictor_matrix.shape[1], num_columns_per_image=full_predictor_matrix.shape[2]) if num_classes == 2: full_target_matrix = ml_utils.binarize_front_images( full_target_matrix) if num_classes == 2: full_target_matrix = ml_utils.dilate_binary_target_images( target_matrix=full_target_matrix, dilation_distance_metres=dilation_distance_metres, verbose=False) else: full_target_matrix = ml_utils.dilate_ternary_target_images( target_matrix=full_target_matrix, dilation_distance_metres=dilation_distance_metres, verbose=False) print 'Creating {0:d} downsized 3-D examples...'.format( len(center_row_indices)) this_target_point_dict = { ml_utils.ROW_INDICES_BY_TIME_KEY: [center_row_indices], ml_utils.COLUMN_INDICES_BY_TIME_KEY: [center_column_indices] } (downsized_predictor_matrix, target_values) = ml_utils.downsize_grids_around_selected_points( predictor_matrix=full_predictor_matrix, target_matrix=full_target_matrix, num_rows_in_half_window=num_rows_in_half_grid, num_columns_in_half_window=num_columns_in_half_grid, target_point_dict=this_target_point_dict, verbose=False)[:2] downsized_predictor_matrix = downsized_predictor_matrix.astype('float32') return (downsized_predictor_matrix, target_values, full_predictor_matrix, full_target_matrix)
def create_downsized_4d_examples(center_row_indices, center_column_indices, num_rows_in_half_grid, num_columns_in_half_grid, full_predictor_matrix=None, full_target_matrix=None, target_time_unix_sec=None, num_lead_time_steps=None, predictor_time_step_offsets=None, top_narr_directory_name=None, top_frontal_grid_dir_name=None, narr_predictor_names=None, pressure_level_mb=None, dilation_distance_metres=None, num_classes=None): """Creates downsized 4-D examples from raw files. This method creates one example for each center pixel, where center pixels are specified by `center_row_indices` and `center_column_indices`. If `full_predictor_matrix` and `full_target_matrix` are specified, all input args thereafter will be ignored. E = number of examples (center pixels) P = number of rows in full NARR grid Q = number of columns in full NARR grid :param center_row_indices: See doc for `create_downsized_3d_examples`. :param center_column_indices: Same. :param num_rows_in_half_grid: Same. :param num_columns_in_half_grid: Same. :param full_predictor_matrix: 1-by-P-by-Q-by-T-by-C numpy array of predictor values. :param full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values. Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`. :param target_time_unix_sec: See doc for `create_downsized_3d_examples`. :param num_lead_time_steps: Number of time steps between target time and latest possible predictor time. :param predictor_time_step_offsets: length-T numpy array of offsets between predictor time and latest possible predictor time (target time minus lead time). :param top_narr_directory_name: See doc for `create_downsized_3d_examples`. :param top_frontal_grid_dir_name: Same. :param narr_predictor_names: Same. :param pressure_level_mb: Same. :param dilation_distance_metres: Same. :param num_classes: Same. :return: downsized_predictor_matrix: E-by-M-by-N-by-T-by-C numpy array of predictor values. :return: target_values: length-E numpy array of target values (integers from the list `front_utils.VALID_INTEGER_IDS`). :return: full_predictor_matrix: 1-by-P-by-Q-by-T-by-C numpy array of predictor values. :return: full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values. Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`. """ if full_predictor_matrix is None or full_target_matrix is None: error_checking.assert_is_integer(num_classes) error_checking.assert_is_geq(num_classes, 2) error_checking.assert_is_leq(num_classes, 3) (narr_file_name_matrix, frontal_grid_file_names ) = trainval_io.find_input_files_for_4d_examples( first_target_time_unix_sec=target_time_unix_sec, last_target_time_unix_sec=target_time_unix_sec, predictor_time_step_offsets=predictor_time_step_offsets, num_lead_time_steps=num_lead_time_steps, top_narr_directory_name=top_narr_directory_name, top_frontal_grid_dir_name=top_frontal_grid_dir_name, narr_predictor_names=narr_predictor_names, pressure_level_mb=pressure_level_mb) narr_file_name_matrix = narr_file_name_matrix[0, ...] frontal_grid_file_name = frontal_grid_file_names[0] num_predictor_times_per_example = len(predictor_time_step_offsets) num_predictors = len(narr_predictor_names) tuple_of_4d_predictor_matrices = () for i in range(num_predictor_times_per_example): tuple_of_3d_predictor_matrices = () for j in range(num_predictors): print 'Reading data from: "{0:s}"...'.format( narr_file_name_matrix[i, j]) this_field_predictor_matrix = ( processed_narr_io.read_fields_from_file( narr_file_name_matrix[i, j]))[0] this_field_predictor_matrix = ( ml_utils.fill_nans_in_predictor_images( this_field_predictor_matrix)) tuple_of_3d_predictor_matrices += ( this_field_predictor_matrix, ) tuple_of_4d_predictor_matrices += ( ml_utils.stack_predictor_variables( tuple_of_3d_predictor_matrices), ) full_predictor_matrix = ml_utils.stack_time_steps( tuple_of_4d_predictor_matrices) print 'Reading data from: "{0:s}"...'.format(frontal_grid_file_name) frontal_grid_table = fronts_io.read_narr_grids_from_file( frontal_grid_file_name) full_predictor_matrix, _ = ml_utils.normalize_predictors( predictor_matrix=full_predictor_matrix) full_target_matrix = ml_utils.front_table_to_images( frontal_grid_table=frontal_grid_table, num_rows_per_image=full_predictor_matrix.shape[1], num_columns_per_image=full_predictor_matrix.shape[2]) if num_classes == 2: full_target_matrix = ml_utils.binarize_front_images( full_target_matrix) if num_classes == 2: full_target_matrix = ml_utils.dilate_binary_target_images( target_matrix=full_target_matrix, dilation_distance_metres=dilation_distance_metres, verbose=False) else: full_target_matrix = ml_utils.dilate_ternary_target_images( target_matrix=full_target_matrix, dilation_distance_metres=dilation_distance_metres, verbose=False) print 'Creating {0:d} downsized 4-D examples...'.format( len(center_row_indices)) this_target_point_dict = { ml_utils.ROW_INDICES_BY_TIME_KEY: [center_row_indices], ml_utils.COLUMN_INDICES_BY_TIME_KEY: [center_column_indices] } (downsized_predictor_matrix, target_values) = ml_utils.downsize_grids_around_selected_points( predictor_matrix=full_predictor_matrix, target_matrix=full_target_matrix, num_rows_in_half_window=num_rows_in_half_grid, num_columns_in_half_window=num_columns_in_half_grid, target_point_dict=this_target_point_dict, verbose=False)[:2] downsized_predictor_matrix = downsized_predictor_matrix.astype('float32') return (downsized_predictor_matrix, target_values, full_predictor_matrix, full_target_matrix)