def _run(top_narr_dir_name, top_front_line_dir_name, top_wpc_bulletin_dir_name,
first_time_string, last_time_string, pressure_level_mb,
thermal_field_name, thermal_colour_map_name,
max_thermal_prctile_for_colours, first_letter_label, letter_interval,
output_dir_name):
"""Plots predictors on full NARR grid.
This is effectively the main method.
:param top_narr_dir_name: See documentation at top of file.
:param top_front_line_dir_name: Same.
:param top_wpc_bulletin_dir_name: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param pressure_level_mb: Same.
:param thermal_field_name: Same.
:param thermal_colour_map_name: Same.
:param max_thermal_prctile_for_colours: Same.
:param first_letter_label: Same.
:param letter_interval: Same.
:param output_dir_name: Same.
:raises: ValueError: if
`thermal_field_name not in VALID_THERMAL_FIELD_NAMES`.
"""
# Check input args.
if top_wpc_bulletin_dir_name in ['', 'None']:
top_wpc_bulletin_dir_name = None
if first_letter_label in ['', 'None']:
first_letter_label = None
if thermal_field_name not in VALID_THERMAL_FIELD_NAMES:
error_string = (
'\n{0:s}\nValid thermal fields (listed above) do not include '
'"{1:s}".'
).format(str(VALID_THERMAL_FIELD_NAMES), thermal_field_name)
raise ValueError(error_string)
thermal_colour_map_object = pyplot.cm.get_cmap(thermal_colour_map_name)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, DEFAULT_TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, DEFAULT_TIME_FORMAT)
valid_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=NARR_TIME_INTERVAL_SEC, include_endpoint=True)
# Read metadata for NARR grid.
narr_latitude_matrix_deg, narr_longitude_matrix_deg = (
nwp_model_utils.get_latlng_grid_point_matrices(
model_name=nwp_model_utils.NARR_MODEL_NAME)
)
narr_rotation_cos_matrix, narr_rotation_sin_matrix = (
nwp_model_utils.get_wind_rotation_angles(
latitudes_deg=narr_latitude_matrix_deg,
longitudes_deg=narr_longitude_matrix_deg,
model_name=nwp_model_utils.NARR_MODEL_NAME)
)
narr_row_limits, narr_column_limits = (
nwp_plotting.latlng_limits_to_rowcol_limits(
min_latitude_deg=MIN_LATITUDE_DEG,
max_latitude_deg=MAX_LATITUDE_DEG,
min_longitude_deg=MIN_LONGITUDE_DEG,
max_longitude_deg=MAX_LONGITUDE_DEG,
model_name=nwp_model_utils.NARR_MODEL_NAME)
)
narr_rotation_cos_matrix = narr_rotation_cos_matrix[
narr_row_limits[0]:(narr_row_limits[1] + 1),
narr_column_limits[0]:(narr_column_limits[1] + 1)
]
narr_rotation_sin_matrix = narr_rotation_sin_matrix[
narr_row_limits[0]:(narr_row_limits[1] + 1),
narr_column_limits[0]:(narr_column_limits[1] + 1)
]
# Do plotting.
narr_field_names = [
processed_narr_io.U_WIND_GRID_RELATIVE_NAME,
processed_narr_io.V_WIND_GRID_RELATIVE_NAME,
thermal_field_name
]
this_letter_label = None
for this_time_unix_sec in valid_times_unix_sec:
this_file_name = fronts_io.find_file_for_one_time(
top_directory_name=top_front_line_dir_name,
file_type=fronts_io.POLYLINE_FILE_TYPE,
valid_time_unix_sec=this_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(this_file_name)
this_polyline_table = fronts_io.read_polylines_from_file(this_file_name)
if top_wpc_bulletin_dir_name is None:
this_high_low_table = None
else:
this_file_name = wpc_bulletin_io.find_file(
top_directory_name=top_wpc_bulletin_dir_name,
valid_time_unix_sec=this_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(this_file_name)
this_high_low_table = wpc_bulletin_io.read_highs_and_lows(
this_file_name)
this_predictor_matrix = None
for this_field_name in narr_field_names:
this_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_dir_name,
field_name=this_field_name,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=this_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(this_file_name)
this_field_matrix = processed_narr_io.read_fields_from_file(
this_file_name
)[0][0, ...]
this_field_matrix = utils.fill_nans(this_field_matrix)
this_field_matrix = this_field_matrix[
narr_row_limits[0]:(narr_row_limits[1] + 1),
narr_column_limits[0]:(narr_column_limits[1] + 1)
]
if this_field_name in [processed_narr_io.TEMPERATURE_NAME,
processed_narr_io.WET_BULB_THETA_NAME]:
this_field_matrix -= ZERO_CELSIUS_IN_KELVINS
if this_field_name == processed_narr_io.SPECIFIC_HUMIDITY_NAME:
this_field_matrix = this_field_matrix * KG_TO_GRAMS
this_field_matrix = numpy.expand_dims(this_field_matrix, axis=-1)
if this_predictor_matrix is None:
this_predictor_matrix = this_field_matrix + 0.
else:
this_predictor_matrix = numpy.concatenate(
(this_predictor_matrix, this_field_matrix), axis=-1)
u_wind_index = narr_field_names.index(
processed_narr_io.U_WIND_GRID_RELATIVE_NAME)
v_wind_index = narr_field_names.index(
processed_narr_io.V_WIND_GRID_RELATIVE_NAME)
(this_predictor_matrix[..., u_wind_index],
this_predictor_matrix[..., v_wind_index]
) = nwp_model_utils.rotate_winds_to_earth_relative(
u_winds_grid_relative_m_s01=this_predictor_matrix[
..., u_wind_index],
v_winds_grid_relative_m_s01=this_predictor_matrix[
..., v_wind_index],
rotation_angle_cosines=narr_rotation_cos_matrix,
rotation_angle_sines=narr_rotation_sin_matrix)
this_title_string = time_conversion.unix_sec_to_string(
this_time_unix_sec, NICE_TIME_FORMAT)
if pressure_level_mb == 1013:
this_title_string += ' at surface'
else:
this_title_string += ' at {0:d} mb'.format(pressure_level_mb)
this_default_time_string = time_conversion.unix_sec_to_string(
this_time_unix_sec, DEFAULT_TIME_FORMAT)
this_output_file_name = '{0:s}/predictors_{1:s}.jpg'.format(
output_dir_name, this_default_time_string)
if first_letter_label is not None:
if this_letter_label is None:
this_letter_label = first_letter_label
else:
this_letter_label = chr(
ord(this_letter_label) + letter_interval
)
_plot_one_time(
predictor_matrix=this_predictor_matrix,
predictor_names=narr_field_names,
front_polyline_table=this_polyline_table,
high_low_table=this_high_low_table,
thermal_colour_map_object=thermal_colour_map_object,
max_thermal_prctile_for_colours=max_thermal_prctile_for_colours,
narr_row_limits=narr_row_limits,
narr_column_limits=narr_column_limits,
title_string=this_title_string, letter_label=this_letter_label,
output_file_name=this_output_file_name)
print '\n'
def _run(valid_time_string, smoothing_radius_pixels, front_percentile,
num_closing_iters, pressure_level_mb, top_narr_directory_name,
narr_mask_file_name, output_dir_name):
"""Plots NFA (numerical frontal analysis) procedure.
This is effectively the main method.
:param valid_time_string: See documentation at top of file.
:param smoothing_radius_pixels: Same.
:param front_percentile: Same.
:param num_closing_iters: Same.
:param pressure_level_mb: Same.
:param top_narr_directory_name: Same.
:param narr_mask_file_name: Same.
:param output_dir_name: Same.
"""
cutoff_radius_pixels = 4 * smoothing_radius_pixels
valid_time_unix_sec = time_conversion.string_to_unix_sec(
valid_time_string, INPUT_TIME_FORMAT)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
if narr_mask_file_name == '':
num_grid_rows, num_grid_columns = nwp_model_utils.get_grid_dimensions(
model_name=nwp_model_utils.NARR_MODEL_NAME)
narr_mask_matrix = numpy.full(
(num_grid_rows, num_grid_columns), 1, dtype=int)
else:
print 'Reading mask from: "{0:s}"...\n'.format(narr_mask_file_name)
narr_mask_matrix = ml_utils.read_narr_mask(narr_mask_file_name)
wet_bulb_theta_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_directory_name,
field_name=processed_narr_io.WET_BULB_THETA_NAME,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(wet_bulb_theta_file_name)
wet_bulb_theta_matrix_kelvins = processed_narr_io.read_fields_from_file(
wet_bulb_theta_file_name)[0][0, ...]
wet_bulb_theta_matrix_kelvins = general_utils.fill_nans(
wet_bulb_theta_matrix_kelvins)
u_wind_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_directory_name,
field_name=processed_narr_io.U_WIND_GRID_RELATIVE_NAME,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(u_wind_file_name)
u_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file(
u_wind_file_name)[0][0, ...]
u_wind_matrix_m_s01 = general_utils.fill_nans(u_wind_matrix_m_s01)
v_wind_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_directory_name,
field_name=processed_narr_io.V_WIND_GRID_RELATIVE_NAME,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(v_wind_file_name)
v_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file(
v_wind_file_name)[0][0, ...]
v_wind_matrix_m_s01 = general_utils.fill_nans(v_wind_matrix_m_s01)
unsmoothed_narr_file_name = '{0:s}/unsmoothed_narr_fields.jpg'.format(
output_dir_name)
_plot_narr_fields(
wet_bulb_theta_matrix_kelvins=wet_bulb_theta_matrix_kelvins,
u_wind_matrix_m_s01=u_wind_matrix_m_s01,
v_wind_matrix_m_s01=v_wind_matrix_m_s01,
title_string='Predictors before smoothing', annotation_string='(a)',
output_file_name=unsmoothed_narr_file_name)
wet_bulb_theta_matrix_kelvins = nfa.gaussian_smooth_2d_field(
field_matrix=wet_bulb_theta_matrix_kelvins,
standard_deviation_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels)
u_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field(
field_matrix=u_wind_matrix_m_s01,
standard_deviation_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels)
v_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field(
field_matrix=v_wind_matrix_m_s01,
standard_deviation_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels)
smoothed_narr_file_name = '{0:s}/smoothed_narr_fields.jpg'.format(
output_dir_name)
_plot_narr_fields(
wet_bulb_theta_matrix_kelvins=wet_bulb_theta_matrix_kelvins,
u_wind_matrix_m_s01=u_wind_matrix_m_s01,
v_wind_matrix_m_s01=v_wind_matrix_m_s01,
title_string='Predictors after smoothing', annotation_string='(b)',
output_file_name=smoothed_narr_file_name)
x_spacing_metres, y_spacing_metres = nwp_model_utils.get_xy_grid_spacing(
model_name=nwp_model_utils.NARR_MODEL_NAME)
tfp_matrix_kelvins_m02 = nfa.get_thermal_front_param(
thermal_field_matrix_kelvins=wet_bulb_theta_matrix_kelvins,
x_spacing_metres=x_spacing_metres, y_spacing_metres=y_spacing_metres)
tfp_matrix_kelvins_m02[narr_mask_matrix == 0] = 0.
tfp_file_name = '{0:s}/tfp.jpg'.format(output_dir_name)
tfp_title_string = (
r'Thermal front parameter ($\times$ 10$^{-10}$ K m$^{-2}$)')
_plot_tfp(tfp_matrix_kelvins_m02=tfp_matrix_kelvins_m02,
title_string=tfp_title_string, annotation_string='(c)',
output_file_name=tfp_file_name)
proj_velocity_matrix_m_s01 = nfa.project_wind_to_thermal_gradient(
u_matrix_grid_relative_m_s01=u_wind_matrix_m_s01,
v_matrix_grid_relative_m_s01=v_wind_matrix_m_s01,
thermal_field_matrix_kelvins=wet_bulb_theta_matrix_kelvins,
x_spacing_metres=x_spacing_metres, y_spacing_metres=y_spacing_metres)
locating_var_matrix_m01_s01 = nfa.get_locating_variable(
tfp_matrix_kelvins_m02=tfp_matrix_kelvins_m02,
projected_velocity_matrix_m_s01=proj_velocity_matrix_m_s01)
locating_var_file_name = '{0:s}/locating_variable.jpg'.format(
output_dir_name)
locating_var_title_string = (
r'Locating variable ($\times$ 10$^{-9}$ K m$^{-1}$ s$^{-1}$)')
_plot_locating_variable(
locating_var_matrix_m01_s01=locating_var_matrix_m01_s01,
title_string=locating_var_title_string, annotation_string='(d)',
output_file_name=locating_var_file_name)
predicted_label_matrix = nfa.get_front_types(
locating_var_matrix_m01_s01=locating_var_matrix_m01_s01,
warm_front_percentile=front_percentile,
cold_front_percentile=front_percentile)
unclosed_fronts_file_name = '{0:s}/unclosed_fronts.jpg'.format(
output_dir_name)
_plot_front_types(
predicted_label_matrix=predicted_label_matrix,
title_string='Frontal regions before closing', annotation_string='(e)',
output_file_name=unclosed_fronts_file_name)
predicted_label_matrix = front_utils.close_frontal_image(
ternary_image_matrix=predicted_label_matrix,
num_iterations=num_closing_iters)
closed_fronts_file_name = '{0:s}/closed_fronts.jpg'.format(output_dir_name)
_plot_front_types(
predicted_label_matrix=predicted_label_matrix,
title_string='Frontal regions after closing', annotation_string='(f)',
output_file_name=closed_fronts_file_name)
concat_file_name = '{0:s}/nfa_procedure.jpg'.format(output_dir_name)
print 'Concatenating figures to: "{0:s}"...'.format(concat_file_name)
panel_file_names = [
unsmoothed_narr_file_name, smoothed_narr_file_name, tfp_file_name,
locating_var_file_name, unclosed_fronts_file_name,
closed_fronts_file_name
]
imagemagick_utils.concatenate_images(
input_file_names=panel_file_names,
output_file_name=concat_file_name, num_panel_rows=3,
num_panel_columns=2)
imagemagick_utils.resize_image(
input_file_name=concat_file_name, output_file_name=concat_file_name,
output_size_pixels=CONCAT_SIZE_PIXELS)
def _run():
"""Plots input example.
This is effectively the main method.
:return: figure_file_name: Path to output file (where the figure was saved).
"""
valid_time_unix_sec = time_conversion.string_to_unix_sec(
VALID_TIME_STRING, TIME_FORMAT)
front_file_name = fronts_io.find_file_for_one_time(
top_directory_name=TOP_FRONT_DIR_NAME,
file_type=fronts_io.POLYLINE_FILE_TYPE,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(front_file_name)
front_line_table = fronts_io.read_polylines_from_file(front_file_name)
num_narr_fields = len(NARR_FIELD_NAMES)
narr_matrix_by_field = [numpy.array([])] * num_narr_fields
for j in range(num_narr_fields):
if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES:
this_directory_name = '{0:s}/earth_relative_wind'.format(
TOP_NARR_DIRECTORY_NAME)
else:
this_directory_name = TOP_NARR_DIRECTORY_NAME + ''
this_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=this_directory_name,
field_name=NARR_FIELD_NAMES[j],
pressure_level_mb=PRESSURE_LEVEL_MB,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(this_file_name)
narr_matrix_by_field[j] = processed_narr_io.read_fields_from_file(
this_file_name)[0][0, ...]
narr_matrix_by_field[j] = utils.fill_nans(narr_matrix_by_field[j])
if NARR_FIELD_NAMES[j] == processed_narr_io.WET_BULB_THETA_NAME:
narr_matrix_by_field[j] = (narr_matrix_by_field[j] -
ZERO_CELSIUS_IN_KELVINS)
# (_, front_centroid_latitude_deg, front_centroid_longitude_deg
# ) = _find_nearest_front(
# front_line_table=front_line_table,
# query_latitude_deg=APPROX_FRONT_LATITUDE_DEG,
# query_longitude_deg=APPROX_FRONT_LONGITUDE_DEG)
front_centroid_latitude_deg = APPROX_FRONT_LATITUDE_DEG + 0.
front_centroid_longitude_deg = APPROX_FRONT_LONGITUDE_DEG + 0.
projection_object = nwp_model_utils.init_model_projection(
nwp_model_utils.NARR_MODEL_NAME)
these_x_metres, these_y_metres = nwp_model_utils.project_latlng_to_xy(
latitudes_deg=numpy.array([front_centroid_latitude_deg]),
longitudes_deg=numpy.array([front_centroid_longitude_deg]),
projection_object=projection_object,
model_name=nwp_model_utils.NARR_MODEL_NAME)
front_centroid_x_metres = these_x_metres[0]
front_centroid_y_metres = these_y_metres[0]
grid_spacing_metres, _ = nwp_model_utils.get_xy_grid_spacing(
model_name=nwp_model_utils.NARR_MODEL_NAME)
center_narr_row_index = int(
numpy.round(front_centroid_y_metres / grid_spacing_metres))
center_narr_column_index = int(
numpy.round(front_centroid_x_metres / grid_spacing_metres))
first_narr_row_index = center_narr_row_index - NUM_ROWS_IN_HALF_GRID
last_narr_row_index = center_narr_row_index + NUM_ROWS_IN_HALF_GRID
first_narr_column_index = (center_narr_column_index -
NUM_COLUMNS_IN_HALF_GRID)
last_narr_column_index = center_narr_column_index + NUM_COLUMNS_IN_HALF_GRID
for j in range(num_narr_fields):
narr_matrix_by_field[j] = narr_matrix_by_field[j][
first_narr_row_index:(last_narr_row_index + 1),
first_narr_column_index:(last_narr_column_index + 1)]
_, axes_object, basemap_object = nwp_plotting.init_basemap(
model_name=nwp_model_utils.NARR_MODEL_NAME,
first_row_in_full_grid=first_narr_row_index,
last_row_in_full_grid=last_narr_row_index,
first_column_in_full_grid=first_narr_column_index,
last_column_in_full_grid=last_narr_column_index,
resolution_string='i')
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
bottom_left_lat_deg=-90.,
upper_right_lat_deg=90.,
parallel_spacing_deg=PARALLEL_SPACING_DEG)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
bottom_left_lng_deg=0.,
upper_right_lng_deg=360.,
meridian_spacing_deg=MERIDIAN_SPACING_DEG)
for j in range(num_narr_fields):
if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES:
continue
min_colour_value = numpy.percentile(narr_matrix_by_field[j],
MIN_COLOUR_PERCENTILE)
max_colour_value = numpy.percentile(narr_matrix_by_field[j],
MAX_COLOUR_PERCENTILE)
nwp_plotting.plot_subgrid(
field_matrix=narr_matrix_by_field[j],
model_name=nwp_model_utils.NARR_MODEL_NAME,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map=THERMAL_COLOUR_MAP_OBJECT,
min_value_in_colour_map=min_colour_value,
max_value_in_colour_map=max_colour_value,
first_row_in_full_grid=first_narr_row_index,
first_column_in_full_grid=first_narr_column_index)
plotting_utils.add_linear_colour_bar(
axes_object_or_list=axes_object,
values_to_colour=narr_matrix_by_field[j],
colour_map=THERMAL_COLOUR_MAP_OBJECT,
colour_min=min_colour_value,
colour_max=max_colour_value,
orientation='horizontal',
extend_min=True,
extend_max=True)
u_wind_index = NARR_FIELD_NAMES.index(
processed_narr_io.U_WIND_EARTH_RELATIVE_NAME)
v_wind_index = NARR_FIELD_NAMES.index(
processed_narr_io.V_WIND_EARTH_RELATIVE_NAME)
nwp_plotting.plot_wind_barbs_on_subgrid(
u_wind_matrix_m_s01=narr_matrix_by_field[u_wind_index],
v_wind_matrix_m_s01=narr_matrix_by_field[v_wind_index],
model_name=nwp_model_utils.NARR_MODEL_NAME,
axes_object=axes_object,
basemap_object=basemap_object,
first_row_in_full_grid=first_narr_row_index,
first_column_in_full_grid=first_narr_column_index,
plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB,
plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB,
barb_length=WIND_BARB_LENGTH,
empty_barb_radius=EMPTY_WIND_BARB_RADIUS,
colour_map=WIND_COLOUR_MAP_OBJECT,
colour_minimum_kt=MIN_COLOUR_WIND_SPEED_KT,
colour_maximum_kt=MAX_COLOUR_WIND_SPEED_KT)
num_fronts = len(front_line_table.index)
for i in range(num_fronts):
this_front_type_string = front_line_table[
front_utils.FRONT_TYPE_COLUMN].values[i]
if this_front_type_string == front_utils.WARM_FRONT_STRING_ID:
this_colour = WARM_FRONT_COLOUR
else:
this_colour = COLD_FRONT_COLOUR
# front_plotting.plot_polyline(
# latitudes_deg=front_line_table[
# front_utils.LATITUDES_COLUMN].values[i],
# longitudes_deg=front_line_table[
# front_utils.LONGITUDES_COLUMN].values[i],
# basemap_object=basemap_object, axes_object=axes_object,
# front_type=front_line_table[
# front_utils.FRONT_TYPE_COLUMN].values[i],
# line_width=FRONT_LINE_WIDTH, line_colour=this_colour)
print 'Saving figure to: "{0:s}"...'.format(OUTPUT_FILE_NAME)
file_system_utils.mkdir_recursive_if_necessary(file_name=OUTPUT_FILE_NAME)
pyplot.savefig(OUTPUT_FILE_NAME, dpi=OUTPUT_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=OUTPUT_FILE_NAME,
output_file_name=OUTPUT_FILE_NAME)
def _plot_observations_one_time(
valid_time_string, title_string, annotation_string, output_file_name):
"""Plots observations (NARR predictors and WPC fronts) for one valid time.
:param valid_time_string: Valid time (format "yyyy-mm-dd-HH").
:param title_string: Title (will be placed above figure).
:param annotation_string: Text annotation (will be placed in top left of
figure).
:param output_file_name: Path to output file (figure will be saved here).
"""
(narr_row_limits, narr_column_limits
) = nwp_plotting.latlng_limits_to_rowcol_limits(
min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG,
min_longitude_deg=MIN_LONGITUDE_DEG,
max_longitude_deg=MAX_LONGITUDE_DEG,
model_name=nwp_model_utils.NARR_MODEL_NAME)
valid_time_unix_sec = time_conversion.string_to_unix_sec(
valid_time_string, INPUT_TIME_FORMAT)
front_file_name = fronts_io.find_file_for_one_time(
top_directory_name=TOP_FRONT_DIR_NAME,
file_type=fronts_io.POLYLINE_FILE_TYPE,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(front_file_name)
front_line_table = fronts_io.read_polylines_from_file(front_file_name)
num_narr_fields = len(NARR_FIELD_NAMES)
narr_matrix_by_field = [numpy.array([])] * num_narr_fields
for j in range(num_narr_fields):
if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES:
this_directory_name = TOP_NARR_WIND_DIR_NAME + ''
else:
this_directory_name = TOP_NARR_DIR_NAME + ''
this_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=this_directory_name,
field_name=NARR_FIELD_NAMES[j], pressure_level_mb=PRESSURE_LEVEL_MB,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(this_file_name)
narr_matrix_by_field[j] = processed_narr_io.read_fields_from_file(
this_file_name)[0][0, ...]
narr_matrix_by_field[j] = utils.fill_nans(narr_matrix_by_field[j])
narr_matrix_by_field[j] = narr_matrix_by_field[j][
narr_row_limits[0]:(narr_row_limits[1] + 1),
narr_column_limits[0]:(narr_column_limits[1] + 1)
]
if NARR_FIELD_NAMES[j] == processed_narr_io.WET_BULB_THETA_NAME:
narr_matrix_by_field[j] = (
narr_matrix_by_field[j] - ZERO_CELSIUS_IN_KELVINS
)
_, axes_object, basemap_object = nwp_plotting.init_basemap(
model_name=nwp_model_utils.NARR_MODEL_NAME,
first_row_in_full_grid=narr_row_limits[0],
last_row_in_full_grid=narr_row_limits[1],
first_column_in_full_grid=narr_column_limits[0],
last_column_in_full_grid=narr_column_limits[1])
plotting_utils.plot_coastlines(
basemap_object=basemap_object, axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(
basemap_object=basemap_object, axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(
basemap_object=basemap_object, axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(
basemap_object=basemap_object, axes_object=axes_object,
bottom_left_lat_deg=-90., upper_right_lat_deg=90.,
parallel_spacing_deg=PARALLEL_SPACING_DEG)
plotting_utils.plot_meridians(
basemap_object=basemap_object, axes_object=axes_object,
bottom_left_lng_deg=0., upper_right_lng_deg=360.,
meridian_spacing_deg=MERIDIAN_SPACING_DEG)
for j in range(num_narr_fields):
if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES:
continue
min_colour_value = numpy.percentile(
narr_matrix_by_field[j], MIN_COLOUR_PERCENTILE)
max_colour_value = numpy.percentile(
narr_matrix_by_field[j], MAX_COLOUR_PERCENTILE)
nwp_plotting.plot_subgrid(
field_matrix=narr_matrix_by_field[j],
model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object,
basemap_object=basemap_object, colour_map=THERMAL_COLOUR_MAP_OBJECT,
min_value_in_colour_map=min_colour_value,
max_value_in_colour_map=max_colour_value,
first_row_in_full_grid=narr_row_limits[0],
first_column_in_full_grid=narr_column_limits[0])
plotting_utils.add_linear_colour_bar(
axes_object_or_list=axes_object,
values_to_colour=narr_matrix_by_field[j],
colour_map=THERMAL_COLOUR_MAP_OBJECT, colour_min=min_colour_value,
colour_max=max_colour_value, orientation='vertical',
extend_min=True, extend_max=True,
fraction_of_axis_length=LENGTH_FRACTION_FOR_THETA_COLOUR_BAR)
u_wind_index = NARR_FIELD_NAMES.index(
processed_narr_io.U_WIND_EARTH_RELATIVE_NAME)
v_wind_index = NARR_FIELD_NAMES.index(
processed_narr_io.V_WIND_EARTH_RELATIVE_NAME)
nwp_plotting.plot_wind_barbs_on_subgrid(
u_wind_matrix_m_s01=narr_matrix_by_field[u_wind_index],
v_wind_matrix_m_s01=narr_matrix_by_field[v_wind_index],
model_name=nwp_model_utils.NARR_MODEL_NAME, axes_object=axes_object,
basemap_object=basemap_object,
first_row_in_full_grid=narr_row_limits[0],
first_column_in_full_grid=narr_column_limits[0],
plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB,
plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB,
barb_length=WIND_BARB_LENGTH, empty_barb_radius=EMPTY_WIND_BARB_RADIUS,
colour_map=WIND_COLOUR_MAP_OBJECT,
colour_minimum_kt=MIN_COLOUR_WIND_SPEED_KT,
colour_maximum_kt=MAX_COLOUR_WIND_SPEED_KT)
num_fronts = len(front_line_table.index)
for i in range(num_fronts):
this_front_type_string = front_line_table[
front_utils.FRONT_TYPE_COLUMN].values[i]
if this_front_type_string == front_utils.WARM_FRONT_STRING_ID:
this_colour = WARM_FRONT_COLOUR
else:
this_colour = COLD_FRONT_COLOUR
front_plotting.plot_polyline(
latitudes_deg=front_line_table[
front_utils.LATITUDES_COLUMN].values[i],
longitudes_deg=front_line_table[
front_utils.LONGITUDES_COLUMN].values[i],
basemap_object=basemap_object, axes_object=axes_object,
front_type=front_line_table[
front_utils.FRONT_TYPE_COLUMN].values[i],
line_width=FRONT_LINE_WIDTH, line_colour=this_colour)
pyplot.title(title_string)
plotting_utils.annotate_axes(
axes_object=axes_object, annotation_string=annotation_string)
print 'Saving figure to: "{0:s}"...'.format(output_file_name)
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=output_file_name,
output_file_name=output_file_name)
def _plot_one_time(valid_time_string, pressure_level_mb, title_string,
annotation_string, narr_rotation_cos_matrix,
narr_rotation_sin_matrix):
"""Plots WPC fronts and NARR fields at one time.
M = number of grid rows in the full NARR
N = number of grid columns in the full NARR
:param valid_time_string: Valid time (format "yyyy-mm-dd-HH").
:param pressure_level_mb: Pressure level (millibars).
:param title_string: Title (will be placed above figure).
:param annotation_string: Annotation (will be placed above and left of
figure).
:param narr_rotation_cos_matrix: M-by-N numpy array of cosines for wind-
rotation angles.
:param narr_rotation_sin_matrix: M-by-N numpy array of sines for wind-
rotation angles.
"""
narr_row_limits, narr_column_limits = (
nwp_plotting.latlng_limits_to_rowcol_limits(
min_latitude_deg=MIN_LATITUDE_DEG,
max_latitude_deg=MAX_LATITUDE_DEG,
min_longitude_deg=MIN_LONGITUDE_DEG,
max_longitude_deg=MAX_LONGITUDE_DEG,
model_name=nwp_model_utils.NARR_MODEL_NAME))
valid_time_unix_sec = time_conversion.string_to_unix_sec(
valid_time_string, DEFAULT_TIME_FORMAT)
front_file_name = fronts_io.find_file_for_one_time(
top_directory_name=TOP_FRONT_DIR_NAME,
file_type=fronts_io.POLYLINE_FILE_TYPE,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(front_file_name)
front_line_table = fronts_io.read_polylines_from_file(front_file_name)
num_narr_fields = len(NARR_FIELD_NAMES)
narr_matrix_by_field = [numpy.array([])] * num_narr_fields
for j in range(num_narr_fields):
this_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=TOP_NARR_DIRECTORY_NAME,
field_name=NARR_FIELD_NAMES[j],
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_time_unix_sec)
print 'Reading data from: "{0:s}"...'.format(this_file_name)
narr_matrix_by_field[j] = processed_narr_io.read_fields_from_file(
this_file_name)[0][0, ...]
narr_matrix_by_field[j] = utils.fill_nans(narr_matrix_by_field[j])
narr_matrix_by_field[j] = narr_matrix_by_field[j][narr_row_limits[0]:(
narr_row_limits[1] +
1), narr_column_limits[0]:(narr_column_limits[1] + 1)]
if NARR_FIELD_NAMES[j] == processed_narr_io.WET_BULB_THETA_NAME:
narr_matrix_by_field[j] = (narr_matrix_by_field[j] -
ZERO_CELSIUS_IN_KELVINS)
_, axes_object, basemap_object = nwp_plotting.init_basemap(
model_name=nwp_model_utils.NARR_MODEL_NAME,
first_row_in_full_grid=narr_row_limits[0],
last_row_in_full_grid=narr_row_limits[1],
first_column_in_full_grid=narr_column_limits[0],
last_column_in_full_grid=narr_column_limits[1])
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
bottom_left_lat_deg=-90.,
upper_right_lat_deg=90.,
parallel_spacing_deg=PARALLEL_SPACING_DEG)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
bottom_left_lng_deg=0.,
upper_right_lng_deg=360.,
meridian_spacing_deg=MERIDIAN_SPACING_DEG)
for j in range(num_narr_fields):
if NARR_FIELD_NAMES[j] in WIND_FIELD_NAMES:
continue
min_colour_value = numpy.percentile(narr_matrix_by_field[j],
MIN_COLOUR_PERCENTILE)
max_colour_value = numpy.percentile(narr_matrix_by_field[j],
MAX_COLOUR_PERCENTILE)
nwp_plotting.plot_subgrid(
field_matrix=narr_matrix_by_field[j],
model_name=nwp_model_utils.NARR_MODEL_NAME,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map=THERMAL_COLOUR_MAP_OBJECT,
min_value_in_colour_map=min_colour_value,
max_value_in_colour_map=max_colour_value,
first_row_in_full_grid=narr_row_limits[0],
first_column_in_full_grid=narr_column_limits[0])
plotting_utils.add_linear_colour_bar(
axes_object_or_list=axes_object,
values_to_colour=narr_matrix_by_field[j],
colour_map=THERMAL_COLOUR_MAP_OBJECT,
colour_min=min_colour_value,
colour_max=max_colour_value,
orientation='horizontal',
extend_min=True,
extend_max=True,
fraction_of_axis_length=0.9)
this_cos_matrix = narr_rotation_cos_matrix[narr_row_limits[0]:(
narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] +
1)]
this_sin_matrix = narr_rotation_sin_matrix[narr_row_limits[0]:(
narr_row_limits[1] + 1), narr_column_limits[0]:(narr_column_limits[1] +
1)]
u_wind_index = NARR_FIELD_NAMES.index(
processed_narr_io.U_WIND_GRID_RELATIVE_NAME)
v_wind_index = NARR_FIELD_NAMES.index(
processed_narr_io.V_WIND_GRID_RELATIVE_NAME)
narr_matrix_by_field[u_wind_index], narr_matrix_by_field[v_wind_index] = (
nwp_model_utils.rotate_winds_to_earth_relative(
u_winds_grid_relative_m_s01=narr_matrix_by_field[u_wind_index],
v_winds_grid_relative_m_s01=narr_matrix_by_field[v_wind_index],
rotation_angle_cosines=this_cos_matrix,
rotation_angle_sines=this_sin_matrix))
nwp_plotting.plot_wind_barbs_on_subgrid(
u_wind_matrix_m_s01=narr_matrix_by_field[u_wind_index],
v_wind_matrix_m_s01=narr_matrix_by_field[v_wind_index],
model_name=nwp_model_utils.NARR_MODEL_NAME,
axes_object=axes_object,
basemap_object=basemap_object,
first_row_in_full_grid=narr_row_limits[0],
first_column_in_full_grid=narr_column_limits[0],
plot_every_k_rows=PLOT_EVERY_KTH_WIND_BARB,
plot_every_k_columns=PLOT_EVERY_KTH_WIND_BARB,
barb_length=WIND_BARB_LENGTH,
empty_barb_radius=EMPTY_WIND_BARB_RADIUS,
fill_empty_barb=False,
colour_map=WIND_COLOUR_MAP_OBJECT,
colour_minimum_kt=MIN_COLOUR_WIND_SPEED_KT,
colour_maximum_kt=MAX_COLOUR_WIND_SPEED_KT)
num_fronts = len(front_line_table.index)
for i in range(num_fronts):
this_front_type_string = front_line_table[
front_utils.FRONT_TYPE_COLUMN].values[i]
if this_front_type_string == front_utils.WARM_FRONT_STRING_ID:
this_colour = WARM_FRONT_COLOUR
else:
this_colour = COLD_FRONT_COLOUR
front_plotting.plot_front_with_markers(
line_latitudes_deg=front_line_table[
front_utils.LATITUDES_COLUMN].values[i],
line_longitudes_deg=front_line_table[
front_utils.LONGITUDES_COLUMN].values[i],
axes_object=axes_object,
basemap_object=basemap_object,
front_type_string=front_line_table[
front_utils.FRONT_TYPE_COLUMN].values[i],
marker_colour=this_colour)
pyplot.title(title_string)
plotting_utils.annotate_axes(axes_object=axes_object,
annotation_string=annotation_string)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=OUTPUT_DIR_NAME)
figure_file_name = '{0:s}/fronts_{1:04d}mb_{2:s}.jpg'.format(
OUTPUT_DIR_NAME, pressure_level_mb, valid_time_string)
print 'Saving figure to: "{0:s}"...'.format(figure_file_name)
pyplot.savefig(figure_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=figure_file_name,
output_file_name=figure_file_name)
return figure_file_name
def _run(first_time_string, last_time_string, randomize_times, num_times,
thermal_field_name, smoothing_radius_pixels, warm_front_percentile,
cold_front_percentile, num_closing_iters, pressure_level_mb,
top_narr_directory_name, narr_mask_file_name, output_dir_name):
"""Uses NFA (numerical frontal analysis) to predict front type at each px.
This is effectively the main method.
:param first_time_string: See documentation at top of file.
:param last_time_string: Same.
:param randomize_times: Same.
:param num_times: Same.
:param thermal_field_name: Same.
:param smoothing_radius_pixels: Same.
:param warm_front_percentile: Same.
:param cold_front_percentile: Same.
:param num_closing_iters: Same.
:param pressure_level_mb: Same.
:param top_narr_directory_name: Same.
:param narr_mask_file_name: Same.
:param output_dir_name: Same.
:raises: ValueError: if
`thermal_field_name not in VALID_THERMAL_FIELD_NAMES`.
"""
if thermal_field_name not in VALID_THERMAL_FIELD_NAMES:
error_string = (
'\n{0:s}\nValid thermal fields (listed above) do not include '
'"{1:s}".').format(str(VALID_THERMAL_FIELD_NAMES),
thermal_field_name)
raise ValueError(error_string)
cutoff_radius_pixels = 4 * smoothing_radius_pixels
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, INPUT_TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, INPUT_TIME_FORMAT)
valid_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=NARR_TIME_INTERVAL_SEC,
include_endpoint=True)
if randomize_times:
error_checking.assert_is_leq(num_times, len(valid_times_unix_sec))
numpy.random.shuffle(valid_times_unix_sec)
valid_times_unix_sec = valid_times_unix_sec[:num_times]
if narr_mask_file_name == '':
num_grid_rows, num_grid_columns = nwp_model_utils.get_grid_dimensions(
model_name=nwp_model_utils.NARR_MODEL_NAME)
narr_mask_matrix = numpy.full((num_grid_rows, num_grid_columns),
1,
dtype=int)
else:
print 'Reading mask from: "{0:s}"...\n'.format(narr_mask_file_name)
narr_mask_matrix = ml_utils.read_narr_mask(narr_mask_file_name)
x_spacing_metres, y_spacing_metres = nwp_model_utils.get_xy_grid_spacing(
model_name=nwp_model_utils.NARR_MODEL_NAME)
num_times = len(valid_times_unix_sec)
for i in range(num_times):
this_thermal_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_directory_name,
field_name=thermal_field_name,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_times_unix_sec[i])
print 'Reading data from: "{0:s}"...'.format(this_thermal_file_name)
this_thermal_matrix_kelvins = processed_narr_io.read_fields_from_file(
this_thermal_file_name)[0][0, ...]
this_thermal_matrix_kelvins = general_utils.fill_nans(
this_thermal_matrix_kelvins)
this_thermal_matrix_kelvins = nfa.gaussian_smooth_2d_field(
field_matrix=this_thermal_matrix_kelvins,
standard_deviation_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels)
this_u_wind_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_directory_name,
field_name=processed_narr_io.U_WIND_GRID_RELATIVE_NAME,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_times_unix_sec[i])
print 'Reading data from: "{0:s}"...'.format(this_u_wind_file_name)
this_u_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file(
this_u_wind_file_name)[0][0, ...]
this_u_wind_matrix_m_s01 = general_utils.fill_nans(
this_u_wind_matrix_m_s01)
this_u_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field(
field_matrix=this_u_wind_matrix_m_s01,
standard_deviation_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels)
this_v_wind_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_narr_directory_name,
field_name=processed_narr_io.V_WIND_GRID_RELATIVE_NAME,
pressure_level_mb=pressure_level_mb,
valid_time_unix_sec=valid_times_unix_sec[i])
print 'Reading data from: "{0:s}"...'.format(this_v_wind_file_name)
this_v_wind_matrix_m_s01 = processed_narr_io.read_fields_from_file(
this_v_wind_file_name)[0][0, ...]
this_v_wind_matrix_m_s01 = general_utils.fill_nans(
this_v_wind_matrix_m_s01)
this_v_wind_matrix_m_s01 = nfa.gaussian_smooth_2d_field(
field_matrix=this_v_wind_matrix_m_s01,
standard_deviation_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels)
this_tfp_matrix_kelvins_m02 = nfa.get_thermal_front_param(
thermal_field_matrix_kelvins=this_thermal_matrix_kelvins,
x_spacing_metres=x_spacing_metres,
y_spacing_metres=y_spacing_metres)
this_tfp_matrix_kelvins_m02[narr_mask_matrix == 0] = 0.
this_proj_velocity_matrix_m_s01 = nfa.project_wind_to_thermal_gradient(
u_matrix_grid_relative_m_s01=this_u_wind_matrix_m_s01,
v_matrix_grid_relative_m_s01=this_v_wind_matrix_m_s01,
thermal_field_matrix_kelvins=this_thermal_matrix_kelvins,
x_spacing_metres=x_spacing_metres,
y_spacing_metres=y_spacing_metres)
this_locating_var_matrix_m01_s01 = nfa.get_locating_variable(
tfp_matrix_kelvins_m02=this_tfp_matrix_kelvins_m02,
projected_velocity_matrix_m_s01=this_proj_velocity_matrix_m_s01)
this_predicted_label_matrix = nfa.get_front_types(
locating_var_matrix_m01_s01=this_locating_var_matrix_m01_s01,
warm_front_percentile=warm_front_percentile,
cold_front_percentile=cold_front_percentile)
this_predicted_label_matrix = front_utils.close_frontal_image(
ternary_image_matrix=this_predicted_label_matrix,
num_iterations=num_closing_iters)
this_prediction_file_name = nfa.find_prediction_file(
directory_name=output_dir_name,
first_valid_time_unix_sec=valid_times_unix_sec[i],
last_valid_time_unix_sec=valid_times_unix_sec[i],
ensembled=False,
raise_error_if_missing=False)
print 'Writing gridded predictions to file: "{0:s}"...\n'.format(
this_prediction_file_name)
nfa.write_gridded_predictions(
pickle_file_name=this_prediction_file_name,
predicted_label_matrix=numpy.expand_dims(
this_predicted_label_matrix, axis=0),
valid_times_unix_sec=valid_times_unix_sec[[i]],
narr_mask_matrix=narr_mask_matrix,
pressure_level_mb=pressure_level_mb,
smoothing_radius_pixels=smoothing_radius_pixels,
cutoff_radius_pixels=cutoff_radius_pixels,
warm_front_percentile=warm_front_percentile,
cold_front_percentile=cold_front_percentile,
num_closing_iters=num_closing_iters)
def _run(top_input_dir_name, first_time_string, last_time_string,
pressure_level_mb, top_output_dir_name):
"""Computes theta_w (wet-bulb potential temperature) for NARR data.
This is effectively the main method.
:param top_input_dir_name: See documentation at top of file.
:param first_time_string: Same.
:param last_time_string: Same.
:param pressure_level_mb: Same.
:param top_output_dir_name: Same.
"""
if pressure_level_mb <= 0:
pressure_level_mb = None
if pressure_level_mb is None:
pressure_in_file_name_mb = DUMMY_PRESSURE_LEVEL_MB + 0
else:
pressure_in_file_name_mb = pressure_level_mb + 0
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, INPUT_TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, INPUT_TIME_FORMAT)
valid_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=TIME_INTERVAL_SECONDS)
num_times = len(valid_times_unix_sec)
this_pressure_matrix_pascals = None
for i in range(num_times):
this_temperature_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_input_dir_name,
field_name=processed_narr_io.TEMPERATURE_NAME,
pressure_level_mb=pressure_in_file_name_mb,
valid_time_unix_sec=valid_times_unix_sec[i])
print 'Reading data from: "{0:s}"...'.format(
this_temperature_file_name)
this_temperature_matrix_kelvins = (
processed_narr_io.read_fields_from_file(
this_temperature_file_name)[0])
this_humidity_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_input_dir_name,
field_name=processed_narr_io.SPECIFIC_HUMIDITY_NAME,
pressure_level_mb=pressure_in_file_name_mb,
valid_time_unix_sec=valid_times_unix_sec[i])
print 'Reading data from: "{0:s}"...'.format(this_humidity_file_name)
this_humidity_matrix_kg_kg01 = (
processed_narr_io.read_fields_from_file(this_humidity_file_name)[0]
)
if pressure_level_mb is None:
this_pressure_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_input_dir_name,
field_name=processed_narr_io.HEIGHT_NAME,
pressure_level_mb=pressure_in_file_name_mb,
valid_time_unix_sec=valid_times_unix_sec[i])
print 'Reading data from: "{0:s}"...'.format(
this_pressure_file_name)
this_pressure_matrix_pascals = (
processed_narr_io.read_fields_from_file(
this_pressure_file_name)[0])
print this_pressure_matrix_pascals[:5, :5]
else:
if this_pressure_matrix_pascals is None:
this_pressure_matrix_pascals = numpy.full(
this_humidity_matrix_kg_kg01.shape,
pressure_level_mb * MB_TO_PASCALS)
this_dewpoint_matrix_kelvins = (
moisture_conversions.specific_humidity_to_dewpoint(
specific_humidities_kg_kg01=this_humidity_matrix_kg_kg01,
total_pressures_pascals=this_pressure_matrix_pascals))
this_wb_temp_matrix_kelvins = (
ge_conversions.dewpoint_to_wet_bulb_temperature(
dewpoints_kelvins=this_dewpoint_matrix_kelvins,
temperatures_kelvins=this_temperature_matrix_kelvins,
total_pressures_pascals=this_pressure_matrix_pascals))
this_theta_w_matrix_kelvins = (
temperature_conversions.temperatures_to_potential_temperatures(
temperatures_kelvins=this_wb_temp_matrix_kelvins,
total_pressures_pascals=this_pressure_matrix_pascals))
this_theta_w_file_name = processed_narr_io.find_file_for_one_time(
top_directory_name=top_output_dir_name,
field_name=processed_narr_io.WET_BULB_THETA_NAME,
pressure_level_mb=pressure_in_file_name_mb,
valid_time_unix_sec=valid_times_unix_sec[i],
raise_error_if_missing=False)
print 'Writing theta_w field to: "{0:s}"...\n'.format(
this_theta_w_file_name)
processed_narr_io.write_fields_to_file(
pickle_file_name=this_theta_w_file_name,
field_matrix=this_theta_w_matrix_kelvins,
field_name=processed_narr_io.WET_BULB_THETA_NAME,
pressure_level_pascals=pressure_in_file_name_mb * MB_TO_PASCALS,
valid_times_unix_sec=valid_times_unix_sec[[i]])
def create_full_size_4d_example(target_time_unix_sec, num_lead_time_steps,
predictor_time_step_offsets,
top_narr_directory_name,
top_frontal_grid_dir_name,
narr_predictor_names, pressure_level_mb,
dilation_distance_metres, num_classes):
"""Creates full-size 4-D examples from raw files.
:param target_time_unix_sec: See doc for `create_downsized_3d_examples`.
:param num_lead_time_steps: See doc for `create_downsized_4d_examples`.
:param predictor_time_step_offsets: Same.
:param top_narr_directory_name: See doc for `create_downsized_3d_examples`.
:param top_frontal_grid_dir_name: Same.
:param narr_predictor_names: Same.
:param pressure_level_mb: Same.
:param dilation_distance_metres: Same.
:param num_classes: Same.
:return: predictor_matrix: 1-by-M-by-N-by-T-by-C numpy array of predictor
values.
:return: target_matrix: 1-by-M-by-N numpy array of target values. Each
value is an integer from the list `front_utils.VALID_INTEGER_IDS`.
"""
error_checking.assert_is_integer(num_classes)
error_checking.assert_is_geq(num_classes, 2)
error_checking.assert_is_leq(num_classes, 3)
(narr_file_name_matrix,
frontal_grid_file_names) = trainval_io.find_input_files_for_4d_examples(
first_target_time_unix_sec=target_time_unix_sec,
last_target_time_unix_sec=target_time_unix_sec,
predictor_time_step_offsets=predictor_time_step_offsets,
num_lead_time_steps=num_lead_time_steps,
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=narr_predictor_names,
pressure_level_mb=pressure_level_mb)
narr_file_name_matrix = narr_file_name_matrix[0, ...]
frontal_grid_file_name = frontal_grid_file_names[0]
num_predictor_times_per_example = len(predictor_time_step_offsets)
num_predictors = len(narr_predictor_names)
tuple_of_4d_predictor_matrices = ()
for i in range(num_predictor_times_per_example):
tuple_of_3d_predictor_matrices = ()
for j in range(num_predictors):
print 'Reading data from: "{0:s}"...'.format(
narr_file_name_matrix[i, j])
this_field_predictor_matrix = (
processed_narr_io.read_fields_from_file(
narr_file_name_matrix[i, j]))[0]
this_field_predictor_matrix = (
ml_utils.fill_nans_in_predictor_images(
this_field_predictor_matrix))
tuple_of_3d_predictor_matrices += (this_field_predictor_matrix, )
tuple_of_4d_predictor_matrices += (ml_utils.stack_predictor_variables(
tuple_of_3d_predictor_matrices), )
predictor_matrix = ml_utils.stack_time_steps(
tuple_of_4d_predictor_matrices)
print 'Reading data from: "{0:s}"...'.format(frontal_grid_file_name)
frontal_grid_table = fronts_io.read_narr_grids_from_file(
frontal_grid_file_name)
print 'Processing full-size 4-D machine-learning example...'
predictor_matrix, _ = ml_utils.normalize_predictors(
predictor_matrix=predictor_matrix)
target_matrix = ml_utils.front_table_to_images(
frontal_grid_table=frontal_grid_table,
num_rows_per_image=predictor_matrix.shape[1],
num_columns_per_image=predictor_matrix.shape[2])
if num_classes == 2:
target_matrix = ml_utils.binarize_front_images(target_matrix)
predictor_matrix = ml_utils.subset_narr_grid_for_fcn_input(
predictor_matrix)
target_matrix = ml_utils.subset_narr_grid_for_fcn_input(target_matrix)
if num_classes == 2:
target_matrix = ml_utils.dilate_binary_target_images(
target_matrix=target_matrix,
dilation_distance_metres=dilation_distance_metres,
verbose=False)
else:
target_matrix = ml_utils.dilate_ternary_target_images(
target_matrix=target_matrix,
dilation_distance_metres=dilation_distance_metres,
verbose=False)
predictor_matrix = predictor_matrix.astype('float32')
print 'Fraction of pixels with a front = {0:.4f}'.format(
numpy.mean(target_matrix > 0))
target_matrix = numpy.expand_dims(target_matrix, axis=-1)
return predictor_matrix, target_matrix
def create_downsized_3d_examples(center_row_indices,
center_column_indices,
num_rows_in_half_grid,
num_columns_in_half_grid,
full_predictor_matrix=None,
full_target_matrix=None,
target_time_unix_sec=None,
top_narr_directory_name=None,
top_frontal_grid_dir_name=None,
narr_predictor_names=None,
pressure_level_mb=None,
dilation_distance_metres=None,
num_classes=None):
"""Creates downsized 3-D examples from raw files.
This method creates one example for each center pixel, where center pixels
are specified by `center_row_indices` and `center_column_indices`.
If `full_predictor_matrix` and `full_target_matrix` are specified, all input
args thereafter will be ignored.
E = number of examples (center pixels)
P = number of rows in full NARR grid
Q = number of columns in full NARR grid
:param center_row_indices: length-E numpy array with row for each center
pixel.
:param center_column_indices: length-E numpy array with column for each
center pixel.
:param num_rows_in_half_grid: Number of rows in half-grid for each example.
Actual number of rows will be 2 * `num_rows_in_half_grid` + 1.
:param num_columns_in_half_grid: Same but for columns.
:param full_predictor_matrix: 1-by-P-by-Q-by-C numpy array of predictor
values.
:param full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values.
Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`.
:param target_time_unix_sec: Target time.
:param top_narr_directory_name: Name of top-level directory with NARR data.
Files therein will be found by
`processed_narr_io.find_file_for_one_time` and read by
`processed_narr_io.read_fields_from_file`.
:param top_frontal_grid_dir_name: Name of top-level directory with target
values (grids of front labels). Files therein will be found by
`fronts_io.find_file_for_one_time` and read by
`fronts_io.read_narr_grids_from_file`.
:param narr_predictor_names: length-C list with names of predictor
variables. Each must be accepted by
`processed_narr_io.check_field_name`.
:param pressure_level_mb: Pressure level (millibars) for predictors.
:param dilation_distance_metres: Dilation distance. Will be used to dilate
WF and CF labels, which effectively creates a distance buffer around
each front, thus accounting for spatial uncertainty in front placement.
:param num_classes: Number of target classes (either 2 or 3).
:return: downsized_predictor_matrix: E-by-M-by-N-by-C numpy array of
predictor values.
:return: target_values: length-E numpy array of target values (integers from
the list `front_utils.VALID_INTEGER_IDS`).
:return: full_predictor_matrix: 1-by-P-by-Q-by-C numpy array of predictor
values.
:return: full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values.
Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`.
"""
if full_predictor_matrix is None or full_target_matrix is None:
error_checking.assert_is_integer(num_classes)
error_checking.assert_is_geq(num_classes, 2)
error_checking.assert_is_leq(num_classes, 3)
(narr_file_name_matrix, frontal_grid_file_names
) = trainval_io.find_input_files_for_3d_examples(
first_target_time_unix_sec=target_time_unix_sec,
last_target_time_unix_sec=target_time_unix_sec,
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=narr_predictor_names,
pressure_level_mb=pressure_level_mb)
narr_file_names = narr_file_name_matrix[0, :]
frontal_grid_file_name = frontal_grid_file_names[0]
num_predictors = len(narr_predictor_names)
tuple_of_full_predictor_matrices = ()
for j in range(num_predictors):
print 'Reading data from: "{0:s}"...'.format(narr_file_names[j])
this_field_predictor_matrix = (
processed_narr_io.read_fields_from_file(narr_file_names[j]))[0]
this_field_predictor_matrix = (
ml_utils.fill_nans_in_predictor_images(
this_field_predictor_matrix))
tuple_of_full_predictor_matrices += (this_field_predictor_matrix, )
print 'Reading data from: "{0:s}"...'.format(frontal_grid_file_name)
frontal_grid_table = fronts_io.read_narr_grids_from_file(
frontal_grid_file_name)
full_predictor_matrix = ml_utils.stack_predictor_variables(
tuple_of_full_predictor_matrices)
full_predictor_matrix, _ = ml_utils.normalize_predictors(
predictor_matrix=full_predictor_matrix)
full_target_matrix = ml_utils.front_table_to_images(
frontal_grid_table=frontal_grid_table,
num_rows_per_image=full_predictor_matrix.shape[1],
num_columns_per_image=full_predictor_matrix.shape[2])
if num_classes == 2:
full_target_matrix = ml_utils.binarize_front_images(
full_target_matrix)
if num_classes == 2:
full_target_matrix = ml_utils.dilate_binary_target_images(
target_matrix=full_target_matrix,
dilation_distance_metres=dilation_distance_metres,
verbose=False)
else:
full_target_matrix = ml_utils.dilate_ternary_target_images(
target_matrix=full_target_matrix,
dilation_distance_metres=dilation_distance_metres,
verbose=False)
print 'Creating {0:d} downsized 3-D examples...'.format(
len(center_row_indices))
this_target_point_dict = {
ml_utils.ROW_INDICES_BY_TIME_KEY: [center_row_indices],
ml_utils.COLUMN_INDICES_BY_TIME_KEY: [center_column_indices]
}
(downsized_predictor_matrix,
target_values) = ml_utils.downsize_grids_around_selected_points(
predictor_matrix=full_predictor_matrix,
target_matrix=full_target_matrix,
num_rows_in_half_window=num_rows_in_half_grid,
num_columns_in_half_window=num_columns_in_half_grid,
target_point_dict=this_target_point_dict,
verbose=False)[:2]
downsized_predictor_matrix = downsized_predictor_matrix.astype('float32')
return (downsized_predictor_matrix, target_values, full_predictor_matrix,
full_target_matrix)
def create_downsized_4d_examples(center_row_indices,
center_column_indices,
num_rows_in_half_grid,
num_columns_in_half_grid,
full_predictor_matrix=None,
full_target_matrix=None,
target_time_unix_sec=None,
num_lead_time_steps=None,
predictor_time_step_offsets=None,
top_narr_directory_name=None,
top_frontal_grid_dir_name=None,
narr_predictor_names=None,
pressure_level_mb=None,
dilation_distance_metres=None,
num_classes=None):
"""Creates downsized 4-D examples from raw files.
This method creates one example for each center pixel, where center pixels
are specified by `center_row_indices` and `center_column_indices`.
If `full_predictor_matrix` and `full_target_matrix` are specified, all input
args thereafter will be ignored.
E = number of examples (center pixels)
P = number of rows in full NARR grid
Q = number of columns in full NARR grid
:param center_row_indices: See doc for `create_downsized_3d_examples`.
:param center_column_indices: Same.
:param num_rows_in_half_grid: Same.
:param num_columns_in_half_grid: Same.
:param full_predictor_matrix: 1-by-P-by-Q-by-T-by-C numpy array of
predictor values.
:param full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values.
Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`.
:param target_time_unix_sec: See doc for `create_downsized_3d_examples`.
:param num_lead_time_steps: Number of time steps between target time and
latest possible predictor time.
:param predictor_time_step_offsets: length-T numpy array of offsets between
predictor time and latest possible predictor time (target time minus
lead time).
:param top_narr_directory_name: See doc for `create_downsized_3d_examples`.
:param top_frontal_grid_dir_name: Same.
:param narr_predictor_names: Same.
:param pressure_level_mb: Same.
:param dilation_distance_metres: Same.
:param num_classes: Same.
:return: downsized_predictor_matrix: E-by-M-by-N-by-T-by-C numpy array of
predictor values.
:return: target_values: length-E numpy array of target values (integers from
the list `front_utils.VALID_INTEGER_IDS`).
:return: full_predictor_matrix: 1-by-P-by-Q-by-T-by-C numpy array of
predictor values.
:return: full_target_matrix: 1-by-P-by-Q-by-C numpy array of target values.
Each value is an integer from the list `front_utils.VALID_INTEGER_IDS`.
"""
if full_predictor_matrix is None or full_target_matrix is None:
error_checking.assert_is_integer(num_classes)
error_checking.assert_is_geq(num_classes, 2)
error_checking.assert_is_leq(num_classes, 3)
(narr_file_name_matrix, frontal_grid_file_names
) = trainval_io.find_input_files_for_4d_examples(
first_target_time_unix_sec=target_time_unix_sec,
last_target_time_unix_sec=target_time_unix_sec,
predictor_time_step_offsets=predictor_time_step_offsets,
num_lead_time_steps=num_lead_time_steps,
top_narr_directory_name=top_narr_directory_name,
top_frontal_grid_dir_name=top_frontal_grid_dir_name,
narr_predictor_names=narr_predictor_names,
pressure_level_mb=pressure_level_mb)
narr_file_name_matrix = narr_file_name_matrix[0, ...]
frontal_grid_file_name = frontal_grid_file_names[0]
num_predictor_times_per_example = len(predictor_time_step_offsets)
num_predictors = len(narr_predictor_names)
tuple_of_4d_predictor_matrices = ()
for i in range(num_predictor_times_per_example):
tuple_of_3d_predictor_matrices = ()
for j in range(num_predictors):
print 'Reading data from: "{0:s}"...'.format(
narr_file_name_matrix[i, j])
this_field_predictor_matrix = (
processed_narr_io.read_fields_from_file(
narr_file_name_matrix[i, j]))[0]
this_field_predictor_matrix = (
ml_utils.fill_nans_in_predictor_images(
this_field_predictor_matrix))
tuple_of_3d_predictor_matrices += (
this_field_predictor_matrix, )
tuple_of_4d_predictor_matrices += (
ml_utils.stack_predictor_variables(
tuple_of_3d_predictor_matrices), )
full_predictor_matrix = ml_utils.stack_time_steps(
tuple_of_4d_predictor_matrices)
print 'Reading data from: "{0:s}"...'.format(frontal_grid_file_name)
frontal_grid_table = fronts_io.read_narr_grids_from_file(
frontal_grid_file_name)
full_predictor_matrix, _ = ml_utils.normalize_predictors(
predictor_matrix=full_predictor_matrix)
full_target_matrix = ml_utils.front_table_to_images(
frontal_grid_table=frontal_grid_table,
num_rows_per_image=full_predictor_matrix.shape[1],
num_columns_per_image=full_predictor_matrix.shape[2])
if num_classes == 2:
full_target_matrix = ml_utils.binarize_front_images(
full_target_matrix)
if num_classes == 2:
full_target_matrix = ml_utils.dilate_binary_target_images(
target_matrix=full_target_matrix,
dilation_distance_metres=dilation_distance_metres,
verbose=False)
else:
full_target_matrix = ml_utils.dilate_ternary_target_images(
target_matrix=full_target_matrix,
dilation_distance_metres=dilation_distance_metres,
verbose=False)
print 'Creating {0:d} downsized 4-D examples...'.format(
len(center_row_indices))
this_target_point_dict = {
ml_utils.ROW_INDICES_BY_TIME_KEY: [center_row_indices],
ml_utils.COLUMN_INDICES_BY_TIME_KEY: [center_column_indices]
}
(downsized_predictor_matrix,
target_values) = ml_utils.downsize_grids_around_selected_points(
predictor_matrix=full_predictor_matrix,
target_matrix=full_target_matrix,
num_rows_in_half_window=num_rows_in_half_grid,
num_columns_in_half_window=num_columns_in_half_grid,
target_point_dict=this_target_point_dict,
verbose=False)[:2]
downsized_predictor_matrix = downsized_predictor_matrix.astype('float32')
return (downsized_predictor_matrix, target_values, full_predictor_matrix,
full_target_matrix)