def test_first_and_last_times_in_year(self):
"""Ensures correct output from first_and_last_times_in_year."""
this_start_time_unix_sec, this_end_time_unix_sec = (
time_conversion.first_and_last_times_in_year(2017))
self.assertTrue(this_start_time_unix_sec == START_TIME_2017_UNIX_SEC)
self.assertTrue(this_end_time_unix_sec == END_TIME_2017_UNIX_SEC)
def _remove_examples_in_wrong_year(example_dict, desired_year):
"""Removes examples in the wrong year.
:param example_dict: Dictionary in format created by `example_io.read_file`.
:param desired_year: Year that belongs in this dictionary (integer).
:return: example_dict: Same as input but maybe with fewer examples.
"""
first_time_unix_sec, last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(desired_year))
num_examples_orig = len(example_dict[example_utils.VALID_TIMES_KEY])
example_dict = example_utils.subset_by_time(
example_dict=example_dict,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec)[0]
num_examples = len(example_dict[example_utils.VALID_TIMES_KEY])
print(
'Removed {0:d} of {1:d} examples for being in the wrong year.'.format(
num_examples_orig - num_examples, num_examples_orig))
return example_dict
def _find_rrtm_files(rrtm_directory_name, year):
"""Finds RRTM files for the given year.
:param rrtm_directory_name: See documentation at top of file.
:param year: Year (integer).
:return: example_file_names: 1-D list of file paths.
:raises: ValueError: if no daily files can be found for the given year.
"""
first_time_unix_sec, last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(year))
num_seconds_in_year = last_time_unix_sec - first_time_unix_sec + 1
num_days_in_year = int(
numpy.round(float(num_seconds_in_year) / DAYS_TO_SECONDS))
day_indices = numpy.linspace(1,
num_days_in_year,
num=num_days_in_year,
dtype=int)
example_file_names = []
for i in day_indices:
this_file_name = '{0:s}/{1:04d}{2:03d}/output_file.{1:04d}.cdf'.format(
rrtm_directory_name, year, i)
if not os.path.isfile(this_file_name):
continue
example_file_names.append(this_file_name)
if len(example_file_names) > 0:
return example_file_names
error_string = (
'Cannot find daily examples for year {0:d} in directory "{1:s}".'
).format(year, rrtm_directory_name)
raise ValueError(error_string)
def _run(tropical_example_dir_name, non_tropical_example_dir_name,
num_histogram_bins, output_dir_name):
"""Plots distribution of each target variable.
This is effectively the main method.
:param tropical_example_dir_name: See documentation at top of file.
:param non_tropical_example_dir_name: Same.
:param num_histogram_bins: Same.
:param output_dir_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
first_time_unix_sec = (
time_conversion.first_and_last_times_in_year(FIRST_YEAR)[0])
last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(LAST_YEAR)[-1])
example_file_names = example_io.find_many_files(
directory_name=tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=True)
example_file_names += example_io.find_many_files(
directory_name=non_tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=True)
example_dicts = []
for this_file_name in example_file_names:
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_example_dict = example_io.read_file(this_file_name)
this_example_dict = example_utils.subset_by_field(
example_dict=this_example_dict, field_names=TARGET_NAMES_IN_FILE)
example_dicts.append(this_example_dict)
example_dict = example_utils.concat_examples(example_dicts)
del example_dicts
letter_label = None
panel_file_names = []
for this_target_name in TARGET_NAMES:
if this_target_name in TARGET_NAMES_IN_FILE:
these_target_values = example_utils.get_field_from_dict(
example_dict=example_dict, field_name=this_target_name)
else:
down_fluxes_w_m02 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.SHORTWAVE_SURFACE_DOWN_FLUX_NAME)
up_fluxes_w_m02 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.SHORTWAVE_TOA_UP_FLUX_NAME)
these_target_values = down_fluxes_w_m02 - up_fluxes_w_m02
these_target_values = numpy.ravel(these_target_values)
if letter_label is None:
letter_label = 'a'
else:
letter_label = chr(ord(letter_label) + 1)
this_file_name = _plot_histogram_one_target(
target_values=these_target_values,
target_name=this_target_name,
num_bins=num_histogram_bins,
letter_label=letter_label,
output_dir_name=output_dir_name)
panel_file_names.append(this_file_name)
concat_file_name = '{0:s}/target_distributions.jpg'.format(output_dir_name)
print('Concatenating panels to: "{0:s}"...'.format(concat_file_name))
imagemagick_utils.concatenate_images(input_file_names=panel_file_names,
output_file_name=concat_file_name,
num_panel_rows=2,
num_panel_columns=2,
border_width_pixels=25)
imagemagick_utils.trim_whitespace(input_file_name=concat_file_name,
output_file_name=concat_file_name)
def _run(tropical_example_dir_name, non_tropical_example_dir_name,
output_file_name):
"""Plots all sites wtih data.
This is effectively the main method.
:param tropical_example_dir_name: See documentation at top of file.
:param non_tropical_example_dir_name: Same.
:param output_file_name: Same.
"""
first_time_unix_sec = (
time_conversion.first_and_last_times_in_year(FIRST_YEAR)[0])
last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(LAST_YEAR)[-1])
tropical_file_names = example_io.find_many_files(
directory_name=tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=False)
non_tropical_file_names = example_io.find_many_files(
directory_name=non_tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=False)
latitudes_deg_n = numpy.array([])
longitudes_deg_e = numpy.array([])
for this_file_name in tropical_file_names:
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_example_dict = example_io.read_file(this_file_name)
these_latitudes_deg_n = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LATITUDE_NAME)
these_longitudes_deg_e = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LONGITUDE_NAME)
latitudes_deg_n = numpy.concatenate(
(latitudes_deg_n, these_latitudes_deg_n))
longitudes_deg_e = numpy.concatenate(
(longitudes_deg_e, these_longitudes_deg_e))
for this_file_name in non_tropical_file_names:
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_example_dict = example_io.read_file(this_file_name)
these_latitudes_deg_n = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LATITUDE_NAME)
these_longitudes_deg_e = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LONGITUDE_NAME)
latitudes_deg_n = numpy.concatenate(
(latitudes_deg_n, these_latitudes_deg_n))
longitudes_deg_e = numpy.concatenate(
(longitudes_deg_e, these_longitudes_deg_e))
coord_matrix = numpy.transpose(
numpy.vstack((latitudes_deg_n, longitudes_deg_e)))
coord_matrix = number_rounding.round_to_nearest(coord_matrix,
LATLNG_TOLERANCE_DEG)
coord_matrix = numpy.unique(coord_matrix, axis=0)
latitudes_deg_n = coord_matrix[:, 0]
longitudes_deg_e = coord_matrix[:, 1]
figure_object, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=MIN_PLOT_LATITUDE_DEG_N,
max_latitude_deg=MAX_PLOT_LATITUDE_DEG_N,
min_longitude_deg=MIN_PLOT_LONGITUDE_DEG_E,
max_longitude_deg=MAX_PLOT_LONGITUDE_DEG_E,
resolution_string='l'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR,
line_width=BORDER_WIDTH)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR,
line_width=BORDER_WIDTH)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS,
line_colour=GRID_LINE_COLOUR,
line_width=GRID_LINE_WIDTH,
font_size=FONT_SIZE)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS,
line_colour=GRID_LINE_COLOUR,
line_width=GRID_LINE_WIDTH,
font_size=FONT_SIZE)
arctic_indices = numpy.where(latitudes_deg_n >= 66.5)[0]
print(len(arctic_indices))
arctic_x_coords, arctic_y_coords = basemap_object(
longitudes_deg_e[arctic_indices], latitudes_deg_n[arctic_indices])
axes_object.plot(arctic_x_coords,
arctic_y_coords,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=ARCTIC_COLOUR,
markeredgecolor=ARCTIC_COLOUR)
mid_latitude_indices = numpy.where(
numpy.logical_and(latitudes_deg_n >= 30., latitudes_deg_n < 66.5))[0]
print(len(mid_latitude_indices))
mid_latitude_x_coords, mid_latitude_y_coords = basemap_object(
longitudes_deg_e[mid_latitude_indices],
latitudes_deg_n[mid_latitude_indices])
axes_object.plot(mid_latitude_x_coords,
mid_latitude_y_coords,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=MID_LATITUDE_COLOUR,
markeredgecolor=MID_LATITUDE_COLOUR)
tropical_indices = numpy.where(latitudes_deg_n < 30.)[0]
print(len(tropical_indices))
tropical_x_coords, tropical_y_coords = basemap_object(
longitudes_deg_e[tropical_indices], latitudes_deg_n[tropical_indices])
axes_object.plot(tropical_x_coords,
tropical_y_coords,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=TROPICAL_COLOUR,
markeredgecolor=TROPICAL_COLOUR)
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
figure_object.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_object)
def get_examples_for_inference(model_metadata_dict, example_file_name,
num_examples, example_dir_name,
example_id_file_name):
"""Returns examples to be used by a model at inference stage.
:param model_metadata_dict: Dictionary read by `neural_net.read_metafile`.
:param example_file_name: [use only if you want random examples]
Path to file with data examples (to be read by `example_io.read_file`).
:param num_examples: [use only if you want random examples]
Number of examples to use. If you want to use all examples in
`example_file_name`, leave this alone.
:param example_dir_name: [use only if you want specific examples]
Name of directory with data examples. Files therein will be found by
`example_io.find_file` and read by `example_io.read_file`.
:param example_id_file_name: [use only if you want specific examples]
Path to file with desired IDs. Will be read by
`read_example_ids_from_netcdf`.
:return: Same output variables as `neural_net.data_generator`.
"""
error_checking.assert_is_string(example_file_name)
use_specific_ids = example_file_name == ''
generator_option_dict = copy.deepcopy(
model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY])
if use_specific_ids:
error_checking.assert_is_string(example_id_file_name)
print('Reading desired example IDs from: "{0:s}"...'.format(
example_id_file_name))
example_id_strings = read_example_ids_from_netcdf(example_id_file_name)
generator_option_dict[neural_net.EXAMPLE_DIRECTORY_KEY] = (
example_dir_name)
predictor_matrix, target_array = (
neural_net.create_data_specific_examples(
option_dict=generator_option_dict,
net_type_string=model_metadata_dict[neural_net.NET_TYPE_KEY],
example_id_strings=example_id_strings))
return predictor_matrix, target_array, example_id_strings
error_checking.assert_is_string(example_dir_name)
error_checking.assert_is_integer(num_examples)
error_checking.assert_is_greater(num_examples, 0)
example_dir_name = os.path.split(example_file_name)[0]
year = example_io.file_name_to_year(example_file_name)
first_time_unix_sec, last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(year))
generator_option_dict[neural_net.EXAMPLE_DIRECTORY_KEY] = (
example_dir_name)
generator_option_dict[neural_net.FIRST_TIME_KEY] = first_time_unix_sec
generator_option_dict[neural_net.LAST_TIME_KEY] = last_time_unix_sec
predictor_matrix, target_array, example_id_strings = neural_net.create_data(
option_dict=generator_option_dict,
for_inference=True,
net_type_string=model_metadata_dict[neural_net.NET_TYPE_KEY],
is_loss_constrained_mse=False)
num_examples_total = len(example_id_strings)
if num_examples >= num_examples_total:
return predictor_matrix, target_array, example_id_strings
good_indices = numpy.linspace(0,
num_examples_total - 1,
num=num_examples_total,
dtype=int)
good_indices = numpy.random.choice(good_indices,
size=num_examples,
replace=False)
predictor_matrix = predictor_matrix[good_indices, ...]
example_id_strings = [example_id_strings[i] for i in good_indices]
if isinstance(target_array, list):
target_array = [t[good_indices, ...] for t in target_array]
else:
target_array = target_array[good_indices, ...]
return predictor_matrix, target_array, example_id_strings