def test_concat_examples_bad_heights(self):
"""Ensures correct output from concat_examples.
In this case, expecting an error due to mismatched heights.
"""
this_second_example_dict = copy.deepcopy(SECOND_EXAMPLE_DICT)
this_second_example_dict[example_utils.HEIGHTS_KEY] += 1
with self.assertRaises(ValueError):
example_utils.concat_examples(
[FIRST_EXAMPLE_DICT, this_second_example_dict])
def test_concat_examples_bad_fields(self):
"""Ensures correct output from concat_examples.
In this case, expecting an error due to mismatched fields.
"""
this_second_example_dict = copy.deepcopy(SECOND_EXAMPLE_DICT)
this_second_example_dict[
example_utils.SCALAR_PREDICTOR_NAMES_KEY].append(
example_utils.ALBEDO_NAME)
with self.assertRaises(ValueError):
example_utils.concat_examples(
[FIRST_EXAMPLE_DICT, this_second_example_dict])
def test_concat_examples_good(self):
"""Ensures correct output from concat_examples.
In this case, not expecting an error.
"""
this_example_dict = example_utils.concat_examples(
[FIRST_EXAMPLE_DICT, SECOND_EXAMPLE_DICT])
self.assertTrue(
_compare_example_dicts(this_example_dict, CONCAT_EXAMPLE_DICT))
def _process_files_one_year(rrtm_directory_name, input_example_dir_name, year):
"""Processes RRTM files for one year.
:param rrtm_directory_name: See documentation at top of file.
:param input_example_dir_name: Same.
:param year: Year (integer).
:return: example_dict: Dictionary with all examples for the given year. For
a list of keys, see `example_io.read_file`.
"""
rrtm_file_names = _find_rrtm_files(rrtm_directory_name=rrtm_directory_name,
year=year)
num_files = len(rrtm_file_names)
example_dicts = [dict()] * num_files
for i in range(num_files):
print('Reading data from: "{0:s}"...'.format(rrtm_file_names[i]))
example_dicts[i] = rrtm_io.read_file(
netcdf_file_name=rrtm_file_names[i], allow_bad_values=True)
if input_example_dir_name is None:
example_file_name = None
else:
example_file_name = rrtm_io.find_file(
directory_name=input_example_dir_name,
year=year,
raise_error_if_missing=False)
if not os.path.isfile(example_file_name):
example_file_name = None
if example_file_name is not None:
print('Reading data from: "{0:s}"...'.format(example_file_name))
example_dicts.append(rrtm_io.read_file(example_file_name))
return example_utils.concat_examples(example_dicts)
def _run(tropical_example_dir_name, non_tropical_example_dir_name,
num_histogram_bins, output_dir_name):
"""Plots distribution of each target variable.
This is effectively the main method.
:param tropical_example_dir_name: See documentation at top of file.
:param non_tropical_example_dir_name: Same.
:param num_histogram_bins: Same.
:param output_dir_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
first_time_unix_sec = (
time_conversion.first_and_last_times_in_year(FIRST_YEAR)[0])
last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(LAST_YEAR)[-1])
example_file_names = example_io.find_many_files(
directory_name=tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=True)
example_file_names += example_io.find_many_files(
directory_name=non_tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=True)
example_dicts = []
for this_file_name in example_file_names:
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_example_dict = example_io.read_file(this_file_name)
this_example_dict = example_utils.subset_by_field(
example_dict=this_example_dict, field_names=TARGET_NAMES_IN_FILE)
example_dicts.append(this_example_dict)
example_dict = example_utils.concat_examples(example_dicts)
del example_dicts
letter_label = None
panel_file_names = []
for this_target_name in TARGET_NAMES:
if this_target_name in TARGET_NAMES_IN_FILE:
these_target_values = example_utils.get_field_from_dict(
example_dict=example_dict, field_name=this_target_name)
else:
down_fluxes_w_m02 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.SHORTWAVE_SURFACE_DOWN_FLUX_NAME)
up_fluxes_w_m02 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.SHORTWAVE_TOA_UP_FLUX_NAME)
these_target_values = down_fluxes_w_m02 - up_fluxes_w_m02
these_target_values = numpy.ravel(these_target_values)
if letter_label is None:
letter_label = 'a'
else:
letter_label = chr(ord(letter_label) + 1)
this_file_name = _plot_histogram_one_target(
target_values=these_target_values,
target_name=this_target_name,
num_bins=num_histogram_bins,
letter_label=letter_label,
output_dir_name=output_dir_name)
panel_file_names.append(this_file_name)
concat_file_name = '{0:s}/target_distributions.jpg'.format(output_dir_name)
print('Concatenating panels to: "{0:s}"...'.format(concat_file_name))
imagemagick_utils.concatenate_images(input_file_names=panel_file_names,
output_file_name=concat_file_name,
num_panel_rows=2,
num_panel_columns=2,
border_width_pixels=25)
imagemagick_utils.trim_whitespace(input_file_name=concat_file_name,
output_file_name=concat_file_name)
def get_raw_examples(example_file_name, num_examples, example_dir_name,
example_id_file_name):
"""Returns raw examples.
The difference between `get_raw_examples` and `get_examples_for_inference`
is that `get_raw_examples` returns examples in their raw form, *not*
pre-processed to be fed through a model for inference.
:param example_file_name: See doc for `get_examples_for_inference`.
:param num_examples: Same.
:param example_dir_name: Same.
:param example_id_file_name: Same.
:return: example_dict: See doc for `example_io.read_file`.
"""
error_checking.assert_is_string(example_file_name)
use_specific_ids = example_file_name == ''
if use_specific_ids:
error_checking.assert_is_string(example_id_file_name)
print('Reading desired example IDs from: "{0:s}"...'.format(
example_id_file_name))
example_id_strings = read_example_ids_from_netcdf(example_id_file_name)
valid_times_unix_sec = example_utils.parse_example_ids(
example_id_strings)[example_utils.VALID_TIMES_KEY]
example_file_names = example_io.find_many_files(
directory_name=example_dir_name,
first_time_unix_sec=numpy.min(valid_times_unix_sec),
last_time_unix_sec=numpy.max(valid_times_unix_sec))
num_files = len(example_file_names)
example_dicts = [dict()] * num_files
for i in range(num_files):
print('Reading data from: "{0:s}"...'.format(
example_file_names[i]))
example_dicts[i] = example_io.read_file(example_file_names[i])
example_dict = example_utils.concat_examples(example_dicts)
good_indices = example_utils.find_examples(
all_id_strings=example_dict[example_utils.EXAMPLE_IDS_KEY],
desired_id_strings=example_id_strings,
allow_missing=False)
example_dict = example_utils.subset_by_index(
example_dict=example_dict, desired_indices=good_indices)
else:
error_checking.assert_is_string(example_dir_name)
error_checking.assert_is_integer(num_examples)
error_checking.assert_is_greater(num_examples, 0)
print('Reading data from: "{0:s}"...'.format(example_file_name))
example_dict = example_io.read_file(example_file_name)
num_examples_total = len(example_dict[example_utils.VALID_TIMES_KEY])
desired_indices = numpy.linspace(0,
num_examples_total - 1,
num=num_examples_total,
dtype=int)
if num_examples < num_examples_total:
desired_indices = numpy.random.choice(desired_indices,
size=num_examples,
replace=False)
example_dict = example_utils.subset_by_index(
example_dict=example_dict, desired_indices=desired_indices)
return example_dict
def _run(tropical_example_dir_name, non_tropical_example_dir_name, year,
assorted1_example_dir_name, assorted2_example_dir_name):
"""Splits examples into Assorted1 and Assorted2 sites.
This is effectively the main method.
:param tropical_example_dir_name: See documentation at top of file.
:param non_tropical_example_dir_name: Same.
:param year: Same.
:param assorted1_example_dir_name: Same.
:param assorted2_example_dir_name: Same.
"""
tropical_example_file_name = example_io.find_file(
directory_name=tropical_example_dir_name, year=year,
raise_error_if_missing=True
)
non_tropical_example_file_name = example_io.find_file(
directory_name=non_tropical_example_dir_name, year=year,
raise_error_if_missing=True
)
print('Reading data from: "{0:s}"...'.format(tropical_example_file_name))
tropical_example_dict = example_io.read_file(tropical_example_file_name)
print('Reading data from: "{0:s}"...'.format(
non_tropical_example_file_name
))
non_tropical_example_dict = example_io.read_file(
non_tropical_example_file_name
)
example_dict = example_utils.concat_examples([
tropical_example_dict, non_tropical_example_dict
])
del tropical_example_dict, non_tropical_example_dict
example_metadata_dict = example_utils.parse_example_ids(
example_dict[example_utils.EXAMPLE_IDS_KEY]
)
example_latitudes_deg_n = example_metadata_dict[example_utils.LATITUDES_KEY]
example_longitudes_deg_e = lng_conversion.convert_lng_positive_in_west(
example_metadata_dict[example_utils.LONGITUDES_KEY]
)
example_coord_matrix = numpy.transpose(numpy.vstack((
example_latitudes_deg_n, example_longitudes_deg_e
)))
assorted2_coord_matrix = numpy.transpose(numpy.vstack((
ASSORTED2_LATITUDES_DEG_N, ASSORTED2_LONGITUDES_DEG_E
)))
distance_matrix_deg2 = euclidean_distances(
X=example_coord_matrix, Y=assorted2_coord_matrix, squared=True
)
assorted2_flags = numpy.any(distance_matrix_deg2 <= TOLERANCE_DEG2, axis=1)
assorted2_example_dict = example_utils.subset_by_index(
example_dict=copy.deepcopy(example_dict),
desired_indices=numpy.where(assorted2_flags)[0]
)
assorted2_example_file_name = example_io.find_file(
directory_name=assorted2_example_dir_name, year=year,
raise_error_if_missing=False
)
print('Writing {0:d} examples in set Assorted2 to: "{1:s}"...'.format(
len(assorted2_example_dict[example_utils.VALID_TIMES_KEY]),
assorted2_example_file_name
))
example_io.write_file(
example_dict=assorted2_example_dict,
netcdf_file_name=assorted2_example_file_name
)
assorted1_example_dict = example_utils.subset_by_index(
example_dict=example_dict,
desired_indices=numpy.where(numpy.invert(assorted2_flags))[0]
)
assorted1_example_file_name = example_io.find_file(
directory_name=assorted1_example_dir_name, year=year,
raise_error_if_missing=False
)
print('Writing {0:d} examples in set Assorted1 to: "{1:s}"...'.format(
len(assorted1_example_dict[example_utils.VALID_TIMES_KEY]),
assorted1_example_file_name
))
example_io.write_file(
example_dict=assorted1_example_dict,
netcdf_file_name=assorted1_example_file_name
)