def test_variable_attribute_requires_variable(self):
"""
Test that an error is raised when an attribute is given to a variable
that does not exist.
"""
# Test on an empty dataset.
writer = NetCDFWriter()
with self.assertRaises(ValueError):
writer.variable_attribute("dummy", "description",
"A dummy variable")
# Test with one dimension of the same name.
writer.dimension("dummy", np.int32, 1)
with self.assertRaises(ValueError):
writer.variable_attribute("dummy", "description",
"A dummy variable")
# Test with a variable with a different name.
writer.variable("fake", np.int32, ["dummy"])
with self.assertRaises(ValueError):
writer.variable_attribute("dummy", "description",
"A dummy variable")
# Test that no errors are raised when a correct variable exists.
writer.variable("dummy", np.int32, ["dummy"])
writer.variable_attribute("dummy", "description", "A dummy variable")
def test_data_requires_variable(self):
"""
Test that an error is raised when data is submitted to a variable
that does not exist.
"""
# Test with no variables.
writer = NetCDFWriter()
with self.assertRaises(KeyError):
writer.data("dummy", np.array([1]))
# Test with no variables but a dimension of the same name.
writer.dimension("dummy", np.int32, 1)
with self.assertRaises(KeyError):
writer.data("dummy", np.array([1]))
# Test with a variable of a different name.
writer.variable("fake", np.int32, ["dummy"])
with self.assertRaises(KeyError):
writer.data("dummy", np.array([1]))
# Test that no errors are raised when a correct variable exists.
writer.variable("dummy", np.int32, ["dummy"])
writer.data("dummy", np.array([1]))
def test_variable_attributes(self):
"""
Test that attributes can be added to an existing variable, and that
they can be successfully read back and accessed in future reads of the
dataset.
"""
writer = NetCDFWriter()
writer.dimension("x", np.int16, 100)
writer.variable("dimensionless", np.int8, [])
writer.variable("dimensional", np.int16, ["x"])
writer.variable("no_attrs", np.int32, ["x"])
# Add arbitrary data to each variable so that the write can be
# successful.
writer.data("dimensionless", np.array(5))
writer.data("dimensional", np.array(range(100)))
writer.data("no_attrs", np.array(range(100, 400, 3)))
writer.variable_attribute("dimensionless", "description",
"No data available")
writer.variable_attribute("dimensionless", "units", "No units")
writer.variable_attribute("dimensional", "description",
"A coordinate on the x-axis")
writer.variable_attribute("dimensional", "units", "meters")
filepath = path.join(WRITE_OUTPUT_DIR, "var_attrs.nc")
writer.write(filepath)
# Read back the three variables using a trusted library.
ds = Dataset(filepath)
dimless = ds.variables["dimensionless"]
dimful = ds.variables["dimensional"]
no_attrs = ds.variables["no_attrs"]
# Ensure each variable has exactly the right number of attributes, and
# that these attributes have the right values.
self.assertEqual(2, len(vars(dimless)))
self.assertEqual("No data available", dimless.description)
self.assertEqual("No units", dimless.units)
self.assertEqual(2, len(vars(dimful)))
self.assertEqual("A coordinate on the x-axis", dimful.description)
self.assertEqual("meters", dimful.units)
self.assertEqual(0, len(vars(no_attrs)))
ds.close()
def test_variable_requires_dimension(self):
"""
Test that an error is raised when a variable is created that refers
to a dimension that does not exist.
"""
# Test with an empty dataset.
writer = NetCDFWriter()
with self.assertRaises(ValueError):
writer.variable("dummy", np.int32, ["dummy"])
# Test with a dimension of a different name.
writer.dimension("fake", np.int32, 5)
with self.assertRaises(ValueError):
writer.variable("dummy", np.int32, ["dummy"])
# Test that no errors are raised when a correct dimension exists.
writer.dimension("dummy", np.int32, 4)
writer.variable("dummy", np.int32, ["dummy"])
def test_variable_dimensions(self):
"""
Test that variables can be created with varying numbers of dimensions,
and that they accept data with the corresponding number of dimensions.
Test reading back data to ensure proper persistence.
"""
writer = NetCDFWriter()
writer.dimension("x", np.int16, 10)
writer.dimension("y", np.int16, 10)
# Examples with 0, 1, and multiple dimensions.
writer.variable("no_dims", np.int16, [])
writer.variable("line", np.int16, ["x"])
writer.variable("plane", np.int16, ["x", "y"])
# Each variable receives data of the same number of dimensions as
# the variable itself.
writer.data("no_dims", np.array(1))
writer.data("line", np.array(range(10)))
writer.data("plane", np.array([range(i, i + 10) for i in range(10)]))
filepath = path.join(WRITE_OUTPUT_DIR, "cartesian2.nc")
writer.write(filepath)
# Read the dataset back in with a trusted library.
ds = Dataset(filepath)
vars = ds.variables
# Ensure only the right variables are present (including two dimension
# variables and the three added manually)
self.assertEqual(5, len(vars))
self.assertIn("x", vars)
self.assertIn("y", vars)
self.assertIn("no_dims", vars)
self.assertIn("line", vars)
self.assertIn("plane", vars)
# Proper variable values for the 0-dimensional variable.
self.assertEqual(1, vars["no_dims"].size)
self.assertEqual(1, vars["no_dims"][:])
# Proper variable values for the 1-dimensional variable.
self.assertEqual(10, vars["line"].size)
self.assertEqual((10, ), vars["line"].shape)
self.assertEqual(0, vars["line"][0])
self.assertEqual(4, vars["line"][4])
self.assertEqual(9, vars["line"][9])
# Proper variable values for the 2-dimensional variable.
self.assertEqual(100, vars["plane"].size)
self.assertEqual((10, 10), vars["plane"].shape)
self.assertEqual(0, vars["plane"][0, 0])
self.assertEqual(5, vars["plane"][0, 5])
self.assertEqual(2, vars["plane"][1, 1])
self.assertEqual(6, vars["plane"][2, 4])
self.assertEqual(12, vars["plane"][5, 7])
self.assertEqual(11, vars["plane"][8, 3])
self.assertEqual(9, vars["plane"][9, 0])
self.assertEqual(18, vars["plane"][9, 9])
ds.close()
class ModelOutput:
"""
A general-purpose center for all forms of output. Responsible for
organization of program output into folders.
The output of a program is defined as the temperature data produced by
a run of the model. This data may be saved in the form of a data file
(such as a NetCDF file) and/or as image representations, and/or in other
data formats.
All of these output types are produced side-by-side, and stored in their
own directory to keep data separate from different model runs.
"""
def __init__(self: 'ModelOutput', data: List['LatLongGrid']) -> None:
"""
Instantiate a new ModelOutput object.
Model data is provided through the data parameter, through a list of
grid objects. Each grid in the list represents a segment of time, such
as a month or a season. All grids must have the same dimensions.
:param data:
A list of latitude-longitude grids of data
"""
# Create output directory if it does not already exist.
parent_out_dir = Path(OUTPUT_FULL_PATH)
parent_out_dir.mkdir(exist_ok=True)
self._data = data
self._grid = data[0].dimensions()
self._dataset = NetCDFWriter()
def write_dataset(self: 'ModelOutput', data: List['LatLongGrid'],
dir_path: str, dataset_name: str) -> None:
"""
Produce a NetCDF dataset, with the name given by dataset_name.nc,
containing the variables in the data parameter that the output
controller allows. The dataset will be written to the specified
path in the file system.
The dataset contains all the dimensions that are used in the data
(e.g. time, latitude, longitude) as well as variables including
final temperature, temperature change, humidity, etc. according
to which of the ReportDatatype output types are enabled in the
current output controller.
:param data:
Output from an Arrhenius model run
:param dir_path:
The directory where the dataset will be written
:param dataset_name:
The name of the dataset
"""
# Write the data out to a NetCDF file in the output directory.
grid_by_count = self._grid.dims_by_count()
output_path = path.join(dir_path, dataset_name)
global_output_center().submit_output(Debug.PRINT_NOTICES,
"Writing NetCDF dataset...")
self._dataset.global_attribute("description", "Output for an"
"Arrhenius model run.")\
.dimension('time', np.int32, len(data), (0, len(data)))\
.dimension('latitude', np.int32, grid_by_count[0], (-90, 90)) \
.dimension('longitude', np.int32, grid_by_count[1], (-180, 180)) \
for output_type in ReportDatatype:
variable_data =\
extract_multidimensional_grid_variable(data,
output_type.value)
global_output_center().submit_output(output_type, variable_data,
output_type.value)
self._dataset.write(output_path)
def write_dataset_variable(self: 'ModelOutput', data: np.ndarray,
data_type: str) -> None:
"""
Prepare to write data into a variable by the name of data_type
in this instance's NetCDF dataset file. Apply this variable's
dimensions and type, along with several attributes.
:param data:
A single-variable grid taken from Arrhenius model output
:param data_type:
The name of the variable as it will appear in the dataset
"""
dims_map = [[], ['latitude'], ['latitude', 'longitude'],
['time', 'latitude', 'longitude'],
['time', 'level', 'latitude', 'longitude']]
global_output_center().submit_output(
Debug.PRINT_NOTICES, "Writing {} to dataset".format(data_type))
variable_type = VARIABLE_METADATA[data_type][VAR_TYPE]
self._dataset.variable(data_type, variable_type, dims_map[data.ndim])
for attr, val in VARIABLE_METADATA[data_type][VAR_ATTRS].items():
self._dataset.variable_attribute(data_type, attr, val)
self._dataset.data(data_type, data)
def write_images(self: 'ModelOutput',
data: List['LatLongGrid'],
output_path: str,
run_id: str = "") -> None:
"""
Produce a series of maps displaying some of the results of an
Arrhenius model run according to what variable the output controller
allows. Images are stored in a directory given by output_path.
One image will be produced per time segment per variable for which
output is allowed by the output controller, based on which
ReportDatatype output types are enabled. The optional argument
img_base_name specifies a prefix that will be added to each of the
image files to identify which model run they belong to.
:param data:
The output from an Arrhenius model run
:param output_path:
The directory where image files will be stored
:param run_id:
A prefix that will start off the names of all the image files
"""
output_controller = global_output_center()
# Attempt to output images for each variable output type.
for output_type in ReportDatatype:
variable_name = output_type.value
variable = extract_multidimensional_grid_variable(
data, variable_name)
img_type_path = path.join(output_path, variable_name)
output_controller.submit_output(output_type, variable,
img_type_path, variable_name,
run_id)
def write_output(self: 'ModelOutput', run_title: str) -> None:
"""
Produce NetCDF data files and image files from the provided data, and
a directory with the name dir_name to hold them.
One image file is created per time segment in the data. In the
case of Arrhenius' model, this is one per season. Only one NetCDF data
file is produced, in which all time segments are present.
"""
# Create a directory for this model output if none exists already.
out_dir_path = path.join(OUTPUT_FULL_PATH, run_title)
out_dir = Path(out_dir_path)
out_dir.mkdir(exist_ok=True)
output_controller = global_output_center()
output_controller.submit_collection_output(
(DATASET_VARS, ), self._data, out_dir_path, run_title + ".nc")
output_controller.submit_collection_output((IMAGES, ), self._data,
out_dir_path, run_title)