def _load_spill_data(saveloc, model):
""" load NetCDF file and add spill data back in """
spill_data = os.path.join(saveloc, "spills_data_arrays.nc")
if not os.path.exists(spill_data):
return
if model.uncertain:
u_spill_data = os.path.join(saveloc, "spills_data_arrays_uncertain.nc")
array_types = {}
for m in model.movers:
array_types.update(m.array_types)
for w in model.weatherers:
array_types.update(w.array_types)
for sc in model.spills.items():
if sc.uncertain:
data = NetCDFOutput.read_data(u_spill_data, time=None, which_data="all")
else:
data = NetCDFOutput.read_data(spill_data, time=None, which_data="all")
sc.current_time_stamp = data.pop("current_time_stamp").item()
sc._data_arrays = data
sc._array_types.update(array_types)
def test_write_output_post_run(model, output_ts_factor):
"""
Create netcdf file post run from the cache. Under the hood, it is simply
calling write_output so no need to check the data is correctly written
test_write_output_standard already checks data is correctly written.
Instead, make sure if output_timestep is not same as model.time_step,
then data is output at correct time stamps
"""
model.rewind()
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.which_data = 'standard'
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
del model.outputters[o_put.id] # remove from list of outputters
_run_model(model)
assert (not os.path.exists(o_put.netcdf_filename))
if o_put._u_netcdf_filename:
assert (not os.path.exists(o_put._u_netcdf_filename))
# now write netcdf output
o_put.write_output_post_run(model.start_time,
model.num_time_steps,
spills=model.spills,
cache=model._cache,
uncertain=model.uncertain)
assert os.path.exists(o_put.netcdf_filename)
if model.uncertain:
assert os.path.exists(o_put._u_netcdf_filename)
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
for step in range(model.num_time_steps, int(output_ts_factor)):
print "step: {0}".format(step)
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
nc_data = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
if o_put.output_last_step:
scp = model._cache.load_timestep(model.num_time_steps - 1)
curr_time = scp.LE('current_time_stamp', uncertain)
nc_data = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
""" at least one matching time found """
print ('\nAll expected timestamps are written out for'
' output_ts_factor: {1}'.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
# add this back in so cleanup script deletes the generated *.nc files
model.outputters += o_put
def _load_spill_data(self, spill_data):
"""
load NetCDF file and add spill data back in - designed for savefiles
"""
if not os.path.exists(spill_data):
return
if self.uncertain:
saveloc, spill_data_fname = os.path.split(spill_data)
spill_data_fname, ext = os.path.splitext(spill_data_fname)
u_spill_data = os.path.join(saveloc,
'{0}_uncertain{1}'.format(spill_data_fname, ext))
array_types = {}
for m in self.movers:
array_types.update(m.array_types)
for w in self.weatherers:
array_types.update(w.array_types)
for sc in self.spills.items():
if sc.uncertain:
data = NetCDFOutput.read_data(u_spill_data, time=None,
which_data='all')
else:
data = NetCDFOutput.read_data(spill_data, time=None,
which_data='all')
sc.current_time_stamp = data.pop('current_time_stamp').item()
sc._data_arrays = data
sc._array_types.update(array_types)
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename, time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename, index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename, index=-1,
which_data='all')[0]
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename,
time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename,
index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename,
index=-1,
which_data='all')[0]
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename, time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename, index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename, index=-1,
which_data='all')[0]
# if init_mass is not there, set it to mass
# fixme: should this be a required data array?
self._init_data.setdefault('init_mass', self._init_data['mass'].copy())
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename,
time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename,
index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename,
index=-1,
which_data='all')[0]
# if init_mass is not there, set it to mass
# fixme: should this be a required data array?
self._init_data.setdefault('init_mass', self._init_data['mass'].copy())
def test_read_data_exception(model):
"""
tests the exception is raised by read_data when file contains more than one
output time and read_data is not given the output time to read
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id]
for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)][0]
_run_model(model)
with raises(ValueError):
NetCDFOutput.read_data(o_put.netcdf_filename)
def test_read_data_exception(model):
"""
tests the exception is raised by read_data when file contains more than one
output time and read_data is not given the output time to read
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id]
for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)][0]
_run_model(model)
with raises(ValueError):
NetCDFOutput.read_data(o_put.filename)
def test_read_standard_arrays(model, output_ts_factor):
"""
tests the data returned by read_data is correct when `which_data` flag is
'standard'. It is only reading the standard_arrays
Test will only verify the data when time_stamp of model matches the
time_stamp of data written out. output_ts_factor means not all data is
written out.
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for step in range(0, model.num_time_steps, int(output_ts_factor)):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
nc_data = NetCDFOutput.read_data(file_, curr_time)
# check time
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
# check standard variables
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
assert np.all(scp.LE('spill_num', uncertain)[:]
== nc_data['spill_num'])
assert np.all(scp.LE('status_codes', uncertain)[:]
== nc_data['status_codes'])
# flag variable is not currently set or checked
if 'mass' in scp.LE_data:
assert np.all(scp.LE('mass', uncertain)[:]
== nc_data['mass'])
if 'age' in scp.LE_data:
assert np.all(scp.LE('age', uncertain)[:]
== nc_data['age'])
if _found_a_matching_time:
""" at least one matching time found """
print ('\ndata in model matches for output in \n{0} \nand'
' output_ts_factor: {1}'.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_read_all_arrays(model):
"""
tests the data returned by read_data is correct
when `which_data` flag is 'all'.
"""
model.rewind()
o_put = [
model.outputters[outputter.id] for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)
][0]
o_put.which_data = 'all'
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for step in range(model.num_time_steps):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_,
curr_time,
which_data='all')
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
for key in scp.LE_data:
if key == 'current_time_stamp':
""" already matched """
continue
elif key == 'positions':
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
elif key == 'mass_balance':
assert scp.LE(key, uncertain) == mb
else:
if key not in ['surface_concentration'
]: # not always there
assert np.all(
scp.LE(key, uncertain)[:] == nc_data[key])
if _found_a_matching_time:
print('\ndata in model matches for output in \n{0}'.format(file_))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_read_all_arrays(model):
"""
tests the data returned by read_data is correct
when `which_data` flag is 'all'.
"""
model.rewind()
o_put = [model.outputters[outputter.id]
for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)][0]
o_put.which_data = 'all'
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for step in range(model.num_time_steps):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time,
which_data='all')
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
for key in scp.LE_data:
if key == 'current_time_stamp':
""" already matched """
continue
elif key == 'positions':
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
elif key == 'mass_balance':
assert scp.LE(key, uncertain) == mb
else:
# if key not in ['last_water_positions',
# 'next_positions']:
assert np.all(scp.LE(key, uncertain)[:] ==
nc_data[key])
if _found_a_matching_time:
print ('\ndata in model matches for output in \n{0}'.format(file_))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_write_output_post_run(model, output_ts_factor):
"""
Create netcdf file post run from the cache. Under the hood, it is simply
calling write_output so no need to check the data is correctly written
test_write_output_standard already checks data is correctly written.
Instead, make sure if output_timestep is not same as model.time_step,
then data is output at correct time stamps
"""
model.rewind()
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.which_data = 'standard'
o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor)
del model.outputters[o_put.id] # remove from list of outputters
_run_model(model)
# clear out old files...
o_put.clean_output_files()
assert not os.path.exists(o_put.netcdf_filename)
if o_put._u_netcdf_filename:
assert (not os.path.exists(o_put._u_netcdf_filename))
# now write netcdf output
o_put.write_output_post_run(model.start_time,
model.num_time_steps,
spills=model.spills,
cache=model._cache,
uncertain=model.uncertain)
assert os.path.exists(o_put.netcdf_filename)
if model.uncertain:
assert os.path.exists(o_put._u_netcdf_filename)
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
ix = 0 # index for grabbing record from NetCDF file
for step in range(0, model.num_time_steps,
int(ceil(output_ts_factor))):
print "step: {0}".format(step)
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
# test to make sure data_by_index is consistent with _cached data
# This is just to double check that getting the data by curr_time
# does infact give the next consecutive index
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
ix += 1
if o_put.output_last_step and step < model.num_time_steps - 1:
'''
Last timestep written to NetCDF wasn't tested - do that here
'''
scp = model._cache.load_timestep(model.num_time_steps - 1)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
# again, check that last time step
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
with pytest.raises(IndexError):
# check that no more data exists in NetCDF
NetCDFOutput.read_data(file_, index=ix + 1)
""" at least one matching time found """
print ('All expected timestamps in {0} for output_ts_factor: {1}'
.format(os.path.split(file_)[1], output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
# add this back in so cleanup script deletes the generated *.nc files
model.outputters += o_put
def test_read_standard_arrays(model, output_ts_factor, use_time):
"""
tests the data returned by read_data is correct when `which_data` flag is
'standard'. It is only reading the standard_arrays
Test will only verify the data when time_stamp of model matches the
time_stamp of data written out. output_ts_factor means not all data is
written out.
The use_time flag says data is read by timestamp. If false, then it is read
by step number - either way, the result should be the same
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for idx, step in enumerate(range(0, model.num_time_steps,
int(ceil(output_ts_factor)))):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
if use_time:
(nc_data, weathering_data) = NetCDFOutput.read_data(file_,
curr_time)
else:
(nc_data, weathering_data) = NetCDFOutput.read_data(file_,
index=idx)
# check time
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
# check standard variables
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
assert np.all(scp.LE('spill_num', uncertain)[:] ==
nc_data['spill_num'])
assert np.all(scp.LE('status_codes', uncertain)[:] ==
nc_data['status_codes'])
# flag variable is not currently set or checked
if 'mass' in scp.LE_data:
assert np.all(scp.LE('mass', uncertain)[:] ==
nc_data['mass'])
if 'age' in scp.LE_data:
assert np.all(scp.LE('age', uncertain)[:] ==
nc_data['age'])
if uncertain:
sc = scp.items()[1]
else:
sc = scp.items()[0]
assert sc.mass_balance == weathering_data
else:
raise Exception('Assertions not tested since no data found '
'in NetCDF file for timestamp: {0}'
.format(curr_time))
if _found_a_matching_time:
print ('\n'
'data in model matches for output in\n'
'{0}\n'
'and output_ts_factor: {1}'
.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_write_output_post_run(model, output_ts_factor):
"""
Create netcdf file post run from the cache. Under the hood, it is simply
calling write_output so no need to check the data is correctly written
test_write_output_standard already checks data is correctly written.
Instead, make sure if output_timestep is not same as model.time_step,
then data is output at correct time stamps
"""
model.rewind()
o_put = [
model.outputters[outputter.id] for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)
][0]
o_put.which_data = 'standard'
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
del model.outputters[o_put.id] # remove from list of outputters
_run_model(model)
# clear out old files...
o_put.clean_output_files()
assert not os.path.exists(o_put.netcdf_filename)
if o_put._u_netcdf_filename:
assert (not os.path.exists(o_put._u_netcdf_filename))
# now write netcdf output
o_put.write_output_post_run(model.start_time,
model.num_time_steps,
spills=model.spills,
cache=model._cache,
uncertain=model.uncertain)
assert os.path.exists(o_put.netcdf_filename)
if model.uncertain:
assert os.path.exists(o_put._u_netcdf_filename)
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
ix = 0 # index for grabbing record from NetCDF file
for step in range(0, model.num_time_steps,
int(ceil(output_ts_factor))):
print "step: {0}".format(step)
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
# test to make sure data_by_index is consistent with _cached data
# This is just to double check that getting the data by curr_time
# does infact give the next consecutive index
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
ix += 1
if o_put.output_last_step and step < model.num_time_steps - 1:
'''
Last timestep written to NetCDF wasn't tested - do that here
'''
scp = model._cache.load_timestep(model.num_time_steps - 1)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
# again, check that last time step
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
with pytest.raises(IndexError):
# check that no more data exists in NetCDF
NetCDFOutput.read_data(file_, index=ix + 1)
""" at least one matching time found """
print(
'All expected timestamps in {0} for output_ts_factor: {1}'.format(
os.path.split(file_)[1], output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
# add this back in so cleanup script deletes the generated *.nc files
model.outputters += o_put
def test_read_standard_arrays(model, output_ts_factor, use_time):
"""
tests the data returned by read_data is correct when `which_data` flag is
'standard'. It is only reading the standard_arrays
Test will only verify the data when time_stamp of model matches the
time_stamp of data written out. output_ts_factor means not all data is
written out.
The use_time flag says data is read by timestamp. If false, then it is read
by step number - either way, the result should be the same
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [
model.outputters[outputter.id] for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)
][0]
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for idx, step in enumerate(
range(0, model.num_time_steps, int(ceil(output_ts_factor)))):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
if use_time:
(nc_data,
weathering_data) = NetCDFOutput.read_data(file_, curr_time)
else:
(nc_data, weathering_data) = NetCDFOutput.read_data(file_,
index=idx)
# check time
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
# check standard variables
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
assert np.all(
scp.LE('spill_num', uncertain)[:] == nc_data['spill_num'])
assert np.all(
scp.LE('status_codes', uncertain)[:] ==
nc_data['status_codes'])
# flag variable is not currently set or checked
if 'mass' in scp.LE_data:
assert np.all(
scp.LE('mass', uncertain)[:] == nc_data['mass'])
if 'age' in scp.LE_data:
assert np.all(
scp.LE('age', uncertain)[:] == nc_data['age'])
if uncertain:
sc = scp.items()[1]
else:
sc = scp.items()[0]
assert sc.mass_balance == weathering_data
else:
raise Exception(
'Assertions not tested since no data found '
'in NetCDF file for timestamp: {0}'.format(curr_time))
if _found_a_matching_time:
print(
'\n'
'data in model matches for output in\n'
'{0}\n'
'and output_ts_factor: {1}'.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True