def _load_spill_data(self, spill_data):
"""
load NetCDF file and add spill data back in - designed for savefiles
"""
if not os.path.exists(spill_data):
return
if self.uncertain:
saveloc, spill_data_fname = os.path.split(spill_data)
spill_data_fname, ext = os.path.splitext(spill_data_fname)
u_spill_data = os.path.join(saveloc,
'{0}_uncertain{1}'.format(spill_data_fname, ext))
array_types = {}
for m in self.movers:
array_types.update(m.array_types)
for w in self.weatherers:
array_types.update(w.array_types)
for sc in self.spills.items():
if sc.uncertain:
data = NetCDFOutput.read_data(u_spill_data, time=None,
which_data='all')
else:
data = NetCDFOutput.read_data(spill_data, time=None,
which_data='all')
sc.current_time_stamp = data.pop('current_time_stamp').item()
sc._data_arrays = data
sc._array_types.update(array_types)
def _save_spill_data(self, datafile):
""" save the data arrays for current timestep to NetCDF """
nc_out = NetCDFOutput(datafile, which_data='all', cache=self._cache)
nc_out.prepare_for_model_run(model_start_time=self.start_time,
uncertain=self.uncertain,
spills=self.spills)
nc_out.write_output(self.current_time_step)
def _load_spill_data(saveloc, model):
""" load NetCDF file and add spill data back in """
spill_data = os.path.join(saveloc, "spills_data_arrays.nc")
if not os.path.exists(spill_data):
return
if model.uncertain:
u_spill_data = os.path.join(saveloc, "spills_data_arrays_uncertain.nc")
array_types = {}
for m in model.movers:
array_types.update(m.array_types)
for w in model.weatherers:
array_types.update(w.array_types)
for sc in model.spills.items():
if sc.uncertain:
data = NetCDFOutput.read_data(u_spill_data, time=None, which_data="all")
else:
data = NetCDFOutput.read_data(spill_data, time=None, which_data="all")
sc.current_time_stamp = data.pop("current_time_stamp").item()
sc._data_arrays = data
sc._array_types.update(array_types)
def test_write_output_post_run(model, output_ts_factor):
"""
Create netcdf file post run from the cache. Under the hood, it is simply
calling write_output so no need to check the data is correctly written
test_write_output_standard already checks data is correctly written.
Instead, make sure if output_timestep is not same as model.time_step,
then data is output at correct time stamps
"""
model.rewind()
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.which_data = 'standard'
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
del model.outputters[o_put.id] # remove from list of outputters
_run_model(model)
assert (not os.path.exists(o_put.netcdf_filename))
if o_put._u_netcdf_filename:
assert (not os.path.exists(o_put._u_netcdf_filename))
# now write netcdf output
o_put.write_output_post_run(model.start_time,
model.num_time_steps,
spills=model.spills,
cache=model._cache,
uncertain=model.uncertain)
assert os.path.exists(o_put.netcdf_filename)
if model.uncertain:
assert os.path.exists(o_put._u_netcdf_filename)
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
for step in range(model.num_time_steps, int(output_ts_factor)):
print "step: {0}".format(step)
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
nc_data = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
if o_put.output_last_step:
scp = model._cache.load_timestep(model.num_time_steps - 1)
curr_time = scp.LE('current_time_stamp', uncertain)
nc_data = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
""" at least one matching time found """
print ('\nAll expected timestamps are written out for'
' output_ts_factor: {1}'.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
# add this back in so cleanup script deletes the generated *.nc files
model.outputters += o_put
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename, time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename, index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename, index=-1,
which_data='all')[0]
def test_init_exceptions():
'''
test exceptions raised during __init__
'''
with raises(ValueError):
# must be filename, not dir name
NetCDFOutput(os.path.abspath(os.path.dirname(__file__)))
with raises(ValueError):
NetCDFOutput('invalid_path_to_file/file.nc')
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename, time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename, index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename, index=-1,
which_data='all')[0]
# if init_mass is not there, set it to mass
# fixme: should this be a required data array?
self._init_data.setdefault('init_mass', self._init_data['mass'].copy())
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename,
time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename,
index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename,
index=-1,
which_data='all')[0]
def _read_data_file(self, filename, index, time):
if time is not None:
self._init_data = NetCDFOutput.read_data(filename,
time,
which_data='all')[0]
elif index is not None:
self._init_data = NetCDFOutput.read_data(filename,
index=index,
which_data='all')[0]
else:
self._init_data = NetCDFOutput.read_data(filename,
index=-1,
which_data='all')[0]
# if init_mass is not there, set it to mass
# fixme: should this be a required data array?
self._init_data.setdefault('init_mass', self._init_data['mass'].copy())
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
start_time = datetime(2012, 9, 15, 12, 0)
mapfile = get_datafile(os.path.join(base_dir, './LongIslandSoundMap.BNA'))
gnome_map = MapFromBNA(mapfile, refloat_halflife=6) # hours
# # the image output renderer
# global renderer
# one hour timestep
model = Model(start_time=start_time,
duration=timedelta(hours=48),
time_step=3600,
map=gnome_map,
uncertain=True,
cache_enabled=True)
netcdf_file = os.path.join(base_dir, 'script_long_island.nc')
scripting.remove_netcdf(netcdf_file)
print 'adding outputters'
model.outputters += Renderer(mapfile, images_dir, size=(800, 600))
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
print 'adding a spill'
spill = point_line_release_spill(num_elements=1000,
start_position=(-72.419992, 41.202120,
0.0),
release_time=start_time)
model.spills += spill
print 'adding a RandomMover:'
model.movers += RandomMover(diffusion_coef=500000, uncertain_factor=2)
print 'adding a wind mover:'
series = np.zeros((5, ), dtype=datetime_value_2d)
series[0] = (start_time, (10, 45))
series[1] = (start_time + timedelta(hours=18), (10, 90))
series[2] = (start_time + timedelta(hours=30), (10, 135))
series[3] = (start_time + timedelta(hours=42), (10, 180))
series[4] = (start_time + timedelta(hours=54), (10, 225))
wind = Wind(timeseries=series, units='m/s')
model.movers += WindMover(wind)
print 'adding a cats mover:'
curr_file = get_datafile(os.path.join(base_dir, r"./LI_tidesWAC.CUR"))
tide_file = get_datafile(os.path.join(base_dir, r"./CLISShio.txt"))
c_mover = CatsMover(curr_file, tide=Tide(tide_file))
model.movers += c_mover
model.environment += c_mover.tide
print 'viewport is:', [
o.viewport for o in model.outputters if isinstance(o, Renderer)
]
return model
def test_read_standard_arrays(model, output_ts_factor):
"""
tests the data returned by read_data is correct when `which_data` flag is
'standard'. It is only reading the standard_arrays
Test will only verify the data when time_stamp of model matches the
time_stamp of data written out. output_ts_factor means not all data is
written out.
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for step in range(0, model.num_time_steps, int(output_ts_factor)):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
nc_data = NetCDFOutput.read_data(file_, curr_time)
# check time
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
# check standard variables
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
assert np.all(scp.LE('spill_num', uncertain)[:]
== nc_data['spill_num'])
assert np.all(scp.LE('status_codes', uncertain)[:]
== nc_data['status_codes'])
# flag variable is not currently set or checked
if 'mass' in scp.LE_data:
assert np.all(scp.LE('mass', uncertain)[:]
== nc_data['mass'])
if 'age' in scp.LE_data:
assert np.all(scp.LE('age', uncertain)[:]
== nc_data['age'])
if _found_a_matching_time:
""" at least one matching time found """
print ('\ndata in model matches for output in \n{0} \nand'
' output_ts_factor: {1}'.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def model(sample_model_fcn, output_filename):
"""
Use fixture model_surface_release_spill and add a few things to it for the
test
"""
model = sample_model_fcn['model']
model.cache_enabled = True
model.spills += \
point_line_release_spill(num_elements=5,
start_position=sample_model_fcn['release_start_pos'],
release_time=model.start_time,
end_release_time=model.start_time + model.duration,
substance=test_oil,
amount=1000,
units='kg')
water = Water()
model.movers += RandomMover(diffusion_coef=100000)
model.movers += constant_wind_mover(1.0, 0.0)
model.weatherers += Evaporation(water=water, wind=model.movers[-1].wind)
model.outputters += NetCDFOutput(output_filename)
model.rewind()
return model
def allWeatherers(timeStep, start_time, duration, weatheringSteps, map, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, tidalFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td):
print 'initializing the model:'
model = Model(time_step=timeStep, start_time=start_time, duration=duration)
print 'adding the map:'
map_folder = os.path.join(data_path, map_path)
if not(os.path.exists(map_folder)):
print('The map folder is incorrectly set:', map_folder)
mapfile = get_datafile( os.path.join(map_folder,map) )
model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife)
print 'adding a renderer'
model.outputters += Renderer(mapfile, output_path, size=(800, 600), output_timestep=timedelta(hours=1))
if save_nc:
nc_outputter = NetCDFOutput(netcdf_file, which_data='most', output_timestep=timedelta(hours=1))
model.outputters += nc_outputter
print 'adding a wind mover:'
wind_file = get_datafile(os.path.join(data_path, wind_path, windFile))
wind = GridWindMover(wind_file)
wind.wind_scale = wind_scale
model.movers += wind
print 'adding a current mover: '
curr_file = get_datafile(os.path.join(data_path, curr_path, currFile))
model.movers += GridCurrentMover(curr_file, num_method='RK4')
if td:
random_mover = RandomMover(diffusion_coef=10000)
model.movers += random_mover
print 'adding spill'
model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration)
print 'adding weatherers'
water = Water(280.92)
wind = constant_wind(20.0, 117, 'knots')
waves = Waves(wind, water)
model.weatherers += Evaporation(water, wind)
model.weatherers += Emulsification(waves)
model.weatherers += NaturalDispersion(waves, water)
return model
def test_read_data_exception(model):
"""
tests the exception is raised by read_data when file contains more than one
output time and read_data is not given the output time to read
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id]
for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)][0]
_run_model(model)
with raises(ValueError):
NetCDFOutput.read_data(o_put.filename)
def CurrentsAndWinds(timeStep, start_time, duration, weatheringSteps, mapfile, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, tidalFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td):
print 'initializing the model:'
model = Model(time_step=timeStep, start_time=start_time, duration=duration)
print 'adding the map:'
print (data_path, map_path, mapfile)
mapfile = get_datafile(os.path.join(data_path, map_path, mapfile))
model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife)
print 'adding a renderer'
model.outputters += Renderer(mapfile, output_path, size=(800, 600), output_timestep=timedelta(hours=timestep_outputs))
if save_nc:
nc_outputter = NetCDFOutput('currentsAndWinds_example.nc', which_data='standard', output_timestep=timedelta(hours=timestep_outputs))
model.outputters += nc_outputter
print 'adding a wind mover:'
wind_file = get_datafile(os.path.join(data_path, wind_path, windFile))
wind = GridWindMover(wind_file)
wind.wind_scale = wind_scale
model.movers += wind
print 'adding a current mover: '
curr_file = get_datafile(os.path.join(data_path, curr_path, currFile))
model.movers += GridCurrentMover(curr_file, num_method='RK4')
if td:
random_mover = RandomMover(diffusion_coef=10000)
model.movers += random_mover
print 'adding spill'
model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration)
return model
def test_read_data_exception(model):
"""
tests the exception is raised by read_data when file contains more than one
output time and read_data is not given the output time to read
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id]
for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)][0]
_run_model(model)
with raises(ValueError):
NetCDFOutput.read_data(o_put.netcdf_filename)
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'creating the maps'
mapfile = get_datafile(os.path.join(base_dir, 'LowerMississippiMap.bna'))
gnome_map = MapFromBNA(mapfile, refloat_halflife=6) # hours
print 'initializing the model'
start_time = datetime(2012, 9, 15, 12, 0)
# default to now, rounded to the nearest hour
model = Model(time_step=600,
start_time=start_time,
duration=timedelta(days=1),
map=gnome_map,
uncertain=True)
print 'adding outputters'
model.outputters += Renderer(mapfile, images_dir, image_size=(800, 600))
netcdf_file = os.path.join(base_dir, 'script_lower_mississippi.nc')
scripting.remove_netcdf(netcdf_file)
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
print 'adding a RandomMover:'
model.movers += RandomMover(diffusion_coef=10000)
print 'adding a wind mover:'
series = np.zeros((5, ), dtype=datetime_value_2d)
series[0] = (start_time, (2, 45))
series[1] = (start_time + timedelta(hours=18), (2, 90))
series[2] = (start_time + timedelta(hours=30), (2, 135))
series[3] = (start_time + timedelta(hours=42), (2, 180))
series[4] = (start_time + timedelta(hours=54), (2, 225))
w_mover = WindMover(Wind(timeseries=series, units='m/s'))
model.movers += w_mover
print 'adding a cats mover:'
curr_file = get_datafile(os.path.join(base_dir, 'LMiss.CUR'))
c_mover = CatsMover(curr_file)
# but do need to scale (based on river stage)
c_mover.scale = True
c_mover.scale_refpoint = (-89.699944, 29.494558)
# based on stage height 10ft (range is 0-18)
c_mover.scale_value = 1.027154
model.movers += c_mover
print 'adding a spill'
spill = point_line_release_spill(num_elements=1000,
start_position=(-89.699944, 29.494558,
0.0),
release_time=start_time)
model.spills += spill
return model
def _make_run_model(spill, nc_name):
'internal funtion'
m = Model()
m.outputters += NetCDFOutput(nc_name)
m.spills += spill
_run_model(m)
return m
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
start_time = datetime(2006, 3, 31, 20, 0)
model = Model(start_time=start_time,
duration=timedelta(days=3),
time_step=30 * 60,
uncertain=True)
print 'adding the map'
mapfile = get_datafile(os.path.join(base_dir, 'coastSF.bna'))
model.map = MapFromBNA(mapfile, refloat_halflife=1) # seconds
renderer = Renderer(mapfile,
images_dir,
image_size=(800, 600),
draw_ontop='forecast')
renderer.viewport = ((-124.5, 37.), (-120.5, 39))
print 'adding outputters'
model.outputters += renderer
netcdf_file = os.path.join(base_dir, 'script_sf_bay.nc')
scripting.remove_netcdf(netcdf_file)
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
print 'adding a spill'
spill = point_line_release_spill(
num_elements=1000,
start_position=(-123.57152, 37.369436, 0.0),
release_time=start_time,
substance=NonWeatheringSubstance(windage_range=(0.01, .04))
#element_type=floating(windage_range=(0.01,
# 0.04)
# )
)
model.spills += spill
# print 'adding a RandomMover:'
# r_mover = gnome.movers.RandomMover(diffusion_coef=50000)
# model.movers += r_mover
print 'adding a grid wind mover:'
wind_file = get_datafile(os.path.join(base_dir, 'WindSpeedDirSubset.nc'))
topology_file = get_datafile(
os.path.join(base_dir, 'WindSpeedDirSubsetTop.dat'))
w_mover = GridWindMover(wind_file, topology_file)
# w_mover.uncertain_time_delay = 6
# w_mover.uncertain_duration = 6
w_mover.uncertain_speed_scale = 1
w_mover.uncertain_angle_scale = 0.2 # default is .4
w_mover.wind_scale = 2
model.movers += w_mover
return model
def _make_run_model(spill, nc_name):
'internal function'
release_time = spill.release.release_time
m = Model(start_time=release_time)
m.outputters += NetCDFOutput(nc_name)
m.spills += spill
_run_model(m)
return m
def test_serialize_deserialize(json_):
'''
todo: this behaves in unexpected ways when using the 'model' testfixture.
For now, define a model in here for the testing - not sure where the
problem lies
'''
s_time = datetime(2014, 1, 1, 1, 1, 1)
model = Model(start_time=s_time)
model.spills += point_line_release_spill(num_elements=5,
start_position=(0, 0, 0),
release_time=model.start_time)
o_put = NetCDFOutput(os.path.join(base_dir, u'xtemp.nc'))
model.outputters += o_put
model.movers += RandomMover(diffusion_coef=100000)
#==========================================================================
# o_put = [model.outputters[outputter.id]
# for outputter in model.outputters
# if isinstance(outputter, NetCDFOutput)][0]
#==========================================================================
model.rewind()
print "step: {0}, _start_idx: {1}".format(-1, o_put._start_idx)
for ix in range(2):
model.step()
print "step: {0}, _start_idx: {1}".format(ix, o_put._start_idx)
#for json_ in ('save', 'webapi'):
dict_ = o_put.deserialize(o_put.serialize(json_))
o_put2 = NetCDFOutput.new_from_dict(dict_)
if json_ == 'save':
assert o_put == o_put2
else:
# _start_idx and _middle_of_run should not match
assert o_put._start_idx != o_put2._start_idx
assert o_put._middle_of_run != o_put2._middle_of_run
assert o_put != o_put2
if os.path.exists(o_put.netcdf_filename):
print '\n{0} exists'.format(o_put.netcdf_filename)
def make_models():
print 'initializing the model'
# start_time = datetime(2015, 12, 18, 06, 01)
# 1 day of data in file
# 1/2 hr in seconds
models = []
start_time = datetime(2012, 10, 27, 0, 30)
duration_hrs=23
time_step=450
num_steps = duration_hrs * 3600 / time_step
names = [
'Euler',
'Trapezoid',
'RK4',
]
mapfile = get_datafile(os.path.join(base_dir, 'long_beach.bna'))
print 'gen map'
map = MapFromBNA(mapfile, refloat_halflife=0.0) # seconds
fn = ('00_dir_roms_display.ncml.nc4')
curr = GridCurrent.from_netCDF(filename=fn)
models = []
for method in names:
mod = Model(start_time=start_time,
duration=timedelta(hours=duration_hrs),
time_step=time_step)
mod.map = map
spill = point_line_release_spill(num_elements=1000,
start_position=(-74.1,
39.7525,
0.0),
release_time=start_time)
mod.spills += spill
mod.movers += RandomMover(diffusion_coef=100)
mod.movers += PyGridCurrentMover(current=curr, default_num_method=method)
images_dir = method + '-' + str(time_step / 60) + 'min-' + str(num_steps) + 'steps'
renderer = Renderer(mapfile, images_dir, image_size=(1024, 768))
renderer.delay = 25
# renderer.add_grid(curr.grid)
mod.outputters += renderer
netCDF_fn = os.path.join(base_dir, images_dir + '.nc')
mod.outputters += NetCDFOutput(netCDF_fn, which_data='all')
models.append(mod)
print 'returning models'
return models
def test_serialize_deserialize(json_, output_filename):
'''
todo: this behaves in unexpected ways when using the 'model' testfixture.
For now, define a model in here for the testing - not sure where the
problem lies
'''
s_time = datetime(2014, 1, 1, 1, 1, 1)
model = Model(start_time=s_time)
model.spills += point_line_release_spill(num_elements=5,
start_position=(0, 0, 0),
release_time=model.start_time)
o_put = NetCDFOutput(output_filename)
model.outputters += o_put
model.movers += RandomMover(diffusion_coef=100000)
# ==========================================================================
# o_put = [model.outputters[outputter.id]
# for outputter in model.outputters
# if isinstance(outputter, NetCDFOutput)][0]
# ==========================================================================
model.rewind()
print "step: {0}, _start_idx: {1}".format(-1, o_put._start_idx)
for ix in range(2):
model.step()
print "step: {0}, _start_idx: {1}".format(ix, o_put._start_idx)
dict_ = o_put.deserialize(o_put.serialize(json_))
o_put2 = NetCDFOutput.new_from_dict(dict_)
if json_ == 'save':
assert o_put == o_put2
else:
# _start_idx and _middle_of_run should not match
assert o_put._start_idx != o_put2._start_idx
assert o_put._middle_of_run != o_put2._middle_of_run
assert o_put != o_put2
if os.path.exists(o_put.netcdf_filename):
print '\n{0} exists'.format(o_put.netcdf_filename)
def test_read_all_arrays(model):
"""
tests the data returned by read_data is correct
when `which_data` flag is 'all'.
"""
model.rewind()
o_put = [
model.outputters[outputter.id] for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)
][0]
o_put.which_data = 'all'
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for step in range(model.num_time_steps):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_,
curr_time,
which_data='all')
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
for key in scp.LE_data:
if key == 'current_time_stamp':
""" already matched """
continue
elif key == 'positions':
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
elif key == 'mass_balance':
assert scp.LE(key, uncertain) == mb
else:
if key not in ['surface_concentration'
]: # not always there
assert np.all(
scp.LE(key, uncertain)[:] == nc_data[key])
if _found_a_matching_time:
print('\ndata in model matches for output in \n{0}'.format(file_))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_read_all_arrays(model):
"""
tests the data returned by read_data is correct
when `which_data` flag is 'all'.
"""
model.rewind()
o_put = [model.outputters[outputter.id]
for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)][0]
o_put.which_data = 'all'
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for step in range(model.num_time_steps):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time,
which_data='all')
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
for key in scp.LE_data:
if key == 'current_time_stamp':
""" already matched """
continue
elif key == 'positions':
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
elif key == 'mass_balance':
assert scp.LE(key, uncertain) == mb
else:
# if key not in ['last_water_positions',
# 'next_positions']:
assert np.all(scp.LE(key, uncertain)[:] ==
nc_data[key])
if _found_a_matching_time:
print ('\ndata in model matches for output in \n{0}'.format(file_))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def make_model(images_dir=os.path.join(base_dir,"images")):
print "initializing the model"
start_time = datetime(2013, 7, 23, 0)
model = Model(start_time = start_time,
duration = timedelta(hours=47), # n+1 of data in file
time_step = 900, # 4 hr in seconds
uncertain = False,
)
mapfile = os.path.join(base_dir, './coast.bna')
print "adding the map"
gnome_map = MapFromBNA(mapfile, refloat_halflife=6) # hours
print "adding renderer"
model.outputters += Renderer(mapfile, images_dir, size=(1800, 1600))
print "adding a wind mover from a time-series"
## this is wind
wind_file=get_datafile(os.path.join(base_dir, 'wind.WND'))
wind = Wind(filename=wind_file)
w_mover = WindMover(wind)
model.movers += w_mover
print "adding a current mover:"
## this is currents
curr_file = get_datafile(os.path.join(base_dir, 'current.txt'))
model.movers += GridCurrentMover(curr_file)
##
## Add some spills (sources of elements)
##
print "adding 13 points in a cluster that has some small initial separation as the source of spill"
for i in range(len(coor)):
aaa=utmToLatLng(14,coor[i][0],coor[i][1],northernHemisphere=True)
model.spills += point_line_release_spill(num_elements=1,
start_position = (aaa[1],aaa[0], 0.0),
release_time = start_time,
)
print "adding netcdf output"
netcdf_output_file = os.path.join(base_dir,'GNOME_output.nc')
scripting.remove_netcdf(netcdf_output_file)
model.outputters += NetCDFOutput(netcdf_output_file, which_data='all')
return model
def only_Winds(timeStep, start_time, duration, weatheringSteps, map, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, tidalFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td):
print 'initializing the model:'
model = Model(time_step=timeStep, start_time=start_time, duration=duration)
print 'adding the map:'
mapfile = get_datafile(os.path.join(data_path, map_path, map))
model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife)
print 'adding a renderer'
model.outputters += Renderer(mapfile, output_path, size=(800, 600), output_timestep=timedelta(hours=timestep_outputs))
if save_nc:
nc_outputter = NetCDFOutput(netcdf_file, which_data='most', output_timestep=timedelta(hours=timestep_outputs))
model.outputters += nc_outputter
print 'adding a wind mover:'
wind_file = get_datafile(os.path.join(data_path, wind_path, windFile))
wind = GridWindMover(wind_file)
wind.wind_scale = wind_scale
model.movers += wind
print 'adding a spill'
model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration)
return model
def make_model(images_dir):
print 'initializing the model'
timestep = timedelta(minutes=15) # this is already default
start_time = datetime(2012, 9, 15, 12, 0)
model = Model(timestep, start_time)
# timeseries for wind data. The value is interpolated if time is between
# the given datapoints
series = np.zeros((4, ), dtype=datetime_value_2d)
series[:] = [(start_time, (5, 180)),
(start_time + timedelta(hours=6), (10, 180)),
(start_time + timedelta(hours=12), (12, 180)),
(start_time + timedelta(hours=18), (8, 180))]
wind = Wind(timeseries=series, units='m/s')
model.environment += wind
# include a wind mover and random diffusion
print 'adding movers'
model.movers += [WindMover(wind), RandomMover()]
# add particles
print 'adding particles'
release = release_from_splot_data(start_time,
'GL.2013267._LE_WHOLELAKE.txt')
model.spills += Spill(release)
# output data as png images and in netcdf format
print 'adding outputters'
netcdf_file = os.path.join(base_dir, 'script_example.nc')
# ignore renderer for now
model.outputters += [
Renderer(images_dir=images_dir,
size=(800, 800),
projection_class=GeoProjection),
NetCDFOutput(netcdf_file)
]
print 'model complete'
return model
def make_modelF(timeStep, start_time, duration, weatheringSteps, map, uncertain, data_path, curr_path, wind_path, map_path, reFloatHalfLife, windFile, currFile, num_elements, depths, lat, lon, output_path, wind_scale, save_nc, timestep_outputs, weatherers, td, dif_coef,temp_water):
print 'initializing the model:'
model = Model(time_step=timeStep, start_time=start_time, duration=duration, uncertain=uncertain)
print 'adding the map:'
mapfile = get_datafile(os.path.join(data_path, map_path, map))
model.map = MapFromBNA(mapfile, refloat_halflife=reFloatHalfLife)
print 'adding a renderer'
if save_nc:
scripting.remove_netcdf(output_path+'/'+'output.nc')
nc_outputter = NetCDFOutput(output_path+'/'+'output.nc', which_data='standard', output_timestep=timedelta(hours=timestep_outputs))
model.outputters += nc_outputter
print 'adding a wind mover:'
wind_file = get_datafile(os.path.join(data_path, wind_path, windFile))
wind = GridWindMover(wind_file)
# wind.wind_scale = wind_scale
model.movers += wind
print 'adding a current mover:'
curr_file = get_datafile(os.path.join(data_path, curr_path, currFile))
model.movers += GridCurrentMover(curr_file, num_method='RK4')
if td:
random_mover = RandomMover(diffusion_coef=dif_coef)
model.movers += random_mover
print 'adding spill'
model.spills += point_line_release_spill(num_elements=num_elements, start_position=(lon, lat, 0), release_time=start_time, end_release_time=start_time + duration)#, substance='AD04001', amount=9600000, units='kg')
if weatherers:
print 'adding weatherers'
water = Water(temp_water)
wind = constant_wind(0.0001, 0, 'knots')
waves = Waves(wind, water)
model.weatherers += Evaporation(water, wind)
# model.weatherers += Emulsification(waves)
model.weatherers += NaturalDispersion(waves, water)
return model
def make_model():
duration_hrs = 48
time_step = 900
num_steps = duration_hrs * 3600 / time_step
mod = Model(start_time=t,
duration=timedelta(hours=duration_hrs),
time_step=time_step)
spill = point_line_release_spill(num_elements=1000,
amount=1600,
units='kg',
start_position=(0.5, 0.5, 0.0),
release_time=t,
end_release_time=t + timedelta(hours=4))
mod.spills += spill
method = 'Trapezoid'
images_dir = method + '-' + str(
time_step / 60) + 'min-' + str(num_steps) + 'steps'
renderer = Renderer(output_dir=images_dir, image_size=(800, 800))
renderer.delay = 5
renderer.add_grid(g)
renderer.add_vec_prop(vg)
renderer.graticule.set_max_lines(max_lines=0)
mod.outputters += renderer
mod.movers += PyCurrentMover(current=vg,
default_num_method=method,
extrapolate=True)
mod.movers += RandomMover(diffusion_coef=10)
netCDF_fn = os.path.join(base_dir, images_dir + '.nc')
mod.outputters += NetCDFOutput(netCDF_fn, which_data='all')
return mod
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
start_time = datetime(2004, 12, 31, 13, 0)
model = Model(start_time=start_time,
duration=timedelta(days=3),
time_step=30 * 60,
uncertain=False)
print 'adding the map'
model.map = GnomeMap()
# draw_ontop can be 'uncertain' or 'forecast'
# 'forecast' LEs are in black, and 'uncertain' are in red
# default is 'forecast' LEs draw on top
renderer = Renderer(
output_dir=images_dir,
# size=(800, 600),
output_timestep=timedelta(hours=1),
draw_ontop='uncertain')
renderer.viewport = ((-76.5, 37.), (-75.8, 38.))
print 'adding outputters'
model.outputters += renderer
netcdf_file = os.path.join(base_dir, 'script_plume.nc')
scripting.remove_netcdf(netcdf_file)
model.outputters += NetCDFOutput(netcdf_file,
which_data='most',
output_timestep=timedelta(hours=2))
print 'adding two spills'
# Break the spill into two spills, first with the larger droplets
# and second with the smaller droplets.
# Split the total spill volume (100 m^3) to have most
# in the larger droplet spill.
# Smaller droplets start at a lower depth than larger
wd = WeibullDistribution(alpha=1.8, lambda_=.00456,
min_=.0002) # 200 micron min
end_time = start_time + timedelta(hours=24)
# spill = point_line_release_spill(num_elements=10,
# amount=90, # default volume_units=m^3
# units='m^3',
# start_position=(-76.126872, 37.680952,
# 1700),
# release_time=start_time,
# end_release_time=end_time,
# element_type=plume(distribution=wd,
# density=600)
# )
spill = subsurface_plume_spill(
num_elements=10,
start_position=(-76.126872, 37.680952, 1700),
release_time=start_time,
distribution=wd,
amount=90, # default volume_units=m^3
units='m^3',
end_release_time=end_time,
density=600)
model.spills += spill
wd = WeibullDistribution(alpha=1.8, lambda_=.00456,
max_=.0002) # 200 micron max
spill = point_line_release_spill(
num_elements=10,
amount=90,
units='m^3',
start_position=(-76.126872, 37.680952, 1800),
release_time=start_time,
element_type=plume(distribution=wd, substance_name='oil_crude'))
model.spills += spill
print 'adding a RandomMover:'
model.movers += RandomMover(diffusion_coef=50000)
print 'adding a RiseVelocityMover:'
model.movers += RiseVelocityMover()
print 'adding a RandomVerticalMover:'
model.movers += RandomVerticalMover(vertical_diffusion_coef_above_ml=5,
vertical_diffusion_coef_below_ml=.11,
mixed_layer_depth=10)
# print 'adding a wind mover:'
# series = np.zeros((2, ), dtype=gnome.basic_types.datetime_value_2d)
# series[0] = (start_time, (30, 90))
# series[1] = (start_time + timedelta(hours=23), (30, 90))
# wind = Wind(timeseries=series, units='knot')
#
# default is .4 radians
# w_mover = gnome.movers.WindMover(wind, uncertain_angle_scale=0)
#
# model.movers += w_mover
print 'adding a simple mover:'
s_mover = SimpleMover(velocity=(0.0, -.3, 0.0))
model.movers += s_mover
return model
def test_exceptions(model):
t_file = os.path.join(base_dir, 'temp.nc')
spill_pair = model.spills
def _del_temp_file():
# clean up temporary file from previous run
try:
print 'remove temporary file {0}'.format(t_file)
os.remove(t_file)
except:
pass
# begin tests
_del_temp_file()
netcdf = NetCDFOutput(t_file, which_data='all')
with raises(TypeError):
# need to pass in model start time
netcdf.prepare_for_model_run(num_time_steps=4)
with raises(TypeError):
# need to pass in model start time and spills
netcdf.prepare_for_model_run()
with raises(ValueError):
# need a cache object
netcdf.write_output(0)
with raises(ValueError):
netcdf.which_data = 'some random string'
with raises(ValueError):
# raise error because file 'temp.nc' should exist after 1st call
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
with raises(AttributeError):
'cannot change after prepare_for_model_run has been called'
netcdf.which_data = 'all'
_del_temp_file()
def test_exceptions(output_filename):
spill_pair = SpillContainerPair()
print "output_filename:", output_filename
# begin tests
netcdf = NetCDFOutput(output_filename, which_data='all')
netcdf.rewind() # delete temporary files
with raises(TypeError):
# need to pass in model start time
netcdf.prepare_for_model_run(num_time_steps=4)
with raises(TypeError):
# need to pass in model start time and spills
netcdf.prepare_for_model_run()
with raises(ValueError):
# need a cache object
netcdf.write_output(0)
with raises(ValueError):
netcdf.which_data = 'some random string'
# changed renderer and netcdf ouputter to delete old files in
# prepare_for_model_run() rather than rewind()
# -- rewind() was getting called a lot
# -- before there was time to change the ouput file names, etc.
# So for this unit test, there should be no exception if we do it twice.
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
with raises(AttributeError):
'cannot change after prepare_for_model_run has been called'
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
netcdf.which_data = 'most'
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
start_time = datetime(1985, 1, 1, 13, 31)
# 1 day of data in file
# 1/2 hr in seconds
model = Model(start_time=start_time,
duration=timedelta(days=4),
time_step=7200)
# mapfile = get_datafile(os.path.join(base_dir, 'ak_arctic.bna'))
mapfile = get_datafile('arctic_coast3.bna')
print 'adding the map'
model.map = MapFromBNA(mapfile, refloat_halflife=0.0) # seconds
print 'adding a spill'
# for now subsurface spill stays on initial layer
# - will need diffusion and rise velocity
# - wind doesn't act
# - start_position = (-76.126872, 37.680952, 5.0),
# spill1 = point_line_release_spill(num_elements=10000,
# start_position=(-163.75,
# 69.75,
# 0.0),
# release_time=start_time)
#
spill1 = point_line_release_spill(num_elements=50000,
start_position=(196.25, 69.75, 0.0),
release_time=start_time)
model.spills += spill1
# model.spills += spill2
print 'adding a wind mover:'
# model.movers += constant_wind_mover(0.5, 0, units='m/s')
print 'adding a current mover:'
fn = ['arctic_avg2_0001_gnome.nc', 'arctic_avg2_0002_gnome.nc']
# fn = ['C:\\Users\\jay.hennen\\Documents\\Code\\pygnome\\py_gnome\\scripts\\script_TAP\\arctic_avg2_0001_gnome.nc',
# 'C:\\Users\\jay.hennen\\Documents\\Code\\pygnome\\py_gnome\\scripts\\script_TAP\\arctic_avg2_0002_gnome.nc']
gt = {'node_lon': 'lon', 'node_lat': 'lat'}
# fn='arctic_avg2_0001_gnome.nc'
wind_method = 'Euler'
method = 'RK2'
print 'adding outputters'
# draw_ontop can be 'uncertain' or 'forecast'
# 'forecast' LEs are in black, and 'uncertain' are in red
# default is 'forecast' LEs draw on top
renderer = Renderer(mapfile, images_dir, image_size=(1024, 768))
model.outputters += renderer
netcdf_file = os.path.join(base_dir,
str(model.time_step / 60) + method + '.nc')
scripting.remove_netcdf(netcdf_file)
print 'adding movers'
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
print 'loading entire current data'
ice_aware_curr = IceAwareCurrent.from_netCDF(filename=fn, grid_topology=gt)
# env1 = get_env_from_netCDF(filename)
# mov = PyCurrentMover.from_netCDF(filename)
ice_aware_curr.ice_velocity.variables[0].dimension_ordering = [
'time', 'x', 'y'
]
ice_aware_wind = IceAwareWind.from_netCDF(
filename=fn,
ice_velocity=ice_aware_curr.ice_velocity,
ice_concentration=ice_aware_curr.ice_concentration,
grid=ice_aware_curr.grid)
curr = GridCurrent.from_netCDF(filename=fn)
# GridCurrent.is_gridded()
# import pprint as pp
# from gnome.utilities.orderedcollection import OrderedCollection
# model.environment = OrderedCollection(dtype=Environment)
# model.environment.add(ice_aware_curr)
# from gnome.environment import WindTS
print 'loading entire wind data'
# i_c_mover = PyCurrentMover(current=ice_aware_curr)
# i_c_mover = PyCurrentMover(current=ice_aware_curr, default_num_method='Euler')
i_c_mover = PyCurrentMover(current=ice_aware_curr,
default_num_method=method,
extrapolate=True)
i_w_mover = PyWindMover(wind=ice_aware_wind,
default_num_method=wind_method)
# ice_aware_curr.grid.node_lon = ice_aware_curr.grid.node_lon[:]-360
# ice_aware_curr.grid.build_celltree()
model.movers += i_c_mover
model.movers += i_w_mover
print 'adding an IceAwareRandomMover:'
model.movers += IceAwareRandomMover(
ice_concentration=ice_aware_curr.ice_concentration,
diffusion_coef=1000)
# renderer.add_grid(ice_aware_curr.grid)
# renderer.add_vec_prop(ice_aware_curr)
# curr_file = get_datafile(os.path.join(base_dir, 'COOPSu_CREOFS24.nc'))
# c_mover = GridCurrentMover(curr_file)
# model.movers += c_mover
# model.environment.add(WindTS.constant(10, 300))
# print('Saving')
# model.environment[0].ice_velocity.variables[0].serialize()
# IceVelocity.deserialize(model.environment[0].ice_velocity.serialize())
# model.save('.')
# from gnome.persist.save_load import load
# print('Loading')
# model2 = load('./Model.zip')
return model
def test_read_standard_arrays(model, output_ts_factor, use_time):
"""
tests the data returned by read_data is correct when `which_data` flag is
'standard'. It is only reading the standard_arrays
Test will only verify the data when time_stamp of model matches the
time_stamp of data written out. output_ts_factor means not all data is
written out.
The use_time flag says data is read by timestamp. If false, then it is read
by step number - either way, the result should be the same
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [
model.outputters[outputter.id] for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)
][0]
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for idx, step in enumerate(
range(0, model.num_time_steps, int(ceil(output_ts_factor)))):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
if use_time:
(nc_data,
weathering_data) = NetCDFOutput.read_data(file_, curr_time)
else:
(nc_data, weathering_data) = NetCDFOutput.read_data(file_,
index=idx)
# check time
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
# check standard variables
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
assert np.all(
scp.LE('spill_num', uncertain)[:] == nc_data['spill_num'])
assert np.all(
scp.LE('status_codes', uncertain)[:] ==
nc_data['status_codes'])
# flag variable is not currently set or checked
if 'mass' in scp.LE_data:
assert np.all(
scp.LE('mass', uncertain)[:] == nc_data['mass'])
if 'age' in scp.LE_data:
assert np.all(
scp.LE('age', uncertain)[:] == nc_data['age'])
if uncertain:
sc = scp.items()[1]
else:
sc = scp.items()[0]
assert sc.mass_balance == weathering_data
else:
raise Exception(
'Assertions not tested since no data found '
'in NetCDF file for timestamp: {0}'.format(curr_time))
if _found_a_matching_time:
print(
'\n'
'data in model matches for output in\n'
'{0}\n'
'and output_ts_factor: {1}'.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_write_output_post_run(model, output_ts_factor):
"""
Create netcdf file post run from the cache. Under the hood, it is simply
calling write_output so no need to check the data is correctly written
test_write_output_standard already checks data is correctly written.
Instead, make sure if output_timestep is not same as model.time_step,
then data is output at correct time stamps
"""
model.rewind()
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.which_data = 'standard'
o_put.output_timestep = timedelta(seconds=model.time_step * output_ts_factor)
del model.outputters[o_put.id] # remove from list of outputters
_run_model(model)
# clear out old files...
o_put.clean_output_files()
assert not os.path.exists(o_put.netcdf_filename)
if o_put._u_netcdf_filename:
assert (not os.path.exists(o_put._u_netcdf_filename))
# now write netcdf output
o_put.write_output_post_run(model.start_time,
model.num_time_steps,
spills=model.spills,
cache=model._cache,
uncertain=model.uncertain)
assert os.path.exists(o_put.netcdf_filename)
if model.uncertain:
assert os.path.exists(o_put._u_netcdf_filename)
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
ix = 0 # index for grabbing record from NetCDF file
for step in range(0, model.num_time_steps,
int(ceil(output_ts_factor))):
print "step: {0}".format(step)
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
# test to make sure data_by_index is consistent with _cached data
# This is just to double check that getting the data by curr_time
# does infact give the next consecutive index
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
ix += 1
if o_put.output_last_step and step < model.num_time_steps - 1:
'''
Last timestep written to NetCDF wasn't tested - do that here
'''
scp = model._cache.load_timestep(model.num_time_steps - 1)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
# again, check that last time step
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
with pytest.raises(IndexError):
# check that no more data exists in NetCDF
NetCDFOutput.read_data(file_, index=ix + 1)
""" at least one matching time found """
print ('All expected timestamps in {0} for output_ts_factor: {1}'
.format(os.path.split(file_)[1], output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
# add this back in so cleanup script deletes the generated *.nc files
model.outputters += o_put
def test_read_standard_arrays(model, output_ts_factor, use_time):
"""
tests the data returned by read_data is correct when `which_data` flag is
'standard'. It is only reading the standard_arrays
Test will only verify the data when time_stamp of model matches the
time_stamp of data written out. output_ts_factor means not all data is
written out.
The use_time flag says data is read by timestamp. If false, then it is read
by step number - either way, the result should be the same
"""
model.rewind()
# check contents of netcdf File at multiple time steps (should only be 1!)
o_put = [model.outputters[outputter.id] for outputter in
model.outputters if isinstance(outputter, NetCDFOutput)][0]
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
_run_model(model)
atol = 1e-5
rtol = 0
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
_found_a_matching_time = False
for idx, step in enumerate(range(0, model.num_time_steps,
int(ceil(output_ts_factor)))):
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
if use_time:
(nc_data, weathering_data) = NetCDFOutput.read_data(file_,
curr_time)
else:
(nc_data, weathering_data) = NetCDFOutput.read_data(file_,
index=idx)
# check time
if curr_time == nc_data['current_time_stamp'].item():
_found_a_matching_time = True
# check standard variables
assert np.allclose(scp.LE('positions', uncertain),
nc_data['positions'], rtol, atol)
assert np.all(scp.LE('spill_num', uncertain)[:] ==
nc_data['spill_num'])
assert np.all(scp.LE('status_codes', uncertain)[:] ==
nc_data['status_codes'])
# flag variable is not currently set or checked
if 'mass' in scp.LE_data:
assert np.all(scp.LE('mass', uncertain)[:] ==
nc_data['mass'])
if 'age' in scp.LE_data:
assert np.all(scp.LE('age', uncertain)[:] ==
nc_data['age'])
if uncertain:
sc = scp.items()[1]
else:
sc = scp.items()[0]
assert sc.mass_balance == weathering_data
else:
raise Exception('Assertions not tested since no data found '
'in NetCDF file for timestamp: {0}'
.format(curr_time))
if _found_a_matching_time:
print ('\n'
'data in model matches for output in\n'
'{0}\n'
'and output_ts_factor: {1}'
.format(file_, output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
def test_write_output_post_run(model, output_ts_factor):
"""
Create netcdf file post run from the cache. Under the hood, it is simply
calling write_output so no need to check the data is correctly written
test_write_output_standard already checks data is correctly written.
Instead, make sure if output_timestep is not same as model.time_step,
then data is output at correct time stamps
"""
model.rewind()
o_put = [
model.outputters[outputter.id] for outputter in model.outputters
if isinstance(outputter, NetCDFOutput)
][0]
o_put.which_data = 'standard'
o_put.output_timestep = timedelta(seconds=model.time_step *
output_ts_factor)
del model.outputters[o_put.id] # remove from list of outputters
_run_model(model)
# clear out old files...
o_put.clean_output_files()
assert not os.path.exists(o_put.netcdf_filename)
if o_put._u_netcdf_filename:
assert (not os.path.exists(o_put._u_netcdf_filename))
# now write netcdf output
o_put.write_output_post_run(model.start_time,
model.num_time_steps,
spills=model.spills,
cache=model._cache,
uncertain=model.uncertain)
assert os.path.exists(o_put.netcdf_filename)
if model.uncertain:
assert os.path.exists(o_put._u_netcdf_filename)
uncertain = False
for file_ in (o_put.netcdf_filename, o_put._u_netcdf_filename):
ix = 0 # index for grabbing record from NetCDF file
for step in range(0, model.num_time_steps,
int(ceil(output_ts_factor))):
print "step: {0}".format(step)
scp = model._cache.load_timestep(step)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
# test to make sure data_by_index is consistent with _cached data
# This is just to double check that getting the data by curr_time
# does infact give the next consecutive index
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
ix += 1
if o_put.output_last_step and step < model.num_time_steps - 1:
'''
Last timestep written to NetCDF wasn't tested - do that here
'''
scp = model._cache.load_timestep(model.num_time_steps - 1)
curr_time = scp.LE('current_time_stamp', uncertain)
(nc_data, mb) = NetCDFOutput.read_data(file_, curr_time)
assert curr_time == nc_data['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
# again, check that last time step
(data_by_index, mb) = NetCDFOutput.read_data(file_, index=ix)
assert curr_time == data_by_index['current_time_stamp'].item()
assert scp.LE('mass_balance', uncertain) == mb
with pytest.raises(IndexError):
# check that no more data exists in NetCDF
NetCDFOutput.read_data(file_, index=ix + 1)
""" at least one matching time found """
print(
'All expected timestamps in {0} for output_ts_factor: {1}'.format(
os.path.split(file_)[1], output_ts_factor))
# 2nd time around, look at uncertain filename so toggle uncertain flag
uncertain = True
# add this back in so cleanup script deletes the generated *.nc files
model.outputters += o_put
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
# set up the modeling environment
start_time = datetime(2016, 9, 23, 0, 0)
model = Model(start_time=start_time,
duration=timedelta(days=2),
time_step=30 * 60,
uncertain=False)
print 'adding the map'
model.map = GnomeMap() # this is a "water world -- no land anywhere"
# renderere is only top-down view on 2d -- but it's something
renderer = Renderer(output_dir=images_dir,
image_size=(1024, 768),
output_timestep=timedelta(hours=1),
)
renderer.viewport = ((196.14, 71.89), (196.18, 71.93))
print 'adding outputters'
model.outputters += renderer
# Also going to write the results out to a netcdf file
netcdf_file = os.path.join(base_dir, 'script_arctic_plume.nc')
scripting.remove_netcdf(netcdf_file)
model.outputters += NetCDFOutput(netcdf_file,
which_data='most',
# output most of the data associated with the elements
output_timestep=timedelta(hours=2))
print "adding Horizontal and Vertical diffusion"
# Horizontal Diffusion
model.movers += RandomMover(diffusion_coef=500)
# vertical diffusion (different above and below the mixed layer)
model.movers += RandomMover3D(vertical_diffusion_coef_above_ml=5,
vertical_diffusion_coef_below_ml=.11,
mixed_layer_depth=10)
print 'adding Rise Velocity'
# droplets rise as a function of their density and radius
model.movers += TamocRiseVelocityMover()
print 'adding a circular current and eastward current'
fn = 'hycom_glb_regp17_2016092300_subset.nc'
fn_ice = 'hycom-cice_ARCu0.08_046_2016092300_subset.nc'
iconc = IceConcentration.from_netCDF(filename=fn_ice)
ivel = IceVelocity.from_netCDF(filename=fn_ice, grid = iconc.grid)
ic = IceAwareCurrent.from_netCDF(ice_concentration = iconc, ice_velocity= ivel, filename=fn)
model.movers += PyCurrentMover(current = ic)
model.movers += SimpleMover(velocity=(0., 0., 0.))
model.movers += constant_wind_mover(20, 315, units='knots')
# Now to add in the TAMOC "spill"
print "Adding TAMOC spill"
model.spills += tamoc_spill.TamocSpill(release_time=start_time,
start_position=(196.16, 71.91, 40.0),
num_elements=1000,
end_release_time=start_time + timedelta(days=1),
name='TAMOC plume',
TAMOC_interval=None, # how often to re-run TAMOC
)
model.spills[0].data_sources['currents'] = ic
return model
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
start_time = datetime(2014, 6, 9, 0, 0)
mapfile = get_datafile(os.path.join(base_dir, 'PassamaquoddyMap.bna'))
gnome_map = MapFromBNA(mapfile, refloat_halflife=1) # hours
# # the image output renderer
# global renderer
# one hour timestep
model = Model(start_time=start_time,
duration=timedelta(hours=24), time_step=360,
map=gnome_map, uncertain=False, cache_enabled=True)
print 'adding outputters'
renderer = Renderer(mapfile, images_dir, size=(800, 600),
# output_timestep=timedelta(hours=1),
draw_ontop='uncertain')
renderer.viewport = ((-67.15, 45.), (-66.9, 45.2))
model.outputters += renderer
netcdf_file = os.path.join(base_dir, 'script_passamaquoddy.nc')
scripting.remove_netcdf(netcdf_file)
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
print 'adding a spill'
spill = point_line_release_spill(num_elements=1000,
start_position=(-66.991344, 45.059316,
0.0),
release_time=start_time)
model.spills += spill
print 'adding a RandomMover:'
model.movers += RandomMover(diffusion_coef=30000, uncertain_factor=2)
print 'adding a wind mover:'
series = np.zeros((5, ), dtype=datetime_value_2d)
series[0] = (start_time, (5, 90))
series[1] = (start_time + timedelta(hours=18), (5, 180))
series[2] = (start_time + timedelta(hours=30), (5, 135))
series[3] = (start_time + timedelta(hours=42), (5, 180))
series[4] = (start_time + timedelta(hours=54), (5, 225))
wind = Wind(timeseries=series, units='m/s')
model.movers += WindMover(wind)
print 'adding a current mover:'
curr_file = get_datafile(os.path.join(base_dir, 'PQBayCur.nc4'))
topology_file = get_datafile(os.path.join(base_dir, 'PassamaquoddyTOP.dat')
)
tide_file = get_datafile(os.path.join(base_dir, 'EstesHead.txt'))
cc_mover = CurrentCycleMover(curr_file, topology_file,
tide=Tide(tide_file))
model.movers += cc_mover
model.environment += cc_mover.tide
print 'viewport is:', [o.viewport
for o in model.outputters
if isinstance(o, Renderer)]
return model
def main(RootDir, Data_Dir, StartSite, RunSite, NumStarts, RunStarts,
ReleaseLength, TrajectoryRunLength, StartTimeFiles, TrajectoriesPath,
NumLEs, MapFileName, refloat, current_files, wind_files,
diffusion_coef, model_timestep, windage_range, windage_persist,
OutputTimestep):
timingRecord = open(os.path.join(RootDir, "timing.txt"), "w")
count = len(StartTimeFiles) * len(RunStarts)
timingRecord.write("This file tracks the time to process " + str(count) +
" gnome runs")
# model timing
release_duration = timedelta(hours=ReleaseLength)
run_time = timedelta(hours=TrajectoryRunLength)
# initiate model
model = Model(duration=run_time, time_step=model_timestep, uncertain=False)
# determine boundary for model
print "Adding the map:", MapFileName
mapfile = get_datafile(os.path.join(Data_Dir, MapFileName))
# model.map = MapFromBNA(mapfile, refloat_halflife=refloat) no, model map needs to inclde mudflats. later
# loop through seasons
for Season in StartTimeFiles:
timer1 = datetime.now()
SeasonName = Season[1]
start_times = open(Season[0], 'r').readlines()[:NumStarts]
SeasonTrajDir = os.path.join(RootDir, TrajectoriesPath, SeasonName)
if not os.path.isdir(SeasonTrajDir):
print "Creating directory: ", SeasonTrajDir
make_dir(SeasonTrajDir)
print " Season:", SeasonName
# get and parse start times in this season
start_dt = []
for start_time in start_times:
start_time = [int(i) for i in start_time.split(',')]
start_time = datetime(start_time[0], start_time[1], start_time[2],
start_time[3], start_time[4])
start_dt.append(start_time)
## loop through start times
for time_idx in RunStarts:
timer2 = datetime.now()
gc.collect()
model.movers.clear()
## set the start location
start_time = start_dt[time_idx]
end_time = start_time + run_time
model.start_time = start_time
print " ", start_time, "to", end_time
## get a list of the only data files needed for the start time (less data used)
## note: requires data files in year increments
#Todo: needs fixing before real run
years = range(start_time.year, end_time.year + 1)
years = [str(i) for i in years]
wind = [s for s in wind_files if any(xs in s for xs in years)]
current = [
s for s in current_files if any(xs in s for xs in years)
]
#Todo: add mudflats. Does it work like this?
topology = {'node_lon': 'x', 'node_lat': 'y'}
## add wind movers
w_mover = PyWindMover(filename=wind)
model.movers += w_mover
## add current movers
current_mover = gs.GridCurrent.from_netCDF(current,
grid_topology=topology)
c_mover = PyCurrentMover(current=current_mover)
model.movers += c_mover
tideflat = Matroos_Mudflats(current, grid_topology=topology)
land_map = gs.MapFromBNA(mapfile)
model.map = TideflatMap(land_map, tideflat)
## add diffusion
model.movers += RandomMover(diffusion_coef=diffusion_coef)
## loop through start locations
timer3 = datetime.now()
#Todo: can it deal with the test.location.txt file??
start_position = [float(i) for i in StartSite.split(',')]
OutDir = os.path.join(RootDir, TrajectoriesPath, SeasonName,
'pos_%03i' % (RunSite + 1))
make_dir(OutDir)
print " ", RunSite, time_idx
print " Running: start time:", start_time,
print "at start location:", start_position
## set the spill to the location
spill = surface_point_line_spill(
num_elements=NumLEs,
start_position=(start_position[0], start_position[1], 0.0),
release_time=start_time,
end_release_time=start_time + release_duration,
windage_range=windage_range,
windage_persist=windage_persist)
# print "adding netcdf output"
netcdf_output_file = os.path.join(
OutDir,
'pos_%03i-t%03i_%08i.nc' %
(RunSite + 1, time_idx, int(start_time.strftime('%y%m%d%H'))),
)
model.outputters.clear()
model.outputters += NetCDFOutput(
netcdf_output_file,
output_timestep=timedelta(hours=OutputTimestep))
model.spills.clear()
model.spills += spill
model.full_run(rewind=True)
timer4 = datetime.now()
diff = round((timer4 - timer3).total_seconds() / 60, 2)
timingRecord.write("\t\t" + str(RunSite) + " took " + str(diff) +
" minutes to complete")
diff = round((timer4 - timer1).total_seconds() / 3600, 2)
count = len(RunStarts)
timingRecord.write("\t" + str(SeasonName) + " took " + str(diff) +
" hours to finish " + str(count) + " Gnome runs")
#OutDir.close
timingRecord.close
def test_exceptions():
t_file = os.path.join(base_dir, 'temp.nc')
# clean up temporary file from previos run
if os.path.exists(t_file):
print 'remove temporary file {0}'.format(t_file)
os.remove(t_file)
netcdf = NetCDFOutput(t_file, which_data='all')
with raises(TypeError):
# need to pass in model start time
netcdf.prepare_for_model_run(num_time_steps=4)
with raises(TypeError):
# need to pass in model start time
netcdf.prepare_for_model_run()
with raises(ValueError):
# needs a spills object for "all"
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
num_time_steps=4,)
with raises(ValueError):
# need a cache object
netcdf.write_output(0)
with raises(ValueError):
# raise error because file 'temp.nc' should already exist
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
num_time_steps=4)
with raises(ValueError):
# which_data is 'all' but spills are not provided so raise an error
netcdf.rewind()
netcdf.which_data = 'all'
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
num_time_steps=4)
with raises(ValueError):
netcdf.which_data = 'some random string'
def test_exceptions(output_filename):
spill_pair = SpillContainerPair()
print "output_filename:", output_filename
# begin tests
netcdf = NetCDFOutput(output_filename, which_data='all')
netcdf.rewind() # delete temporary files
with raises(TypeError):
# need to pass in model start time
netcdf.prepare_for_model_run(num_time_steps=4)
with raises(TypeError):
# need to pass in model start time and spills
netcdf.prepare_for_model_run()
with raises(ValueError):
# need a cache object
netcdf.write_output(0)
with raises(ValueError):
netcdf.which_data = 'some random string'
# changed renderer and netcdf ouputter to delete old files in
# prepare_for_model_run() rather than rewind()
# -- rewind() was getting called a lot
# -- before there was time to change the ouput file names, etc.
# So for this unit test, there should be no exception if we do it twice.
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
with raises(AttributeError):
'cannot change after prepare_for_model_run has been called'
netcdf.prepare_for_model_run(model_start_time=datetime.now(),
spills=spill_pair,
num_time_steps=4)
netcdf.which_data = 'most'
def make_model(images_dir=os.path.join(base_dir, 'images')):
# create the maps:
print 'creating the maps'
mapfile = get_datafile(os.path.join(base_dir, 'DelawareRiverMap.bna'))
gnome_map = MapFromBNA(mapfile, refloat_halflife=1) # hours
renderer = Renderer(mapfile, images_dir, image_size=(800, 800),
projection_class=GeoProjection)
print 'initializing the model'
start_time = datetime(2012, 8, 20, 13, 0)
# 15 minutes in seconds
# Default to now, rounded to the nearest hour
model = Model(time_step=900, start_time=start_time,
duration=timedelta(days=1),
map=gnome_map, uncertain=False)
print 'adding outputters'
model.outputters += renderer
netcdf_file = os.path.join(base_dir, 'script_delaware_bay.nc')
scripting.remove_netcdf(netcdf_file)
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
print 'adding a RandomMover:'
model.movers += RandomMover(diffusion_coef=100000)
print 'adding a wind mover:'
# wind_file = get_datafile(os.path.join(base_dir, 'ConstantWind.WND'))
# wind = Wind(filename=wind_file)
series = np.zeros((2, ), dtype=datetime_value_2d)
series[0] = (start_time, (5, 270))
series[1] = (start_time + timedelta(hours=25), (5, 270))
wind = Wind(timeseries=series, units='m/s')
# w_mover = WindMover(Wind(timeseries=series, units='knots'))
w_mover = WindMover(wind)
model.movers += w_mover
print 'adding a cats shio mover:'
curr_file = get_datafile(os.path.join(base_dir, 'FloodTides.cur'))
tide_file = get_datafile(os.path.join(base_dir, 'FloodTidesShio.txt'))
c_mover = CatsMover(curr_file, tide=Tide(tide_file))
# this is the value in the file (default)
c_mover.scale_refpoint = (-75.081667, 38.7995)
c_mover.scale = True
c_mover.scale_value = 1
model.movers += c_mover
# TODO: cannot add this till environment base class is created
model.environment += c_mover.tide
print 'adding a cats mover:'
curr_file = get_datafile(os.path.join(base_dir, 'Offshore.cur'))
c_mover = CatsMover(curr_file)
# but do need to scale (based on river stage)
c_mover.scale = True
c_mover.scale_refpoint = (-74.7483333, 38.898333)
c_mover.scale_value = .03
model.movers += c_mover
#
# these are from windows they don't match Mac values...
# pat1Angle 315;
# pat1Speed 30; pat1SpeedUnits knots;
# pat1ScaleToValue 0.314426
#
# pat2Angle 225;
# pat2Speed 30; pat2SpeedUnits knots;
# pat2ScaleToValue 0.032882
# scaleBy WindStress
print 'adding a component mover:'
# if only using one current pattern
# comp_mover = ComponentMover(curr_file1, None, wind)
#
# todo: following is not working when model is saved out - fix
# comp_mover = ComponentMover(curr_file1, curr_file2,
# Wind(timeseries=series, units='m/s'))
# comp_mover = ComponentMover(curr_file1, curr_file2,
# wind=Wind(filename=wind_file))
curr_file1 = get_datafile(os.path.join(base_dir, 'NW30ktwinds.cur'))
curr_file2 = get_datafile(os.path.join(base_dir, 'SW30ktwinds.cur'))
comp_mover = ComponentMover(curr_file1, curr_file2, wind)
comp_mover.scale_refpoint = (-75.263166, 39.1428333)
comp_mover.pat1_angle = 315
comp_mover.pat1_speed = 30
comp_mover.pat1_speed_units = 1
# comp_mover.pat1ScaleToValue = .314426
comp_mover.pat1_scale_to_value = .502035
comp_mover.pat2_angle = 225
comp_mover.pat2_speed = 30
comp_mover.pat2_speed_units = 1
# comp_mover.pat2ScaleToValue = .032882
comp_mover.pat2_scale_to_value = .021869
model.movers += comp_mover
print 'adding a spill'
end_time = start_time + timedelta(hours=12)
spill = point_line_release_spill(num_elements=1000,
release_time=start_time,
# end_release_time=end_time,
start_position=(-75.262319,
39.142987, 0.0),
)
model.spills += spill
return model
def make_model(images_dir=os.path.join(base_dir, 'images')):
print 'initializing the model'
start_time = datetime(1985, 1, 1, 13, 31)
# 1 day of data in file
# 1/2 hr in seconds
model = Model(start_time=start_time,
duration=timedelta(days=4),
time_step=3600 * 2)
# mapfile = get_datafile(os.path.join(base_dir, 'ak_arctic.bna'))
mapfile = get_datafile('arctic_coast3.bna')
print 'adding the map'
model.map = MapFromBNA(mapfile, refloat_halflife=0.0) # seconds
print 'adding a spill'
# for now subsurface spill stays on initial layer
# - will need diffusion and rise velocity
# - wind doesn't act
# - start_position = (-76.126872, 37.680952, 5.0),
# spill1 = point_line_release_spill(num_elements=10000,
# start_position=(-163.75,
# 69.75,
# 0.0),
# release_time=start_time)
#
spill1 = point_line_release_spill(num_elements=100,
start_position=(196.25, 69.75, 0.0),
release_time=start_time)
model.spills += spill1
# model.spills += spill2
print 'adding a wind mover:'
# model.movers += constant_wind_mover(0.5, 0, units='m/s')
print 'adding a current mover:'
fn = ['arctic_avg2_0001_gnome.nc', 'arctic_avg2_0002_gnome.nc']
gt = {'node_lon': 'lon', 'node_lat': 'lat'}
# fn='arctic_avg2_0001_gnome.nc'
wind_method = 'Euler'
method = 'Trapezoid'
print 'adding outputters'
# draw_ontop can be 'uncertain' or 'forecast'
# 'forecast' LEs are in black, and 'uncertain' are in red
# default is 'forecast' LEs draw on top
# renderer = Renderer(mapfile, images_dir, image_size=(1024, 768))
# model.outputters += renderer
netcdf_file = os.path.join(base_dir,
str(model.time_step / 60) + method + '.nc')
scripting.remove_netcdf(netcdf_file)
print 'adding movers'
model.outputters += NetCDFOutput(netcdf_file, which_data='all')
ice_aware_curr = IceAwareCurrent.from_netCDF(filename=fn, grid_topology=gt)
ice_aware_wind = IceAwareWind.from_netCDF(
filename=fn,
ice_var=ice_aware_curr.ice_var,
ice_conc_var=ice_aware_curr.ice_conc_var,
grid=ice_aware_curr.grid,
)
# i_c_mover = PyGridCurrentMover(current=ice_aware_curr)
# i_c_mover = PyGridCurrentMover(current=ice_aware_curr, default_num_method='Euler')
i_c_mover = PyGridCurrentMover(current=ice_aware_curr,
default_num_method=method)
i_w_mover = PyWindMover(wind=ice_aware_wind,
default_num_method=wind_method)
# ice_aware_curr.grid.node_lon = ice_aware_curr.grid.node_lon[:]-360
# ice_aware_curr.grid.build_celltree()
model.movers += i_c_mover
model.movers += i_w_mover
print 'adding an IceAwareRandomMover:'
model.movers += IceAwareRandomMover(
ice_conc_var=ice_aware_curr.ice_conc_var, diffusion_coef=1000)
# renderer.add_grid(ice_aware_curr.grid)
# renderer.add_vec_prop(ice_aware_curr)
# renderer.set_viewport(((-190.9, 60), (-72, 89)))
# curr_file = get_datafile(os.path.join(base_dir, 'COOPSu_CREOFS24.nc'))
# c_mover = GridCurrentMover(curr_file)
# model.movers += c_mover
return model
