def test_add_replace_callback(self):
"register one callback with multiple events (add, replace)"
# lets work with a mutable type
oc = OrderedCollection(dtype=ObjToAdd)
oc.register_callback(self._add_callback, events=("add", "replace"))
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
rep = ObjToAdd()
oc[s_id(self.to_add[0])] = rep
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
def test_remove_callback(self):
"test remove callback is invoked after removing an object"
oc = OrderedCollection(dtype=ObjToAdd) # lets work with a mutable type
oc.register_callback(self._rm_callback, events="remove")
oc.register_callback(self._add_callback, events="add")
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
del oc[s_id(self.to_add[0])]
assert self.to_add[0].rm_callback
assert self.to_add[0].add_callback
assert not self.to_add[0].replace_callback
self.to_add[0].reset() # reset all to false
oc += self.to_add[0] # let's add this back in
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
def test_remove_callback(self):
'test remove callback is invoked after removing an object'
oc = OrderedCollection(dtype=ObjToAdd) # lets work with a mutable type
oc.register_callback(self._rm_callback, events='remove')
oc.register_callback(self._add_callback, events='add')
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
del oc[s_id(self.to_add[0])]
assert self.to_add[0].rm_callback
assert self.to_add[0].add_callback
assert not self.to_add[0].replace_callback
self.to_add[0].reset() # reset all to false
oc += self.to_add[0] # let's add this back in
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
def test_add_replace_callback(self):
'register one callback with multiple events (add, replace)'
# lets work with a mutable type
oc = OrderedCollection(dtype=ObjToAdd)
oc.register_callback(self._add_callback, events=('add', 'replace'))
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
rep = ObjToAdd()
oc[s_id(self.to_add[0])] = rep
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
def test_add_callback(self):
'''
test add callback is invoked after adding an object or
list of objects
'''
# lets work with a mutable type
oc = OrderedCollection(dtype=ObjToAdd)
oc.register_callback(self._add_callback, events='add')
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
oc += ObjToAdd()
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
def test_add_callback(self):
'''
test add callback is invoked after adding an object or
list of objects
'''
# lets work with a mutable type
oc = OrderedCollection(dtype=ObjToAdd)
oc.register_callback(self._add_callback, events='add')
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
oc += ObjToAdd()
for obj in oc:
assert obj.add_callback
assert not obj.rm_callback
assert not obj.replace_callback
def test_replace_callback(self):
"test replace callback is invoked after replacing an object"
# lets work with a mutable type
oc = OrderedCollection(dtype=ObjToAdd)
oc.register_callback(self._replace_callback, events="replace")
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
rep = ObjToAdd()
oc[s_id(self.to_add[0])] = rep
for obj in oc:
assert not obj.add_callback
assert not obj.rm_callback
if id(obj) == id(rep):
assert obj.replace_callback
else:
assert not obj.replace_callback
def test_replace_callback(self):
'test replace callback is invoked after replacing an object'
# lets work with a mutable type
oc = OrderedCollection(dtype=ObjToAdd)
oc.register_callback(self._replace_callback, events='replace')
# check everything if False initially
self._reset_ObjToAdd_init_state()
oc += self.to_add
rep = ObjToAdd()
oc[s_id(self.to_add[0])] = rep
for obj in oc:
assert not obj.add_callback
assert not obj.rm_callback
if id(obj) == id(rep):
assert obj.replace_callback
else:
assert not obj.replace_callback
class SpillContainer(AddLogger, SpillContainerData):
"""
Container class for all spills -- it takes care of capturing the released
LEs from all the spills, putting them all in a single set of arrays.
Many of the "fields" associated with a collection of elements are optional,
or used only by some movers, so only the ones required will be requested
by each mover.
The data for the elements is stored in the _data_arrays dict. They can be
accessed by indexing. For example:
positions = spill_container['positions'] : returns a (num_LEs, 3) array of
world_point_types
"""
def __init__(self, uncertain=False):
super(SpillContainer, self).__init__(uncertain=uncertain)
self.spills = OrderedCollection(dtype=gnome.spill.Spill)
self.spills.register_callback(self._spills_changed,
('add', 'replace', 'remove'))
self.rewind()
def __setitem__(self, data_name, array):
"""
Invoke base class __setitem__ method so the _data_array is set
correctly. In addition, create the appropriate ArrayType if it wasn't
created by the user.
"""
super(SpillContainer, self).__setitem__(data_name, array)
if data_name not in self._array_types:
shape = self._data_arrays[data_name].shape[1:]
dtype = self._data_arrays[data_name].dtype.type
self._array_types[data_name] = ArrayType(shape, dtype,
name=data_name)
def _reset_arrays(self):
'''
reset _array_types dict so it contains default keys/values
'''
gnome.array_types.reset_to_defaults(['spill_num', 'id'])
self._array_types = {'positions': positions,
'next_positions': next_positions,
'last_water_positions': last_water_positions,
'status_codes': status_codes,
'spill_num': spill_num,
'id': id,
'mass': mass,
'age': age}
self._data_arrays = {}
def _reset__substances_spills(self):
'''
reset internal attributes to None and empty list []:
1. _substances_spills: data structure to contain spills per substance
2. _oil_comp_array_len: max number of psuedocomponents - relevant if
more than one substance is used.
3. _fate_data_list: list of FateDataView() objects. One object per
substance if substance is not None
'''
# Initialize following either the first time it is used or in
# prepare_for_model_run() -- it could change with each new spill
self._substances_spills = None
self._oil_comp_array_len = None
def _reset__fate_data_list(self):
# define the fate view of the data if 'fate_status' is in data arrays
# 'fate_status' is included if weathering is on
self._fate_data_list = []
def reset_fate_dataview(self):
'''
reset data arrays for each fate_dataviewer. Each substance that is not
None has a fate_dataviewer object.
'''
for viewer in self._fate_data_list:
viewer.reset()
def _set_substancespills(self):
'''
_substances could change when spills are added/deleted
using _spills_changed callback to reset self._substance_spills to None
If 'substance' is None, we still include it in this data structure -
all spills that are 'on' are included. A spill that is off isn't really
being modeled so ignore it.
.. note::
Should not be called in middle of run. prepare_for_model_run()
will invoke this if self._substance_spills is None. This is another
view of the data - it doesn't contain any state that needs to be
persisted.
'''
subs = []
spills = []
if self._oil_comp_array_len is None:
self._oil_comp_array_len = 1
for spill in self.spills:
if not spill.on:
continue
new_subs = spill.get('substance')
if new_subs in subs:
# substance already defined for another spill
ix = subs.index(new_subs)
spills[ix].append(spill)
else:
# new substance not yet defined
subs.append(new_subs)
spills.append([spill])
# also set _oil_comp_array_len to substance with most
# components? -- *not* being used right now, but make it so
# it works correctly for testing multiple substances
if (hasattr(new_subs, 'num_components') and
new_subs.num_components > self._oil_comp_array_len):
self._oil_comp_array_len = new_subs.num_components
# let's reorder subs so None is in the end:
if None in subs:
ix = subs.index(None)
spills.append(spills.pop(ix))
subs.append(subs.pop(ix))
s_id = range(len(subs))
# 'data' will be updated when weatherers ask for arrays they need
# define the substances list and the list of spills for each substance
self._substances_spills = \
substances_spills(substances=subs, s_id=s_id, spills=spills)
if len(self.get_substances()) > 1:
# add an arraytype for substance if more than one substance
self._array_types.update({'substance': substance})
self.logger.info('{0} - number of substances: {1}'.
format(os.getpid(), len(self.get_substances())))
def _set_fate_data_list(self):
'''
For each substance that is not None, initialize
FateDataView(substance_id) object. The substance_id corresponds with
self._substance_spills.s_id for each substance.
'''
self._fate_data_list = []
for s_id, subs in zip(self._substances_spills.s_id,
self._substances_spills.substances):
if subs is None:
continue
self._fate_data_list.append(FateDataView(s_id))
def _spills_changed(self, *args):
'''
call back called on spills add/delete/replace
Callback simply resets the internal _substance_spills attribute to None
since the old _substance_spills value could now be invalid.
'''
self._substances_spills = None
def _get_s_id(self, substance):
'''
Look in the _substances_spills data structure of substance and return
the corresponding s_id
'''
try:
ix = self._substances_spills.substances.index(substance)
except ValueError:
'substance is not in list'
self.logger.debug('{0} - Substance named: {1}, not found in data '
'structure'.format(os.getpid(), substance.name))
return None
return self._substances_spills.s_id[ix]
def _get_fatedataview(self, substance):
'''
return the FateDataView object associated with substance
'''
ix = self._get_s_id(substance)
if ix is None:
msg = "substance named {0} not found".format(substance.name)
self.logger.info(msg)
return
# check
view = self._fate_data_list[ix]
if view.substance_id != ix:
msg = "substance_id did not match as expected. Check!"
raise ValueError(msg)
return view
def _array_name(self, at):
'''
given an array type, return the name of the array. This can be string,
in which case, it is the name of the array so return it. If its not
a string, then return the at.name attribute.
'''
if isinstance(at, basestring):
return at
else:
return at.name
def _append_array_types(self, array_types):
'''
append to self.array_types the input array_types.
:param array_types: set of array_types to be appended
:type array_types: set()
The set contains either a name as a string, say: 'rise_vel'
In this case, get the ArrayType from gnome.array_types.rise_vel
Set elements could also be tuples, say: ('rise_vel': ArrayType())
In this case the user name of the data_array and its array_type is
specified by the tuple so append it.
.. note:: If a tuple: ('name', ArrayType()), is given and an ArrayType
with that name already exists in self._array_types, then it is
overwritten.
'''
for array in array_types:
if isinstance(array, basestring):
# allow user to override an array_type that might already exist
# in self._array_types
try:
array = getattr(gat, array)
except AttributeError:
msg = ("Skipping {0} - not found in gnome.array_types;"
" and ArrayType is not provided.").format(array)
self.logger.error(msg)
raise GnomeRuntimeError(msg)
# must be an ArrayType of an object
self._array_types[array.name] = array
def _append_initializer_array_types(self, array_types):
# for each array_types, use the name to get the associated initializer
for name in array_types:
for spill in self.spills:
if spill.has_initializer(name):
self._append_array_types(spill.get_initializer(name).
array_types)
def _append_data_arrays(self, num_released):
"""
initialize data arrays once spill has spawned particles
Data arrays are set to their initial_values
:param int num_released: number of particles released
"""
for name, atype in self._array_types.iteritems():
# initialize all arrays even if 0 length
if atype.shape is None:
# assume array type is for weather data, provide it the shape
# per the number of components used to model the oil
# currently, we only have one type of oil, so all spills will
# model same number of oil_components
a_append = atype.initialize(num_released,
shape=(self._oil_comp_array_len,),
initial_value=tuple([0] * self._oil_comp_array_len))
else:
a_append = atype.initialize(num_released)
self._data_arrays[name] = np.r_[self._data_arrays[name], a_append]
def _set_substance_array(self, subs_idx, num_rel_by_substance):
'''
-. update 'substance' array if more than one substance present. The
value of array is the index of 'substance' in _substances_spills
data structure
'''
if 'substance' in self:
if num_rel_by_substance > 0:
self['substance'][-num_rel_by_substance:] = subs_idx
def substancefatedata(self,
substance,
array_types,
fate='surface_weather'):
'''
todo: fix this so it works for type of fate requested
return the data for specified substance
data must contain array names specified in 'array_types'
'''
view = self._get_fatedataview(substance)
return view.get_data(self, array_types, fate)
def iterspillsbysubstance(self):
'''
iterate through the substances spills datastructure and return the
spills associated with each substance. This is used by release_elements
DataStructure contains all spills. If some spills contain None for
substance, these will be returned
'''
if self._substances_spills is None:
self._set_substancespills()
return self._substances_spills.spills
def itersubstancedata(self, array_types, fate='surface_weather'):
'''
iterates through and returns the following for each iteration:
(substance, substance_data)
This is used by weatherers - if a substance is None, omit it from
the iteration.
:param array_types: iterable containing array that should be in the
data. This could be a set of strings corresponding with array names
or ArrayType objects which have a name attribute
:param select='select': a string stating the type of data to be
returned. Default if 'surface', so all elements with
status_codes==oil_status.in_water and z == 0 in positions array
:returns: (substance, substance_data) for each iteration
substance: substance object
substance_data: dict of numpy arrays associated with substance with
elements in_water and on surface if select == 'surface' or
subsurface if select == 'subsurface'
'''
if self._substances_spills is None:
self._set_substancespills()
return zip(self.get_substances(complete=False),
[view.get_data(self, array_types, fate) for view in
self._fate_data_list])
def update_from_fatedataview(self, substance=None,
fate='surface_weather'):
'''
let's only update the arrays that were changed
only update if a copy of 'data' exists. This is the case if there are
more then one substances
'''
if substance is not None:
view = self._get_fatedataview(substance)
view.update_sc(self, fate)
else:
# do for all substances
for view in self._fate_data_list:
view.update_sc(self, fate)
def get_substances(self, complete=True):
'''
return substances stored in _substances_spills structure.
Include None if complete is True. Default is complete=True.
'''
if self._substances_spills is None:
self._set_substancespills()
if complete:
return self._substances_spills.substances
else:
return filter(None, self._substances_spills.substances)
@property
def total_mass(self):
'''
return total mass spilled in 'kg'
'''
mass = 0
for spill in self.spills:
if spill.get_mass() is not None:
mass += spill.get_mass()
if mass == 0:
return None
else:
return mass
@property
def substances(self):
'''
Returns list of substances for weathering - not including None since
that is non-weathering.
Currently, only one weathering substance is supported
'''
return self.get_substances(complete=False)
@property
def array_types(self):
"""
user can modify ArrayType initial_value in middle of run. Changing
the shape should throw an error. Change the dtype at your own risk.
This returns a new dict so user cannot add/delete an ArrayType in
middle of run. Use prepare_for_model_run() to do add an ArrayType.
"""
return dict(self._array_types)
def rewind(self):
"""
In the rewind operation, we:
- rewind all the spills
- restore _array_types to contain only defaults
- movers/weatherers could have been deleted and we don't want to
carry associated data_arrays
- prepare_for_model_run() will be called before the next run and
new arrays can be given
- purge the data arrays
- we gather data arrays for each contained spill
- the stored arrays are cleared, then replaced with appropriate
empty arrays
"""
for spill in self.spills:
spill.rewind()
# create a full set of zero-sized arrays. If we rewound, something
# must have changed so let's get back to default _array_types
self._reset_arrays()
self._reset__substances_spills()
self._reset__fate_data_list()
self.initialize_data_arrays()
self.mass_balance = {} # reset to empty array
def get_spill_mask(self, spill):
return self['spill_num'] == self.spills.index(spill)
def uncertain_copy(self):
"""
Returns a copy of the spill_container suitable for uncertainty
It has all the same spills, with the same ids, and the uncertain
flag set to True
"""
u_sc = SpillContainer(uncertain=True)
for sp in self.spills:
u_sc.spills += sp.uncertain_copy()
return u_sc
def prepare_for_model_run(self, array_types=set()):
"""
called when setting up the model prior to 1st time step
This is considered 0th timestep by model
Make current_time optional since SpillContainer doesn't require it
especially for 0th step; however, the model needs to set it because
it will write_output() after each step. The data_arrays along with
the current_time_stamp must be set in order to write_output()
:param model_start_time: model_start_time to initialize
current_time_stamp. This is the time_stamp associated with 0-th
step so initial conditions for data arrays
:param array_types: a set of additional names and/or array_types to
append to standard array_types attribute. Set can contain only
strings or a tuple with (string, ArrayType). See Note below.
.. note:: set can contains strings or tuples. If set contains only
strings, say: {'mass', 'windages'},
then SpillContainer looks for corresponding ArrayType object
defined in gnome.array_types for 'mass' and 'windages'.
If set contains a tuple, say: {('mass', gnome.array_types.mass)},
then SpillContainer uses the ArrayType defined in the tuple.
.. note:: The SpillContainer iterates through each of the item in
array_types and checks to see if there is an associated initializer
in any Spill. If corresponding initializer is found, it gets the
array_types from initializer and appends them to its own list. This
was added for the case where 'droplet_diameter' array is
defined/used by initializer (InitRiseVelFromDropletSizeFromDist)
and we would like to see it in output, but no Mover/Weatherer needs
it.
"""
# Question - should we purge any new arrays that were added in previous
# call to prepare_for_model_run()?
# No! If user made modifications to _array_types before running model,
# let's keep those. A rewind will reset data_arrays.
self._append_array_types(array_types)
self._append_initializer_array_types(array_types)
if self._substances_spills is None:
self._set_substancespills()
# also create fate_dataview if 'fate_status' is part of arrays
if 'fate_status' in self.array_types:
self._set_fate_data_list()
# 'substance' data_array may have been added so initialize after
# _set_substancespills() is invoked
self.initialize_data_arrays()
# todo: maybe better to let map do this, but it does not have a
# prepare_for_model_run() yet so can't do it there
# need 'amount_released' here as well
self.mass_balance['beached'] = 0.0
self.mass_balance['off_maps'] = 0.0
def initialize_data_arrays(self):
"""
initialize_data_arrays() is called without input data during rewind
and prepare_for_model_run to define all data arrays.
At this time the arrays are empty.
"""
for name, atype in self._array_types.iteritems():
# Initialize data_arrays with 0 elements
if atype.shape is None:
num_comp = self._oil_comp_array_len
self._data_arrays[name] = \
atype.initialize_null(shape=(num_comp, ))
else:
self._data_arrays[name] = atype.initialize_null()
def release_elements(self, time_step, model_time):
"""
Called at the end of a time step
This calls release_elements on all of the contained spills, and adds
the elements to the data arrays
:returns: total number of particles released
todo: may need to update the 'mass' array to use a default of 1.0 but
will need to define it in particle units or something along those lines
"""
total_released = 0
# substance index - used label elements from same substance
# used internally only by SpillContainer - could be a strided array.
# Simpler to define it only in SpillContainer as opposed to ArrayTypes
# 'substance': ((), np.uint8, 0)
for ix, spills in enumerate(self.iterspillsbysubstance()):
num_rel_by_substance = 0
for spill in spills:
# only spills that are included here - no need to check
# spill.on flag
num_rel = spill.num_elements_to_release(model_time, time_step)
if num_rel > 0:
# update 'spill_num' ArrayType's initial_value so it
# corresponds with spill number for this set of released
# particles - just another way to set value of spill_num
# correctly
self._array_types['spill_num'].initial_value = \
self.spills.index(spill)
if len(self['spill_num']) > 0:
# unique identifier for each new element released
# this adjusts the _array_types initial_value since the
# initialize function just calls:
# range(initial_value, num_released + initial_value)
self._array_types['id'].initial_value = \
self['id'][-1] + 1
else:
# always reset value of first particle released to 0!
# The array_types are shared globally. To initialize
# uncertain spills correctly, reset this to 0.
# To be safe, always reset to 0 when no
# particles are released
self._array_types['id'].initial_value = 0
# append to data arrays - number of oil components is
# currently the same for all spills
self._append_data_arrays(num_rel)
spill.set_newparticle_values(num_rel,
model_time,
time_step,
self._data_arrays)
num_rel_by_substance += num_rel
# always reset data arrays else the changing arrays are stale
self._set_substance_array(ix, num_rel_by_substance)
# reset fate_dataview at each step - do it after release elements
self.reset_fate_dataview()
# update total elements released for substance
total_released += num_rel_by_substance
return total_released
def split_element(self, ix, num, l_frac=None):
'''
split an element into specified number.
For data, like mass, that gets divided, l_frac can be optionally
provided. l_frac is a list containing fraction of component's value
given to each new element. len(l_frac) must be equal to num and
sum(l_frac) == 1.0
:param ix: id of element to be split - before splitting each element
has a unique 'id' defined in 'id' data array
:type ix: int
:param num: split ix into 'num' number of elements
:type num: int
:param l_frac: list containing fractions that sum to 1.0 with
len(l_frac) == num
:type l_frac: list or tuple or numpy array
'''
# split the first location where 'id' matches
try:
idx = np.where(self['id'] == ix)[0][0]
except IndexError:
msg = "no element with id = {0} found".format(ix)
self.logger.warning(msg)
raise
for name, at in self.array_types.iteritems():
data = self[name]
split_elems = at.split_element(num, self[name][idx], l_frac)
data = np.insert(data, idx, split_elems[:-1], 0)
data[idx + len(split_elems) - 1] = split_elems[-1]
self._data_arrays[name] = data
# update fate_dataview which contains this LE
# for now we only have one type of substance
if len(self._fate_data_list) > 1:
msg = "split_elements assumes only one substance is being modeled"
self.logger.error(msg)
self._fate_data_list[0]._reset_fatedata(self, ix)
def model_step_is_done(self):
'''
Called at the end of a time step
Need to remove particles marked as to_be_removed...
'''
if len(self._data_arrays) == 0:
return # nothing to do - arrays are not yet defined.
# LEs are marked as to_be_removed
# C++ might care about this so leave as is
to_be_removed = np.where(self['status_codes'] ==
oil_status.to_be_removed)[0]
if len(to_be_removed) > 0:
for key in self._array_types.keys():
self._data_arrays[key] = np.delete(self[key], to_be_removed,
axis=0)
def __str__(self):
return ('gnome.spill_container.SpillContainer\n'
'spill LE attributes: {0}'
.format(sorted(self._data_arrays.keys())))
__repr__ = __str__
class SpillContainer(AddLogger, SpillContainerData):
"""
Container class for all spills -- it takes care of capturing the released
LEs from all the spills, putting them all in a single set of arrays.
Many of the "fields" associated with a collection of elements are optional,
or used only by some movers, so only the ones required will be requested
by each mover.
The data for the elements is stored in the _data_arrays dict. They can be
accessed by indexing. For example:
positions = spill_container['positions'] : returns a (num_LEs, 3) array of
world_point_types
"""
def __init__(self, uncertain=False):
super(SpillContainer, self).__init__(uncertain=uncertain)
self.spills = OrderedCollection(dtype=gnome.spill.Spill)
self.spills.register_callback(self._spills_changed,
('add', 'replace', 'remove'))
self.rewind()
def __setitem__(self, data_name, array):
"""
Invoke base class __setitem__ method so the _data_array is set
correctly. In addition, create the appropriate ArrayType if it wasn't
created by the user.
"""
super(SpillContainer, self).__setitem__(data_name, array)
if data_name not in self._array_types:
shape = self._data_arrays[data_name].shape[1:]
dtype = self._data_arrays[data_name].dtype.type
self._array_types[data_name] = ArrayType(shape,
dtype,
name=data_name)
def _reset_arrays(self):
'''
reset _array_types dict so it contains default keys/values
'''
gnome.array_types.reset_to_defaults(['spill_num', 'id'])
self._array_types = {
'positions': positions,
'next_positions': next_positions,
'last_water_positions': last_water_positions,
'status_codes': status_codes,
'spill_num': spill_num,
'id': id,
'mass': mass,
'age': age
}
self._data_arrays = {}
def _reset__substances_spills(self):
'''
reset internal attributes to None and empty list []:
1. _substances_spills: data structure to contain spills per substance
2. _oil_comp_array_len: max number of psuedocomponents - relevant if
more than one substance is used.
3. _fate_data_list: list of FateDataView() objects. One object per
substance if substance is not None
'''
# Initialize following either the first time it is used or in
# prepare_for_model_run() -- it could change with each new spill
self._substances_spills = None
self._oil_comp_array_len = None
def _reset__fate_data_list(self):
# define the fate view of the data if 'fate_status' is in data arrays
# 'fate_status' is included if weathering is on
self._fate_data_list = []
def reset_fate_dataview(self):
'''
reset data arrays for each fate_dataviewer. Each substance that is not
None has a fate_dataviewer object.
'''
for viewer in self._fate_data_list:
viewer.reset()
def _set_substancespills(self):
'''
_substances could change when spills are added/deleted
using _spills_changed callback to reset self._substance_spills to None
If 'substance' is None, we still include it in this data structure -
all spills that are 'on' are included. A spill that is off isn't really
being modeled so ignore it.
.. note::
Should not be called in middle of run. prepare_for_model_run()
will invoke this if self._substance_spills is None. This is another
view of the data - it doesn't contain any state that needs to be
persisted.
'''
subs = []
spills = []
if self._oil_comp_array_len is None:
self._oil_comp_array_len = 1
for spill in self.spills:
if not spill.on:
continue
new_subs = spill.get('substance')
if new_subs in subs:
# substance already defined for another spill
ix = subs.index(new_subs)
spills[ix].append(spill)
else:
# new substance not yet defined
subs.append(new_subs)
spills.append([spill])
# also set _oil_comp_array_len to substance with most
# components? -- *not* being used right now, but make it so
# it works correctly for testing multiple substances
if (hasattr(new_subs, 'num_components') and
new_subs.num_components > self._oil_comp_array_len):
self._oil_comp_array_len = new_subs.num_components
# let's reorder subs so None is in the end:
if None in subs:
ix = subs.index(None)
spills.append(spills.pop(ix))
subs.append(subs.pop(ix))
s_id = range(len(subs))
# 'data' will be updated when weatherers ask for arrays they need
# define the substances list and the list of spills for each substance
self._substances_spills = \
substances_spills(substances=subs, s_id=s_id, spills=spills)
if len(self.get_substances()) > 1:
# add an arraytype for substance if more than one substance
self._array_types.update({'substance': substance})
self.logger.info('{0} - number of substances: {1}'.format(
os.getpid(), len(self.get_substances())))
def _set_fate_data_list(self):
'''
For each substance that is not None, initialize
FateDataView(substance_id) object. The substance_id corresponds with
self._substance_spills.s_id for each substance.
'''
self._fate_data_list = []
for s_id, subs in zip(self._substances_spills.s_id,
self._substances_spills.substances):
if subs is None:
continue
self._fate_data_list.append(FateDataView(s_id))
def _spills_changed(self, *args):
'''
call back called on spills add/delete/replace
Callback simply resets the internal _substance_spills attribute to None
since the old _substance_spills value could now be invalid.
'''
self._substances_spills = None
def _get_s_id(self, substance):
'''
Look in the _substances_spills data structure of substance and return
the corresponding s_id
'''
try:
ix = self._substances_spills.substances.index(substance)
except ValueError:
'substance is not in list'
self.logger.debug('{0} - Substance named: {1}, not found in data '
'structure'.format(os.getpid(), substance.name))
return None
return self._substances_spills.s_id[ix]
def _get_fatedataview(self, substance):
'''
return the FateDataView object associated with substance
'''
ix = self._get_s_id(substance)
if ix is None:
msg = "substance named {0} not found".format(substance.name)
self.logger.info(msg)
return
# check
view = self._fate_data_list[ix]
if view.substance_id != ix:
msg = "substance_id did not match as expected. Check!"
raise ValueError(msg)
return view
def _array_name(self, at):
'''
given an array type, return the name of the array. This can be string,
in which case, it is the name of the array so return it. If its not
a string, then return the at.name attribute.
'''
if isinstance(at, basestring):
return at
else:
return at.name
def _append_array_types(self, array_types):
'''
append to self.array_types the input array_types.
:param array_types: set of array_types to be appended
:type array_types: set()
The set contains either a name as a string, say: 'rise_vel'
In this case, get the ArrayType from gnome.array_types.rise_vel
Set elements could also be tuples, say: ('rise_vel': ArrayType())
In this case the user name of the data_array and its array_type is
specified by the tuple so append it.
.. note:: If a tuple: ('name', ArrayType()), is given and an ArrayType
with that name already exists in self._array_types, then it is
overwritten.
'''
for array in array_types:
if isinstance(array, basestring):
# allow user to override an array_type that might already exist
# in self._array_types
try:
array = getattr(gat, array)
except AttributeError:
msg = ("Skipping {0} - not found in gnome.array_types;"
" and ArrayType is not provided.").format(array)
self.logger.error(msg)
raise GnomeRuntimeError(msg)
# must be an ArrayType of an object
self._array_types[array.name] = array
def _append_initializer_array_types(self, array_types):
# for each array_types, use the name to get the associated initializer
for name in array_types:
for spill in self.spills:
if spill.has_initializer(name):
self._append_array_types(
spill.get_initializer(name).array_types)
def _append_data_arrays(self, num_released):
"""
initialize data arrays once spill has spawned particles
Data arrays are set to their initial_values
:param int num_released: number of particles released
"""
for name, atype in self._array_types.iteritems():
# initialize all arrays even if 0 length
if atype.shape is None:
# assume array type is for weather data, provide it the shape
# per the number of components used to model the oil
# currently, we only have one type of oil, so all spills will
# model same number of oil_components
a_append = atype.initialize(
num_released,
shape=(self._oil_comp_array_len, ),
initial_value=tuple([0] * self._oil_comp_array_len))
else:
a_append = atype.initialize(num_released)
self._data_arrays[name] = np.r_[self._data_arrays[name], a_append]
def _set_substance_array(self, subs_idx, num_rel_by_substance):
'''
-. update 'substance' array if more than one substance present. The
value of array is the index of 'substance' in _substances_spills
data structure
'''
if 'substance' in self:
if num_rel_by_substance > 0:
self['substance'][-num_rel_by_substance:] = subs_idx
def substancefatedata(self,
substance,
array_types,
fate='surface_weather'):
'''
todo: fix this so it works for type of fate requested
return the data for specified substance
data must contain array names specified in 'array_types'
'''
view = self._get_fatedataview(substance)
return view.get_data(self, array_types, fate)
def iterspillsbysubstance(self):
'''
iterate through the substances spills datastructure and return the
spills associated with each substance. This is used by release_elements
DataStructure contains all spills. If some spills contain None for
substance, these will be returned
'''
if self._substances_spills is None:
self._set_substancespills()
return self._substances_spills.spills
def itersubstancedata(self, array_types, fate='surface_weather'):
'''
iterates through and returns the following for each iteration:
(substance, substance_data)
This is used by weatherers - if a substance is None, omit it from
the iteration.
:param array_types: iterable containing array that should be in the
data. This could be a set of strings corresponding with array names
or ArrayType objects which have a name attribute
:param select='select': a string stating the type of data to be
returned. Default if 'surface', so all elements with
status_codes==oil_status.in_water and z == 0 in positions array
:returns: (substance, substance_data) for each iteration
substance: substance object
substance_data: dict of numpy arrays associated with substance with
elements in_water and on surface if select == 'surface' or
subsurface if select == 'subsurface'
'''
if self._substances_spills is None:
self._set_substancespills()
return zip(self.get_substances(complete=False), [
view.get_data(self, array_types, fate)
for view in self._fate_data_list
])
def update_from_fatedataview(self, substance=None, fate='surface_weather'):
'''
let's only update the arrays that were changed
only update if a copy of 'data' exists. This is the case if there are
more then one substances
'''
if substance is not None:
view = self._get_fatedataview(substance)
view.update_sc(self, fate)
else:
# do for all substances
for view in self._fate_data_list:
view.update_sc(self, fate)
def get_substances(self, complete=True):
'''
return substances stored in _substances_spills structure.
Include None if complete is True. Default is complete=True.
'''
if self._substances_spills is None:
self._set_substancespills()
if complete:
return self._substances_spills.substances
else:
return filter(None, self._substances_spills.substances)
@property
def total_mass(self):
'''
return total mass spilled in 'kg'
'''
mass = 0
for spill in self.spills:
if spill.get_mass() is not None:
mass += spill.get_mass()
if mass == 0:
return None
else:
return mass
@property
def substances(self):
'''
Returns list of substances for weathering - not including None since
that is non-weathering.
Currently, only one weathering substance is supported
'''
return self.get_substances(complete=False)
@property
def array_types(self):
"""
user can modify ArrayType initial_value in middle of run. Changing
the shape should throw an error. Change the dtype at your own risk.
This returns a new dict so user cannot add/delete an ArrayType in
middle of run. Use prepare_for_model_run() to do add an ArrayType.
"""
return dict(self._array_types)
def rewind(self):
"""
In the rewind operation, we:
- rewind all the spills
- restore _array_types to contain only defaults
- movers/weatherers could have been deleted and we don't want to
carry associated data_arrays
- prepare_for_model_run() will be called before the next run and
new arrays can be given
- purge the data arrays
- we gather data arrays for each contained spill
- the stored arrays are cleared, then replaced with appropriate
empty arrays
"""
for spill in self.spills:
spill.rewind()
# create a full set of zero-sized arrays. If we rewound, something
# must have changed so let's get back to default _array_types
self._reset_arrays()
self._reset__substances_spills()
self._reset__fate_data_list()
self.initialize_data_arrays()
self.mass_balance = {} # reset to empty array
def get_spill_mask(self, spill):
return self['spill_num'] == self.spills.index(spill)
def uncertain_copy(self):
"""
Returns a copy of the spill_container suitable for uncertainty
It has all the same spills, with the same ids, and the uncertain
flag set to True
"""
u_sc = SpillContainer(uncertain=True)
for sp in self.spills:
u_sc.spills += sp.uncertain_copy()
return u_sc
def prepare_for_model_run(self, array_types=set()):
"""
called when setting up the model prior to 1st time step
This is considered 0th timestep by model
Make current_time optional since SpillContainer doesn't require it
especially for 0th step; however, the model needs to set it because
it will write_output() after each step. The data_arrays along with
the current_time_stamp must be set in order to write_output()
:param model_start_time: model_start_time to initialize
current_time_stamp. This is the time_stamp associated with 0-th
step so initial conditions for data arrays
:param array_types: a set of additional names and/or array_types to
append to standard array_types attribute. Set can contain only
strings or a tuple with (string, ArrayType). See Note below.
.. note:: set can contains strings or tuples. If set contains only
strings, say: {'mass', 'windages'},
then SpillContainer looks for corresponding ArrayType object
defined in gnome.array_types for 'mass' and 'windages'.
If set contains a tuple, say: {('mass', gnome.array_types.mass)},
then SpillContainer uses the ArrayType defined in the tuple.
.. note:: The SpillContainer iterates through each of the item in
array_types and checks to see if there is an associated initializer
in any Spill. If corresponding initializer is found, it gets the
array_types from initializer and appends them to its own list. This
was added for the case where 'droplet_diameter' array is
defined/used by initializer (InitRiseVelFromDropletSizeFromDist)
and we would like to see it in output, but no Mover/Weatherer needs
it.
"""
# Question - should we purge any new arrays that were added in previous
# call to prepare_for_model_run()?
# No! If user made modifications to _array_types before running model,
# let's keep those. A rewind will reset data_arrays.
self._append_array_types(array_types)
self._append_initializer_array_types(array_types)
if self._substances_spills is None:
self._set_substancespills()
# also create fate_dataview if 'fate_status' is part of arrays
if 'fate_status' in self.array_types:
self._set_fate_data_list()
# 'substance' data_array may have been added so initialize after
# _set_substancespills() is invoked
self.initialize_data_arrays()
# todo: maybe better to let map do this, but it does not have a
# prepare_for_model_run() yet so can't do it there
# need 'amount_released' here as well
self.mass_balance['beached'] = 0.0
self.mass_balance['off_maps'] = 0.0
def initialize_data_arrays(self):
"""
initialize_data_arrays() is called without input data during rewind
and prepare_for_model_run to define all data arrays.
At this time the arrays are empty.
"""
for name, atype in self._array_types.iteritems():
# Initialize data_arrays with 0 elements
if atype.shape is None:
num_comp = self._oil_comp_array_len
self._data_arrays[name] = \
atype.initialize_null(shape=(num_comp, ))
else:
self._data_arrays[name] = atype.initialize_null()
def release_elements(self, time_step, model_time):
"""
Called at the end of a time step
This calls release_elements on all of the contained spills, and adds
the elements to the data arrays
:returns: total number of particles released
todo: may need to update the 'mass' array to use a default of 1.0 but
will need to define it in particle units or something along those lines
"""
total_released = 0
# substance index - used label elements from same substance
# used internally only by SpillContainer - could be a strided array.
# Simpler to define it only in SpillContainer as opposed to ArrayTypes
# 'substance': ((), np.uint8, 0)
for ix, spills in enumerate(self.iterspillsbysubstance()):
num_rel_by_substance = 0
for spill in spills:
# only spills that are included here - no need to check
# spill.on flag
num_rel = spill.num_elements_to_release(model_time, time_step)
if num_rel > 0:
# update 'spill_num' ArrayType's initial_value so it
# corresponds with spill number for this set of released
# particles - just another way to set value of spill_num
# correctly
self._array_types['spill_num'].initial_value = \
self.spills.index(spill)
if len(self['spill_num']) > 0:
# unique identifier for each new element released
# this adjusts the _array_types initial_value since the
# initialize function just calls:
# range(initial_value, num_released + initial_value)
self._array_types['id'].initial_value = \
self['id'][-1] + 1
else:
# always reset value of first particle released to 0!
# The array_types are shared globally. To initialize
# uncertain spills correctly, reset this to 0.
# To be safe, always reset to 0 when no
# particles are released
self._array_types['id'].initial_value = 0
# append to data arrays - number of oil components is
# currently the same for all spills
self._append_data_arrays(num_rel)
spill.set_newparticle_values(num_rel, model_time,
time_step, self._data_arrays)
num_rel_by_substance += num_rel
# always reset data arrays else the changing arrays are stale
self._set_substance_array(ix, num_rel_by_substance)
# reset fate_dataview at each step - do it after release elements
self.reset_fate_dataview()
# update total elements released for substance
total_released += num_rel_by_substance
return total_released
def split_element(self, ix, num, l_frac=None):
'''
split an element into specified number.
For data, like mass, that gets divided, l_frac can be optionally
provided. l_frac is a list containing fraction of component's value
given to each new element. len(l_frac) must be equal to num and
sum(l_frac) == 1.0
:param ix: id of element to be split - before splitting each element
has a unique 'id' defined in 'id' data array
:type ix: int
:param num: split ix into 'num' number of elements
:type num: int
:param l_frac: list containing fractions that sum to 1.0 with
len(l_frac) == num
:type l_frac: list or tuple or numpy array
'''
# split the first location where 'id' matches
try:
idx = np.where(self['id'] == ix)[0][0]
except IndexError:
msg = "no element with id = {0} found".format(ix)
self.logger.warning(msg)
raise
for name, at in self.array_types.iteritems():
data = self[name]
split_elems = at.split_element(num, self[name][idx], l_frac)
data = np.insert(data, idx, split_elems[:-1], 0)
data[idx + len(split_elems) - 1] = split_elems[-1]
self._data_arrays[name] = data
# update fate_dataview which contains this LE
# for now we only have one type of substance
if len(self._fate_data_list) > 1:
msg = "split_elements assumes only one substance is being modeled"
self.logger.error(msg)
self._fate_data_list[0]._reset_fatedata(self, ix)
def model_step_is_done(self):
'''
Called at the end of a time step
Need to remove particles marked as to_be_removed...
'''
if len(self._data_arrays) == 0:
return # nothing to do - arrays are not yet defined.
# LEs are marked as to_be_removed
# C++ might care about this so leave as is
to_be_removed = np.where(
self['status_codes'] == oil_status.to_be_removed)[0]
if len(to_be_removed) > 0:
for key in self._array_types.keys():
self._data_arrays[key] = np.delete(self[key],
to_be_removed,
axis=0)
def __str__(self):
return ('gnome.spill_container.SpillContainer\n'
'spill LE attributes: {0}'.format(
sorted(self._data_arrays.keys())))
__repr__ = __str__
