def test_add_replace_callback(self): 'register one callback with multiple events (add, replace)' # lets work with a mutable type oc = OrderedCollection(dtype=ObjToAdd) oc.register_callback(self._add_callback, events=('add', 'replace')) # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback rep = ObjToAdd() oc[s_id(self.to_add[0])] = rep for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback
def test_add_replace_callback(self): "register one callback with multiple events (add, replace)" # lets work with a mutable type oc = OrderedCollection(dtype=ObjToAdd) oc.register_callback(self._add_callback, events=("add", "replace")) # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback rep = ObjToAdd() oc[s_id(self.to_add[0])] = rep for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback
def test_index(self): oc = OrderedCollection([1, 2, 3, 4, 5]) assert oc.index(id(3)) == 2 oc[id(3)] = 6 assert oc.index(id(6)) == 2 del oc[id(6)] assert oc.index(id(4)) == 2
def test_remove_callback(self): "test remove callback is invoked after removing an object" oc = OrderedCollection(dtype=ObjToAdd) # lets work with a mutable type oc.register_callback(self._rm_callback, events="remove") oc.register_callback(self._add_callback, events="add") # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add del oc[s_id(self.to_add[0])] assert self.to_add[0].rm_callback assert self.to_add[0].add_callback assert not self.to_add[0].replace_callback self.to_add[0].reset() # reset all to false oc += self.to_add[0] # let's add this back in for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback
def __restore__(self, time_step, start_time, duration, weathering_substeps, map, uncertain, cache_enabled): """ Take out initialization that does not register the callback here. This is because new_from_dict will use this to restore the model _state when doing a midrun persistence. """ # making sure basic stuff is in place before properties are set self.environment = OrderedCollection(dtype=Environment) self.movers = OrderedCollection(dtype=Mover) self.weatherers = OrderedCollection(dtype=Weatherer) # contains both certain/uncertain spills self.spills = SpillContainerPair(uncertain) self._cache = gnome.utilities.cache.ElementCache() self._cache.enabled = cache_enabled # list of output objects self.outputters = OrderedCollection(dtype=Outputter) # default to now, rounded to the nearest hour self._start_time = start_time self._duration = duration self.weathering_substeps = weathering_substeps self._map = map self.time_step = time_step # this calls rewind() !
def test_with_movers(self): mover_1 = SimpleMover(velocity=(1.0, -1.0, 0.0)) mover_2 = SimpleMover(velocity=(1.0, -1.0, 0.0)) mover_3 = SimpleMover(velocity=(1.0, -1.0, 0.0)) mover_4 = SimpleMover(velocity=(1.0, -1.0, 0.0)) # test our init, iter, get, and len methods mymovers = OrderedCollection([mover_1, mover_2], dtype=Mover) assert [m for m in mymovers] == [mover_1, mover_2] assert mymovers[mover_1.id] == mover_1 assert len(mymovers) == 2 # test our add methods mymovers = OrderedCollection(dtype=Mover) mymovers += mover_1 mymovers += mover_2 assert [m for m in mymovers] == [mover_1, mover_2] mymovers = OrderedCollection(dtype=Mover) mymovers += [mover_1, mover_2] assert [m for m in mymovers] == [mover_1, mover_2] # test our del method mymovers = OrderedCollection([mover_1, mover_2, mover_3], dtype=Mover) del mymovers[mover_2.id] assert [m for m in mymovers] == [mover_1, mover_3] # test our replace method mymovers = OrderedCollection([mover_1, mover_2, mover_3], dtype=Mover) mymovers[mover_2.id] = mover_4 assert [m for m in mymovers] == [mover_1, mover_4, mover_3] assert mymovers[mover_4.id] == mover_4
def test_values(): 'OrderedCollection().values() works like a dict.values()' x = range(5) oc = OrderedCollection(x) del x[-2] del oc[-2] for ix, v in enumerate(oc.values()): assert x[ix] == v
def test_values(): 'OrderedCollection().values() works like a dict.values()' x = range(5) oc = OrderedCollection(x) del x[-2] del oc[-2] for ix, v in enumerate(oc.values()): assert x[ix] == v
def test_remake(): 'delete automatically remakes internal lists without None' oc = OrderedCollection(['p', 'q', 'ab', 'adsf', 'ss']) del oc[0] del oc[2] oc.remake() for ix, elem in enumerate(oc._elems): assert elem is not None assert oc._d_index[s_id(elem)] == ix
def test_remake(): 'delete automatically remakes internal lists without None' oc = OrderedCollection(['p', 'q', 'ab', 'adsf', 'ss']) del oc[0] del oc[2] oc.remake() for ix, elem in enumerate(oc._elems): assert elem is not None assert oc._d_index[s_id(elem)] == ix
def test_ne(self): 'Test comparison operator (not equal)' assert (OrderedCollection([1, 2, 3, 4, 5]) != OrderedCollection( [2, 1, 3, 4, 5])) assert (OrderedCollection([1, 2, 3, 4, 5]) != OrderedCollection( [1, 2, 3, 4])) assert OrderedCollection([1, 2, 3, 4, 5]) != [1, 2, 3, 4, 5]
def test_clear(): 'test clear()' oc = OrderedCollection(range(4)) oc.clear() assert len(oc) == 0 assert oc._elems == [] # there should be no None's assert oc._d_index == {} assert oc.dtype is int with raises(TypeError): oc += 1.0
def test_remake(): "remakes internal lists without None enteries" oc = OrderedCollection(["p", "q", "ab", "adsf", "ss"]) del oc[0] del oc[3] assert oc._elems[0] is None assert oc._elems[3] is None oc.remake() for ix, elem in enumerate(oc._elems): assert elem is not None assert oc._index[s_id(elem)] == ix
def test_clear(): 'test clear()' oc = OrderedCollection(range(4)) oc.clear() assert len(oc) == 0 assert oc._elems == [] # there should be no None's assert oc._d_index == {} assert oc.dtype is int with raises(TypeError): oc += 1.0
def test_int_to_dict(self): '''added a to_dict() method - test this method for int dtype. Tests the try, except is working correctly''' items = range(5) oc = OrderedCollection(items) dict_ = oc.to_dict() assert dict_['dtype'] == int for (i, item) in enumerate(items): assert dict_['items'][i][0] \ == '{0}'.format(item.__class__.__name__) assert dict_['items'][i][1] == i
def test_add(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc.add(6) assert [i for i in oc] == [ 1, 2, 3, 4, 5, 6, ] with pytest.raises(TypeError): oc.add('not an int')
def test_to_dict(self): 'added a to_dict() method - test this method' items = [SimpleMover(velocity=(i * 0.5, -1.0, 0.0)) for i in range(2)] items.extend([RandomMover() for i in range(2)]) mymovers = OrderedCollection(items, dtype=Mover) dict_ = mymovers.to_dict() assert dict_['dtype'] == mymovers.dtype for (i, mv) in enumerate(items): assert dict_['items'][i][0] \ == '{0}.{1}'.format(mv.__module__, mv.__class__.__name__) assert dict_['items'][i][1] == i
def test_update(self): int_oc = OrderedCollection([1, 2, 3, 4, 5]) upd_cstruct = [2, 3, 4] int_oc.update(upd_cstruct) assert int_oc._elems == [2, 3, 4] obj_oc = OrderedCollection( [DemoObj.demo(), DemoObj.demo(), DemoObj.demo()]) obj0 = obj_oc[0] obj1 = obj_oc[1] assert obj_oc[0] is not obj_oc[1] upd_list = [o.serialize() for o in obj_oc ] #this is like the payload from the web client upd_list[0]['name'] = '2nd DemoObj' upd_list[1]['foo_float'] = 55 temp = upd_list[0] upd_list[0] = upd_list[1] upd_list[1] = temp del upd_list[2] obj_oc.update(upd_list) assert obj_oc[0] is obj1 assert obj_oc[1] is obj0 assert obj_oc[0].foo_float == 55 assert obj_oc[0].id == upd_list[0]['id'] assert obj_oc[1].name == '2nd DemoObj' assert obj_oc[1].id == upd_list[1]['id'] assert len(obj_oc) == 2
def __init__(self, grid=None, depth=None, grid_file=None, data_file=None, dataset=None, varnames=None, **kwargs): super(GridVectorProp, self).__init__(**kwargs) if isinstance(self.variables, list): self.variables = OrderedCollection(elems=self.variables, dtype=EnvProp) if isinstance(self.variables[0], GriddedProp): self.grid = self.variables[0].grid if grid is None else grid self.depth = self.variables[0].depth if depth is None else depth self.grid_file = self.variables[ 0].grid_file if grid_file is None else grid_file self.data_file = self.variables[ 0].data_file if data_file is None else data_file # self._check_consistency() self._result_memo = OrderedDict() for i, comp in enumerate(self.__class__.comp_order): setattr(self, comp, self.variables[i])
def test_getitem_byindex(self): oc = OrderedCollection(['x', 'a', 'p', 'd']) assert oc[1] == 'a' oc[s_id('a')] = 'b' assert oc[1] == 'b' del oc[1] assert oc[1] == 'p'
def test_iadd(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc += 6 assert [i for i in oc] == [1, 2, 3, 4, 5, 6] oc += [7, 8, 9] assert [i for i in oc] == [1, 2, 3, 4, 5, 6, 7, 8, 9]
def test_remove(self): oc = OrderedCollection([1, 2, 3, 4, 5]) with raises(KeyError): oc.remove(s_id(6)) oc.remove(s_id(4)) assert [i for i in oc] == [1, 2, 3, 5] oc.remove(2) assert [i for i in oc] == [1, 2, 5]
def test_to_dict(self, json_): 'added a to_dict() method - test this method' items = [SimpleMover(velocity=(i * 0.5, -1.0, 0.0)) for i in range(2)] items.extend([RandomMover() for i in range(2)]) mymovers = OrderedCollection(items, dtype=Mover) self._to_dict_assert(mymovers, items, json_)
def test_setitem_exceptions(self): 'Use add to add an element' oc = OrderedCollection([1, 2, 3, 4, 5]) with raises(KeyError): oc[s_id(6)] = 6 with raises(IndexError): oc[5] = 6
def test_getslice(): # getting a slice returns a new list l_ = range(6) oc = OrderedCollection(l_) b = oc[:3] b[0] = 10 assert b != l_ assert l_[::2] == oc[::2]
def test_replace(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc.replace(id(6), 6) assert [i for i in oc] == [ 1, 2, 3, 4, 5, 6, ] oc.replace(id(4), 7) assert [i for i in oc] == [ 1, 2, 3, 7, 5, 6, ] assert oc[id(7)] == 7 with pytest.raises(KeyError): # our key should also be gone after the delete oc[id(4)] with pytest.raises(TypeError): oc.replace(id(7), 'not an int')
def test_init(self): oc = OrderedCollection([1, 2, 3, 4, 5]) assert oc.dtype == int oc = OrderedCollection([1, 2, 3, 4, 5], int) assert oc.dtype == int oc = OrderedCollection(dtype=int) assert oc.dtype == int with raises(TypeError): # either a populated list or a dtype is required oc = OrderedCollection() with raises(TypeError): oc = OrderedCollection('not a list') with raises(TypeError): # either a populated list or a dtype is required oc = OrderedCollection([]) with raises(TypeError): oc = OrderedCollection([1, 2, 3, 4, 5], float)
def test_remove(self): oc = OrderedCollection([1, 2, 3, 4, 5]) with raises(KeyError): oc.remove(s_id(6)) oc.remove(s_id(4)) assert [i for i in oc] == [1, 2, 3, 5] oc.remove(2) assert [i for i in oc] == [1, 2, 5]
def test_replace(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc.replace(s_id(4), 7) # replace by object ID oc.replace(0, 0) # replace by index assert [i for i in oc] == [0, 2, 3, 7, 5] assert oc[s_id(7)] == 7 with raises(KeyError): # our key should also be gone after the delete oc[s_id(4)] with raises(TypeError): oc.replace(s_id(7), 'not an int')
def __init__(self, uncertain=False): super(SpillContainer, self).__init__(uncertain=uncertain) self.spills = OrderedCollection(dtype=gnome.spill.Spill) self.rewind() # don't want user to add to array_types in middle of run. Since its # not possible to throw an error in this case, let's just make it a # bit difficult to do. # dict must be updated via prepar_for_model_run() only at beginning of # run. Make self._array_types an an instance variable self._reset_arrays()
def test_index(self): 'behaves like index for a list' oc = OrderedCollection([1, 2, 3, 4, 5]) assert oc.index(3) == 2 assert oc.index(s_id(3)) == 2 oc[s_id(3)] = 6 assert oc.index(6) == 2 assert oc.index(s_id(6)) == 2 del oc[s_id(6)] assert oc.index(4) == 2 assert oc.index(s_id(4)) == 2
def test_add_callback(self): ''' test add callback is invoked after adding an object or list of objects ''' # lets work with a mutable type oc = OrderedCollection(dtype=ObjToAdd) oc.register_callback(self._add_callback, events='add') # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add oc += ObjToAdd() for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback
def test_add_callback(self): ''' test add callback is invoked after adding an object or list of objects ''' # lets work with a mutable type oc = OrderedCollection(dtype=ObjToAdd) oc.register_callback(self._add_callback, events='add') # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add oc += ObjToAdd() for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback
def test_remove_callback(self): 'test remove callback is invoked after removing an object' oc = OrderedCollection(dtype=ObjToAdd) # lets work with a mutable type oc.register_callback(self._rm_callback, events='remove') oc.register_callback(self._add_callback, events='add') # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add del oc[s_id(self.to_add[0])] assert self.to_add[0].rm_callback assert self.to_add[0].add_callback assert not self.to_add[0].replace_callback self.to_add[0].reset() # reset all to false oc += self.to_add[0] # let's add this back in for obj in oc: assert obj.add_callback assert not obj.rm_callback assert not obj.replace_callback
def test_update(self): int_oc = OrderedCollection([1, 2, 3, 4, 5]) upd_cstruct = [2,3,4] int_oc.update(upd_cstruct) assert int_oc._elems == [2,3,4] obj_oc = OrderedCollection([DemoObj.demo(), DemoObj.demo(), DemoObj.demo()]) obj0 = obj_oc[0] obj1 = obj_oc[1] assert obj_oc[0] is not obj_oc[1] upd_list = [o.serialize() for o in obj_oc] #this is like the payload from the web client upd_list[0]['name'] = '2nd DemoObj' upd_list[1]['foo_float'] = 55 temp = upd_list[0] upd_list[0] = upd_list[1] upd_list[1] = temp del upd_list[2] obj_oc.update(upd_list) assert obj_oc[0] is obj1 assert obj_oc[1] is obj0 assert obj_oc[0].foo_float == 55 assert obj_oc[0].id == upd_list[0]['id'] assert obj_oc[1].name == '2nd DemoObj' assert obj_oc[1].id == upd_list[1]['id'] assert len(obj_oc) == 2
def test_index(self): "behaves like index for a list" oc = OrderedCollection([1, 2, 3, 4, 5]) assert oc.index(3) == 2 assert oc.index(s_id(3)) == 2 oc[s_id(3)] = 6 assert oc.index(6) == 2 assert oc.index(s_id(6)) == 2 del oc[s_id(6)] assert oc.index(4) == 2 assert oc.index(s_id(4)) == 2
def test_replace(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc.replace(s_id(4), 7) # replace by object ID oc.replace(0, 0) # replace by index assert [i for i in oc] == [0, 2, 3, 7, 5] assert oc[s_id(7)] == 7 with raises(KeyError): # our key should also be gone after the delete oc[s_id(4)] with raises(TypeError): oc.replace(s_id(7), "not an int")
def test_replace_callback(self): 'test replace callback is invoked after replacing an object' # lets work with a mutable type oc = OrderedCollection(dtype=ObjToAdd) oc.register_callback(self._replace_callback, events='replace') # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add rep = ObjToAdd() oc[s_id(self.to_add[0])] = rep for obj in oc: assert not obj.add_callback assert not obj.rm_callback if id(obj) == id(rep): assert obj.replace_callback else: assert not obj.replace_callback
def test_replace_callback(self): "test replace callback is invoked after replacing an object" # lets work with a mutable type oc = OrderedCollection(dtype=ObjToAdd) oc.register_callback(self._replace_callback, events="replace") # check everything if False initially self._reset_ObjToAdd_init_state() oc += self.to_add rep = ObjToAdd() oc[s_id(self.to_add[0])] = rep for obj in oc: assert not obj.add_callback assert not obj.rm_callback if id(obj) == id(rep): assert obj.replace_callback else: assert not obj.replace_callback
def test_str(): """ __str__ and __repr__ methods should return something handy not really a test, but it runs the code """ mymovers = OrderedCollection([ SimpleMover(velocity=(1.0, -1.0, 0.0)), SimpleMover(velocity=(2.0, -1.0, 0.0)), SimpleMover(velocity=(3.0, -1.0, 0.0)), SimpleMover(velocity=(4.0, -1.0, 0.0)) ]) print repr(mymovers) print str(mymovers) assert repr(mymovers).startswith("OrderedCollection(") assert str(mymovers).startswith("OrderedCollection(")
def test_remake_emptyoc(): "empty OC" oc = OrderedCollection(dtype=int) oc.remake()
class Model(serializable.Serializable): """ PyGNOME Model Class """ _update = [ 'time_step', 'start_time', 'duration', 'uncertain', 'movers', 'environment', 'spills', 'map', 'outputters', 'cache_enabled', ] _create = [] _create.extend(_update) state = copy.deepcopy(serializable.Serializable.state) state.add(create=_create, update=_update) # no need to copy parent's state in tis case @classmethod def new_from_dict(cls, dict_): """ Restore model from previously persisted state """ l_env = dict_.pop('environment') l_out = dict_.pop('outputters') l_movers = dict_.pop('movers') c_spills = dict_.pop('certain_spills') if 'uncertain_spills' in dict_.keys(): u_spills = dict_.pop('uncertain_spills') l_spills = zip(c_spills, u_spills) else: l_spills = c_spills model = object.__new__(cls) model.__restore__(**dict_) [model.environment.add(obj) for obj in l_env] [model.outputters.add(obj) for obj in l_out] [model.spills.add(obj) for obj in l_spills] [model.movers.add(obj) for obj in l_movers] # register callback with OrderedCollection model.movers.register_callback(model._callback_add_mover, ('add' , 'replace')) return model def __init__( self, time_step=timedelta(minutes=15), start_time=round_time(datetime.now(), 3600), duration=timedelta(days=1), map=gnome.map.GnomeMap(), uncertain=False, cache_enabled=False, id=None, ): """ Initializes a model. All arguments have a default. :param time_step=timedelta(minutes=15): model time step in seconds or as a timedelta object :param start_time=datetime.now(): start time of model, datetime object. default to now, rounded to the nearest hour :param duration=timedelta(days=1): how long to run the model, a timedelta object :param map=gnome.map.GnomeMap(): the land-water map, default is a map with no land-water :param uncertain=False: flag for setting uncertainty :param cache_enabled=False: flag for setting whether the mocel should cache results to disk. :param id: Unique Id identifying the newly created mover (a UUID as a string). This is used when loading an object from a persisted model """ self.__restore__( time_step, start_time, duration, map, uncertain, cache_enabled, id, ) # register callback with OrderedCollection self.movers.register_callback(self._callback_add_mover, ('add', 'replace')) def __restore__( self, time_step, start_time, duration, map, uncertain, cache_enabled, id, ): """ Take out initialization that does not register the callback here. This is because new_from_dict will use this to restore the model state when doing a midrun persistence. """ # making sure basic stuff is in place before properties are set self.environment = OrderedCollection(dtype=Environment) self.movers = OrderedCollection(dtype=Mover) self.spills = SpillContainerPair(uncertain) # contains both certain/uncertain spills self._cache = gnome.utilities.cache.ElementCache() self._cache.enabled = cache_enabled self.outputters = \ OrderedCollection(dtype=gnome.outputter.Outputter) # list of output objects self._start_time = start_time # default to now, rounded to the nearest hour self._duration = duration self._map = map self.time_step = time_step # this calls rewind() ! self._gnome_id = gnome.GnomeId(id) def reset(self, **kwargs): """ Resets model to defaults -- Caution -- clears all movers, spills, etc. Takes same keyword arguments as __init__ """ self.__init__(**kwargs) def rewind(self): """ Rewinds the model to the beginning (start_time) """ # # fixme: do the movers need re-setting? -- or wait for prepare_for_model_run? self.current_time_step = -1 # start at -1 self.model_time = self._start_time # # note: this may be redundant -- they will get reset in setup_model_run() anyway.. self.spills.rewind() gnome.utilities.rand.seed(1) # set rand before each call so windages are set correctly # clear the cache: self._cache.rewind() for outputter in self.outputters: outputter.rewind() # def write_from_cache(self, filetype='netcdf', time_step='all'): # """ # write the already-cached data to an output files. # """ # ## Assorted properties @property def uncertain(self): return self.spills.uncertain @uncertain.setter def uncertain(self, uncertain_value): """ only if uncertainty switch is toggled, then restart model """ if self.spills.uncertain != uncertain_value: self.spills.uncertain = uncertain_value # update uncertainty self.rewind() @property def cache_enabled(self): return self._cache.enabled @cache_enabled.setter def cache_enabled(self, enabled): self._cache.enabled = enabled @property def id(self): return self._gnome_id.id @property def start_time(self): return self._start_time @start_time.setter def start_time(self, start_time): self._start_time = start_time self.rewind() @property def time_step(self): return self._time_step @time_step.setter def time_step(self, time_step): """ sets the time step, and rewinds the model :param time_step: the timestep as a timedelta object or integer seconds. """ try: self._time_step = time_step.total_seconds() except AttributeError: # not a timedelta object -- assume it's in seconds. self._time_step = int(time_step) # there is a zeroth time step self._num_time_steps = int(self._duration.total_seconds() // self._time_step) + 1 self.rewind() @property def current_time_step(self): return self._current_time_step @current_time_step.setter def current_time_step(self, step): self.model_time = self._start_time + timedelta(seconds=step * self.time_step) self._current_time_step = step @property def duration(self): return self._duration @duration.setter def duration(self, duration): if duration < self._duration: # only need to rewind if shorter than it was... # # fixme: actually, only need to rewide is current model time is byond new time... self.rewind() self._duration = duration self._num_time_steps = int(self._duration.total_seconds() // self.time_step) + 1 # there is a zeroth time step @property def map(self): return self._map @map.setter def map(self, map_in): self._map = map_in self.rewind() @property def num_time_steps(self): return self._num_time_steps def setup_model_run(self): """ Sets up each mover for the model run """ self.spills.rewind() # why is rewind for spills here? for outputter in self.outputters: outputter.prepare_for_model_run(model_start_time=self.start_time, cache=self._cache, uncertain=self.uncertain, spills=self.spills) array_types = {} for mover in self.movers: mover.prepare_for_model_run() array_types.update(mover.array_types) for sc in self.spills.items(): sc.prepare_for_model_run(array_types) def setup_time_step(self): """ sets up everything for the current time_step: right now only prepares the movers -- maybe more later?. """ # initialize movers differently if model uncertainty is on for mover in self.movers: for sc in self.spills.items(): mover.prepare_for_model_step(sc, self.time_step, self.model_time) for outputter in self.outputters: outputter.prepare_for_model_step(self.time_step, self.model_time) def move_elements(self): """ Moves elements: - loops through all the movers. and moves the elements - sets new_position array for each spill - calls the beaching code to beach the elements that need beaching. - sets the new position """ # # if there are no spills, there is nothing to do: if len(self.spills) > 0: # can this check be removed? for sc in self.spills.items(): if sc.num_released > 0: # can this check be removed? # possibly refloat elements self.map.refloat_elements(sc, self.time_step) # reset next_positions (sc['next_positions'])[:] = sc['positions'] # loop through the movers for mover in self.movers: delta = mover.get_move(sc, self.time_step, self.model_time) sc['next_positions'] += delta self.map.beach_elements(sc) # the final move to the new positions (sc['positions'])[:] = sc['next_positions'] def step_is_done(self): """ Loop through movers and call model_step_is_done """ for mover in self.movers: for sc in self.spills.items(): mover.model_step_is_done(sc) for sc in self.spills.items(): sc.model_step_is_done() for outputter in self.outputters: outputter.model_step_is_done() def write_output(self): output_info = {'step_num': self.current_time_step} for outputter in self.outputters: if self.current_time_step == self.num_time_steps - 1: output = outputter.write_output(self.current_time_step, True) else: output = outputter.write_output(self.current_time_step) if output is not None: output_info.update(output) return output_info def step(self): """ Steps the model forward (or backward) in time. Needs testing for hind casting. """ for sc in self.spills.items(): # set the current time stamp only after current_time_step is # incremented and before the output is written. Set it to None here # just so we're not carrying around the old time_stamp sc.current_time_stamp = None # it gets incremented after this check if self.current_time_step >= self._num_time_steps - 1: raise StopIteration if self.current_time_step == -1: # that's all we need to do for the zeroth time step self.setup_model_run() else: self.setup_time_step() self.move_elements() self.step_is_done() self.current_time_step += 1 # this is where the new step begins! # the elements released are during the time period: # self.model_time + self.time_step # The else part of the loop computes values for data_arrays that # correspond with time_stamp: # self.model_time + self.time_step # This is the current_time_stamp attribute of the SpillContainer # [sc.current_time_stamp for sc in self.spills.items()] for sc in self.spills.items(): sc.current_time_stamp = self.model_time sc.release_elements(self.time_step, self.model_time) # cache the results - current_time_step is incremented but the # current_time_stamp in spill_containers (self.spills) is not updated # till we go through the prepare_for_model_step self._cache.save_timestep(self.current_time_step, self.spills) output_info = self.write_output() return output_info def __iter__(self): """ for compatibility with Python's iterator protocol rewinds the model and returns itself so it can be iterated over. """ self.rewind() return self def next(self): """ (This method here to satisfy Python's iterator and generator protocols) Simply calls model.step() :return: the step number """ return self.step() def full_run(self, rewind=True, log=False): """ Do a full run of the model. :param rewind=True: whether to rewind the model first -- defaults to True if set to false, model will be run from the current step to the end :returns: list of outputter info dicts """ if rewind: self.rewind() # run the model output_data = [] while True: try: results = self.step() if log: print results output_data.append(results) except StopIteration: print 'Done with the model run' break return output_data def movers_to_dict(self): """ call to_dict method of OrderedCollection object """ return self.movers.to_dict() def environment_to_dict(self): """ call to_dict method of OrderedCollection object """ return self.environment.to_dict() def spills_to_dict(self): return self.spills.to_dict() def outputters_to_dict(self): """ call to_dict method of OrderedCollection object """ return self.outputters.to_dict() def map_to_dict(self): """ create a tuple that contains: (type, object.id) """ # dict_ = {'map': ("{0}.{1}".format(self.map.__module__, self.map.__class__.__name__), self.map.id)} return ('{0}.{1}'.format(self.map.__module__, self.map.__class__.__name__), self.map.id) # if self.output_map is not None: # dict_.update({'output_map': ("{0}.{1}".format(self.output_map.__module__, self.output_map.__class__.__name__), self.output_map.id)}) def _callback_add_mover(self, obj_added): """ callback after mover has been added """ if isinstance(obj_added, WindMover): if obj_added.wind.id not in self.environment: self.environment += obj_added.wind if isinstance(obj_added, CatsMover): if obj_added.tide is not None and obj_added.tide.id \ not in self.environment: self.environment += obj_added.tide self.rewind() # rewind model if a new mover is added def __eq__(self, other): check = super(Model, self).__eq__(other) if check: # also check the data in spill_container object if type(self.spills) != type(other.spills): return False if self.spills != other.spills: return False return check def __ne__(self, other): """ Compare inequality (!=) of two objects """ if self == other: return False else: return True
class Model(Serializable): ''' PyGnome Model Class ''' _update = ['time_step', 'weathering_substeps', 'start_time', 'duration', 'time_step', 'uncertain', 'cache_enabled', 'weathering_substeps', 'map', 'movers', 'weatherers', 'environment', 'outputters' ] _create = [] _create.extend(_update) _state = copy.deepcopy(Serializable._state) _schema = ModelSchema # no need to copy parent's _state in this case _state.add(save=_create, update=_update) # override __eq__ since 'spills' and 'uncertain_spills' need to be checked # They both have _to_dict() methods to return underlying ordered # collections and that would not be the correct way to check equality _state += [Field('spills', save=True, update=True, test_for_eq=False), Field('uncertain_spills', save=True, test_for_eq=False)] # list of OrderedCollections _oc_list = ['movers', 'weatherers', 'environment', 'outputters'] @classmethod def new_from_dict(cls, dict_): 'Restore model from previously persisted _state' json_ = dict_.pop('json_') l_env = dict_.pop('environment', []) l_out = dict_.pop('outputters', []) l_movers = dict_.pop('movers', []) l_weatherers = dict_.pop('weatherers', []) c_spills = dict_.pop('spills', []) if 'uncertain_spills' in dict_: u_spills = dict_.pop('uncertain_spills') l_spills = zip(c_spills, u_spills) else: l_spills = c_spills # define defaults for properties that a location file may not contain kwargs = inspect.getargspec(cls.__init__) default_restore = dict(zip(kwargs[0][1:], kwargs[3])) if json_ == 'webapi': # default is to enable cache default_restore['cache_enabled'] = True for key in default_restore: default_restore[key] = dict_.pop(key, default_restore[key]) model = object.__new__(cls) model.__restore__(**default_restore) # if there are other values in dict_, setattr if json_ == 'webapi': model.update_from_dict(dict_) else: cls._restore_attr_from_save(model, dict_) [model.environment.add(obj) for obj in l_env] [model.outputters.add(obj) for obj in l_out] [model.spills.add(obj) for obj in l_spills] [model.movers.add(obj) for obj in l_movers] [model.weatherers.add(obj) for obj in l_weatherers] # register callback with OrderedCollection model.movers.register_callback(model._callback_add_mover, ('add', 'replace')) model.weatherers.register_callback(model._callback_add_weatherer, ('add', 'replace')) # restore the spill data outside this method - let's not try to find # the saveloc here return model def __init__(self, time_step=timedelta(minutes=15), start_time=round_time(datetime.now(), 3600), duration=timedelta(days=1), weathering_substeps=1, map=None, uncertain=False, cache_enabled=False, name=None): ''' Initializes a model. All arguments have a default. :param time_step=timedelta(minutes=15): model time step in seconds or as a timedelta object :param start_time=datetime.now(): start time of model, datetime object. Rounded to the nearest hour. :param duration=timedelta(days=1): How long to run the model, a timedelta object. :param int weathering_substeps=1: How many weathering substeps to run inside a single model time step. :param map=gnome.map.GnomeMap(): The land-water map. :param uncertain=False: Flag for setting uncertainty. :param cache_enabled=False: Flag for setting whether the model should cache results to disk. ''' self.__restore__(time_step, start_time, duration, weathering_substeps, uncertain, cache_enabled, map, name) # register callback with OrderedCollection self.movers.register_callback(self._callback_add_mover, ('add', 'replace')) self.weatherers.register_callback(self._callback_add_weatherer, ('add', 'replace')) def __restore__(self, time_step, start_time, duration, weathering_substeps, uncertain, cache_enabled, map, name): ''' Take out initialization that does not register the callback here. This is because new_from_dict will use this to restore the model _state when doing a midrun persistence. ''' # making sure basic stuff is in place before properties are set self.environment = OrderedCollection(dtype=Environment) self.movers = OrderedCollection(dtype=Mover) self.weatherers = OrderedCollection(dtype=Weatherer) # contains both certain/uncertain spills self.spills = SpillContainerPair(uncertain) self._cache = gnome.utilities.cache.ElementCache() self._cache.enabled = cache_enabled # list of output objects self.outputters = OrderedCollection(dtype=Outputter) # default to now, rounded to the nearest hour self._start_time = start_time self._duration = duration self.weathering_substeps = weathering_substeps if not map: map = gnome.map.GnomeMap() if name: self.name = name self._map = map self.time_step = time_step # this calls rewind() ! def reset(self, **kwargs): ''' Resets model to defaults -- Caution -- clears all movers, spills, etc. Takes same keyword arguments as :meth:`__init__()` ''' self.__init__(**kwargs) def rewind(self): ''' Rewinds the model to the beginning (start_time) ''' # fixme: do the movers need re-setting? -- or wait for # prepare_for_model_run? self.current_time_step = -1 self.model_time = self._start_time # note: This may be redundant. They will get reset in # setup_model_run() anyway.. self.spills.rewind() # set rand before each call so windages are set correctly gnome.utilities.rand.seed(1) # clear the cache: self._cache.rewind() for outputter in self.outputters: outputter.rewind() # def write_from_cache(self, filetype='netcdf', time_step='all'): # """ # write the already-cached data to an output files. # """ @property def uncertain(self): ''' Uncertainty attribute of the model. If flag is toggled, rewind model ''' return self.spills.uncertain @uncertain.setter def uncertain(self, uncertain_value): ''' Uncertainty attribute of the model ''' if self.spills.uncertain != uncertain_value: self.spills.uncertain = uncertain_value # update uncertainty self.rewind() @property def cache_enabled(self): ''' If True, then generated data is cached ''' return self._cache.enabled @cache_enabled.setter def cache_enabled(self, enabled): self._cache.enabled = enabled @property def start_time(self): ''' Start time of the simulation ''' return self._start_time @start_time.setter def start_time(self, start_time): self._start_time = start_time self.rewind() @property def time_step(self): ''' time step over which the dynamics is computed ''' return self._time_step @time_step.setter def time_step(self, time_step): ''' Sets the time step, and rewinds the model :param time_step: The timestep can be a timedelta object or integer seconds. ''' try: self._time_step = time_step.total_seconds() except AttributeError: self._time_step = int(time_step) # there is a zeroth time step self._num_time_steps = int(self._duration.total_seconds() // self._time_step) + 1 self.rewind() @property def current_time_step(self): ''' Current timestep of the simulation ''' return self._current_time_step @current_time_step.setter def current_time_step(self, step): self.model_time = self._start_time + timedelta(seconds=step * self.time_step) self._current_time_step = step @property def duration(self): ''' total duration of the model run ''' return self._duration @duration.setter def duration(self, duration): if duration < self._duration: # only need to rewind if shorter than it was... # fixme: actually, only need to rewind if current model time # is beyond new time... self.rewind() self._duration = duration # there is a zeroth time step self._num_time_steps = int(self._duration.total_seconds() // self.time_step) + 1 @property def map(self): ''' land water map used for simulation ''' return self._map @map.setter def map(self, map_in): self._map = map_in self.rewind() @property def num_time_steps(self): ''' Read only attribute computed number of timesteps based on py:attribute:`duration` and py:attribute:`time_step` ''' return self._num_time_steps def setup_model_run(self): ''' Sets up each mover for the model run ''' self.spills.rewind() # why is rewind for spills here? # remake orderedcollections defined by model for oc in [self.movers, self.weatherers, self.outputters, self.environment]: oc.remake() array_types = {} for mover in self.movers: mover.prepare_for_model_run() array_types.update(mover.array_types) for w in self.weatherers: w.prepare_for_model_run() array_types.update(w.array_types) for sc in self.spills.items(): sc.prepare_for_model_run(array_types) # outputters need array_types, so this needs to come after those # have been updated. for outputter in self.outputters: outputter.prepare_for_model_run(model_start_time=self.start_time, cache=self._cache, uncertain=self.uncertain, spills=self.spills) def setup_time_step(self): ''' sets up everything for the current time_step: ''' # initialize movers differently if model uncertainty is on for m in self.movers: for sc in self.spills.items(): m.prepare_for_model_step(sc, self.time_step, self.model_time) for w in self.weatherers: for sc in self.spills.items(): # maybe we will setup a super-sampling step here??? w.prepare_for_model_step(sc, self.time_step, self.model_time) for outputter in self.outputters: outputter.prepare_for_model_step(self.time_step, self.model_time) def move_elements(self): ''' Moves elements: - loops through all the movers. and moves the elements - sets new_position array for each spill - calls the beaching code to beach the elements that need beaching. - sets the new position ''' for sc in self.spills.items(): if sc.num_released > 0: # can this check be removed? # possibly refloat elements self.map.refloat_elements(sc, self.time_step) # reset next_positions (sc['next_positions'])[:] = sc['positions'] # loop through the movers for m in self.movers: delta = m.get_move(sc, self.time_step, self.model_time) sc['next_positions'] += delta self.map.beach_elements(sc) # the final move to the new positions (sc['positions'])[:] = sc['next_positions'] def weather_elements(self): ''' Weathers elements: - loops through all the weatherers, passing in the spill_container and the time range - a weatherer modifies the data arrays in the spill container, so a particular time range should not be run multiple times. It is expected that we are processing a sequence of contiguous time ranges. - Note: If there are multiple sequential weathering processes, some inaccuracy could occur. A proposed solution is to 'super-sample' the model time step so that it will be replaced with many smaller time steps. We'll have to see if this pans out in practice. ''' for sc in self.spills.items(): for w in self.weatherers: for model_time, time_step in self._split_into_substeps(): w.weather_elements(sc, time_step, model_time) def _split_into_substeps(self): ''' :return: sequence of (datetime, timestep) (Note: we divide evenly on second boundaries. Thus, there will likely be a remainder that needs to be included. We include this remainder, which results in 1 more sub-step than we requested.) ''' time_step = int(self._time_step) sub_step = time_step / self.weathering_substeps indexes = [idx for idx in range(0, time_step + 1, sub_step)] res = [(idx, next_idx - idx) for idx, next_idx in zip(indexes, indexes[1:])] if sum(res[-1]) < time_step: # collect the remaining slice res.append((sum(res[-1]), time_step % sub_step)) res = [(self.model_time + timedelta(seconds=idx), delta) for idx, delta in res] return res def step_is_done(self): ''' Loop through movers and call model_step_is_done ''' for mover in self.movers: for sc in self.spills.items(): mover.model_step_is_done(sc) for w in self.weatherers: w.model_step_is_done() for sc in self.spills.items(): 'removes elements with oil_status.to_be_removed' sc.model_step_is_done() # age remaining particles sc['age'][:] = sc['age'][:] + self.time_step for outputter in self.outputters: outputter.model_step_is_done() def write_output(self): output_info = {} for outputter in self.outputters: if self.current_time_step == self.num_time_steps - 1: output = outputter.write_output(self.current_time_step, True) else: output = outputter.write_output(self.current_time_step) if output is not None: output_info.update(output) if not output_info: return {'step_num': self.current_time_step} return output_info def step(self): ''' Steps the model forward (or backward) in time. Needs testing for hind casting. ''' for sc in self.spills.items(): # Set the current time stamp only after current_time_step is # incremented and before the output is written. Set it to None here # just so we're not carrying around the old time_stamp sc.current_time_stamp = None # it gets incremented after this check if self.current_time_step >= self._num_time_steps - 1: raise StopIteration if self.current_time_step == -1: # that's all we need to do for the zeroth time step self.setup_model_run() else: self.setup_time_step() self.move_elements() self.weather_elements() self.step_is_done() self.current_time_step += 1 # this is where the new step begins! # the elements released are during the time period: # self.model_time + self.time_step # The else part of the loop computes values for data_arrays that # correspond with time_stamp: # self.model_time + self.time_step # This is the current_time_stamp attribute of the SpillContainer # [sc.current_time_stamp for sc in self.spills.items()] for sc in self.spills.items(): sc.current_time_stamp = self.model_time # release particles for next step - these particles will be aged # in the next step sc.release_elements(self.time_step, self.model_time) # cache the results - current_time_step is incremented but the # current_time_stamp in spill_containers (self.spills) is not updated # till we go through the prepare_for_model_step self._cache.save_timestep(self.current_time_step, self.spills) output_info = self.write_output() return output_info def __iter__(self): ''' Rewinds the model and returns itself so it can be iterated over. ''' self.rewind() return self def next(self): ''' (This method satisfies Python's iterator and generator protocols) :return: the step number ''' return self.step() def full_run(self, rewind=True, log=False): ''' Do a full run of the model. :param rewind=True: whether to rewind the model first -- if set to false, model will be run from the current step to the end :returns: list of outputter info dicts ''' if rewind: self.rewind() # run the model output_data = [] while True: try: results = self.step() if log: print results output_data.append(results) except StopIteration: print 'Done with the model run' break return output_data def _callback_add_mover(self, obj_added): 'Callback after mover has been added' if hasattr(obj_added, 'wind'): if obj_added.wind.id not in self.environment: self.environment += obj_added.wind if hasattr(obj_added, 'tide') and obj_added.tide is not None: if obj_added.tide.id not in self.environment: self.environment += obj_added.tide self.rewind() # rewind model if a new mover is added def _callback_add_weatherer(self, obj_added): 'Callback after weatherer has been added' if isinstance(obj_added, Weatherer): # not sure what kind of dependencies we have just yet. pass self.rewind() # rewind model if a new weatherer is added def __eq__(self, other): check = super(Model, self).__eq__(other) print 'Model.__eq__(): super check =', check if check: # also check the data in ordered collections if type(self.spills) != type(other.spills): print 'Model.__eq__(): spill types:', (type(self.spills), type(other.spills)) return False if self.spills != other.spills: print 'Model.__eq__(): spills:' pp.pprint((self.spills, other.spills)) return False return check def __ne__(self, other): return not self == other ''' Following methods are for saving a Model instance or creating a new model instance from a saved location ''' def spills_to_dict(self): ''' return the spills ordered collection for serialization ''' return self.spills.to_dict()['spills'] def uncertain_spills_to_dict(self): ''' return the uncertain_spills ordered collection for serialization/save files ''' if self.uncertain: dict_ = self.spills.to_dict() return dict_['uncertain_spills'] return None def save(self, saveloc, references=None, name=None): # Note: Defining references=References() in the function definition # keeps the references object in memory between tests - it changes the # scope of Referneces() to be outside the Model() instance. We don't # want this references = (references, References())[references is None] self._make_saveloc(saveloc) self._empty_save_dir(saveloc) json_ = self.serialize('save') # map is the only nested structure - let's manually call # _move_data_file on it self.map._move_data_file(saveloc, json_['map']) for oc in self._oc_list: coll_ = getattr(self, oc) self._save_collection(saveloc, coll_, references, json_[oc]) for sc in self.spills.items(): if sc.uncertain: key = 'uncertain_spills' else: key = 'spills' self._save_collection(saveloc, sc.spills, references, json_[key]) if self.current_time_step > -1: ''' hard code the filename - can make this an attribute if user wants to change it - but not sure if that will ever be needed? ''' self._save_spill_data(os.path.join(saveloc, 'spills_data_arrays.nc')) # there should be no more references self._json_to_saveloc(json_, saveloc, references, name) if name and references.reference(self) != name: # todo: want a warning here instead of an exception raise Exception("{0} already exists, cannot name " "the model's json file: {0}".format(name)) pass return references def _save_collection(self, saveloc, coll_, refs, coll_json): """ Reference objects inside OrderedCollections. Since the OC itself isn't a reference but the objects in the list are a reference, do something a little differently here :param OrderedCollection coll_: ordered collection to be saved """ for count, obj in enumerate(coll_): json_ = obj.serialize('save') for field in obj._state: if field.save_reference: 'attribute is stored as a reference to environment list' if getattr(obj, field.name) is not None: ref_obj = getattr(obj, field.name) index = self.environment.index(ref_obj) json_[field.name] = index obj_ref = refs.get_reference(obj) if obj_ref is None: # try following name - if 'fname' already exists in references, # then obj.save() assigns a different name to file fname = '{0.__class__.__name__}_{1}.json'.format(obj, count) obj.save(saveloc, refs, fname) coll_json[count]['id'] = refs.reference(obj) else: coll_json[count]['id'] = obj_ref def _save_spill_data(self, datafile): """ save the data arrays for current timestep to NetCDF """ nc_out = NetCDFOutput(datafile, which_data='all', cache=self._cache) nc_out.prepare_for_model_run(model_start_time=self.start_time, uncertain=self.uncertain, spills=self.spills) nc_out.write_output(self.current_time_step) def _load_spill_data(self, spill_data): """ load NetCDF file and add spill data back in - designed for savefiles """ if not os.path.exists(spill_data): return if self.uncertain: saveloc, spill_data_fname = os.path.split(spill_data) spill_data_fname, ext = os.path.splitext(spill_data_fname) u_spill_data = os.path.join(saveloc, '{0}_uncertain{1}'.format(spill_data_fname, ext)) array_types = {} for m in self.movers: array_types.update(m.array_types) for w in self.weatherers: array_types.update(w.array_types) for sc in self.spills.items(): if sc.uncertain: data = NetCDFOutput.read_data(u_spill_data, time=None, which_data='all') else: data = NetCDFOutput.read_data(spill_data, time=None, which_data='all') sc.current_time_stamp = data.pop('current_time_stamp').item() sc._data_arrays = data sc._array_types.update(array_types) def _empty_save_dir(self, saveloc): ''' Remove all files, directories under saveloc First clean out directory, then add new save files This should only be called by self.save() ''' (dirpath, dirnames, filenames) = os.walk(saveloc).next() if dirnames: for dir_ in dirnames: shutil.rmtree(os.path.join(dirpath, dir_)) if filenames: for file_ in filenames: os.remove(os.path.join(dirpath, file_)) def serialize(self, json_='webapi'): ''' Serialize Model object treat special-case attributes of Model. ''' toserial = self.to_serialize(json_) schema = self.__class__._schema() o_json_ = schema.serialize(toserial) o_json_['map'] = self.map.serialize(json_) if json_ == 'webapi': for attr in ('environment', 'outputters', 'weatherers', 'movers', 'spills'): o_json_[attr] = self.serialize_oc(attr, json_) return o_json_ def serialize_oc(self, attr, json_='webapi'): ''' Serialize Model attributes of type ordered collection ''' json_out = [] attr = getattr(self, attr) if isinstance(attr, (OrderedCollection, SpillContainerPair)): for item in attr: json_out.append(item.serialize(json_)) return json_out @classmethod def deserialize(cls, json_): ''' treat special-case attributes of Model. ''' deserial = cls._schema().deserialize(json_) if 'map' in json_ and json_['map']: deserial['map'] = json_['map'] if json_['json_'] == 'webapi': for attr in ('environment', 'outputters', 'weatherers', 'movers', 'spills'): if attr in json_ and json_[attr]: deserial[attr] = cls.deserialize_oc(json_[attr]) return deserial @classmethod def deserialize_oc(cls, json_): ''' check contents of orderered collections to figure out what schema to use. Basically, the json serialized ordered collection looks like a regular list. ''' deserial = [] for item in json_: fqn = item['obj_type'] name, scope = (list(reversed(fqn.rsplit('.', 1))) if fqn.find('.') >= 0 else [fqn, '']) my_module = __import__(scope, globals(), locals(), [str(name)], -1) py_class = getattr(my_module, name) deserial.append(py_class.deserialize(item)) return deserial @classmethod def load(cls, saveloc, json_data, references=None): ''' ''' references = (references, References())[references is None] # model has no datafiles or 'save_reference' attributes so no need to # do anything special for it. But let's add this as a check anyway datafiles = cls._state.get_field_by_attribute('isdatafile') ref_fields = cls._state.get_field_by_attribute('save_reference') if (datafiles or ref_fields): raise Exception("Model.load() assumes none of the attributes " "defining the state 'isdatafile' or is 'save_reference'. " "If this changes, then we need to make this more robust.") # deserialize after removing references _to_dict = cls.deserialize(json_data) # load nested map object and add it - currently, 'load' is only used # for laoding save files/location files, so it assumes: # json_data['json_'] == 'save' if ('map' in json_data): map_obj = eval(json_data['map']['obj_type']).load(saveloc, json_data['map'], references) _to_dict['map'] = map_obj # load collections for oc in cls._oc_list: if oc in _to_dict: _to_dict[oc] = cls._load_collection(saveloc, _to_dict[oc], references) for spill in ['spills', 'uncertain_spills']: if spill in _to_dict: _to_dict[spill] = cls._load_collection(saveloc, _to_dict[spill], references) # also need to load spill data for mid-run save! model = cls.new_from_dict(_to_dict) model._load_spill_data(os.path.join(saveloc, 'spills_data_arrays.nc')) return model @classmethod def _load_collection(cls, saveloc, l_coll_dict, refs): ''' doesn't need to be classmethod of the Model, but its only used by Model at present ''' l_coll = [] for item in l_coll_dict: i_ref = item['id'] if refs.retrieve(i_ref): l_coll.append(refs.retrieve(i_ref)) else: f_name = os.path.join(saveloc, item['id']) obj = load(f_name, refs) # will add obj to refs l_coll.append(obj) return (l_coll)
def __init__(self, uncertain=False): super(SpillContainer, self).__init__(uncertain=uncertain) self.spills = OrderedCollection(dtype=gnome.spill.Spill) self.spills.register_callback(self._spills_changed, ('add', 'replace', 'remove')) self.rewind()
class SpillContainer(AddLogger, SpillContainerData): """ Container class for all spills -- it takes care of capturing the released LEs from all the spills, putting them all in a single set of arrays. Many of the "fields" associated with a collection of elements are optional, or used only by some movers, so only the ones required will be requested by each mover. The data for the elements is stored in the _data_arrays dict. They can be accessed by indexing. For example: positions = spill_container['positions'] : returns a (num_LEs, 3) array of world_point_types """ def __init__(self, uncertain=False): super(SpillContainer, self).__init__(uncertain=uncertain) self.spills = OrderedCollection(dtype=gnome.spill.Spill) self.spills.register_callback(self._spills_changed, ('add', 'replace', 'remove')) self.rewind() def __setitem__(self, data_name, array): """ Invoke base class __setitem__ method so the _data_array is set correctly. In addition, create the appropriate ArrayType if it wasn't created by the user. """ super(SpillContainer, self).__setitem__(data_name, array) if data_name not in self._array_types: shape = self._data_arrays[data_name].shape[1:] dtype = self._data_arrays[data_name].dtype.type self._array_types[data_name] = ArrayType(shape, dtype, name=data_name) def _reset_arrays(self): ''' reset _array_types dict so it contains default keys/values ''' gnome.array_types.reset_to_defaults(['spill_num', 'id']) self._array_types = {'positions': positions, 'next_positions': next_positions, 'last_water_positions': last_water_positions, 'status_codes': status_codes, 'spill_num': spill_num, 'id': id, 'mass': mass, 'age': age} self._data_arrays = {} def _reset__substances_spills(self): ''' reset internal attributes to None and empty list []: 1. _substances_spills: data structure to contain spills per substance 2. _oil_comp_array_len: max number of psuedocomponents - relevant if more than one substance is used. 3. _fate_data_list: list of FateDataView() objects. One object per substance if substance is not None ''' # Initialize following either the first time it is used or in # prepare_for_model_run() -- it could change with each new spill self._substances_spills = None self._oil_comp_array_len = None def _reset__fate_data_list(self): # define the fate view of the data if 'fate_status' is in data arrays # 'fate_status' is included if weathering is on self._fate_data_list = [] def reset_fate_dataview(self): ''' reset data arrays for each fate_dataviewer. Each substance that is not None has a fate_dataviewer object. ''' for viewer in self._fate_data_list: viewer.reset() def _set_substancespills(self): ''' _substances could change when spills are added/deleted using _spills_changed callback to reset self._substance_spills to None If 'substance' is None, we still include it in this data structure - all spills that are 'on' are included. A spill that is off isn't really being modeled so ignore it. .. note:: Should not be called in middle of run. prepare_for_model_run() will invoke this if self._substance_spills is None. This is another view of the data - it doesn't contain any state that needs to be persisted. ''' subs = [] spills = [] if self._oil_comp_array_len is None: self._oil_comp_array_len = 1 for spill in self.spills: if not spill.on: continue new_subs = spill.get('substance') if new_subs in subs: # substance already defined for another spill ix = subs.index(new_subs) spills[ix].append(spill) else: # new substance not yet defined subs.append(new_subs) spills.append([spill]) # also set _oil_comp_array_len to substance with most # components? -- *not* being used right now, but make it so # it works correctly for testing multiple substances if (hasattr(new_subs, 'num_components') and new_subs.num_components > self._oil_comp_array_len): self._oil_comp_array_len = new_subs.num_components # let's reorder subs so None is in the end: if None in subs: ix = subs.index(None) spills.append(spills.pop(ix)) subs.append(subs.pop(ix)) s_id = range(len(subs)) # 'data' will be updated when weatherers ask for arrays they need # define the substances list and the list of spills for each substance self._substances_spills = \ substances_spills(substances=subs, s_id=s_id, spills=spills) if len(self.get_substances()) > 1: # add an arraytype for substance if more than one substance self._array_types.update({'substance': substance}) self.logger.info('{0} - number of substances: {1}'. format(os.getpid(), len(self.get_substances()))) def _set_fate_data_list(self): ''' For each substance that is not None, initialize FateDataView(substance_id) object. The substance_id corresponds with self._substance_spills.s_id for each substance. ''' self._fate_data_list = [] for s_id, subs in zip(self._substances_spills.s_id, self._substances_spills.substances): if subs is None: continue self._fate_data_list.append(FateDataView(s_id)) def _spills_changed(self, *args): ''' call back called on spills add/delete/replace Callback simply resets the internal _substance_spills attribute to None since the old _substance_spills value could now be invalid. ''' self._substances_spills = None def _get_s_id(self, substance): ''' Look in the _substances_spills data structure of substance and return the corresponding s_id ''' try: ix = self._substances_spills.substances.index(substance) except ValueError: 'substance is not in list' self.logger.debug('{0} - Substance named: {1}, not found in data ' 'structure'.format(os.getpid(), substance.name)) return None return self._substances_spills.s_id[ix] def _get_fatedataview(self, substance): ''' return the FateDataView object associated with substance ''' ix = self._get_s_id(substance) if ix is None: msg = "substance named {0} not found".format(substance.name) self.logger.info(msg) return # check view = self._fate_data_list[ix] if view.substance_id != ix: msg = "substance_id did not match as expected. Check!" raise ValueError(msg) return view def _array_name(self, at): ''' given an array type, return the name of the array. This can be string, in which case, it is the name of the array so return it. If its not a string, then return the at.name attribute. ''' if isinstance(at, basestring): return at else: return at.name def _append_array_types(self, array_types): ''' append to self.array_types the input array_types. :param array_types: set of array_types to be appended :type array_types: set() The set contains either a name as a string, say: 'rise_vel' In this case, get the ArrayType from gnome.array_types.rise_vel Set elements could also be tuples, say: ('rise_vel': ArrayType()) In this case the user name of the data_array and its array_type is specified by the tuple so append it. .. note:: If a tuple: ('name', ArrayType()), is given and an ArrayType with that name already exists in self._array_types, then it is overwritten. ''' for array in array_types: if isinstance(array, basestring): # allow user to override an array_type that might already exist # in self._array_types try: array = getattr(gat, array) except AttributeError: msg = ("Skipping {0} - not found in gnome.array_types;" " and ArrayType is not provided.").format(array) self.logger.error(msg) raise GnomeRuntimeError(msg) # must be an ArrayType of an object self._array_types[array.name] = array def _append_initializer_array_types(self, array_types): # for each array_types, use the name to get the associated initializer for name in array_types: for spill in self.spills: if spill.has_initializer(name): self._append_array_types(spill.get_initializer(name). array_types) def _append_data_arrays(self, num_released): """ initialize data arrays once spill has spawned particles Data arrays are set to their initial_values :param int num_released: number of particles released """ for name, atype in self._array_types.iteritems(): # initialize all arrays even if 0 length if atype.shape is None: # assume array type is for weather data, provide it the shape # per the number of components used to model the oil # currently, we only have one type of oil, so all spills will # model same number of oil_components a_append = atype.initialize(num_released, shape=(self._oil_comp_array_len,), initial_value=tuple([0] * self._oil_comp_array_len)) else: a_append = atype.initialize(num_released) self._data_arrays[name] = np.r_[self._data_arrays[name], a_append] def _set_substance_array(self, subs_idx, num_rel_by_substance): ''' -. update 'substance' array if more than one substance present. The value of array is the index of 'substance' in _substances_spills data structure ''' if 'substance' in self: if num_rel_by_substance > 0: self['substance'][-num_rel_by_substance:] = subs_idx def substancefatedata(self, substance, array_types, fate='surface_weather'): ''' todo: fix this so it works for type of fate requested return the data for specified substance data must contain array names specified in 'array_types' ''' view = self._get_fatedataview(substance) return view.get_data(self, array_types, fate) def iterspillsbysubstance(self): ''' iterate through the substances spills datastructure and return the spills associated with each substance. This is used by release_elements DataStructure contains all spills. If some spills contain None for substance, these will be returned ''' if self._substances_spills is None: self._set_substancespills() return self._substances_spills.spills def itersubstancedata(self, array_types, fate='surface_weather'): ''' iterates through and returns the following for each iteration: (substance, substance_data) This is used by weatherers - if a substance is None, omit it from the iteration. :param array_types: iterable containing array that should be in the data. This could be a set of strings corresponding with array names or ArrayType objects which have a name attribute :param select='select': a string stating the type of data to be returned. Default if 'surface', so all elements with status_codes==oil_status.in_water and z == 0 in positions array :returns: (substance, substance_data) for each iteration substance: substance object substance_data: dict of numpy arrays associated with substance with elements in_water and on surface if select == 'surface' or subsurface if select == 'subsurface' ''' if self._substances_spills is None: self._set_substancespills() return zip(self.get_substances(complete=False), [view.get_data(self, array_types, fate) for view in self._fate_data_list]) def update_from_fatedataview(self, substance=None, fate='surface_weather'): ''' let's only update the arrays that were changed only update if a copy of 'data' exists. This is the case if there are more then one substances ''' if substance is not None: view = self._get_fatedataview(substance) view.update_sc(self, fate) else: # do for all substances for view in self._fate_data_list: view.update_sc(self, fate) def get_substances(self, complete=True): ''' return substances stored in _substances_spills structure. Include None if complete is True. Default is complete=True. ''' if self._substances_spills is None: self._set_substancespills() if complete: return self._substances_spills.substances else: return filter(None, self._substances_spills.substances) @property def total_mass(self): ''' return total mass spilled in 'kg' ''' mass = 0 for spill in self.spills: if spill.get_mass() is not None: mass += spill.get_mass() if mass == 0: return None else: return mass @property def substances(self): ''' Returns list of substances for weathering - not including None since that is non-weathering. Currently, only one weathering substance is supported ''' return self.get_substances(complete=False) @property def array_types(self): """ user can modify ArrayType initial_value in middle of run. Changing the shape should throw an error. Change the dtype at your own risk. This returns a new dict so user cannot add/delete an ArrayType in middle of run. Use prepare_for_model_run() to do add an ArrayType. """ return dict(self._array_types) def rewind(self): """ In the rewind operation, we: - rewind all the spills - restore _array_types to contain only defaults - movers/weatherers could have been deleted and we don't want to carry associated data_arrays - prepare_for_model_run() will be called before the next run and new arrays can be given - purge the data arrays - we gather data arrays for each contained spill - the stored arrays are cleared, then replaced with appropriate empty arrays """ for spill in self.spills: spill.rewind() # create a full set of zero-sized arrays. If we rewound, something # must have changed so let's get back to default _array_types self._reset_arrays() self._reset__substances_spills() self._reset__fate_data_list() self.initialize_data_arrays() self.mass_balance = {} # reset to empty array def get_spill_mask(self, spill): return self['spill_num'] == self.spills.index(spill) def uncertain_copy(self): """ Returns a copy of the spill_container suitable for uncertainty It has all the same spills, with the same ids, and the uncertain flag set to True """ u_sc = SpillContainer(uncertain=True) for sp in self.spills: u_sc.spills += sp.uncertain_copy() return u_sc def prepare_for_model_run(self, array_types=set()): """ called when setting up the model prior to 1st time step This is considered 0th timestep by model Make current_time optional since SpillContainer doesn't require it especially for 0th step; however, the model needs to set it because it will write_output() after each step. The data_arrays along with the current_time_stamp must be set in order to write_output() :param model_start_time: model_start_time to initialize current_time_stamp. This is the time_stamp associated with 0-th step so initial conditions for data arrays :param array_types: a set of additional names and/or array_types to append to standard array_types attribute. Set can contain only strings or a tuple with (string, ArrayType). See Note below. .. note:: set can contains strings or tuples. If set contains only strings, say: {'mass', 'windages'}, then SpillContainer looks for corresponding ArrayType object defined in gnome.array_types for 'mass' and 'windages'. If set contains a tuple, say: {('mass', gnome.array_types.mass)}, then SpillContainer uses the ArrayType defined in the tuple. .. note:: The SpillContainer iterates through each of the item in array_types and checks to see if there is an associated initializer in any Spill. If corresponding initializer is found, it gets the array_types from initializer and appends them to its own list. This was added for the case where 'droplet_diameter' array is defined/used by initializer (InitRiseVelFromDropletSizeFromDist) and we would like to see it in output, but no Mover/Weatherer needs it. """ # Question - should we purge any new arrays that were added in previous # call to prepare_for_model_run()? # No! If user made modifications to _array_types before running model, # let's keep those. A rewind will reset data_arrays. self._append_array_types(array_types) self._append_initializer_array_types(array_types) if self._substances_spills is None: self._set_substancespills() # also create fate_dataview if 'fate_status' is part of arrays if 'fate_status' in self.array_types: self._set_fate_data_list() # 'substance' data_array may have been added so initialize after # _set_substancespills() is invoked self.initialize_data_arrays() # todo: maybe better to let map do this, but it does not have a # prepare_for_model_run() yet so can't do it there # need 'amount_released' here as well self.mass_balance['beached'] = 0.0 self.mass_balance['off_maps'] = 0.0 def initialize_data_arrays(self): """ initialize_data_arrays() is called without input data during rewind and prepare_for_model_run to define all data arrays. At this time the arrays are empty. """ for name, atype in self._array_types.iteritems(): # Initialize data_arrays with 0 elements if atype.shape is None: num_comp = self._oil_comp_array_len self._data_arrays[name] = \ atype.initialize_null(shape=(num_comp, )) else: self._data_arrays[name] = atype.initialize_null() def release_elements(self, time_step, model_time): """ Called at the end of a time step This calls release_elements on all of the contained spills, and adds the elements to the data arrays :returns: total number of particles released todo: may need to update the 'mass' array to use a default of 1.0 but will need to define it in particle units or something along those lines """ total_released = 0 # substance index - used label elements from same substance # used internally only by SpillContainer - could be a strided array. # Simpler to define it only in SpillContainer as opposed to ArrayTypes # 'substance': ((), np.uint8, 0) for ix, spills in enumerate(self.iterspillsbysubstance()): num_rel_by_substance = 0 for spill in spills: # only spills that are included here - no need to check # spill.on flag num_rel = spill.num_elements_to_release(model_time, time_step) if num_rel > 0: # update 'spill_num' ArrayType's initial_value so it # corresponds with spill number for this set of released # particles - just another way to set value of spill_num # correctly self._array_types['spill_num'].initial_value = \ self.spills.index(spill) if len(self['spill_num']) > 0: # unique identifier for each new element released # this adjusts the _array_types initial_value since the # initialize function just calls: # range(initial_value, num_released + initial_value) self._array_types['id'].initial_value = \ self['id'][-1] + 1 else: # always reset value of first particle released to 0! # The array_types are shared globally. To initialize # uncertain spills correctly, reset this to 0. # To be safe, always reset to 0 when no # particles are released self._array_types['id'].initial_value = 0 # append to data arrays - number of oil components is # currently the same for all spills self._append_data_arrays(num_rel) spill.set_newparticle_values(num_rel, model_time, time_step, self._data_arrays) num_rel_by_substance += num_rel # always reset data arrays else the changing arrays are stale self._set_substance_array(ix, num_rel_by_substance) # reset fate_dataview at each step - do it after release elements self.reset_fate_dataview() # update total elements released for substance total_released += num_rel_by_substance return total_released def split_element(self, ix, num, l_frac=None): ''' split an element into specified number. For data, like mass, that gets divided, l_frac can be optionally provided. l_frac is a list containing fraction of component's value given to each new element. len(l_frac) must be equal to num and sum(l_frac) == 1.0 :param ix: id of element to be split - before splitting each element has a unique 'id' defined in 'id' data array :type ix: int :param num: split ix into 'num' number of elements :type num: int :param l_frac: list containing fractions that sum to 1.0 with len(l_frac) == num :type l_frac: list or tuple or numpy array ''' # split the first location where 'id' matches try: idx = np.where(self['id'] == ix)[0][0] except IndexError: msg = "no element with id = {0} found".format(ix) self.logger.warning(msg) raise for name, at in self.array_types.iteritems(): data = self[name] split_elems = at.split_element(num, self[name][idx], l_frac) data = np.insert(data, idx, split_elems[:-1], 0) data[idx + len(split_elems) - 1] = split_elems[-1] self._data_arrays[name] = data # update fate_dataview which contains this LE # for now we only have one type of substance if len(self._fate_data_list) > 1: msg = "split_elements assumes only one substance is being modeled" self.logger.error(msg) self._fate_data_list[0]._reset_fatedata(self, ix) def model_step_is_done(self): ''' Called at the end of a time step Need to remove particles marked as to_be_removed... ''' if len(self._data_arrays) == 0: return # nothing to do - arrays are not yet defined. # LEs are marked as to_be_removed # C++ might care about this so leave as is to_be_removed = np.where(self['status_codes'] == oil_status.to_be_removed)[0] if len(to_be_removed) > 0: for key in self._array_types.keys(): self._data_arrays[key] = np.delete(self[key], to_be_removed, axis=0) def __str__(self): return ('gnome.spill_container.SpillContainer\n' 'spill LE attributes: {0}' .format(sorted(self._data_arrays.keys()))) __repr__ = __str__
def test_setitem(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc[s_id(4)] = 7 oc[0] = 0 assert [i for i in oc] == [0, 2, 3, 7, 5]
class Model(Serializable): "PyGNOME Model Class" _update = [ "time_step", "weathering_substeps", "start_time", "duration", "uncertain", "movers", "weatherers", "environment", "spills", "map", "outputters", "cache_enabled", ] _create = [] _create.extend(_update) _state = copy.deepcopy(Serializable._state) # no need to copy parent's _state in this case _state.add(create=_create, update=_update) @classmethod def new_from_dict(cls, dict_): "Restore model from previously persisted _state" l_env = dict_.pop("environment") l_out = dict_.pop("outputters") l_movers = dict_.pop("movers") l_weatherers = dict_.pop("weatherers") c_spills = dict_.pop("certain_spills") if "uncertain_spills" in dict_: u_spills = dict_.pop("uncertain_spills") l_spills = zip(c_spills, u_spills) else: l_spills = c_spills model = object.__new__(cls) model.__restore__(**dict_) [model.environment.add(obj) for obj in l_env] [model.outputters.add(obj) for obj in l_out] [model.spills.add(obj) for obj in l_spills] [model.movers.add(obj) for obj in l_movers] [model.weatherers.add(obj) for obj in l_weatherers] # register callback with OrderedCollection model.movers.register_callback(model._callback_add_mover, ("add", "replace")) model.weatherers.register_callback(model._callback_add_weatherer, ("add", "replace")) return model def __init__( self, time_step=timedelta(minutes=15), start_time=round_time(datetime.now(), 3600), duration=timedelta(days=1), weathering_substeps=1, map=gnome.map.GnomeMap(), uncertain=False, cache_enabled=False, id=None, ): """ Initializes a model. All arguments have a default. :param time_step=timedelta(minutes=15): model time step in seconds or as a timedelta object :param start_time=datetime.now(): start time of model, datetime object. Rounded to the nearest hour. :param duration=timedelta(days=1): How long to run the model, a timedelta object. :param int weathering_substeps=1: How many weathering substeps to run inside a single model time step. :param map=gnome.map.GnomeMap(): The land-water map. :param uncertain=False: Flag for setting uncertainty. :param cache_enabled=False: Flag for setting whether the model should cache results to disk. :param id: Unique Id identifying the newly created mover (a UUID as a string). This is used when loading an object from a persisted model """ self.__restore__(time_step, start_time, duration, weathering_substeps, map, uncertain, cache_enabled) self._gnome_id = gnome.GnomeId(id) # register callback with OrderedCollection self.movers.register_callback(self._callback_add_mover, ("add", "replace")) self.weatherers.register_callback(self._callback_add_weatherer, ("add", "replace")) def __restore__(self, time_step, start_time, duration, weathering_substeps, map, uncertain, cache_enabled): """ Take out initialization that does not register the callback here. This is because new_from_dict will use this to restore the model _state when doing a midrun persistence. """ # making sure basic stuff is in place before properties are set self.environment = OrderedCollection(dtype=Environment) self.movers = OrderedCollection(dtype=Mover) self.weatherers = OrderedCollection(dtype=Weatherer) # contains both certain/uncertain spills self.spills = SpillContainerPair(uncertain) self._cache = gnome.utilities.cache.ElementCache() self._cache.enabled = cache_enabled # list of output objects self.outputters = OrderedCollection(dtype=Outputter) # default to now, rounded to the nearest hour self._start_time = start_time self._duration = duration self.weathering_substeps = weathering_substeps self._map = map self.time_step = time_step # this calls rewind() ! def reset(self, **kwargs): """ Resets model to defaults -- Caution -- clears all movers, spills, etc. Takes same keyword arguments as __init__ """ self.__init__(**kwargs) def rewind(self): """ Rewinds the model to the beginning (start_time) """ # fixme: do the movers need re-setting? -- or wait for # prepare_for_model_run? self.current_time_step = -1 self.model_time = self._start_time # note: This may be redundant. They will get reset in # setup_model_run() anyway.. self.spills.rewind() # set rand before each call so windages are set correctly gnome.utilities.rand.seed(1) # clear the cache: self._cache.rewind() for outputter in self.outputters: outputter.rewind() # def write_from_cache(self, filetype='netcdf', time_step='all'): # """ # write the already-cached data to an output files. # """ @property def uncertain(self): return self.spills.uncertain @uncertain.setter def uncertain(self, uncertain_value): """ only if uncertainty switch is toggled, then restart model """ if self.spills.uncertain != uncertain_value: self.spills.uncertain = uncertain_value # update uncertainty self.rewind() @property def cache_enabled(self): return self._cache.enabled @cache_enabled.setter def cache_enabled(self, enabled): self._cache.enabled = enabled @property def id(self): return self._gnome_id.id @property def start_time(self): return self._start_time @start_time.setter def start_time(self, start_time): self._start_time = start_time self.rewind() @property def time_step(self): return self._time_step @time_step.setter def time_step(self, time_step): """ Sets the time step, and rewinds the model :param time_step: The timestep can be a timedelta object or integer seconds. """ try: self._time_step = time_step.total_seconds() except AttributeError: self._time_step = int(time_step) # there is a zeroth time step self._num_time_steps = int(self._duration.total_seconds() // self._time_step) + 1 self.rewind() @property def current_time_step(self): return self._current_time_step @current_time_step.setter def current_time_step(self, step): self.model_time = self._start_time + timedelta(seconds=step * self.time_step) self._current_time_step = step @property def duration(self): return self._duration @duration.setter def duration(self, duration): if duration < self._duration: # only need to rewind if shorter than it was... # fixme: actually, only need to rewind if current model time # is beyond new time... self.rewind() self._duration = duration # there is a zeroth time step self._num_time_steps = int(self._duration.total_seconds() // self.time_step) + 1 @property def map(self): return self._map @map.setter def map(self, map_in): self._map = map_in self.rewind() @property def num_time_steps(self): return self._num_time_steps def setup_model_run(self): """ Sets up each mover for the model run """ self.spills.rewind() # why is rewind for spills here? array_types = {} for mover in self.movers: mover.prepare_for_model_run() array_types.update(mover.array_types) for w in self.weatherers: w.prepare_for_model_run() array_types.update(w.array_types) for sc in self.spills.items(): sc.prepare_for_model_run(array_types) # outputters need array_types, so this needs to come after those # have been updated. for outputter in self.outputters: outputter.prepare_for_model_run( model_start_time=self.start_time, cache=self._cache, uncertain=self.uncertain, spills=self.spills ) def setup_time_step(self): """ sets up everything for the current time_step: """ # initialize movers differently if model uncertainty is on for m in self.movers: for sc in self.spills.items(): m.prepare_for_model_step(sc, self.time_step, self.model_time) for w in self.weatherers: for sc in self.spills.items(): # maybe we will setup a super-sampling step here??? w.prepare_for_model_step(sc, self.time_step, self.model_time) for outputter in self.outputters: outputter.prepare_for_model_step(self.time_step, self.model_time) def move_elements(self): """ Moves elements: - loops through all the movers. and moves the elements - sets new_position array for each spill - calls the beaching code to beach the elements that need beaching. - sets the new position """ for sc in self.spills.items(): if sc.num_released > 0: # can this check be removed? # possibly refloat elements self.map.refloat_elements(sc, self.time_step) # reset next_positions (sc["next_positions"])[:] = sc["positions"] # loop through the movers for m in self.movers: delta = m.get_move(sc, self.time_step, self.model_time) sc["next_positions"] += delta self.map.beach_elements(sc) # the final move to the new positions (sc["positions"])[:] = sc["next_positions"] def weather_elements(self): """ Weathers elements: - loops through all the weatherers, passing in the spill_container and the time range - a weatherer modifies the data arrays in the spill container, so a particular time range should not be run multiple times. It is expected that we are processing a sequence of contiguous time ranges. - Note: If there are multiple sequential weathering processes, some inaccuracy could occur. A proposed solution is to 'super-sample' the model time step so that it will be replaced with many smaller time steps. We'll have to see if this pans out in practice. """ for sc in self.spills.items(): for w in self.weatherers: for model_time, time_step in self._split_into_substeps(): w.weather_elements(sc, time_step, model_time) def _split_into_substeps(self): """ :return: sequence of (datetime, timestep) (Note: we divide evenly on second boundaries. Thus, there will likely be a remainder that needs to be included. We include this remainder, which results in 1 more sub-step than we requested.) """ time_step = int(self._time_step) sub_step = time_step / self.weathering_substeps indexes = [idx for idx in range(0, time_step + 1, sub_step)] res = [(idx, next_idx - idx) for idx, next_idx in zip(indexes, indexes[1:])] if sum(res[-1]) < time_step: # collect the remaining slice res.append((sum(res[-1]), time_step % sub_step)) res = [(self.model_time + timedelta(seconds=idx), delta) for idx, delta in res] return res def step_is_done(self): """ Loop through movers and call model_step_is_done """ for mover in self.movers: for sc in self.spills.items(): mover.model_step_is_done(sc) for w in self.weatherers: w.model_step_is_done() for sc in self.spills.items(): "removes elements with oil_status.to_be_removed" sc.model_step_is_done() # age remaining particles sc["age"][:] = sc["age"][:] + self.time_step for outputter in self.outputters: outputter.model_step_is_done() def write_output(self): output_info = {"step_num": self.current_time_step} for outputter in self.outputters: if self.current_time_step == self.num_time_steps - 1: output = outputter.write_output(self.current_time_step, True) else: output = outputter.write_output(self.current_time_step) if output is not None: output_info.update(output) return output_info def step(self): """ Steps the model forward (or backward) in time. Needs testing for hind casting. """ for sc in self.spills.items(): # Set the current time stamp only after current_time_step is # incremented and before the output is written. Set it to None here # just so we're not carrying around the old time_stamp sc.current_time_stamp = None # it gets incremented after this check if self.current_time_step >= self._num_time_steps - 1: raise StopIteration if self.current_time_step == -1: # that's all we need to do for the zeroth time step self.setup_model_run() else: self.setup_time_step() self.move_elements() self.weather_elements() self.step_is_done() self.current_time_step += 1 # this is where the new step begins! # the elements released are during the time period: # self.model_time + self.time_step # The else part of the loop computes values for data_arrays that # correspond with time_stamp: # self.model_time + self.time_step # This is the current_time_stamp attribute of the SpillContainer # [sc.current_time_stamp for sc in self.spills.items()] for sc in self.spills.items(): sc.current_time_stamp = self.model_time # release particles for next step - these particles will be aged # in the next step sc.release_elements(self.time_step, self.model_time) # cache the results - current_time_step is incremented but the # current_time_stamp in spill_containers (self.spills) is not updated # till we go through the prepare_for_model_step self._cache.save_timestep(self.current_time_step, self.spills) output_info = self.write_output() return output_info def __iter__(self): """ Rewinds the model and returns itself so it can be iterated over. """ self.rewind() return self def next(self): """ (This method satisfies Python's iterator and generator protocols) :return: the step number """ return self.step() def full_run(self, rewind=True, log=False): """ Do a full run of the model. :param rewind=True: whether to rewind the model first -- if set to false, model will be run from the current step to the end :returns: list of outputter info dicts """ if rewind: self.rewind() # run the model output_data = [] while True: try: results = self.step() if log: print results output_data.append(results) except StopIteration: print "Done with the model run" break return output_data def movers_to_dict(self): """ Call to_dict method of OrderedCollection object """ return self.movers.to_dict() def weatherers_to_dict(self): """ Call to_dict method of OrderedCollection object """ return self.weatherers.to_dict() def environment_to_dict(self): """ Call to_dict method of OrderedCollection object """ return self.environment.to_dict() def spills_to_dict(self): return self.spills.to_dict() def outputters_to_dict(self): """ Call to_dict method of OrderedCollection object """ return self.outputters.to_dict() def map_to_dict(self): """ returns the gnome object type as a string """ return "{0}.{1}".format(self.map.__module__, self.map.__class__.__name__) def _callback_add_mover(self, obj_added): "Callback after mover has been added" if isinstance(obj_added, WindMover): if obj_added.wind.id not in self.environment: self.environment += obj_added.wind if isinstance(obj_added, CatsMover): if obj_added.tide is not None and obj_added.tide.id not in self.environment: self.environment += obj_added.tide self.rewind() # rewind model if a new mover is added def _callback_add_weatherer(self, obj_added): "Callback after weatherer has been added" if isinstance(obj_added, Weatherer): # not sure what kind of dependencies we have just yet. pass self.rewind() # rewind model if a new weatherer is added def __eq__(self, other): check = super(Model, self).__eq__(other) if check: # also check the data in spill_container object if type(self.spills) != type(other.spills): return False if self.spills != other.spills: return False return check def __ne__(self, other): "Compare inequality (!=) of two objects" if self == other: return False else: return True """ Following methods are for saving a Model instance or creating a new model instance from a saved location """ def save(self, saveloc): """ save model in json format to user specified saveloc :param saveloc: A valid directory. Model files are either persisted here or a new model is re-created from the files stored here. The files are clobbered when save() is called. :type saveloc: A path as a string or unicode """ path_, savedir = os.path.split(saveloc) if path_ == "": path_ = "." if not os.path.exists(path_): raise ValueError('"{0}" does not exist. \nCannot create "{1}"'.format(path_, savedir)) if not os.path.exists(saveloc): os.mkdir(saveloc) self._empty_save_dir(saveloc) json_ = self.serialize("create") self._save_json_to_file(saveloc, json_, "{0}.json".format(self.__class__.__name__)) json_ = self.map.serialize("create") self._save_json_to_file(saveloc, json_, "{0}.json".format(self.map.__class__.__name__)) self._save_collection(saveloc, self.movers) self._save_collection(saveloc, self.weatherers) self._save_collection(saveloc, self.environment) self._save_collection(saveloc, self.outputters) for sc in self.spills.items(): self._save_collection(saveloc, sc.spills) # persist model _state since middle of run if self.current_time_step > -1: self._save_spill_data(os.path.join(saveloc, "spills_data_arrays.nc")) def _save_collection(self, saveloc, coll_): """ Function loops over an orderedcollection or any other iterable containing a list of objects. It calls the to_dict method for each object, then converts it o valid json (dict_to_json), and finally saves it to file (_save_json_to_file) :param OrderedCollection coll_: ordered collection to be saved Note: The movers and weatherer objects reference the environment collection. If a field is saved as reference (field.save_reference is True), then this function adds json_[field.name] = index where index is the index into the environment array for the reference object. Currently, only objects in the environment collection are referenced by movers. """ for count, obj in enumerate(coll_): json_ = obj.serialize("create") for field in obj._state: if field.save_reference: "attribute is stored as a reference to environment list" if getattr(obj, field.name) is not None: obj_id = getattr(obj, field.name).id index = self.environment.index(obj_id) json_[field.name] = index self._save_json_to_file(saveloc, json_, "{0}_{1}.json".format(obj.__class__.__name__, count)) def _save_json_to_file(self, saveloc, data, name): """ write json data to file :param dict data: JSON data to be saved :param obj: gnome object corresponding w/ data """ fname = os.path.join(saveloc, name) data = self._move_data_file(saveloc, data) # if there is a with open(fname, "w") as outfile: json.dump(data, outfile, indent=True) def _move_data_file(self, saveloc, json_): """ Look at _state attribute of object. Find all fields with 'isdatafile' attribute as True. If there is a key in to_json corresponding with 'name' of the fields with True 'isdatafile' attribute then move that datafile and update the key in the to_json to point to new location todo: maybe this belongs in serializable base class? Revisit this """ _state = eval("{0}._state".format(json_["obj_type"])) fields = _state.get_field_by_attribute("isdatafile") for field in fields: if field.name not in json_: continue value = json_[field.name] if os.path.exists(value) and os.path.isfile(value): shutil.copy(value, saveloc) json_[field.name] = os.path.split(json_[field.name])[1] return json_ def _save_spill_data(self, datafile): """ save the data arrays for current timestep to NetCDF """ nc_out = NetCDFOutput(datafile, which_data="all", cache=self._cache) nc_out.prepare_for_model_run(model_start_time=self.start_time, uncertain=self.uncertain, spills=self.spills) nc_out.write_output(self.current_time_step) def _empty_save_dir(self, saveloc): """ Remove all files, directories under saveloc First clean out directory, then add new save files This should only be called by self.save() """ (dirpath, dirnames, filenames) = os.walk(saveloc).next() if dirnames: for dir_ in dirnames: shutil.rmtree(os.path.join(dirpath, dir_)) if filenames: for file_ in filenames: os.remove(os.path.join(dirpath, file_))
def test_delitem_byindex(self): oc = OrderedCollection([1, 2, 3, 4, 5]) with raises(IndexError): del oc[5] del oc[3] assert [i for i in oc] == [1, 2, 3, 5]
def test_delitem(self): oc = OrderedCollection([1, 2, 3, 4, 5]) with raises(KeyError): del oc[s_id(6)] del oc[s_id(4)] assert [i for i in oc] == [1, 2, 3, 5]
def test_contains(self): oc = OrderedCollection([1, 2, 3, 4, 5]) assert 5 in oc
class SpillContainer(SpillContainerData): """ Container class for all spills -- it takes care of capturing the released LEs from all the spills, putting them all in a single set of arrays. Many of the "fields" associated with a collection of elements are optional, or used only by some movers, so only the ones required will be requested by each mover. The data for the elements is stored in the _data_arrays dict. They can be accessed by indexing. For example: positions = spill_container['positions'] : returns a (num_LEs, 3) array of world_point_types """ def __init__(self, uncertain=False): super(SpillContainer, self).__init__(uncertain=uncertain) self.spills = OrderedCollection(dtype=gnome.spill.Spill) self.rewind() # don't want user to add to array_types in middle of run. Since its # not possible to throw an error in this case, let's just make it a # bit difficult to do. # dict must be updated via prepar_for_model_run() only at beginning of # run. Make self._array_types an an instance variable self._reset_arrays() def __setitem__(self, data_name, array): """ Invoke baseclass __setitem__ method so the _data_array is set correctly In addition, create the appropriate ArrayType if it wasn't created by the user. """ super(SpillContainer, self).__setitem__(data_name, array) if data_name not in self._array_types: shape = self._data_arrays[data_name].shape[1:] dtype = self._data_arrays[data_name].dtype.type self._array_types[data_name] = gnome.array_types.ArrayType(shape, dtype) def _reset_arrays(self): """ reset _array_types dict so it contains default keys/values """ gnome.array_types.reset_to_defaults(["spill_num", "id"]) self._array_types = { "positions": gnome.array_types.positions, "next_positions": gnome.array_types.next_positions, "last_water_positions": gnome.array_types.last_water_positions, "status_codes": gnome.array_types.status_codes, "spill_num": gnome.array_types.spill_num, "id": gnome.array_types.id, "mass": gnome.array_types.mass, } self._data_arrays = {} @property def array_types(self): """ user can modify ArrayType initial_value in middle of run. Changing the shape should throw an error. Change the dtype at your own risk. This returns a new dict so user cannot add/delete an ArrayType in middle of run. Use prepare_for_model_run() to do add an ArrayType. """ return dict(self._array_types) def rewind(self): """ In the rewind operation, we: - rewind all the spills - restore _array_types to contain only defaults - movers/weatherers could have been deleted and we don't want to carry associated data_arrays - prepare_for_model_run() will be called before the next run and new arrays can be given - purge the data arrays - we gather data arrays for each contained spill - the stored arrays are cleared, then replaced with appropriate empty arrays """ for spill in self.spills: spill.rewind() # create a full set of zero-sized arrays. If we rewound, something # must have changed so let's get back to default _array_types self._reset_arrays() self.initialize_data_arrays() def get_spill_mask(self, spill): return self["spill_num"] == self.spills.index(spill.id) def uncertain_copy(self): """ Returns a copy of the spill_container suitable for uncertainty It has all the same spills, with the same ids, and the uncertain flag set to True """ u_sc = SpillContainer(uncertain=True) for sp in self.spills: u_sc.spills += sp.uncertain_copy() return u_sc def prepare_for_model_run(self, array_types={}): """ called when setting up the model prior to 1st time step This is considered 0th timestep by model Make current_time optional since SpillContainer doesn't require it especially for 0th step; however, the model needs to set it because it will write_output() after each step. The data_arrays along with the current_time_stamp must be set in order to write_output() :param model_start_time: model_start_time to initialize current_time_stamp. This is the time_stamp associated with 0-th step so initial conditions for data arrays :param array_types: a dict of additional array_types to append to standard array_types attribute. The data_arrays are initialized and appended based on the values of array_types attribute """ # Question - should we purge any new arrays that were added in previous # call to prepare_for_model_run()? # No! If user made modifications to _array_types before running model, # let's keep those. A rewind will reset data_arrays. self._array_types.update(array_types) self.initialize_data_arrays() def initialize_data_arrays(self): """ initialize_data_arrays() is called without input data during rewind and prepare_for_model_run to define all data arrays. At this time the arrays are empty. """ for name, elem in self._array_types.iteritems(): # Initialize data_arrays with 0 elements self._data_arrays[name] = elem.initialize_null() def _append_data_arrays(self, num_released): """ initialize data arrays once spill has spawned particles Data arrays are set to their initial_values :param num_released: number of particles released :type num_released: int """ for name, array_type in self._array_types.iteritems(): # initialize all arrays even if 0 length self._data_arrays[name] = np.r_[self._data_arrays[name], array_type.initialize(num_released)] def release_elements(self, time_step, model_time): """ Called at the end of a time step This calls release_elements on all of the contained spills, and adds the elements to the data arrays """ for spill in self.spills: if spill.on: num_released = spill.num_elements_to_release(model_time, time_step) if num_released > 0: # update 'spill_num' ArrayType's initial_value so it # corresponds with spill number for this set of released # particles - just another way to set value of spill_num # correctly self._array_types["spill_num"].initial_value = self.spills.index(spill.id, renumber=False) if len(self["spill_num"]) > 0: # unique identifier for each new element released # this adjusts the _array_types initial_value since the # initialize function just calls: # range(initial_value, num_released + initial_value) self._array_types["id"].initial_value = self["id"][-1] + 1 # else: # self._array_types['id'].initial_value = 0 # append to data arrays self._append_data_arrays(num_released) spill.set_newparticle_values(num_released, model_time, time_step, self._data_arrays) def model_step_is_done(self): """ Called at the end of a time step Need to remove particles marked as to_be_removed... """ if len(self._data_arrays) == 0: return # nothing to do - arrays are not yet defined. to_be_removed = np.where(self["status_codes"] == oil_status.to_be_removed)[0] if len(to_be_removed) > 0: for key in self._array_types.keys(): self._data_arrays[key] = np.delete(self[key], to_be_removed, axis=0) def __str__(self): msg = "gnome.spill_container.SpillContainer\nspill LE attributes: %s" % self._data_arrays.keys() return msg __repr__ = __str__
def test_iter(self): oc = OrderedCollection([1, 2, 3, 4, 5]) assert [i for i in oc] == [1, 2, 3, 4, 5]
def test_len(self): oc = OrderedCollection([1, 2, 3, 4, 5]) assert len(oc) == 5
def test_add(self): oc = OrderedCollection([1, 2, 3, 4, 5]) oc.add(6) assert [i for i in oc] == [1, 2, 3, 4, 5, 6] with raises(TypeError): oc.add('not an int')
def test_eq(self): 'Test comparison operator __eq__' assert (OrderedCollection([1, 2, 3, 4, 5]) == OrderedCollection([1, 2, 3, 4, 5]))
def test_int_to_dict(self, json_): '''added a to_dict() method - test this method for int dtype. Tests the try, except is working correctly''' items = range(5) oc = OrderedCollection(items) self._to_dict_assert(oc, items, json_)