def test_load_fails(saveloc_):
'''
if load fails on map or any of the collections, no model is created
'''
model = make_model()
model.zipsave = False
model.save(saveloc_)
model_json = json.load(open(os.path.join(saveloc_, 'Model.json'), 'r'))
model_json['map']['filename'] = 'junk.bna'
with open(os.path.join(saveloc_, 'Model.json'), 'w') as fd:
json.dump(model_json, fd, indent=True)
with pytest.raises(Exception):
load(saveloc_)
def test_load_fails(saveloc_):
'''
if load fails on map or any of the collections, no model is created
'''
model = make_model()
model.zipsave = False
model.save(saveloc_)
model_json = json.load(open(os.path.join(saveloc_, 'Model.json'), 'r'))
model_json['map']['filename'] = 'junk.bna'
with open(os.path.join(saveloc_, 'Model.json'), 'w') as fd:
json.dump(model_json, fd, indent=True)
with pytest.raises(Exception):
load(saveloc_)
def test_save_load_midrun_no_movers(uncertain, zipsave, saveloc_):
"""
create model, save it after 1step, then load and check equality of original
model and persisted model
Remove all movers and ensure it still works as expected
"""
model = make_model(uncertain)
for mover in model.movers:
del model.movers[mover.id]
model.step()
print 'saving scenario ..'
model.save(saveloc_)
print 'loading scenario ..'
model2 = load(zipname(saveloc_, model))
for sc in zip(model.spills.items(), model2.spills.items()):
# need to change both atol since reading persisted data
sc[0]._array_allclose_atol = 1e-5
sc[1]._array_allclose_atol = 1e-5
assert model.spills == model2.spills
assert model == model2
def test_load_location_file(self, saveloc_, model):
'''
create a model
load save file from script_boston which contains a spill. Then merge
the created model into the model loaded from save file
'''
m = Model()
m.environment += [Water(), constant_wind(1., 0.)]
m.weatherers += Evaporation(m.environment[0], m.environment[-1])
m.spills += point_line_release_spill(10, (0, 0, 0),
datetime(2014, 1, 1, 12, 0))
# create save model
sample_save_file = os.path.join(saveloc_, 'SampleSaveModel.zip')
model.save(saveloc_, name='SampleSaveModel.zip')
if os.path.exists(sample_save_file):
model = load(sample_save_file)
model.merge(m)
for oc in m._oc_list:
for item in getattr(m, oc):
model_oc = getattr(model, oc)
assert item is model_oc[item.id]
for spill in m.spills:
assert spill is model.spills[spill.id]
# merge the other way and ensure model != m
m.merge(model)
assert model != m
def test_save_load_midrun_no_movers(images_dir, uncertain):
"""
create model, save it after 1step, then load and check equality of original
model and persisted model
Remove all movers and ensure it still works as expected
"""
model = make_model(images_dir, uncertain)
for mover in model.movers:
del model.movers[mover.id]
model.step()
print 'saving scenario ..'
model.save(saveloc_)
print 'loading scenario ..'
model2 = load(os.path.join(saveloc_, 'Model.json'))
for sc in zip(model.spills.items(), model2.spills.items()):
# need to change both atol since reading persisted data
sc[0]._array_allclose_atol = 1e-5
sc[1]._array_allclose_atol = 1e-5
assert model.spills == model2.spills
assert model == model2
def load_location_file(location_file, request):
'''
We would like to merge the current active model into the new model
created by our location file prior to clearing our session
'''
if isdir(location_file):
active_model = get_active_model(request)
new_model = load(location_file)
new_model._cache.enabled = False
if active_model is not None:
active_model._map = new_model._map
active_model._time_step = new_model._time_step
active_model._num_time_steps = new_model._num_time_steps
active_model.merge(new_model)
else:
active_model = new_model
name = split(location_file)[1]
if name != '':
active_model.name = name
init_session_objects(request, force=True)
log.debug("model loaded - begin registering objects")
RegisterObject(active_model, request)
set_active_model(request, active_model.id)
def test_load_location_file(self, saveloc_, model):
'''
create a model
load save file from script_boston which contains a spill. Then merge
the created model into the model loaded from save file
'''
m = Model()
m.environment += [Water(), constant_wind(1., 0.)]
m.weatherers += Evaporation(m.environment[0], m.environment[-1])
m.spills += point_line_release_spill(10, (0, 0, 0),
datetime(2014, 1, 1, 12, 0))
# create save model
sample_save_file = os.path.join(saveloc_, 'SampleSaveModel.zip')
model.save(saveloc_, name='SampleSaveModel.zip')
if os.path.exists(sample_save_file):
model = load(sample_save_file)
model.merge(m)
for oc in m._oc_list:
for item in getattr(m, oc):
model_oc = getattr(model, oc)
assert item is model_oc[item.id]
for spill in m.spills:
assert spill is model.spills[spill.id]
# merge the other way and ensure model != m
m.merge(model)
assert model != m
def test_save_load2(obj):
'test save/load functionality'
temp = os.path.join(base_dir, 'temp')
for dir_ in (temp, os.path.relpath(temp, base_dir)):
refs = obj.save(dir_)
obj2 = load(os.path.join(dir_, refs.reference(obj)))
assert obj == obj2
def run_from_save(saveloc_model):
if not os.path.isfile(saveloc_model):
raise ValueError('{0} does not appear to be a valid'
' json file'.format(saveloc_model))
model = load(saveloc_model)
model.rewind()
run(model)
def run_from_save(saveloc_model):
if not os.path.isfile(saveloc_model):
raise ValueError('{0} does not appear to be a valid'
' json file'.format(saveloc_model))
model = load(saveloc_model)
model.rewind()
run(model)
def test_save_load(saveloc_, test_obj):
'''
test save/load for initializers and for ElementType objects containing
each initializer. Tests serialize/deserialize as well.
These are stored as nested objects in the Spill but this should also work
so test it here
'''
refs = test_obj.save(saveloc_)
test_obj2 = load(os.path.join(saveloc_, refs.reference(test_obj)))
assert test_obj == test_obj2
def test_save_load(saveloc_, test_obj):
'''
test save/load for initializers and for ElementType objects containing
each initializer. Tests serialize/deserialize as well.
These are stored as nested objects in the Spill but this should also work
so test it here
'''
refs = test_obj.save(saveloc_)
test_obj2 = load(os.path.join(saveloc_, refs.reference(test_obj)))
assert test_obj == test_obj2
def _load_collection(cls, saveloc, l_coll_dict, refs):
'''
doesn't need to be classmethod of the Model, but its only used by
Model at present
'''
l_coll = []
for item in l_coll_dict:
i_ref = item['id']
if refs.retrieve(i_ref):
l_coll.append(refs.retrieve(i_ref))
else:
f_name = os.path.join(saveloc, item['id'])
obj = load(f_name, refs) # will add obj to refs
l_coll.append(obj)
return (l_coll)
def test_save_load_model(images_dir, uncertain):
'''
create a model, save it, then load it back up and check it is equal to
original model
'''
model = make_model(images_dir, uncertain)
print 'saving scenario ..'
#model.save(saveloc_, name='Model.json')
model.save(saveloc_)
print 'loading scenario ..'
#model2 = load(os.path.join(saveloc_, 'Model.json'))
model2 = load(saveloc_)
assert model == model2
def upload_model(request):
'''
Uploads a new model in the form of a zipfile and registers it as the
current active model.
We are generating our own filename instead of trusting
the incoming filename since that might result in insecure paths.
We may want to eventually use something other than /tmp,
and if you write to an untrusted location you will need to do
some extra work to prevent symlink attacks.
'''
clean_session_dir(request)
file_path = process_upload(request, 'new_model')
# Now that we have our file, we will now try to load the model into
# memory.
# Now that we have our file, is it a zipfile?
if not is_savezip_valid(file_path):
raise cors_response(
request, HTTPBadRequest('Incoming file is not a '
'valid zipfile!'))
# now we try to load our model from the zipfile.
gnome_sema = request.registry.settings['py_gnome_semaphore']
gnome_sema.acquire()
log.info('semaphore acquired.')
try:
log.info('loading our model from zip...')
new_model = load(file_path)
new_model._cache.enabled = False
init_session_objects(request, force=True)
RegisterObject(new_model, request)
log.info('setting active model...')
set_active_model(request, new_model.id)
except:
raise cors_exception(request, HTTPBadRequest, with_stacktrace=True)
finally:
gnome_sema.release()
log.info('semaphore released.')
# We will want to clean up our tempfile when we are done.
os.remove(file_path)
return cors_response(request, Response('OK'))
def upload_model(request):
'''
Uploads a new model in the form of a zipfile and registers it as the
current active model.
We are generating our own filename instead of trusting
the incoming filename since that might result in insecure paths.
We may want to eventually use something other than /tmp,
and if you write to an untrusted location you will need to do
some extra work to prevent symlink attacks.
'''
clean_session_dir(request)
file_path = process_upload(request, 'new_model')
# Now that we have our file, we will now try to load the model into
# memory.
# Now that we have our file, is it a zipfile?
if not is_savezip_valid(file_path):
raise cors_response(request, HTTPBadRequest('Incoming file is not a '
'valid zipfile!'))
# now we try to load our model from the zipfile.
gnome_sema = request.registry.settings['py_gnome_semaphore']
gnome_sema.acquire()
log.info('semaphore acquired.')
try:
log.info('loading our model from zip...')
new_model = load(file_path)
new_model._cache.enabled = False
init_session_objects(request, force=True)
RegisterObject(new_model, request)
log.info('setting active model...')
set_active_model(request, new_model.id)
except:
raise cors_exception(request, HTTPBadRequest, with_stacktrace=True)
finally:
gnome_sema.release()
log.info('semaphore released.')
# We will want to clean up our tempfile when we are done.
os.remove(file_path)
return cors_response(request, Response('OK'))
def test_load_midrun_ne_rewound_model(images_dir, uncertain):
"""
Load the same model that was persisted previously after 1 step
This time rewind the original model and test that the two are not equal.
The data arrays in the spill container must not match
"""
# data arrays in model.spills no longer equal
model = make_model(images_dir, uncertain)
model.step()
print 'saving scenario ..'
model.save(saveloc_)
model.rewind()
model2 = load(os.path.join(saveloc_, 'Model.json'))
assert model.spills != model2.spills
assert model != model2
def test_save_load_model(uncertain, zipsave, saveloc_):
'''
create a model, save it, then load it back up and check it is equal to
original model
'''
model = make_model(uncertain)
ice_mover = IceMover(testdata['IceMover']['ice_curr_curv'],
testdata['IceMover']['ice_top_curv'])
model.movers += ice_mover
model.outputters += IceJsonOutput([ice_mover])
model.zipsave = zipsave
print 'saving scenario ..'
model.save(saveloc_)
print 'loading scenario ..'
model2 = load(zipname(saveloc_, model))
assert model == model2
def test_save_load_model(uncertain, zipsave, saveloc_):
'''
create a model, save it, then load it back up and check it is equal to
original model
'''
model = make_model(uncertain)
ice_mover = IceMover(testdata['IceMover']['ice_curr_curv'],
testdata['IceMover']['ice_top_curv'])
model.movers += ice_mover
model.outputters += IceJsonOutput([ice_mover])
model.zipsave = zipsave
print 'saving scenario ..'
model.save(saveloc_)
print 'loading scenario ..'
model2 = load(zipname(saveloc_, model))
assert model == model2
def test_load_midrun_ne_rewound_model(uncertain, saveloc_):
"""
Load the same model that was persisted previously after 1 step
This time rewind the original model and test that the two are not equal.
The data arrays in the spill container must not match
"""
# data arrays in model.spills no longer equal
model = make_model(uncertain)
model.step()
print 'saving scenario ..'
model.save(saveloc_)
model.rewind()
model2 = load(zipname(saveloc_, model))
assert model.spills != model2.spills
assert model != model2
def test_save_load_midrun_scenario(images_dir, uncertain):
"""
create model, save it after 1step, then load and check equality of original
model and persisted model
"""
model = make_model(images_dir, uncertain)
model.step()
print 'saving scnario ..'
model.save(saveloc_)
print 'loading scenario ..'
model2 = load(os.path.join(saveloc_, 'Model.json'))
for sc in zip(model.spills.items(), model2.spills.items()):
sc[0]._array_allclose_atol = 1e-5 # need to change both atol
sc[1]._array_allclose_atol = 1e-5
assert model.spills == model2.spills
assert model == model2
def test_save_load_midrun_scenario(uncertain, zipsave, saveloc_):
"""
create model, save it after 1step, then load and check equality of original
model and persisted model
"""
model = make_model(uncertain)
model.step()
print 'saving scnario ..'
model.save(saveloc_)
print 'loading scenario ..'
model2 = load(zipname(saveloc_, model))
for sc in zip(model.spills.items(), model2.spills.items()):
sc[0]._array_allclose_atol = 1e-5 # need to change both atol
sc[1]._array_allclose_atol = 1e-5
assert model.spills == model2.spills
assert model == model2
def test_save_load(save_ref, saveloc_):
"""
tests and illustrates the functionality of save/load for
WindMover
"""
wind = Wind(filename=file_)
wm = WindMover(wind)
wm_fname = 'WindMover_save_test.json'
refs = None
if save_ref:
w_fname = 'Wind.json'
refs = References()
refs.reference(wind, w_fname)
wind.save(saveloc_, refs, w_fname)
wm.save(saveloc_, references=refs, name=wm_fname)
l_refs = References()
obj = load(os.path.join(saveloc_, wm_fname), l_refs)
assert (obj == wm and obj is not wm)
assert (obj.wind == wind and obj.wind is not wind)
shutil.rmtree(saveloc_) # clean-up
def test_save_load(save_ref, saveloc_):
"""
tests and illustrates the functionality of save/load for
WindMover
"""
wind = Wind(filename=file_)
wm = WindMover(wind)
wm_fname = 'WindMover_save_test.json'
refs = None
if save_ref:
w_fname = 'Wind.json'
refs = References()
refs.reference(wind, w_fname)
wind.save(saveloc_, refs, w_fname)
wm.save(saveloc_, references=refs, name=wm_fname)
l_refs = References()
obj = load(os.path.join(saveloc_, wm_fname), l_refs)
assert (obj == wm and obj is not wm)
assert (obj.wind == wind and obj.wind is not wind)
shutil.rmtree(saveloc_) # clean-up
def test_save_load_grids(saveloc_, obj):
'test save/load functionality'
refs = obj.save(saveloc_)
obj2 = load(os.path.join(saveloc_, refs.reference(obj)))
assert obj == obj2
def test_save_load_netcdf(clean_temp, obj):
'test save/load functionality'
refs = obj.save(clean_temp)
obj2 = load(os.path.join(clean_temp, refs.reference(obj)))
assert obj == obj2
#!/usr/bin/env python
import os
from gnome.persist import load
'''
very simple script to test if location files load
'''
loc_files = os.path.dirname(os.path.abspath(__file__))
dirs = os.listdir(loc_files)
for d in dirs:
save_dir = os.path.join(d, '{0}_save'.format(d))
model = os.path.join(loc_files, save_dir, 'Model.json')
if not os.path.exists(model):
continue
try:
m = load(model)
print "successfully loaded: {0}".format(model)
except:
print "FAILED: {0}".format(model)
def test_model_load(self):
_m = load('Model.zip')
def test_model_load(self):
_m = load('Model.zip')
def load_location_file(location_file, session):
if isdir(location_file):
obj = load(location_file)
RegisterObject(obj, session)
set_active_model(session, obj.id)
def test_model_load(self):
m = load('./Model.zip')
def test_save_load_grids(saveloc_, obj):
'test save/load functionality'
refs = obj.save(saveloc_)
obj2 = load(os.path.join(saveloc_, refs.reference(obj)))
assert obj == obj2
