def _load_container(storage, sf):
container_list = [
obj for obj in storage.simulation_objects if obj.name == 'container'
]
assert len(container_list) == 1
container = container_list[0]
assert get_uuid(container.cv) == get_uuid(sf)
assert container.cv is not sf # should not be the same obj in memory
assert container.cv.func.call_count == 1
assert len(container.cv.local_cache) == 0
return container
def _store_via_container(backend, container, inp, call_count, backend_count):
storage = GeneralStorage(backend=backend,
class_info=_serialization,
schema=_schema)
assert container(inp) == 'f'
assert container.cv.func.call_count == call_count
storage.save(container)
sf_uuid = str(get_uuid(container.cv))
assert storage.backend.has_table(sf_uuid)
assert storage.backend.table_len(sf_uuid) == backend_count
return storage
def test_closing(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
sf_uuid = str(get_uuid(sf))
st1_fname = tmpdir.join("st1.db")
be_1w = SQLStorageBackend(st1_fname, mode='w')
storage = _store_via_container(be_1w,
container,
inp1,
call_count=1,
backend_count=1)
assert sf.has_handler
storage.close()
assert not sf.has_handler
def test_multiple_storage_inside_other_object(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
sf_uuid = str(get_uuid(sf))
be_1w = SQLStorageBackend(tmpdir.join("st1.db"), mode='w')
_store_via_container(be_1w, container, inp1, call_count=1, backend_count=1)
# different store, first store one result then store the other
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='r'),
class_info=_serialization,
schema=_schema)
container = _load_container(storage, sf)
st2 = GeneralStorage(backend=SQLStorageBackend(tmpdir.join('st2.db'),
mode='w'),
class_info=_serialization,
schema=_schema)
# store container before using it: should not add table
st2.save(container)
assert not st2.backend.has_table(sf_uuid)
assert len(container.cv.local_cache) == 0
assert container.cv.func.call_count == 1
# now we use container on inp2; should call CV; should disk-cache
assert container(inp2) == 'f'
assert container.cv.func.call_count == 2
assert len(container.cv.local_cache) == 1
st2.save([container, inp2])
assert st2.backend.has_table(sf_uuid)
assert st2.backend.table_len(sf_uuid) == 1
# now we calculate and save the value with inp1; should load from
# existing storage (st1)
assert container.cv.func.call_count == 2
assert len(container.cv.local_cache) == 1
assert container(inp1) == 'f'
assert container.cv.func.call_count == 2
assert len(container.cv.local_cache) == 2
st2.save([container, inp1])
assert st2.backend.table_len(sf_uuid) == 2
# now clear the local cache and load inp2; should load from st2 without
# calculating
container.cv.local_cache.clear()
assert container.cv.func.call_count == 2
assert container(inp2) == 'f'
assert container.cv.func.call_count == 2
def test_storage_inside_other_object(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
be_1w = SQLStorageBackend(tmpdir.join("st1.db"), mode='w')
_store_via_container(be_1w, container, inp1, call_count=1, backend_count=1)
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='a'),
class_info=_serialization,
schema=_schema)
container = _load_container(storage, sf)
# same store; store new result should work
assert container(inp2) == 'f'
assert container.cv.func.call_count == 2
storage.save([container, inp2])
sf_uuid = str(get_uuid(sf))
assert storage.backend.table_len(sf_uuid) == 2
def test_setup():
# test that we can store and load at all
backend = MemoryStorageBackend()
storage = GeneralStorage(backend=backend,
class_info=_serialization,
schema=_schema)
obj = InputObj()
assert storage.backend.table_len('input_objs') == 0
storage.save(obj)
assert storage.backend.table_len('input_objs') == 1
func = Mock(return_value='foo')
sf = StorableFunction(func).named('foo-return')
sf_uuid = get_uuid(sf)
assert not storage.backend.has_table(sf_uuid)
assert len(sf.local_cache) == 0
assert sf(obj) == 'foo'
assert len(sf.local_cache) == 1
storage.save(sf)
assert storage.backend.has_table(sf_uuid)
def test_storage_file_problem(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
sf_uuid = str(get_uuid(sf))
st1_fname = tmpdir.join("st1.db")
be_1w = SQLStorageBackend(st1_fname, mode='w')
_store_via_container(be_1w, container, inp1, call_count=1, backend_count=1)
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(st1_fname, mode='r'),
class_info=_serialization,
schema=_schema)
container = _load_container(storage, sf)
assert container.cv.func.call_count == 1
os.remove(st1_fname) # naughty user behavior
with pytest.warns(UserWarning):
assert container(inp1) == 'f'
assert container.cv.func.call_count == 2
def test_multiple_storage(tmpdir):
# TODO: This should probably be refactored into several individual tests
# that are each a little smaller. However, the main goal is to ensure
# that this functionality gets tested
inp1 = InputObj()
inp2 = InputObj()
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='w'),
class_info=_serialization,
schema=_schema)
func = Mock(return_value='foo')
sf = StorableFunction(func).named('foo-return')
out1 = sf(inp1)
storage.save(sf)
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='r'),
class_info=_serialization,
schema=_schema)
sf = storage.storable_functions[0]
assert sf.name == 'foo-return'
assert len(sf.local_cache) == 0
assert sf.func.call_count == 1
# load result from storage: don't call func; do cache result
_ = sf(inp1)
assert sf.func.call_count == 1
assert len(sf.local_cache) == 1
# save to a new storage: this should save the cached result, too!
_ = sf(inp2)
assert sf.func.call_count == 2
assert len(sf.local_cache) == 2
st2 = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st2.db"),
mode='w'),
class_info=_serialization,
schema=_schema)
st2.save(sf)
assert st2.backend.table_len(get_uuid(sf)) == 2
# save one result to one storage, the other result to another storage;
# can we get both results without calling the function?
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='r'),
class_info=_serialization,
schema=_schema)
sf = storage.storable_functions[0]
assert sf.name == 'foo-return'
assert len(sf.local_cache) == 0
assert sf.func.call_count == 1
st2 = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st2.db"),
mode='w'),
class_info=_serialization,
schema=_schema)
_ = sf(inp2)
st2.save(sf)
assert sf.func.call_count == 2
res1 = sf(inp1)
assert sf.func.call_count == 2
res2 = sf(inp2)
assert sf.func.call_count == 2
