def test_numpy_sfrs(minimal_sfr_serialization, tmpdir):
schema, serialization = minimal_sfr_serialization
# create and save
backend = SQLStorageBackend(tmpdir / 'tmp.db', mode='w')
storage = GeneralStorage(backend, serialization, schema)
inp = MockUUIDObject('foo', 1)
func = StorableFunction(stupid_func).named('stupid')
eval_result = func(inp)
storage.save([func, inp])
storage.close()
# reload
storage._known_storages = {}
backend = SQLStorageBackend(tmpdir / 'tmp.db', mode='r')
storage = GeneralStorage(backend, serialization, schema)
sfs = list(storage.backend.table_iterator('storable_functions'))
func = storage.load([sfs[0].uuid])[0]
store_result = func(inp)
assert store_result is not eval_result # not same in memory
assert isinstance(store_result, eval_result.__class__)
np.testing.assert_array_equal(store_result, eval_result)
def test_multiple_storage_inside_other_object(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
sf_uuid = str(get_uuid(sf))
be_1w = SQLStorageBackend(tmpdir.join("st1.db"), mode='w')
_store_via_container(be_1w, container, inp1, call_count=1, backend_count=1)
# different store, first store one result then store the other
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='r'),
class_info=_serialization,
schema=_schema)
container = _load_container(storage, sf)
st2 = GeneralStorage(backend=SQLStorageBackend(tmpdir.join('st2.db'),
mode='w'),
class_info=_serialization,
schema=_schema)
# store container before using it: should not add table
st2.save(container)
assert not st2.backend.has_table(sf_uuid)
assert len(container.cv.local_cache) == 0
assert container.cv.func.call_count == 1
# now we use container on inp2; should call CV; should disk-cache
assert container(inp2) == 'f'
assert container.cv.func.call_count == 2
assert len(container.cv.local_cache) == 1
st2.save([container, inp2])
assert st2.backend.has_table(sf_uuid)
assert st2.backend.table_len(sf_uuid) == 1
# now we calculate and save the value with inp1; should load from
# existing storage (st1)
assert container.cv.func.call_count == 2
assert len(container.cv.local_cache) == 1
assert container(inp1) == 'f'
assert container.cv.func.call_count == 2
assert len(container.cv.local_cache) == 2
st2.save([container, inp1])
assert st2.backend.table_len(sf_uuid) == 2
# now clear the local cache and load inp2; should load from st2 without
# calculating
container.cv.local_cache.clear()
assert container.cv.func.call_count == 2
assert container(inp2) == 'f'
assert container.cv.func.call_count == 2
def _make_storage(self, mode):
Storage._known_storages = {}
backend = SQLStorageBackend("test.sql", mode=mode)
storage = Storage.from_backend(
backend=backend,
schema=ops_schema,
class_info=ops_class_info
)
return storage
def test_storage_inside_other_object(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
be_1w = SQLStorageBackend(tmpdir.join("st1.db"), mode='w')
_store_via_container(be_1w, container, inp1, call_count=1, backend_count=1)
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='a'),
class_info=_serialization,
schema=_schema)
container = _load_container(storage, sf)
# same store; store new result should work
assert container(inp2) == 'f'
assert container.cv.func.call_count == 2
storage.save([container, inp2])
sf_uuid = str(get_uuid(sf))
assert storage.backend.table_len(sf_uuid) == 2
def test_storage_file_problem(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
sf_uuid = str(get_uuid(sf))
st1_fname = tmpdir.join("st1.db")
be_1w = SQLStorageBackend(st1_fname, mode='w')
_store_via_container(be_1w, container, inp1, call_count=1, backend_count=1)
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(st1_fname, mode='r'),
class_info=_serialization,
schema=_schema)
container = _load_container(storage, sf)
assert container.cv.func.call_count == 1
os.remove(st1_fname) # naughty user behavior
with pytest.warns(UserWarning):
assert container(inp1) == 'f'
assert container.cv.func.call_count == 2
def test_closing(tmpdir, inputs_and_func):
inp1, inp2, func, sf, container = inputs_and_func
sf_uuid = str(get_uuid(sf))
st1_fname = tmpdir.join("st1.db")
be_1w = SQLStorageBackend(st1_fname, mode='w')
storage = _store_via_container(be_1w,
container,
inp1,
call_count=1,
backend_count=1)
assert sf.has_handler
storage.close()
assert not sf.has_handler
def get(self, name):
if self._is_simstore(name):
import openpathsampling as paths
from openpathsampling.experimental.storage import \
Storage, monkey_patch_all
if not self.has_simstore_patch:
paths = monkey_patch_all(paths)
paths.InterfaceSet.simstore = True
StorageLoader.has_simstore_patch = True
from openpathsampling.experimental.simstore import \
SQLStorageBackend
backend = SQLStorageBackend(name, mode=self.mode)
storage = Storage.from_backend(backend)
else:
from openpathsampling import Storage
self._workaround(name)
storage = Storage(name, self.mode)
return storage
def test_multiple_storage(tmpdir):
# TODO: This should probably be refactored into several individual tests
# that are each a little smaller. However, the main goal is to ensure
# that this functionality gets tested
inp1 = InputObj()
inp2 = InputObj()
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='w'),
class_info=_serialization,
schema=_schema)
func = Mock(return_value='foo')
sf = StorableFunction(func).named('foo-return')
out1 = sf(inp1)
storage.save(sf)
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='r'),
class_info=_serialization,
schema=_schema)
sf = storage.storable_functions[0]
assert sf.name == 'foo-return'
assert len(sf.local_cache) == 0
assert sf.func.call_count == 1
# load result from storage: don't call func; do cache result
_ = sf(inp1)
assert sf.func.call_count == 1
assert len(sf.local_cache) == 1
# save to a new storage: this should save the cached result, too!
_ = sf(inp2)
assert sf.func.call_count == 2
assert len(sf.local_cache) == 2
st2 = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st2.db"),
mode='w'),
class_info=_serialization,
schema=_schema)
st2.save(sf)
assert st2.backend.table_len(get_uuid(sf)) == 2
# save one result to one storage, the other result to another storage;
# can we get both results without calling the function?
GeneralStorage._known_storages = {}
storage = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st1.db"),
mode='r'),
class_info=_serialization,
schema=_schema)
sf = storage.storable_functions[0]
assert sf.name == 'foo-return'
assert len(sf.local_cache) == 0
assert sf.func.call_count == 1
st2 = GeneralStorage(backend=SQLStorageBackend(tmpdir.join("st2.db"),
mode='w'),
class_info=_serialization,
schema=_schema)
_ = sf(inp2)
st2.save(sf)
assert sf.func.call_count == 2
res1 = sf(inp1)
assert sf.func.call_count == 2
res2 = sf(inp2)
assert sf.func.call_count == 2