def test_list_raw(self):
data = {'a': list(range(1, 10)), 'b': list(range(1, 10))}
df = pd.DataFrame(data)
store = OmegaStore()
meta = store.put(df, 'hdfdf', as_hdf=True)
# list with pattern
entries = store.list(pattern='hdf*', raw=True)
self.assertTrue(isinstance(entries[0], Metadata))
self.assertEqual('hdfdf', entries[0].name)
self.assertEqual(len(entries), 1)
# list with regexp
entries = store.list(regexp='hdf.*', raw=True)
self.assertTrue(isinstance(entries[0], Metadata))
self.assertEqual('hdfdf', entries[0].name)
self.assertEqual(len(entries), 1)
# list without pattern nor regexp
entries = store.list('hdfdf', kind=Metadata.PANDAS_HDF, raw=True)
self.assertTrue(isinstance(entries[0], Metadata))
self.assertEqual('hdfdf', entries[0].name)
self.assertEqual(len(entries), 1)
# subset kind
entries = store.list('hdfdf', raw=True, kind=Metadata.PANDAS_DFROWS)
self.assertEqual(len(entries), 0)
entries = store.list('hdfdf', raw=True, kind=Metadata.PANDAS_HDF)
self.assertEqual(len(entries), 1)
def test_hidden_temp_handling(self):
foo_store = OmegaStore(bucket='foo')
foo_store.put({}, '_temp')
self.assertNotIn('_temp', foo_store.list(include_temp=False))
self.assertIn('_temp', foo_store.list(include_temp=True))
foo_store.put({}, '.hidden')
self.assertNotIn('.hidden', foo_store.list(hidden=False))
self.assertIn('.hidden', foo_store.list(hidden=True))
def test_prefix_store(self):
"""
this is to test if store prefixes work
"""
df = pd.DataFrame({'a': list(range(1, 10)), 'b': list(range(1, 10))})
datasets = OmegaStore(prefix='teststore')
models = OmegaStore(prefix='models', kind=Metadata.SKLEARN_JOBLIB)
datasets.put(df, 'test')
self.assertEqual(len(datasets.list()), 1)
self.assertEqual(len(models.list()), 0)
def test_existing_arbitrary_collection_mdataframe(self):
data = {
'foo': 'bar',
'bax': {
'fox': 'fax',
}
}
store = OmegaStore()
store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend)
foo_coll = store.mongodb['foo']
foo_coll.insert(data)
store.make_metadata('myfoo', collection='foo',
kind='pandas.rawdict').save()
self.assertIn('myfoo', store.list())
# test we get back _id column if raw=True
mdf = store.getl('myfoo', raw=True)
self.assertIsInstance(mdf, MDataFrame)
data_df = mdf.value
data_raw = store.collection('myfoo').find_one()
assert_frame_equal(json_normalize(data_raw), data_df)
# test we get just the data column
mdf = store.getl('myfoo', raw=False)
self.assertIsInstance(mdf, MDataFrame)
data_df = mdf.value
data_raw = store.collection('myfoo').find_one()
cols = ['foo', 'bax.fox']
assert_frame_equal(json_normalize(data)[cols], data_df[cols])
def test_custom_levels(self):
"""
this is to test if custom path and levels can be provided ok
"""
df = pd.DataFrame({'a': list(range(1, 10)), 'b': list(range(1, 10))})
datasets = OmegaStore(prefix='data')
models = OmegaStore(prefix='models', kind=Metadata.SKLEARN_JOBLIB)
# directory-like levels
datasets.put(df, 'data/is/mypath/test')
datasets.put(df, 'data/is/mypath/test2')
self.assertEqual(len(datasets.list('data/*/mypath/*')), 2)
self.assertEqual(len(datasets.list('data/*/test')), 1)
# namespace-like levels
datasets.put(df, 'my.namespace.module.test')
datasets.put(df, 'my.namespace.module.test2')
self.assertEqual(len(datasets.list('*.module.*')), 2)
self.assertEqual(len(datasets.list('*.module.test2')), 1)
class DashAppBackendTests(TestCase):
def setUp(self):
self.app = myapp.app
self.store = OmegaStore(prefix='scripts/')
self.backend = DashAppBackend(model_store=self.store,
data_store=self.store)
[self.store.drop(name) for name in self.store.list()]
def tearDown(self):
pass
def test_supports(self):
""" test DashAppBackend supports a jupyter dash app """
self.assertTrue(DashAppBackend.supports(self.app, 'myapp'))
def test_put_get_dashapp(self):
""" test a jupyter dash app can be stored and restored"""
app = self.app
self.backend.put(self.app, 'myapp')
self.assertIn('myapp', self.store.list())
restored_app = self.backend.get('myapp')
self.assertIsInstance(restored_app, DashserveApp)
self.assertEqual(restored_app.config.external_stylesheets,
app.config.external_stylesheets)
def test_existing_arbitrary_collection_flat(self):
data = {'foo': 'bar', 'bax': 'fox'}
store = OmegaStore()
store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend)
foo_coll = store.mongodb['foo']
foo_coll.insert(data)
store.make_metadata('myfoo', collection='foo',
kind='pandas.rawdict').save()
self.assertIn('myfoo', store.list())
# test we get back _id column if raw=True
data_ = store.get('myfoo', raw=True)
assert_frame_equal(json_normalize(data), data_)
# test we get just the data column
data_ = store.get('myfoo', raw=False)
cols = ['foo', 'bax']
assert_frame_equal(json_normalize(data)[cols], data_[cols])
def test_arbitrary_collection_new(self):
data = {'foo': 'bar', 'bax': 'fox'}
store = OmegaStore()
store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend)
# create the collection
foo_coll = store.mongodb['foo']
foo_coll.insert(data)
# store the collection as is
store.put(foo_coll, 'myfoo').save()
self.assertIn('myfoo', store.list())
# test we get back _id column if raw=True
data_ = store.get('myfoo', raw=True)
assert_frame_equal(data_, json_normalize(data))
# test we get just the data column
data_ = store.get('myfoo', raw=False)
cols = ['foo', 'bax']
assert_frame_equal(data_[cols], json_normalize(data)[cols])
def test_drop(self):
data = {'a': list(range(1, 10)), 'b': list(range(1, 10))}
df = pd.DataFrame(data)
store = OmegaStore()
meta = store.put(df, 'hdfdf', as_hdf=True)
self.assertTrue(store.drop('hdfdf'))
meta = store.put(df, 'datadf')
self.assertTrue(store.drop('datadf'))
self.assertEqual(store.list('datadf'), [],
'expected the store to be empty')
with self.assertRaises(DoesNotExist):
store.drop('nxstore', force=False)
try:
store.drop('nxstore', force=True)
raised = False
except:
raised = True
self.assertFalse(raised)
