def test_list_raw(self): data = {'a': list(range(1, 10)), 'b': list(range(1, 10))} df = pd.DataFrame(data) store = OmegaStore() meta = store.put(df, 'hdfdf', as_hdf=True) # list with pattern entries = store.list(pattern='hdf*', raw=True) self.assertTrue(isinstance(entries[0], Metadata)) self.assertEqual('hdfdf', entries[0].name) self.assertEqual(len(entries), 1) # list with regexp entries = store.list(regexp='hdf.*', raw=True) self.assertTrue(isinstance(entries[0], Metadata)) self.assertEqual('hdfdf', entries[0].name) self.assertEqual(len(entries), 1) # list without pattern nor regexp entries = store.list('hdfdf', kind=Metadata.PANDAS_HDF, raw=True) self.assertTrue(isinstance(entries[0], Metadata)) self.assertEqual('hdfdf', entries[0].name) self.assertEqual(len(entries), 1) # subset kind entries = store.list('hdfdf', raw=True, kind=Metadata.PANDAS_DFROWS) self.assertEqual(len(entries), 0) entries = store.list('hdfdf', raw=True, kind=Metadata.PANDAS_HDF) self.assertEqual(len(entries), 1)
def test_hidden_temp_handling(self): foo_store = OmegaStore(bucket='foo') foo_store.put({}, '_temp') self.assertNotIn('_temp', foo_store.list(include_temp=False)) self.assertIn('_temp', foo_store.list(include_temp=True)) foo_store.put({}, '.hidden') self.assertNotIn('.hidden', foo_store.list(hidden=False)) self.assertIn('.hidden', foo_store.list(hidden=True))
def test_prefix_store(self): """ this is to test if store prefixes work """ df = pd.DataFrame({'a': list(range(1, 10)), 'b': list(range(1, 10))}) datasets = OmegaStore(prefix='teststore') models = OmegaStore(prefix='models', kind=Metadata.SKLEARN_JOBLIB) datasets.put(df, 'test') self.assertEqual(len(datasets.list()), 1) self.assertEqual(len(models.list()), 0)
def test_existing_arbitrary_collection_mdataframe(self): data = { 'foo': 'bar', 'bax': { 'fox': 'fax', } } store = OmegaStore() store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend) foo_coll = store.mongodb['foo'] foo_coll.insert(data) store.make_metadata('myfoo', collection='foo', kind='pandas.rawdict').save() self.assertIn('myfoo', store.list()) # test we get back _id column if raw=True mdf = store.getl('myfoo', raw=True) self.assertIsInstance(mdf, MDataFrame) data_df = mdf.value data_raw = store.collection('myfoo').find_one() assert_frame_equal(json_normalize(data_raw), data_df) # test we get just the data column mdf = store.getl('myfoo', raw=False) self.assertIsInstance(mdf, MDataFrame) data_df = mdf.value data_raw = store.collection('myfoo').find_one() cols = ['foo', 'bax.fox'] assert_frame_equal(json_normalize(data)[cols], data_df[cols])
def test_custom_levels(self): """ this is to test if custom path and levels can be provided ok """ df = pd.DataFrame({'a': list(range(1, 10)), 'b': list(range(1, 10))}) datasets = OmegaStore(prefix='data') models = OmegaStore(prefix='models', kind=Metadata.SKLEARN_JOBLIB) # directory-like levels datasets.put(df, 'data/is/mypath/test') datasets.put(df, 'data/is/mypath/test2') self.assertEqual(len(datasets.list('data/*/mypath/*')), 2) self.assertEqual(len(datasets.list('data/*/test')), 1) # namespace-like levels datasets.put(df, 'my.namespace.module.test') datasets.put(df, 'my.namespace.module.test2') self.assertEqual(len(datasets.list('*.module.*')), 2) self.assertEqual(len(datasets.list('*.module.test2')), 1)
class DashAppBackendTests(TestCase): def setUp(self): self.app = myapp.app self.store = OmegaStore(prefix='scripts/') self.backend = DashAppBackend(model_store=self.store, data_store=self.store) [self.store.drop(name) for name in self.store.list()] def tearDown(self): pass def test_supports(self): """ test DashAppBackend supports a jupyter dash app """ self.assertTrue(DashAppBackend.supports(self.app, 'myapp')) def test_put_get_dashapp(self): """ test a jupyter dash app can be stored and restored""" app = self.app self.backend.put(self.app, 'myapp') self.assertIn('myapp', self.store.list()) restored_app = self.backend.get('myapp') self.assertIsInstance(restored_app, DashserveApp) self.assertEqual(restored_app.config.external_stylesheets, app.config.external_stylesheets)
def test_existing_arbitrary_collection_flat(self): data = {'foo': 'bar', 'bax': 'fox'} store = OmegaStore() store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend) foo_coll = store.mongodb['foo'] foo_coll.insert(data) store.make_metadata('myfoo', collection='foo', kind='pandas.rawdict').save() self.assertIn('myfoo', store.list()) # test we get back _id column if raw=True data_ = store.get('myfoo', raw=True) assert_frame_equal(json_normalize(data), data_) # test we get just the data column data_ = store.get('myfoo', raw=False) cols = ['foo', 'bax'] assert_frame_equal(json_normalize(data)[cols], data_[cols])
def test_arbitrary_collection_new(self): data = {'foo': 'bar', 'bax': 'fox'} store = OmegaStore() store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend) # create the collection foo_coll = store.mongodb['foo'] foo_coll.insert(data) # store the collection as is store.put(foo_coll, 'myfoo').save() self.assertIn('myfoo', store.list()) # test we get back _id column if raw=True data_ = store.get('myfoo', raw=True) assert_frame_equal(data_, json_normalize(data)) # test we get just the data column data_ = store.get('myfoo', raw=False) cols = ['foo', 'bax'] assert_frame_equal(data_[cols], json_normalize(data)[cols])
def test_drop(self): data = {'a': list(range(1, 10)), 'b': list(range(1, 10))} df = pd.DataFrame(data) store = OmegaStore() meta = store.put(df, 'hdfdf', as_hdf=True) self.assertTrue(store.drop('hdfdf')) meta = store.put(df, 'datadf') self.assertTrue(store.drop('datadf')) self.assertEqual(store.list('datadf'), [], 'expected the store to be empty') with self.assertRaises(DoesNotExist): store.drop('nxstore', force=False) try: store.drop('nxstore', force=True) raised = False except: raised = True self.assertFalse(raised)