def test_bucket(self): # test different buckets actually separate objects by the same name # -- data foo_store = OmegaStore(bucket='foo') bar_store = OmegaStore(bucket='bar') foo_store.register_backend(PythonRawFileBackend.KIND, PythonRawFileBackend) bar_store.register_backend(PythonRawFileBackend.KIND, PythonRawFileBackend) foo_data = {'foo': 'bar', 'bax': 'fox'} bar_data = {'foo': 'bax', 'bax': 'foz'} foo_store.put(foo_data, 'data') bar_store.put(bar_data, 'data') self.assertEqual(foo_store.get('data')[0], foo_data) self.assertEqual(bar_store.get('data')[0], bar_data) # -- files foo_data = "some data" file_like = BytesIO(foo_data.encode('utf-8')) foo_store.put(file_like, 'myfile') bar_data = "some other data" file_like = BytesIO(bar_data.encode('utf-8')) bar_store.put(file_like, 'myfile') self.assertNotEqual( foo_store.get('myfile').read(), bar_store.get('myfile').read())
def test_existing_arbitrary_collection_mdataframe(self): data = { 'foo': 'bar', 'bax': { 'fox': 'fax', } } store = OmegaStore() store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend) foo_coll = store.mongodb['foo'] foo_coll.insert(data) store.make_metadata('myfoo', collection='foo', kind='pandas.rawdict').save() self.assertIn('myfoo', store.list()) # test we get back _id column if raw=True mdf = store.getl('myfoo', raw=True) self.assertIsInstance(mdf, MDataFrame) data_df = mdf.value data_raw = store.collection('myfoo').find_one() assert_frame_equal(json_normalize(data_raw), data_df) # test we get just the data column mdf = store.getl('myfoo', raw=False) self.assertIsInstance(mdf, MDataFrame) data_df = mdf.value data_raw = store.collection('myfoo').find_one() cols = ['foo', 'bax.fox'] assert_frame_equal(json_normalize(data)[cols], data_df[cols])
def test_raw_files(self): store = OmegaStore() store.register_backend(PythonRawFileBackend.KIND, PythonRawFileBackend) # test we can write from a file-like object data = "some data" file_like = BytesIO(data.encode('utf-8')) store.put(file_like, 'myfile') self.assertEqual(data.encode('utf-8'), store.get('myfile').read()) # test we can write from an actual file data = "some other data" file_like = BytesIO(data.encode('utf-8')) with open('/tmp/testfile.txt', 'wb') as fout: fout.write(file_like.read()) store.put('/tmp/testfile.txt', 'myfile') self.assertEqual(data.encode('utf-8'), store.get('myfile').read())
def test_existing_arbitrary_collection_flat(self): data = {'foo': 'bar', 'bax': 'fox'} store = OmegaStore() store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend) foo_coll = store.mongodb['foo'] foo_coll.insert(data) store.make_metadata('myfoo', collection='foo', kind='pandas.rawdict').save() self.assertIn('myfoo', store.list()) # test we get back _id column if raw=True data_ = store.get('myfoo', raw=True) assert_frame_equal(json_normalize(data), data_) # test we get just the data column data_ = store.get('myfoo', raw=False) cols = ['foo', 'bax'] assert_frame_equal(json_normalize(data)[cols], data_[cols])
def test_arbitrary_collection_new(self): data = {'foo': 'bar', 'bax': 'fox'} store = OmegaStore() store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend) # create the collection foo_coll = store.mongodb['foo'] foo_coll.insert(data) # store the collection as is store.put(foo_coll, 'myfoo').save() self.assertIn('myfoo', store.list()) # test we get back _id column if raw=True data_ = store.get('myfoo', raw=True) assert_frame_equal(data_, json_normalize(data)) # test we get just the data column data_ = store.get('myfoo', raw=False) cols = ['foo', 'bax'] assert_frame_equal(data_[cols], json_normalize(data)[cols])