def test_bucket(self):
# test different buckets actually separate objects by the same name
# -- data
foo_store = OmegaStore(bucket='foo')
bar_store = OmegaStore(bucket='bar')
foo_store.register_backend(PythonRawFileBackend.KIND,
PythonRawFileBackend)
bar_store.register_backend(PythonRawFileBackend.KIND,
PythonRawFileBackend)
foo_data = {'foo': 'bar', 'bax': 'fox'}
bar_data = {'foo': 'bax', 'bax': 'foz'}
foo_store.put(foo_data, 'data')
bar_store.put(bar_data, 'data')
self.assertEqual(foo_store.get('data')[0], foo_data)
self.assertEqual(bar_store.get('data')[0], bar_data)
# -- files
foo_data = "some data"
file_like = BytesIO(foo_data.encode('utf-8'))
foo_store.put(file_like, 'myfile')
bar_data = "some other data"
file_like = BytesIO(bar_data.encode('utf-8'))
bar_store.put(file_like, 'myfile')
self.assertNotEqual(
foo_store.get('myfile').read(),
bar_store.get('myfile').read())
def test_existing_arbitrary_collection_mdataframe(self):
data = {
'foo': 'bar',
'bax': {
'fox': 'fax',
}
}
store = OmegaStore()
store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend)
foo_coll = store.mongodb['foo']
foo_coll.insert(data)
store.make_metadata('myfoo', collection='foo',
kind='pandas.rawdict').save()
self.assertIn('myfoo', store.list())
# test we get back _id column if raw=True
mdf = store.getl('myfoo', raw=True)
self.assertIsInstance(mdf, MDataFrame)
data_df = mdf.value
data_raw = store.collection('myfoo').find_one()
assert_frame_equal(json_normalize(data_raw), data_df)
# test we get just the data column
mdf = store.getl('myfoo', raw=False)
self.assertIsInstance(mdf, MDataFrame)
data_df = mdf.value
data_raw = store.collection('myfoo').find_one()
cols = ['foo', 'bax.fox']
assert_frame_equal(json_normalize(data)[cols], data_df[cols])
def test_raw_files(self):
store = OmegaStore()
store.register_backend(PythonRawFileBackend.KIND, PythonRawFileBackend)
# test we can write from a file-like object
data = "some data"
file_like = BytesIO(data.encode('utf-8'))
store.put(file_like, 'myfile')
self.assertEqual(data.encode('utf-8'), store.get('myfile').read())
# test we can write from an actual file
data = "some other data"
file_like = BytesIO(data.encode('utf-8'))
with open('/tmp/testfile.txt', 'wb') as fout:
fout.write(file_like.read())
store.put('/tmp/testfile.txt', 'myfile')
self.assertEqual(data.encode('utf-8'), store.get('myfile').read())
def test_existing_arbitrary_collection_flat(self):
data = {'foo': 'bar', 'bax': 'fox'}
store = OmegaStore()
store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend)
foo_coll = store.mongodb['foo']
foo_coll.insert(data)
store.make_metadata('myfoo', collection='foo',
kind='pandas.rawdict').save()
self.assertIn('myfoo', store.list())
# test we get back _id column if raw=True
data_ = store.get('myfoo', raw=True)
assert_frame_equal(json_normalize(data), data_)
# test we get just the data column
data_ = store.get('myfoo', raw=False)
cols = ['foo', 'bax']
assert_frame_equal(json_normalize(data)[cols], data_[cols])
def test_arbitrary_collection_new(self):
data = {'foo': 'bar', 'bax': 'fox'}
store = OmegaStore()
store.register_backend(PandasRawDictBackend.KIND, PandasRawDictBackend)
# create the collection
foo_coll = store.mongodb['foo']
foo_coll.insert(data)
# store the collection as is
store.put(foo_coll, 'myfoo').save()
self.assertIn('myfoo', store.list())
# test we get back _id column if raw=True
data_ = store.get('myfoo', raw=True)
assert_frame_equal(data_, json_normalize(data))
# test we get just the data column
data_ = store.get('myfoo', raw=False)
cols = ['foo', 'bax']
assert_frame_equal(data_[cols], json_normalize(data)[cols])
