class FileStorageTest(unittest.TestCase):
def setUp(self):
self.tdir = tempfile.mkdtemp()
self.mgr = FileStorage('fileio:%s' % self.tdir)
data = {
"int": 1,
"float": 1.2,
"list": [1, 2, 3],
"dict": {
"dname": "foo",
"dval": 1
},
"listdict": [{
"lname": "foo"
}],
"str": "string"
}
self.bare_data = dict(data)
data['wmaid'] = wmaHash(data)
data['stype'] = self.mgr.stype
self.data = data
def tearDown(self):
"Tear down content of temp dir"
for fname in os.listdir(self.tdir):
os.remove(os.path.join(self.tdir, fname))
os.rmdir(self.tdir)
def test_write(self):
"Test write functionality"
wmaids = self.mgr.write(self.data)
self.assertEqual(len(wmaids), 1)
data = self.mgr.read(wmaids[0])
self.assertEqual(data[0], self.bare_data)
class STSManager(object):
"Short-Term Storage manager based on Mongo/File/Avro storage back-end"
def __init__(self, uri):
"ctor with STS uri"
if uri.startswith('mongo'):
self.mgr = MongoStorage(uri)
elif uri.startswith('file'):
self.mgr = FileStorage(uri)
elif uri.startswith('avro'):
self.mgr = AvroStorage(uri)
else:
self.mgr = FileStorage(
os.getenv('WMA_STORAGE_ROOT', '/tmp/wma_storage'))
self.stype = self.mgr.stype # determine storage type
def qmap(self, spec, fields):
"map input spec/fields into ones suitable for STS QL"
return spec, fields
def write(self, data, safe=None):
"Write API for STS"
return self.mgr.write(data, safe)
def read(self, spec, fields=None):
"Read API for STS"
return self.mgr.read(spec, fields)
def jobs(self):
"Fetch jobs ID from underlying storage"
return self.mgr.jobsids()
def stats(self):
"Return statistics about underlying storage"
return self.mgr.stats()
def adocs(self):
"Return aggregated documents from underlying storage"
return self.mgr.adocs()
def status(self):
"Return status api"
return {
'sts': {
'mgr': str(self.mgr),
'stype': self.stype,
'stats': self.stats()
}
}
def setUp(self):
self.tdir = tempfile.mkdtemp()
self.mgr = FileStorage('fileio:%s' % self.tdir)
data = {"int":1, "float":1.2, "list":[1,2,3],
"dict":{"dname": "foo", "dval":1},
"listdict":[{"lname":"foo"}], "str":"string"}
self.bare_data = dict(data)
data['wmaid'] = wmaHash(data)
data['stype'] = self.mgr.stype
self.data = data
def setUp(self):
self.tdir = tempfile.mkdtemp()
self.mgr = FileStorage('fileio:%s' % self.tdir)
data = {
"int": 1,
"float": 1.2,
"list": [1, 2, 3],
"dict": {
"dname": "foo",
"dval": 1
},
"listdict": [{
"lname": "foo"
}],
"str": "string"
}
self.bare_data = dict(data)
data['wmaid'] = wmaHash(data)
data['stype'] = self.mgr.stype
self.data = data
class STSManager(object):
"Short-Term Storage manager based on Mongo/File/Avro storage back-end"
def __init__(self, uri):
"ctor with STS uri"
if uri.startswith('mongo'):
self.mgr = MongoStorage(uri)
elif uri.startswith('file'):
self.mgr = FileStorage(uri)
elif uri.startswith('avro'):
self.mgr = AvroStorage(uri)
else:
self.mgr = FileStorage(os.getenv('WMA_STORAGE_ROOT', '/tmp/wma_storage'))
self.stype = self.mgr.stype # determine storage type
def qmap(self, spec, fields):
"map input spec/fields into ones suitable for STS QL"
return spec, fields
def write(self, data, safe=None):
"Write API for STS"
return self.mgr.write(data, safe)
def read(self, spec, fields=None):
"Read API for STS"
return self.mgr.read(spec, fields)
def jobs(self):
"Fetch jobs ID from underlying storage"
return self.mgr.jobsids()
def stats(self):
"Return statistics about underlying storage"
return self.mgr.stats()
def performance(self, **kwargs):
"Return aggregated documents from underlying storage"
return self.mgr.performance(**kwargs)
def status(self):
"Return status api"
return {'sts':{'mgr':str(self.mgr), 'stype':self.stype, 'stats':self.stats()}}
class FileStorageTest(unittest.TestCase):
def setUp(self):
self.tdir = tempfile.mkdtemp()
self.mgr = FileStorage('fileio:%s' % self.tdir)
data = {"int":1, "float":1.2, "list":[1,2,3],
"dict":{"dname": "foo", "dval":1},
"listdict":[{"lname":"foo"}], "str":"string"}
self.bare_data = dict(data)
data['wmaid'] = wmaHash(data)
data['stype'] = self.mgr.stype
self.data = data
def tearDown(self):
"Tear down content of temp dir"
for fname in os.listdir(self.tdir):
os.remove(os.path.join(self.tdir, fname))
os.rmdir(self.tdir)
def test_write(self):
"Test write functionality"
wmaids = self.mgr.write(self.data)
self.assertEqual(len(wmaids), 1)
data = self.mgr.read(wmaids[0])
self.assertEqual(data[0], self.bare_data)
def __init__(self, uri):
"ctor with STS uri"
if uri.startswith('mongo'):
self.mgr = MongoStorage(uri)
elif uri.startswith('file'):
self.mgr = FileStorage(uri)
elif uri.startswith('avro'):
self.mgr = AvroStorage(uri)
else:
self.mgr = FileStorage(os.getenv('WMA_STORAGE_ROOT', '/tmp/wma_storage'))
self.stype = self.mgr.stype # determine storage type
