Example #1
0
class FileStorageTest(unittest.TestCase):
    def setUp(self):
        self.tdir = tempfile.mkdtemp()
        self.mgr = FileStorage('fileio:%s' % self.tdir)
        data = {
            "int": 1,
            "float": 1.2,
            "list": [1, 2, 3],
            "dict": {
                "dname": "foo",
                "dval": 1
            },
            "listdict": [{
                "lname": "foo"
            }],
            "str": "string"
        }
        self.bare_data = dict(data)
        data['wmaid'] = wmaHash(data)
        data['stype'] = self.mgr.stype
        self.data = data

    def tearDown(self):
        "Tear down content of temp dir"
        for fname in os.listdir(self.tdir):
            os.remove(os.path.join(self.tdir, fname))
        os.rmdir(self.tdir)

    def test_write(self):
        "Test write functionality"
        wmaids = self.mgr.write(self.data)
        self.assertEqual(len(wmaids), 1)
        data = self.mgr.read(wmaids[0])
        self.assertEqual(data[0], self.bare_data)
Example #2
0
class STSManager(object):
    "Short-Term Storage manager based on Mongo/File/Avro storage back-end"

    def __init__(self, uri):
        "ctor with STS uri"
        if uri.startswith('mongo'):
            self.mgr = MongoStorage(uri)
        elif uri.startswith('file'):
            self.mgr = FileStorage(uri)
        elif uri.startswith('avro'):
            self.mgr = AvroStorage(uri)
        else:
            self.mgr = FileStorage(
                os.getenv('WMA_STORAGE_ROOT', '/tmp/wma_storage'))
        self.stype = self.mgr.stype  # determine storage type

    def qmap(self, spec, fields):
        "map input spec/fields into ones suitable for STS QL"
        return spec, fields

    def write(self, data, safe=None):
        "Write API for STS"
        return self.mgr.write(data, safe)

    def read(self, spec, fields=None):
        "Read API for STS"
        return self.mgr.read(spec, fields)

    def jobs(self):
        "Fetch jobs ID from underlying storage"
        return self.mgr.jobsids()

    def stats(self):
        "Return statistics about underlying storage"
        return self.mgr.stats()

    def adocs(self):
        "Return aggregated documents from underlying storage"
        return self.mgr.adocs()

    def status(self):
        "Return status api"
        return {
            'sts': {
                'mgr': str(self.mgr),
                'stype': self.stype,
                'stats': self.stats()
            }
        }
Example #3
0
    def setUp(self):
        self.tdir = tempfile.mkdtemp()
        self.mgr = FileStorage('fileio:%s' % self.tdir)
	data = {"int":1, "float":1.2, "list":[1,2,3],
		"dict":{"dname": "foo", "dval":1},
		"listdict":[{"lname":"foo"}], "str":"string"}
        self.bare_data = dict(data)
        data['wmaid'] = wmaHash(data)
        data['stype'] = self.mgr.stype
        self.data = data
Example #4
0
 def setUp(self):
     self.tdir = tempfile.mkdtemp()
     self.mgr = FileStorage('fileio:%s' % self.tdir)
     data = {
         "int": 1,
         "float": 1.2,
         "list": [1, 2, 3],
         "dict": {
             "dname": "foo",
             "dval": 1
         },
         "listdict": [{
             "lname": "foo"
         }],
         "str": "string"
     }
     self.bare_data = dict(data)
     data['wmaid'] = wmaHash(data)
     data['stype'] = self.mgr.stype
     self.data = data
Example #5
0
class STSManager(object):
    "Short-Term Storage manager based on Mongo/File/Avro storage back-end"
    def __init__(self, uri):
        "ctor with STS uri"
        if  uri.startswith('mongo'):
            self.mgr = MongoStorage(uri)
        elif uri.startswith('file'):
            self.mgr = FileStorage(uri)
        elif uri.startswith('avro'):
            self.mgr = AvroStorage(uri)
        else:
            self.mgr = FileStorage(os.getenv('WMA_STORAGE_ROOT', '/tmp/wma_storage'))
        self.stype = self.mgr.stype # determine storage type

    def qmap(self, spec, fields):
        "map input spec/fields into ones suitable for STS QL"
        return spec, fields

    def write(self, data, safe=None):
        "Write API for STS"
        return self.mgr.write(data, safe)

    def read(self, spec, fields=None):
        "Read API for STS"
        return self.mgr.read(spec, fields)

    def jobs(self):
        "Fetch jobs ID from underlying storage"
        return self.mgr.jobsids()

    def stats(self):
        "Return statistics about underlying storage"
        return self.mgr.stats()

    def performance(self, **kwargs):
        "Return aggregated documents from underlying storage"
        return self.mgr.performance(**kwargs)

    def status(self):
        "Return status api"
        return {'sts':{'mgr':str(self.mgr), 'stype':self.stype, 'stats':self.stats()}}
Example #6
0
class FileStorageTest(unittest.TestCase):
    def setUp(self):
        self.tdir = tempfile.mkdtemp()
        self.mgr = FileStorage('fileio:%s' % self.tdir)
	data = {"int":1, "float":1.2, "list":[1,2,3],
		"dict":{"dname": "foo", "dval":1},
		"listdict":[{"lname":"foo"}], "str":"string"}
        self.bare_data = dict(data)
        data['wmaid'] = wmaHash(data)
        data['stype'] = self.mgr.stype
        self.data = data

    def tearDown(self):
        "Tear down content of temp dir"
        for fname in os.listdir(self.tdir):
            os.remove(os.path.join(self.tdir, fname))
        os.rmdir(self.tdir)

    def test_write(self):
        "Test write functionality"
        wmaids = self.mgr.write(self.data)
        self.assertEqual(len(wmaids), 1)
        data = self.mgr.read(wmaids[0])
        self.assertEqual(data[0], self.bare_data)
Example #7
0
 def __init__(self, uri):
     "ctor with STS uri"
     if  uri.startswith('mongo'):
         self.mgr = MongoStorage(uri)
     elif uri.startswith('file'):
         self.mgr = FileStorage(uri)
     elif uri.startswith('avro'):
         self.mgr = AvroStorage(uri)
     else:
         self.mgr = FileStorage(os.getenv('WMA_STORAGE_ROOT', '/tmp/wma_storage'))
     self.stype = self.mgr.stype # determine storage type