class MongoStorageTest(unittest.TestCase):
def setUp(self):
uri = os.environ.get("WMA_MONGODB", "mongodb://localhost:8230")
self.dbname = "test_fwjr"
try:
self.mgr = MongoStorage(uri, dbname=self.dbname)
self.mgr.remove()
except:
self.mgr = None
print("WARNING: cannot connect to %s" % uri)
data = {
"int": 1,
"float": 1.2,
"list": [1, 2, 3],
"dict": {"dname": "foo", "dval": 1},
"listdict": [{"lname": "foo"}],
"str": "string",
}
self.bare_data = dict(data)
data["wmaid"] = wmaHash(data)
data["stype"] = "mongodb"
self.data = data
def tearDown(self):
"Tear down content of temp dir"
self.mgr.remove()
self.mgr.dropdb(self.dbname)
def test_write(self):
"Test write functionality"
if self.mgr:
wmaids = self.mgr.write(self.data)
self.assertEqual(len(wmaids), 1)
data = self.mgr.read(wmaids[0])
record = data[0]
for key in ["wmaid", "stype"]:
if key in record:
del record[key]
self.assertEqual(record, self.bare_data)
data = self.mgr.read(wmaids[0], ["dict"])
self.assertEqual(1, len(data))
self.assertEqual(data[0]["dict"], self.bare_data["dict"])
def cleanup(muri, tst, stype):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}}
rdocs = mstg.ndocs(query)
tdocs = time.time()-time0
print(tstamp('mongo2avro'), 'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0)))
time0 = time.time()
response = mstg.remove(query)
print(tstamp('mongo2avro'), 'remove query %s in %s' % (query, elapsed_time(time0)))
def cleanup(muri, tst, stype, verbose):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}}
if verbose:
print("Clean-up records in MongoDB: %s" % muri)
print("MongoDB cleanup spec:", query)
response = mstg.remove(query)
if verbose:
print("response: %s" % response)
print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype, verbose):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}}
if verbose:
print("Clean-up records in MongoDB: %s" % muri)
print("MongoDB cleanup spec:", query)
response = mstg.remove(query)
if verbose:
print("response: %s" % response)
print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}}
rdocs = mstg.ndocs(query)
tdocs = time.time() - time0
print(tstamp('mongo2avro'),
'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0)))
time0 = time.time()
response = mstg.remove(query)
print(tstamp('mongo2avro'),
'remove query %s in %s' % (query, elapsed_time(time0)))
