def test_dateformat(self):
"Test dateformat function"
date1 = "20150101"
res = dateformat(date1)
self.assertEqual(len(str(res)), 10) # it should be 10 digits
wrong = "Some weird format"
self.assertRaises(Exception, dateformat, wrong)
wrong = "2012"
self.assertRaises(Exception, dateformat, wrong)
date2 = "2d"
res = dateformat(date1)
self.assertEqual(len(str(res)), 10) # it should be 10 digits
def test_dateformat(self):
"Test dateformat function"
date1 = '20150101'
res = dateformat(date1)
self.assertEqual(len(str(res)), 10) # it should be 10 digits
wrong = 'Some weird format'
self.assertRaises(Exception, dateformat, wrong)
wrong = '2012'
self.assertRaises(Exception, dateformat, wrong)
date2 = '2d'
res = dateformat(date1)
self.assertEqual(len(str(res)), 10) # it should be 10 digits
def use_lts(trange, thr):
"""
Helper function to determine based on given time range either
to use Short Term Storage or Long Term Storage
"""
# check if max time is less than given threshold
maxt = dateformat(trange[1])
if time.time()-maxt > thr:
return True
return False
def use_lts(trange, thr):
"""
Helper function to determine based on given time range either
to use Short Term Storage or Long Term Storage
"""
# check if max time is less than given threshold
maxt = dateformat(trange[1])
if time.time() - maxt > thr:
return True
return False
def cleanup(muri, tst, stype):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}}
rdocs = mstg.ndocs(query)
tdocs = time.time()-time0
print(tstamp('mongo2avro'), 'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0)))
time0 = time.time()
response = mstg.remove(query)
print(tstamp('mongo2avro'), 'remove query %s in %s' % (query, elapsed_time(time0)))
def cleanup(muri, tst, stype, verbose):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}}
if verbose:
print("Clean-up records in MongoDB: %s" % muri)
print("MongoDB cleanup spec:", query)
response = mstg.remove(query)
if verbose:
print("response: %s" % response)
print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype, verbose):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}}
if verbose:
print("Clean-up records in MongoDB: %s" % muri)
print("MongoDB cleanup spec:", query)
response = mstg.remove(query)
if verbose:
print("response: %s" % response)
print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}}
rdocs = mstg.ndocs(query)
tdocs = time.time() - time0
print(tstamp('mongo2avro'),
'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0)))
time0 = time.time()
response = mstg.remove(query)
print(tstamp('mongo2avro'),
'remove query %s in %s' % (query, elapsed_time(time0)))
