def test_dateformat(self): "Test dateformat function" date1 = "20150101" res = dateformat(date1) self.assertEqual(len(str(res)), 10) # it should be 10 digits wrong = "Some weird format" self.assertRaises(Exception, dateformat, wrong) wrong = "2012" self.assertRaises(Exception, dateformat, wrong) date2 = "2d" res = dateformat(date1) self.assertEqual(len(str(res)), 10) # it should be 10 digits
def test_dateformat(self): "Test dateformat function" date1 = '20150101' res = dateformat(date1) self.assertEqual(len(str(res)), 10) # it should be 10 digits wrong = 'Some weird format' self.assertRaises(Exception, dateformat, wrong) wrong = '2012' self.assertRaises(Exception, dateformat, wrong) date2 = '2d' res = dateformat(date1) self.assertEqual(len(str(res)), 10) # it should be 10 digits
def use_lts(trange, thr): """ Helper function to determine based on given time range either to use Short Term Storage or Long Term Storage """ # check if max time is less than given threshold maxt = dateformat(trange[1]) if time.time()-maxt > thr: return True return False
def use_lts(trange, thr): """ Helper function to determine based on given time range either to use Short Term Storage or Long Term Storage """ # check if max time is less than given threshold maxt = dateformat(trange[1]) if time.time() - maxt > thr: return True return False
def cleanup(muri, tst, stype): "Cleanup data in MongoDB (muri) for given timestamp (tst)" time0 = time.time() mstg = MongoStorage(muri) # remove records whose type is hdfsio, i.e. already migrated to HDFS, # and whose time stamp is less than provided one query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}} rdocs = mstg.ndocs(query) tdocs = time.time()-time0 print(tstamp('mongo2avro'), 'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0))) time0 = time.time() response = mstg.remove(query) print(tstamp('mongo2avro'), 'remove query %s in %s' % (query, elapsed_time(time0)))
def cleanup(muri, tst, stype, verbose): "Cleanup data in MongoDB (muri) for given timestamp (tst)" time0 = time.time() mstg = MongoStorage(muri) # remove records whose type is hdfsio, i.e. already migrated to HDFS, # and whose time stamp is less than provided one query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}} if verbose: print("Clean-up records in MongoDB: %s" % muri) print("MongoDB cleanup spec:", query) response = mstg.remove(query) if verbose: print("response: %s" % response) print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype, verbose): "Cleanup data in MongoDB (muri) for given timestamp (tst)" time0 = time.time() mstg = MongoStorage(muri) # remove records whose type is hdfsio, i.e. already migrated to HDFS, # and whose time stamp is less than provided one query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}} if verbose: print("Clean-up records in MongoDB: %s" % muri) print("MongoDB cleanup spec:", query) response = mstg.remove(query) if verbose: print("response: %s" % response) print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype): "Cleanup data in MongoDB (muri) for given timestamp (tst)" time0 = time.time() mstg = MongoStorage(muri) # remove records whose type is hdfsio, i.e. already migrated to HDFS, # and whose time stamp is less than provided one query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}} rdocs = mstg.ndocs(query) tdocs = time.time() - time0 print(tstamp('mongo2avro'), 'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0))) time0 = time.time() response = mstg.remove(query) print(tstamp('mongo2avro'), 'remove query %s in %s' % (query, elapsed_time(time0)))