def cleanup(muri, tst, stype):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}}
rdocs = mstg.ndocs(query)
tdocs = time.time()-time0
print(tstamp('mongo2avro'), 'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0)))
time0 = time.time()
response = mstg.remove(query)
print(tstamp('mongo2avro'), 'remove query %s in %s' % (query, elapsed_time(time0)))
def cleanup(muri, tst, stype):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}}
rdocs = mstg.ndocs(query)
tdocs = time.time() - time0
print(tstamp('mongo2avro'),
'found %s docs (in %s) to be removed' % (rdocs, elapsed_time(time0)))
time0 = time.time()
response = mstg.remove(query)
print(tstamp('mongo2avro'),
'remove query %s in %s' % (query, elapsed_time(time0)))
def cleanup(muri, tst, stype, verbose):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats': {'$lt': dateformat(tst)}}
if verbose:
print("Clean-up records in MongoDB: %s" % muri)
print("MongoDB cleanup spec:", query)
response = mstg.remove(query)
if verbose:
print("response: %s" % response)
print("Elapsed time: %s" % elapsed_time(time0))
def cleanup(muri, tst, stype, verbose):
"Cleanup data in MongoDB (muri) for given timestamp (tst)"
time0 = time.time()
mstg = MongoStorage(muri)
# remove records whose type is hdfsio, i.e. already migrated to HDFS,
# and whose time stamp is less than provided one
query = {'stype': stype, 'wmats':{'$lt': dateformat(tst)}}
if verbose:
print("Clean-up records in MongoDB: %s" % muri)
print("MongoDB cleanup spec:", query)
response = mstg.remove(query)
if verbose:
print("response: %s" % response)
print("Elapsed time: %s" % elapsed_time(time0))
