def handle(self, args):
# Connect to database
db.connect()
# Expand vars and user on the location passed
root = os.path.expanduser(args.location[0])
root = os.path.expandvars(root)
# Create the exporter object
exporter = MongoExporter(root,
categories=args.categories,
scheme=args.scheme)
# If list categories is true, list them and exit.
if args.list_categories:
return "\n".join(sorted(exporter.categories))
with Timer() as t:
exporter.export()
return ("Baleen corpus export complete in {}\n"
"Exported {} posts in {} categories\n"
"More information is in README in {}").format(
t, sum(exporter.counts.values()), len(exporter.categories),
root)
def timer_wrapper(*args, **kwargs):
"""
Inner function that uses the Timer context object
"""
with Timer() as timer:
result = func(*args, **kwargs)
return result, timer
def ingest(self):
"""
Subclasses do not typically override the ingest method. Instead they
will override the process hooks for start, failed, and finish, or the
process method directly.
"""
# Set a unique job id for every time run is called.
# The job id is based on the hostname and a time sequence.
self.jobid = uuid.uuid1()
# Call the started hook for logging and notification.
self.started()
# Time how long it takes to perform the processing
with Timer() as self.timer:
try:
self.process()
except Exception as e:
# If something goes wrong, call the failed hook, then raise.
self.failed(e)
raise
# Call the finished hook for logging and notification.
self.finished()