class ProgressLogger:
def __init__(self, name='work', logger=None):
self.name = name
self.total_items = None
self.start_time = None
self.end_time = None
self.counter = AtomicCounter()
self.log_percentage_step = 5
self.log_items_step = 1000
if logger is not None:
self.logger = logger
else:
self.logger = logging.getLogger('ProgressLogger')
def start(self, total_items=None):
self.total_items = total_items
self.start_time = datetime.now()
start_message = 'Started {}.'.format(self.name)
if self.total_items is not None:
start_message = start_message + ' Items to process: {}.'.format(self.total_items)
self.logger.info(start_message)
# Check race conditions
def track(self, item_count=1):
processed_items = self.counter.increment(item_count)
processed_items_before = processed_items - item_count
track_message = None
if self.total_items is None:
if int(processed_items_before / self.log_items_step) != int(processed_items / self.log_items_step):
track_message = '{} items processed.'.format(processed_items)
else:
percentage = processed_items * 100 / self.total_items
percentage_before = processed_items_before * 100 / self.total_items
if int(percentage_before / self.log_percentage_step) != int(percentage / self.log_percentage_step):
track_message = '{} items processed. Progress is {}%'.format(processed_items, int(percentage)) + \
('!!!' if int(percentage) > 100 else '.')
if track_message is not None:
self.logger.info(track_message)
def finish(self):
duration = None
if self.start_time is not None:
self.end_time = datetime.now()
duration = self.end_time - self.start_time
finish_message = 'Finished {}. Total items processed: {}.'.format(self.name, self.counter.increment() - 1)
if duration is not None:
finish_message = finish_message + ' Took {}.'.format(str(duration))
self.logger.info(finish_message)
def __init__(self, name='work', logger=None):
self.name = name
self.total_items = None
self.start_time = None
self.end_time = None
self.counter = AtomicCounter()
self.log_percentage_step = 5
self.log_items_step = 1000
if logger is not None:
self.logger = logger
else:
self.logger = logging.getLogger('ProgressLogger')
def open(self):
for item_type, filename in self.filename_mapping.items():
file = get_file_handle(filename, binary=True)
fields = self.field_mapping[item_type]
self.file_mapping[item_type] = file
if str(filename).endswith('.json'):
item_exporter = JsonLinesItemExporter(file, fields_to_export=fields)
else:
item_exporter = CsvItemExporter(file, fields_to_export=fields)
self.exporter_mapping[item_type] = item_exporter
self.counter_mapping[item_type] = AtomicCounter()