def main(self):
self.es_storage = self.config.elasticsearch_storage_class(self.config)
hb_client = HBaseConnectionForCrashReports(
self.config.hbase_host,
self.config.hbase_port,
self.config.hbase_timeout,
)
current_date = self.config.end_date
one_day = datetime.timedelta(days=1)
for i in range(self.config.duration):
es_index = self.get_index_for_date(current_date)
day = current_date.strftime('%y%m%d')
self.config.logger.info('backfilling crashes for %s', day)
# First create the index if it doesn't already exist
self.es_storage.create_index(es_index)
reports = hb_client.get_list_of_processed_json_for_date(
day, number_of_retries=5)
crashes_to_index = []
for report in reports:
processed_crash = self.format_dates_in_crash(
json.loads(report))
# print 'storing %s' % processed_crash['uuid']
if len(crashes_to_index) > self.config.index_doc_number:
# print 'now indexing crashes! '
self.index_crashes(es_index, crashes_to_index)
crashes_to_index = []
crashes_to_index.append(processed_crash)
if len(crashes_to_index) > 0:
self.index_crashes(es_index, crashes_to_index)
current_date -= one_day
return 0
