def process_aws_account(self, data, dtime_from, dtime_to):
try:
# Iterate over months from dtime_from to dtime_to
dtime = dtime_from
while dtime.month <= dtime_to.month:
try:
csv_zip_file = self.download_aws_billing_file(
data['cred'], data['bucket_name'], date=dtime.date())
if csv_zip_file is None:
continue
with zipfile.ZipFile(csv_zip_file, 'r') as f:
f.extract(f.infolist()[0], self.tmp_dir)
csv_file = os.path.join(
self.tmp_dir,
os.path.basename(csv_zip_file.strip('.zip')))
for rows in self.csv_reader(csv_file, dtime_from=dtime):
records = self.get_aws_records(rows)
records = [
rec for rec in records
if int(rec['env_id']) in data['envs_ids']
]
if records:
with self._lock:
min_records_dtime = min(
[record['dtime'] for record in records])
if self.aws_billing_dtime_from:
self.aws_billing_dtime_from = min(
self.aws_billing_dtime_from,
min_records_dtime)
else:
self.aws_billing_dtime_from = min_records_dtime
for record in records:
self.pool.wait()
if self.args['--recalculate']:
self.pool.apply_async(
self.analytics.update_record, (record, ))
else:
self.pool.apply_async(
self.analytics.insert_record, (record, ))
gevent.sleep(0) # force switch
except:
msg = 'AWS billing for environments {0}, month {1} failed'
msg = msg.format(data['envs_ids'], dtime.month)
LOG.exception(msg)
finally:
dtime = helper.next_month(dtime)
except:
msg = 'AWS billing for environments {0} failed'
msg = msg.format(data['envs_ids'])
LOG.exception(msg)
def process_envs(self, envs, dtime_from, dtime_to):
try:
dtime = dtime_from
while dtime <= dtime_to:
for csv_file, csv_file_envs in self.csv_files(envs, date=dtime.date()):
try:
self.process_csv_file(csv_file, csv_file_envs, dtime_from=dtime, dtime_to=dtime_to)
except:
msg = 'Processing CSV file: {}, environments: {} failed'
msg = msg.format(csv_file, [env['id'] for env in csv_file_envs])
helper.handle_error(message=msg)
dtime = helper.next_month(dtime)
except:
msg = 'AWS billing for environments {} failed'
msg = msg.format([env['id'] for env in envs])
helper.handle_error(message=msg)
def process_aws_account(self, data, dtime_from, dtime_to):
try:
# Iterate over months from dtime_from to dtime_to
dtime = dtime_from
while dtime.month <= dtime_to.month:
try:
csv_zip_file = self.download_aws_billing_file(data['cred'],
data['bucket_name'],
date=dtime.date())
if csv_zip_file is None:
continue
with zipfile.ZipFile(csv_zip_file, 'r') as f:
f.extract(f.infolist()[0], self.tmp_dir)
csv_file = os.path.join(self.tmp_dir,
os.path.basename(csv_zip_file.strip('.zip')))
for rows in self.csv_reader(csv_file, dtime_from=dtime):
records = self.get_aws_records(rows)
records = [rec for rec in records if int(rec['env_id']) in data['envs_ids']]
if records:
with self._lock:
min_records_dtime = min([record['dtime'] for record in records])
if self.aws_billing_dtime_from:
self.aws_billing_dtime_from = min(self.aws_billing_dtime_from,
min_records_dtime)
else:
self.aws_billing_dtime_from = min_records_dtime
for record in records:
self.pool.wait()
if self.args['--recalculate']:
self.pool.apply_async(self.analytics.update_record, (record,))
else:
self.pool.apply_async(self.analytics.insert_record, (record,))
gevent.sleep(0) # force switch
except:
msg = 'AWS billing for environments {0}, month {1} failed'
msg = msg.format(data['envs_ids'], dtime.month)
LOG.exception(msg)
finally:
dtime = helper.next_month(dtime)
except:
msg = 'AWS billing for environments {0} failed'
msg = msg.format(data['envs_ids'])
LOG.exception(msg)