def process_aws_account(self, data, dtime_from, dtime_to): try: # Iterate over months from dtime_from to dtime_to dtime = dtime_from while dtime.month <= dtime_to.month: try: csv_zip_file = self.download_aws_billing_file( data['cred'], data['bucket_name'], date=dtime.date()) if csv_zip_file is None: continue with zipfile.ZipFile(csv_zip_file, 'r') as f: f.extract(f.infolist()[0], self.tmp_dir) csv_file = os.path.join( self.tmp_dir, os.path.basename(csv_zip_file.strip('.zip'))) for rows in self.csv_reader(csv_file, dtime_from=dtime): records = self.get_aws_records(rows) records = [ rec for rec in records if int(rec['env_id']) in data['envs_ids'] ] if records: with self._lock: min_records_dtime = min( [record['dtime'] for record in records]) if self.aws_billing_dtime_from: self.aws_billing_dtime_from = min( self.aws_billing_dtime_from, min_records_dtime) else: self.aws_billing_dtime_from = min_records_dtime for record in records: self.pool.wait() if self.args['--recalculate']: self.pool.apply_async( self.analytics.update_record, (record, )) else: self.pool.apply_async( self.analytics.insert_record, (record, )) gevent.sleep(0) # force switch except: msg = 'AWS billing for environments {0}, month {1} failed' msg = msg.format(data['envs_ids'], dtime.month) LOG.exception(msg) finally: dtime = helper.next_month(dtime) except: msg = 'AWS billing for environments {0} failed' msg = msg.format(data['envs_ids']) LOG.exception(msg)
def process_envs(self, envs, dtime_from, dtime_to): try: dtime = dtime_from while dtime <= dtime_to: for csv_file, csv_file_envs in self.csv_files(envs, date=dtime.date()): try: self.process_csv_file(csv_file, csv_file_envs, dtime_from=dtime, dtime_to=dtime_to) except: msg = 'Processing CSV file: {}, environments: {} failed' msg = msg.format(csv_file, [env['id'] for env in csv_file_envs]) helper.handle_error(message=msg) dtime = helper.next_month(dtime) except: msg = 'AWS billing for environments {} failed' msg = msg.format([env['id'] for env in envs]) helper.handle_error(message=msg)
def process_aws_account(self, data, dtime_from, dtime_to): try: # Iterate over months from dtime_from to dtime_to dtime = dtime_from while dtime.month <= dtime_to.month: try: csv_zip_file = self.download_aws_billing_file(data['cred'], data['bucket_name'], date=dtime.date()) if csv_zip_file is None: continue with zipfile.ZipFile(csv_zip_file, 'r') as f: f.extract(f.infolist()[0], self.tmp_dir) csv_file = os.path.join(self.tmp_dir, os.path.basename(csv_zip_file.strip('.zip'))) for rows in self.csv_reader(csv_file, dtime_from=dtime): records = self.get_aws_records(rows) records = [rec for rec in records if int(rec['env_id']) in data['envs_ids']] if records: with self._lock: min_records_dtime = min([record['dtime'] for record in records]) if self.aws_billing_dtime_from: self.aws_billing_dtime_from = min(self.aws_billing_dtime_from, min_records_dtime) else: self.aws_billing_dtime_from = min_records_dtime for record in records: self.pool.wait() if self.args['--recalculate']: self.pool.apply_async(self.analytics.update_record, (record,)) else: self.pool.apply_async(self.analytics.insert_record, (record,)) gevent.sleep(0) # force switch except: msg = 'AWS billing for environments {0}, month {1} failed' msg = msg.format(data['envs_ids'], dtime.month) LOG.exception(msg) finally: dtime = helper.next_month(dtime) except: msg = 'AWS billing for environments {0} failed' msg = msg.format(data['envs_ids']) LOG.exception(msg)