Esempio n. 1
0
    def process_aws_account(self, data, dtime_from, dtime_to):
        try:
            # Iterate over months from dtime_from to dtime_to
            dtime = dtime_from
            while dtime.month <= dtime_to.month:
                try:
                    csv_zip_file = self.download_aws_billing_file(
                        data['cred'], data['bucket_name'], date=dtime.date())
                    if csv_zip_file is None:
                        continue

                    with zipfile.ZipFile(csv_zip_file, 'r') as f:
                        f.extract(f.infolist()[0], self.tmp_dir)
                    csv_file = os.path.join(
                        self.tmp_dir,
                        os.path.basename(csv_zip_file.strip('.zip')))

                    for rows in self.csv_reader(csv_file, dtime_from=dtime):
                        records = self.get_aws_records(rows)
                        records = [
                            rec for rec in records
                            if int(rec['env_id']) in data['envs_ids']
                        ]
                        if records:
                            with self._lock:
                                min_records_dtime = min(
                                    [record['dtime'] for record in records])
                                if self.aws_billing_dtime_from:
                                    self.aws_billing_dtime_from = min(
                                        self.aws_billing_dtime_from,
                                        min_records_dtime)
                                else:
                                    self.aws_billing_dtime_from = min_records_dtime
                        for record in records:
                            self.pool.wait()
                            if self.args['--recalculate']:
                                self.pool.apply_async(
                                    self.analytics.update_record, (record, ))
                            else:
                                self.pool.apply_async(
                                    self.analytics.insert_record, (record, ))
                            gevent.sleep(0)  # force switch
                except:
                    msg = 'AWS billing for environments {0}, month {1} failed'
                    msg = msg.format(data['envs_ids'], dtime.month)
                    LOG.exception(msg)
                finally:
                    dtime = helper.next_month(dtime)
        except:
            msg = 'AWS billing for environments {0} failed'
            msg = msg.format(data['envs_ids'])
            LOG.exception(msg)
Esempio n. 2
0
 def process_envs(self, envs, dtime_from, dtime_to):
     try:
         dtime = dtime_from
         while dtime <= dtime_to:
             for csv_file, csv_file_envs in self.csv_files(envs, date=dtime.date()):
                 try:
                     self.process_csv_file(csv_file, csv_file_envs, dtime_from=dtime, dtime_to=dtime_to)
                 except:
                     msg = 'Processing CSV file: {}, environments: {} failed'
                     msg = msg.format(csv_file, [env['id'] for env in csv_file_envs])
                     helper.handle_error(message=msg)
             dtime = helper.next_month(dtime)
     except:
         msg = 'AWS billing for environments {} failed'
         msg = msg.format([env['id'] for env in envs])
         helper.handle_error(message=msg)
    def process_aws_account(self, data, dtime_from, dtime_to):
        try:
            # Iterate over months from dtime_from to dtime_to
            dtime = dtime_from
            while dtime.month <= dtime_to.month:
                try:
                    csv_zip_file = self.download_aws_billing_file(data['cred'],
                                                                  data['bucket_name'],
                                                                  date=dtime.date())
                    if csv_zip_file is None:
                        continue

                    with zipfile.ZipFile(csv_zip_file, 'r') as f:
                        f.extract(f.infolist()[0], self.tmp_dir)
                    csv_file = os.path.join(self.tmp_dir,
                                            os.path.basename(csv_zip_file.strip('.zip')))

                    for rows in self.csv_reader(csv_file, dtime_from=dtime):
                        records = self.get_aws_records(rows)
                        records = [rec for rec in records if int(rec['env_id']) in data['envs_ids']]
                        if records:
                            with self._lock:
                                min_records_dtime = min([record['dtime'] for record in records])
                                if self.aws_billing_dtime_from:
                                    self.aws_billing_dtime_from = min(self.aws_billing_dtime_from,
                                                                      min_records_dtime)
                                else:
                                    self.aws_billing_dtime_from = min_records_dtime
                        for record in records:
                            self.pool.wait()
                            if self.args['--recalculate']:
                                self.pool.apply_async(self.analytics.update_record, (record,))
                            else:
                                self.pool.apply_async(self.analytics.insert_record, (record,))
                            gevent.sleep(0)  # force switch
                except:
                    msg = 'AWS billing for environments {0}, month {1} failed'
                    msg = msg.format(data['envs_ids'], dtime.month)
                    LOG.exception(msg)
                finally:
                    dtime = helper.next_month(dtime)
        except:
            msg = 'AWS billing for environments {0} failed'
            msg = msg.format(data['envs_ids'])
            LOG.exception(msg)