def do_iteration(self): for envs in self.analytics.load_envs(): try: self.analytics.load_env_credentials(envs) unique = {} for env in envs: try: credentials = self.analytics.get_credentials([env]) for cred in credentials: if cred.platform == 'ec2' and env.get( 'ec2.detailed_billing.enabled', '0') == '1': continue unique.setdefault(cred.unique, { 'envs_ids': [], 'cred': cred }) unique[cred.unique]['envs_ids'].append(env['id']) except: msg = 'Processing environment: {} failed'.format( env['id']) LOG.exception(msg) for data in unique.values(): while len(self.pool) > self.config['pool_size'] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(process_credential, args=(data['cred'], ), kwds={'envs_ids': data['envs_ids']}) gevent.sleep(0) # force switch except: msg = 'Processing environments: {} failed'.format( [env['id'] for env in envs]) LOG.exception(msg) self.pool.join()
def _handle_exception(e, msg): if isinstance(e, boto.exception.EC2ResponseError) and e.status in (401, 403): LOG.warning(msg) elif isinstance(e, (libcloud.common.types.InvalidCredsError, libcloud.common.types.LibcloudError, libcloud.common.types.MalformedResponseError, oauth2client.client.AccessTokenRefreshError, gevent.timeout.Timeout, socket.timeout, socket.gaierror)): LOG.warning(msg) elif isinstance(e, socket.error) and e.errno in (110, 111, 113): LOG.warning(msg) elif isinstance(e, googleapiclient.errors.HttpError) and e.resp['status'] in ( '403', ): LOG.warning(msg) elif isinstance(e, ssl.SSLError): LOG.warning(msg) elif isinstance(e, greenlet.GreenletExit): pass elif 'userDisabled' in str(e): LOG.warning(msg) elif isinstance(e, exceptions.MissingCredentialsError): LOG.debug(msg) else: LOG.exception(msg)
def do_iteration(self): for envs in self.analytics.load_envs(): msg = "Processing environments: {}".format([env["id"] for env in envs]) LOG.debug(msg) try: self.analytics.load_env_credentials(envs) unique = {} for env in envs: try: credentials = self.analytics.get_credentials([env]) for cred in credentials: if cred.platform == "ec2" and env.get("ec2.detailed_billing.enabled", "0") == "1": continue unique.setdefault(cred.unique, {"envs_ids": [], "cred": cred}) unique[cred.unique]["envs_ids"].append(env["id"]) except: msg = "Processing environment: {} failed".format(env["id"]) LOG.exception(msg) for data in unique.values(): while len(self.pool) > self.config["pool_size"] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(process_credential, args=(data["cred"],), kwds={"envs_ids": data["envs_ids"]}) gevent.sleep(0) # force switch except: msg = "Processing environments: {} failed".format([env["id"] for env in envs]) LOG.exception(msg) self.pool.join()
def do_iteration(self): for envs in self.analytics.load_envs(): msg = "Processing environments: {}".format([env['id'] for env in envs]) LOG.debug(msg) try: self.analytics.load_env_credentials(envs) unique = {} for env in envs: try: credentials = self.analytics.get_credentials([env]) for cred in credentials: if cred.platform == 'ec2' and env.get('ec2.detailed_billing.enabled', '0') == '1': continue unique.setdefault(cred.unique, {'envs_ids': [], 'cred': cred}) unique[cred.unique]['envs_ids'].append(env['id']) except: msg = 'Processing environment: {} failed'.format(env['id']) LOG.exception(msg) for data in unique.values(): while len(self.pool) > self.config['pool_size'] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(process_credential, args=(data['cred'],), kwds={'envs_ids': data['envs_ids']}) gevent.sleep(0) # force switch except: msg = 'Processing environments: {} failed'.format([env['id'] for env in envs]) LOG.exception(msg) self.pool.join()
def _handle_exception(e, msg): if isinstance(e, boto.exception.EC2ResponseError) and e.status in (401, 403): LOG.warning(msg) elif isinstance( e, ( libcloud.common.types.InvalidCredsError, libcloud.common.types.LibcloudError, libcloud.common.types.MalformedResponseError, libcloud.common.exceptions.BaseHTTPError, oauth2client.client.AccessTokenRefreshError, gevent.timeout.Timeout, socket.timeout, socket.gaierror, ), ): LOG.warning(msg) elif isinstance(e, socket.error): LOG.warning(msg) elif isinstance(e, googleapiclient.errors.HttpError) and e.resp["status"] in ("403",): LOG.warning(msg) elif isinstance(e, ssl.SSLError): LOG.warning(msg) elif isinstance(e, greenlet.GreenletExit): pass elif "userDisabled" in str(e): LOG.warning(msg) elif isinstance(e, exceptions.MissingCredentialsError): LOG.debug(msg) else: LOG.exception(msg)
def __call__(self): self.change_permissions() while True: try: self.iteration_timestamp = time.time() self.before_iteration() g = self._do_iteration() try: g.get(timeout=self.iteration_timeout) except: self.on_iteration_error() raise finally: if not g.ready(): g.kill() self.after_iteration() iteration_time = time.time() - self.iteration_timestamp msg = 'End iteration: {0:.1f} seconds'.format(iteration_time) LOG.debug(msg) except: LOG.exception('Iteration failed') time.sleep(self.error_sleep) finally: if self.config['interval']: next_iteration_time = self.iteration_timestamp + self.config['interval'] time.sleep(next_iteration_time - time.time())
def process_aws_billing(self): if self.args['--recalculate']: return dtime_from, dtime_to = self.get_aws_billing_interval() msg = 'AWS billing interval: {0} - {1}' msg = msg.format(dtime_from, dtime_to) LOG.info(msg) for envs in self.analytics.load_envs(): unique = {} for env in envs: if env.get('ec2.detailed_billing.enabled', '0') != '1': continue bucket_name = env['ec2.detailed_billing.bucket'] creds = self.analytics.get_creds([env]) cred = next(cred for cred in creds if cred.platform == 'ec2') unique.setdefault(cred.unique, { 'envs_ids': [], 'cred': cred, 'bucket_name': bucket_name }) unique[cred.unique]['envs_ids'].append(env['id']) for data in unique.values(): while len(self.pool) > self.config['pool_size'] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(self.process_aws_account, args=(data, dtime_from, dtime_to)) self.pool.join() if not self.aws_billing_dtime_from: return dtime_from = self.aws_billing_dtime_from if self.config['dtime_to']: dtime_to = self.config['dtime_to'] else: dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta( hours=1) dtime_to = dtime_hour_ago.replace(minute=59, second=59, microsecond=999999) # fill farm_usage_d dtime_cur = dtime_from msg = 'AWS fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour, platform='ec2') except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format( date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1)
def process_aws_billing(self): if self.args['--recalculate']: return dtime_from, dtime_to = self.get_aws_billing_interval() msg = 'AWS billing interval: {0} - {1}' msg = msg.format(dtime_from, dtime_to) LOG.debug(msg) with self._lock: if not self.aws_billing_dtime_from: self.aws_billing_dtime_from = dtime_from else: self.aws_billing_dtime_from = min(self.aws_billing_dtime_from, dtime_from) for envs in self.analytics.load_envs(): unique = {} for env in envs: if env.get('ec2.detailed_billing.enabled', '0') != '1': continue bucket_name = env['ec2.detailed_billing.bucket'] creds = self.analytics.get_creds([env]) cred = next(cred for cred in creds if cred.platform == 'ec2') unique.setdefault(cred.unique, {'envs_ids': [], 'cred': cred, 'bucket_name': bucket_name}) unique[cred.unique]['envs_ids'].append(env['id']) for data in unique.values(): while len(self.pool) > self.config['pool_size'] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(self.process_aws_account, args=(data, dtime_from, dtime_to)) self.pool.join() if not self.aws_billing_dtime_from: return dtime_from = self.aws_billing_dtime_from if self.config['dtime_to']: dtime_to = self.config['dtime_to'] else: dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=1) dtime_to = dtime_hour_ago.replace(minute=59, second=59, microsecond=999999) # fill farm_usage_d dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour) except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1)
def main(): app = AnalyticsProcessing() try: app.load_config() app.configure() app.run() except exceptions.AlreadyRunningError: LOG.info(helper.exc_info()) except (SystemExit, KeyboardInterrupt): pass except: LOG.exception('Oops')
def process_aws_account(self, data, dtime_from, dtime_to): try: # Iterate over months from dtime_from to dtime_to dtime = dtime_from while dtime.month <= dtime_to.month: try: csv_zip_file = self.download_aws_billing_file( data['cred'], data['bucket_name'], date=dtime.date()) if csv_zip_file is None: continue with zipfile.ZipFile(csv_zip_file, 'r') as f: f.extract(f.infolist()[0], self.tmp_dir) csv_file = os.path.join( self.tmp_dir, os.path.basename(csv_zip_file.strip('.zip'))) for rows in self.csv_reader(csv_file, dtime_from=dtime): records = self.get_aws_records(rows) records = [ rec for rec in records if int(rec['env_id']) in data['envs_ids'] ] if records: with self._lock: min_records_dtime = min( [record['dtime'] for record in records]) if self.aws_billing_dtime_from: self.aws_billing_dtime_from = min( self.aws_billing_dtime_from, min_records_dtime) else: self.aws_billing_dtime_from = min_records_dtime for record in records: self.pool.wait() if self.args['--recalculate']: self.pool.apply_async( self.analytics.update_record, (record, )) else: self.pool.apply_async( self.analytics.insert_record, (record, )) gevent.sleep(0) # force switch except: msg = 'AWS billing for environments {0}, month {1} failed' msg = msg.format(data['envs_ids'], dtime.month) LOG.exception(msg) finally: dtime = helper.next_month(dtime) except: msg = 'AWS billing for environments {0} failed' msg = msg.format(data['envs_ids']) LOG.exception(msg)
def main(): global app app = SzrUpdService() try: app.load_config() app.configure() app.run() except exceptions.AlreadyRunningError: LOG.info(helper.exc_info(where=False)) except (SystemExit, KeyboardInterrupt): pass except: LOG.exception('Oops')
def main(): global app app = LoadStatistics() try: app.load_config() app.configure() app.run() except exceptions.AlreadyRunningError: LOG.info(helper.exc_info(where=False)) except (SystemExit, KeyboardInterrupt): pass except: LOG.exception('Oops')
def _do_iteration(self): try: return self.do_iteration() except SystemExit: if sys.exc_info()[1].args[0] != 0: raise except exceptions.QuitError: return except KeyboardInterrupt: raise except exceptions.NothingToDoError: time.sleep(self.nothing_to_do_sleep) except: if logging.getLevelName(self.args['--verbosity']) < logging.ERROR: LOG.exception(helper.exc_info(where=False)) raise
def do_iteration(self): try: self.process_poller_billing() except: msg = 'Unable to process poller_sessions table, reason: {0}' msg = msg.format(helper.exc_info(where=False)) LOG.exception(msg) try: self.process_aws_billing() except: msg = 'Unable to process AWS billing information, reason: {0}' msg = msg.format(helper.exc_info(where=False)) LOG.exception(msg) if self.args['--recalculate']: sys.exit(0)
def run(self): try: self.configure() t = self._serve_forever() time.sleep(5) # change permissions if self.config['group']: helper.set_gid(self.config['group']) if self.config['user']: helper.set_uid(self.config['user']) LOG.info('Plotter started') t.join() except: LOG.exception(helper.exc_info())
def process_aws_account(self, data, dtime_from, dtime_to): try: # Iterate over months from dtime_from to dtime_to dtime = dtime_from while dtime.month <= dtime_to.month: try: csv_zip_file = self.download_aws_billing_file(data['cred'], data['bucket_name'], date=dtime.date()) if csv_zip_file is None: continue with zipfile.ZipFile(csv_zip_file, 'r') as f: f.extract(f.infolist()[0], self.tmp_dir) csv_file = os.path.join(self.tmp_dir, os.path.basename(csv_zip_file.strip('.zip'))) for rows in self.csv_reader(csv_file, dtime_from=dtime): records = self.get_aws_records(rows) records = [rec for rec in records if int(rec['env_id']) in data['envs_ids']] if records: with self._lock: min_records_dtime = min([record['dtime'] for record in records]) if self.aws_billing_dtime_from: self.aws_billing_dtime_from = min(self.aws_billing_dtime_from, min_records_dtime) else: self.aws_billing_dtime_from = min_records_dtime for record in records: self.pool.wait() if self.args['--recalculate']: self.pool.apply_async(self.analytics.update_record, (record,)) else: self.pool.apply_async(self.analytics.insert_record, (record,)) gevent.sleep(0) # force switch except: msg = 'AWS billing for environments {0}, month {1} failed' msg = msg.format(data['envs_ids'], dtime.month) LOG.exception(msg) finally: dtime = helper.next_month(dtime) except: msg = 'AWS billing for environments {0} failed' msg = msg.format(data['envs_ids']) LOG.exception(msg)
def get_szr_ver_from_repo(self, devel_branch=None, force=False): out = {} if devel_branch: if 'devel_repos' not in self.scalr_config['scalarizr_update']: return out if not force: try: return self.get_szr_ver_from_repo.im_func.devel_cache[ devel_branch] except AttributeError: self.get_szr_ver_from_repo.im_func.devel_cache = {} except KeyError: pass norm_branch = devel_branch.replace('/', '-').replace('_', '-') repos = self.scalr_config['scalarizr_update']['devel_repos'] else: if not force: try: return self.get_szr_ver_from_repo.im_func.cache except AttributeError: pass norm_branch = None repos = self.scalr_config['scalarizr_update']['repos'] for repo_type, repo in repos.iteritems(): for k, func in { 'deb': self.ver_from_deb_repo, 'rpm': self.ver_from_rpm_repo, 'win': self.ver_from_win_repo }.iteritems(): try: data = func(repo, branch=norm_branch) if data: out.update(data) out.setdefault(repo_type, {})[k] = data.values()[0] except: msg = '{0} repository {1} failed'.format(k, repo_type) LOG.exception(msg) if devel_branch: self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out else: self.get_szr_ver_from_repo.im_func.cache = out return out
def get_szr_ver_from_repo(self, devel_branch=None, force=False): out = {} if devel_branch: if "devel_repos" not in self.scalr_config["scalarizr_update"]: return out if not force: try: return self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] except AttributeError: self.get_szr_ver_from_repo.im_func.devel_cache = {} except KeyError: pass norm_branch = devel_branch.replace("/", "-").replace("_", "-") repos = self.scalr_config["scalarizr_update"]["devel_repos"] else: if not force: try: return self.get_szr_ver_from_repo.im_func.cache except AttributeError: pass norm_branch = None repos = self.scalr_config["scalarizr_update"]["repos"] for repo_type, repo in repos.iteritems(): for k, func in { "deb": self.ver_from_deb_repo, "rpm": self.ver_from_rpm_repo, "win": self.ver_from_win_repo, }.iteritems(): try: data = func(repo, branch=norm_branch) if data: out.update(data) out.setdefault(repo_type, {})[k] = data.values()[0] except: msg = "{0} repository {1} failed".format(k, repo_type) LOG.exception(msg) if devel_branch: self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out else: self.get_szr_ver_from_repo.im_func.cache = out return out
def get_szr_ver_from_repo(self, devel_branch=None, force=False): out = {} if devel_branch: if 'devel_repos' not in self.scalr_config['scalarizr_update']: return out if not force: try: return self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] except AttributeError: self.get_szr_ver_from_repo.im_func.devel_cache = {} except KeyError: pass norm_branch = devel_branch.replace('/', '-').replace('_', '-') repos = self.scalr_config['scalarizr_update']['devel_repos'] else: if not force: try: return self.get_szr_ver_from_repo.im_func.cache except AttributeError: pass norm_branch = None repos = self.scalr_config['scalarizr_update']['repos'] for repo_type, repo in repos.iteritems(): for k, func in { 'deb': self.ver_from_deb_repo, 'rpm': self.ver_from_rpm_repo, 'win': self.ver_from_win_repo}.iteritems(): try: data = func(repo, branch=norm_branch) if data: out.update(data) out.setdefault(repo_type, {})[k] = data.values()[0] except: msg = '{0} repository {1} failed'.format(k, repo_type) LOG.exception(msg) if devel_branch: self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out else: self.get_szr_ver_from_repo.im_func.cache = out return out
def run(self): try: self.configure() t = self._serve_forever() while not t.is_alive(): time.sleep(0.5) # wait before change permissions to allow cherrypy read certificates time.sleep(2) # change permissions if self.config['group']: helper.set_gid(self.config['group']) if self.config['user']: helper.set_uid(self.config['user']) LOG.info('Plotter started') t.join() except: LOG.exception(helper.exc_info())
def _handle_exception(e, msg): if isinstance(e, boto.exception.EC2ResponseError) and e.status in (401, 403): LOG.warning(msg) elif isinstance(e, (libcloud.common.types.InvalidCredsError, libcloud.common.types.LibcloudError, libcloud.common.types.MalformedResponseError, oauth2client.client.AccessTokenRefreshError, gevent.timeout.Timeout, socket.timeout, socket.gaierror)): LOG.warning(msg) elif isinstance(e, socket.error) and e.errno in (110, 111, 113): LOG.warning(msg) elif isinstance(e, googleapiclient.errors.HttpError) and e.resp['status'] in ('403'): LOG.warning(msg) elif isinstance(e, ssl.SSLError): LOG.warning(msg) elif isinstance(e, greenlet.GreenletExit): pass elif 'userDisabled' in str(e): LOG.warning(msg) else: LOG.exception(msg)
def wrapper(*args, **kwds): try: return f(*args, **kwds) except: LOG.exception('Exception') raise
def db_update(sorted_data, envs_ids, cred): platform = cred.platform for env_id in envs_ids: for region_data in sorted_data: try: sid = uuid.uuid4() if platform == 'ec2' and 'account_id' in cred: cloud_account = cryptotool.decrypt_scalr(app.crypto_key, cred['account_id']) else: cloud_account = None if analytics.url_key_map[platform]: url = urlparse.urlparse(cryptotool.decrypt_scalr( app.crypto_key, cred[analytics.url_key_map[platform]]).rstrip('/')) url = '%s%s' % (url.netloc, url.path) else: url = '' query = ( "SELECT client_id " "FROM client_environments " "WHERE id={env_id}" ).format(env_id=env_id) results = app.scalr_db.execute(query, retries=1) account_id = results[0]['client_id'] query = ( "INSERT IGNORE INTO poller_sessions " "(sid, account_id, env_id, dtime, platform, url, cloud_location, cloud_account) " "VALUES " "(UNHEX('{sid}'), {account_id}, {env_id}, '{dtime}', '{platform}', '{url}'," "'{cloud_location}', '{cloud_account}')" ).format( sid=sid.hex, account_id=account_id, env_id=env_id, dtime=time.strftime( "%Y-%m-%d %H:%M:%S", time.gmtime(region_data['timestamp'])), platform=platform, url=url, cloud_location=region_data['region'], cloud_account=cloud_account ) app.analytics_db.execute(query, retries=1) # managed for managed in region_data['managed']: if managed['env_id'] != env_id: continue query = ( "INSERT IGNORE INTO managed " "(sid, server_id, instance_type, os) VALUES " "(UNHEX('{sid}'), UNHEX('{server_id}'), '{instance_type}', {os})" ).format( sid=sid.hex, server_id=uuid.UUID(managed['server_id']).hex, instance_type=managed['instance_type'], os=managed['os']) LOG.debug(query) app.analytics_db.execute(query, retries=1) ## not_managed #if region_data['not_managed']: # base_query = ( # "INSERT IGNORE INTO notmanaged " # "(sid, instance_id, instance_type, os) VALUES %s") # values_template = "(UNHEX('{sid}'), '{instance_id}', '{instance_type}', {os})" # i, chunk_size = 0, 20 # while True: # chunk_not_managed = region_data['not_managed'][ # i * chunk_size:(i + 1) * chunk_size] # if not chunk_not_managed: # break # query = base_query % ','.join( # [ # values_template.format( # sid=sid.hex, # instance_id=not_managed['instance_id'], # instance_type=not_managed['instance_type'], # os=not_managed['os'] # ) # for not_managed in chunk_not_managed # ] # ) # app.analytics_db.execute(query, retries=1) # i += 1 except: msg = 'Database update failed, reason: {0}' msg = msg.format(helper.exc_info()) LOG.exception(msg)
def do_iteration(self): try: dtime_from, dtime_to = self._get_processing_dtime() dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: if self.config['recalculate']: self.recalculate(date, hour) else: self.calculate(date, hour) except KeyboardInterrupt: raise except: LOG.error(helper.exc_info()) dtime_cur += datetime.timedelta(seconds=3600) self._pool.join() if not self.config['recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.info(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) try: self.analytics.recalculate_nm_usage_d(date, self.config['platform']) except: msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}" msg = msg.format(year, quarter, helper.exc_info()) LOG.warning(msg) except: if self.config['recalculate']: LOG.exception(helper.exc_info()) sys.exit(1) else: raise # quit from iteration loop raise exceptions.QuitError()
def process_poller_billing(self): dtime_from, dtime_to = self.get_poller_billing_interval() LOG.info('Poller billing interval: {0} - {1}'.format(dtime_from, dtime_to)) # process poller_session table dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: msg = "Process poller data, date {0}, hour {1}".format(date, hour) LOG.info(msg) if self.args['--recalculate']: platform = self.config['platform'] generator = self.analytics.get_records(date, hour, platform) else: generator = self.analytics.get_servers(date, hour) for records in generator: LOG.debug('Records for processing: %s' % len(records)) prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices(record, prices) or 0 self.pool.wait() if self.args['--recalculate']: record['cost'] = float(cost) * int(record['num']) self.pool.apply_async(self.analytics.update_record, (record,)) else: record['cost'] = cost record['num'] = 1.0 record['cost_distr_type'] = 1 self.pool.apply_async(self.analytics.insert_record, (record,)) gevent.sleep(0) # force switch self.pool.join() except: msg = "Unable to process date {0}, hour {1}".format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) # fill farm_usage_d dtime_cur = dtime_from msg = 'Poller fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour) except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) if not self.args['--recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.debug(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, error: {1}".format( date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed" msg = msg.format(year, quarter, helper.exc_info()) LOG.exception(msg)
def do_iteration(self): try: dtime_from, dtime_to = self._get_processing_dtime() dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: if self.config['recalculate']: self.recalculate(date, hour) else: self.calculate(date, hour) except KeyboardInterrupt: raise except: LOG.error(helper.exc_info()) dtime_cur += datetime.timedelta(seconds=3600) self._pool.join() if not self.config['recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.info(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) try: self.analytics.recalculate_nm_usage_d( date, self.config['platform']) except: msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date( dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}" msg = msg.format(year, quarter, helper.exc_info()) LOG.warning(msg) except: if self.config['recalculate']: LOG.exception(helper.exc_info()) sys.exit(1) else: raise # quit from iteration loop raise exceptions.QuitError()
def process_poller_billing(self): dtime_from, dtime_to = self.get_poller_billing_interval() LOG.info('Poller billing interval: {0} - {1}'.format( dtime_from, dtime_to)) # process poller_session table dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: msg = "Process poller data, date {0}, hour {1}".format( date, hour) LOG.info(msg) if self.args['--recalculate']: platform = self.config['platform'] generator = self.analytics.get_records( date, hour, platform) else: generator = self.analytics.get_servers(date, hour) for records in generator: LOG.debug('Records for processing: %s' % len(records)) prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices( record, prices) or 0 self.pool.wait() if self.args['--recalculate']: record['cost'] = float(cost) * int(record['num']) self.pool.apply_async(self.analytics.update_record, (record, )) else: record['cost'] = cost record['num'] = 1.0 record['cost_distr_type'] = 1 self.pool.apply_async(self.analytics.insert_record, (record, )) gevent.sleep(0) # force switch self.pool.join() except: msg = "Unable to process date {0}, hour {1}".format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) # fill farm_usage_d dtime_cur = dtime_from msg = 'Poller fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour) except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format( date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) if not self.args['--recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.debug(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, error: {1}".format( date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed" msg = msg.format(year, quarter, helper.exc_info()) LOG.exception(msg)
def db_update(sorted_data, envs_ids, cred): platform = cred.platform for env_id in envs_ids: for region_data in sorted_data: try: sid = uuid.uuid4() if platform == 'ec2' and 'account_id' in cred: cloud_account = cryptotool.decrypt_scalr( app.crypto_key, cred['account_id']) else: cloud_account = None if analytics.url_key_map[platform]: url = urlparse.urlparse( cryptotool.decrypt_scalr( app.crypto_key, cred[analytics.url_key_map[platform]]).rstrip('/')) url = '%s%s' % (url.netloc, url.path) else: url = '' query = ("SELECT client_id " "FROM client_environments " "WHERE id={env_id}").format(env_id=env_id) results = app.scalr_db.execute(query, retries=1) account_id = results[0]['client_id'] query = ( "INSERT IGNORE INTO poller_sessions " "(sid, account_id, env_id, dtime, platform, url, cloud_location, cloud_account) " "VALUES " "(UNHEX('{sid}'), {account_id}, {env_id}, '{dtime}', '{platform}', '{url}'," "'{cloud_location}', '{cloud_account}')").format( sid=sid.hex, account_id=account_id, env_id=env_id, dtime=time.strftime( "%Y-%m-%d %H:%M:%S", time.gmtime(region_data['timestamp'])), platform=platform, url=url, cloud_location=region_data['region'], cloud_account=cloud_account) app.analytics_db.execute(query, retries=1) # managed for managed in region_data['managed']: if managed['env_id'] != env_id: continue query = ( "INSERT IGNORE INTO managed " "(sid, server_id, instance_type, os) VALUES " "(UNHEX('{sid}'), UNHEX('{server_id}'), '{instance_type}', {os})" ).format(sid=sid.hex, server_id=uuid.UUID(managed['server_id']).hex, instance_type=managed['instance_type'], os=managed['os']) LOG.debug(query) app.analytics_db.execute(query, retries=1) ## not_managed #if region_data['not_managed']: # base_query = ( # "INSERT IGNORE INTO notmanaged " # "(sid, instance_id, instance_type, os) VALUES %s") # values_template = "(UNHEX('{sid}'), '{instance_id}', '{instance_type}', {os})" # i, chunk_size = 0, 20 # while True: # chunk_not_managed = region_data['not_managed'][ # i * chunk_size:(i + 1) * chunk_size] # if not chunk_not_managed: # break # query = base_query % ','.join( # [ # values_template.format( # sid=sid.hex, # instance_id=not_managed['instance_id'], # instance_type=not_managed['instance_type'], # os=not_managed['os'] # ) # for not_managed in chunk_not_managed # ] # ) # app.analytics_db.execute(query, retries=1) # i += 1 except: msg = 'Database update failed, reason: {0}' msg = msg.format(helper.exc_info()) LOG.exception(msg)