def do_iteration(self):
for envs in self.analytics.load_envs():
try:
self.analytics.load_env_credentials(envs)
unique = {}
for env in envs:
try:
credentials = self.analytics.get_credentials([env])
for cred in credentials:
if cred.platform == 'ec2' and env.get(
'ec2.detailed_billing.enabled',
'0') == '1':
continue
unique.setdefault(cred.unique, {
'envs_ids': [],
'cred': cred
})
unique[cred.unique]['envs_ids'].append(env['id'])
except:
msg = 'Processing environment: {} failed'.format(
env['id'])
LOG.exception(msg)
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(process_credential,
args=(data['cred'], ),
kwds={'envs_ids': data['envs_ids']})
gevent.sleep(0) # force switch
except:
msg = 'Processing environments: {} failed'.format(
[env['id'] for env in envs])
LOG.exception(msg)
self.pool.join()
def _handle_exception(e, msg):
if isinstance(e,
boto.exception.EC2ResponseError) and e.status in (401, 403):
LOG.warning(msg)
elif isinstance(e,
(libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
oauth2client.client.AccessTokenRefreshError,
gevent.timeout.Timeout, socket.timeout, socket.gaierror)):
LOG.warning(msg)
elif isinstance(e, socket.error) and e.errno in (110, 111, 113):
LOG.warning(msg)
elif isinstance(e,
googleapiclient.errors.HttpError) and e.resp['status'] in (
'403', ):
LOG.warning(msg)
elif isinstance(e, ssl.SSLError):
LOG.warning(msg)
elif isinstance(e, greenlet.GreenletExit):
pass
elif 'userDisabled' in str(e):
LOG.warning(msg)
elif isinstance(e, exceptions.MissingCredentialsError):
LOG.debug(msg)
else:
LOG.exception(msg)
def do_iteration(self):
for envs in self.analytics.load_envs():
msg = "Processing environments: {}".format([env["id"] for env in envs])
LOG.debug(msg)
try:
self.analytics.load_env_credentials(envs)
unique = {}
for env in envs:
try:
credentials = self.analytics.get_credentials([env])
for cred in credentials:
if cred.platform == "ec2" and env.get("ec2.detailed_billing.enabled", "0") == "1":
continue
unique.setdefault(cred.unique, {"envs_ids": [], "cred": cred})
unique[cred.unique]["envs_ids"].append(env["id"])
except:
msg = "Processing environment: {} failed".format(env["id"])
LOG.exception(msg)
for data in unique.values():
while len(self.pool) > self.config["pool_size"] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(process_credential, args=(data["cred"],), kwds={"envs_ids": data["envs_ids"]})
gevent.sleep(0) # force switch
except:
msg = "Processing environments: {} failed".format([env["id"] for env in envs])
LOG.exception(msg)
self.pool.join()
def do_iteration(self):
for envs in self.analytics.load_envs():
msg = "Processing environments: {}".format([env['id'] for env in envs])
LOG.debug(msg)
try:
self.analytics.load_env_credentials(envs)
unique = {}
for env in envs:
try:
credentials = self.analytics.get_credentials([env])
for cred in credentials:
if cred.platform == 'ec2' and env.get('ec2.detailed_billing.enabled', '0') == '1':
continue
unique.setdefault(cred.unique, {'envs_ids': [], 'cred': cred})
unique[cred.unique]['envs_ids'].append(env['id'])
except:
msg = 'Processing environment: {} failed'.format(env['id'])
LOG.exception(msg)
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(process_credential,
args=(data['cred'],),
kwds={'envs_ids': data['envs_ids']})
gevent.sleep(0) # force switch
except:
msg = 'Processing environments: {} failed'.format([env['id'] for env in envs])
LOG.exception(msg)
self.pool.join()
def _handle_exception(e, msg):
if isinstance(e, boto.exception.EC2ResponseError) and e.status in (401, 403):
LOG.warning(msg)
elif isinstance(
e,
(
libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
libcloud.common.exceptions.BaseHTTPError,
oauth2client.client.AccessTokenRefreshError,
gevent.timeout.Timeout,
socket.timeout,
socket.gaierror,
),
):
LOG.warning(msg)
elif isinstance(e, socket.error):
LOG.warning(msg)
elif isinstance(e, googleapiclient.errors.HttpError) and e.resp["status"] in ("403",):
LOG.warning(msg)
elif isinstance(e, ssl.SSLError):
LOG.warning(msg)
elif isinstance(e, greenlet.GreenletExit):
pass
elif "userDisabled" in str(e):
LOG.warning(msg)
elif isinstance(e, exceptions.MissingCredentialsError):
LOG.debug(msg)
else:
LOG.exception(msg)
def __call__(self):
self.change_permissions()
while True:
try:
self.iteration_timestamp = time.time()
self.before_iteration()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
except:
self.on_iteration_error()
raise
finally:
if not g.ready():
g.kill()
self.after_iteration()
iteration_time = time.time() - self.iteration_timestamp
msg = 'End iteration: {0:.1f} seconds'.format(iteration_time)
LOG.debug(msg)
except:
LOG.exception('Iteration failed')
time.sleep(self.error_sleep)
finally:
if self.config['interval']:
next_iteration_time = self.iteration_timestamp + self.config['interval']
time.sleep(next_iteration_time - time.time())
def process_aws_billing(self):
if self.args['--recalculate']:
return
dtime_from, dtime_to = self.get_aws_billing_interval()
msg = 'AWS billing interval: {0} - {1}'
msg = msg.format(dtime_from, dtime_to)
LOG.info(msg)
for envs in self.analytics.load_envs():
unique = {}
for env in envs:
if env.get('ec2.detailed_billing.enabled', '0') != '1':
continue
bucket_name = env['ec2.detailed_billing.bucket']
creds = self.analytics.get_creds([env])
cred = next(cred for cred in creds if cred.platform == 'ec2')
unique.setdefault(cred.unique, {
'envs_ids': [],
'cred': cred,
'bucket_name': bucket_name
})
unique[cred.unique]['envs_ids'].append(env['id'])
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(self.process_aws_account,
args=(data, dtime_from, dtime_to))
self.pool.join()
if not self.aws_billing_dtime_from:
return
dtime_from = self.aws_billing_dtime_from
if self.config['dtime_to']:
dtime_to = self.config['dtime_to']
else:
dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(
hours=1)
dtime_to = dtime_hour_ago.replace(minute=59,
second=59,
microsecond=999999)
# fill farm_usage_d
dtime_cur = dtime_from
msg = 'AWS fill_farm_usage_d interval: {0} - {1}'
LOG.info(msg.format(dtime_cur, dtime_to))
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
self.analytics.fill_farm_usage_d(date, hour, platform='ec2')
except:
msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(
date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
def process_aws_billing(self):
if self.args['--recalculate']:
return
dtime_from, dtime_to = self.get_aws_billing_interval()
msg = 'AWS billing interval: {0} - {1}'
msg = msg.format(dtime_from, dtime_to)
LOG.debug(msg)
with self._lock:
if not self.aws_billing_dtime_from:
self.aws_billing_dtime_from = dtime_from
else:
self.aws_billing_dtime_from = min(self.aws_billing_dtime_from, dtime_from)
for envs in self.analytics.load_envs():
unique = {}
for env in envs:
if env.get('ec2.detailed_billing.enabled', '0') != '1':
continue
bucket_name = env['ec2.detailed_billing.bucket']
creds = self.analytics.get_creds([env])
cred = next(cred for cred in creds if cred.platform == 'ec2')
unique.setdefault(cred.unique,
{'envs_ids': [], 'cred': cred, 'bucket_name': bucket_name})
unique[cred.unique]['envs_ids'].append(env['id'])
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(self.process_aws_account, args=(data, dtime_from, dtime_to))
self.pool.join()
if not self.aws_billing_dtime_from:
return
dtime_from = self.aws_billing_dtime_from
if self.config['dtime_to']:
dtime_to = self.config['dtime_to']
else:
dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
dtime_to = dtime_hour_ago.replace(minute=59, second=59, microsecond=999999)
# fill farm_usage_d
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
self.analytics.fill_farm_usage_d(date, hour)
except:
msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
def main():
app = AnalyticsProcessing()
try:
app.load_config()
app.configure()
app.run()
except exceptions.AlreadyRunningError:
LOG.info(helper.exc_info())
except (SystemExit, KeyboardInterrupt):
pass
except:
LOG.exception('Oops')
def main():
app = AnalyticsProcessing()
try:
app.load_config()
app.configure()
app.run()
except exceptions.AlreadyRunningError:
LOG.info(helper.exc_info())
except (SystemExit, KeyboardInterrupt):
pass
except:
LOG.exception('Oops')
def process_aws_account(self, data, dtime_from, dtime_to):
try:
# Iterate over months from dtime_from to dtime_to
dtime = dtime_from
while dtime.month <= dtime_to.month:
try:
csv_zip_file = self.download_aws_billing_file(
data['cred'], data['bucket_name'], date=dtime.date())
if csv_zip_file is None:
continue
with zipfile.ZipFile(csv_zip_file, 'r') as f:
f.extract(f.infolist()[0], self.tmp_dir)
csv_file = os.path.join(
self.tmp_dir,
os.path.basename(csv_zip_file.strip('.zip')))
for rows in self.csv_reader(csv_file, dtime_from=dtime):
records = self.get_aws_records(rows)
records = [
rec for rec in records
if int(rec['env_id']) in data['envs_ids']
]
if records:
with self._lock:
min_records_dtime = min(
[record['dtime'] for record in records])
if self.aws_billing_dtime_from:
self.aws_billing_dtime_from = min(
self.aws_billing_dtime_from,
min_records_dtime)
else:
self.aws_billing_dtime_from = min_records_dtime
for record in records:
self.pool.wait()
if self.args['--recalculate']:
self.pool.apply_async(
self.analytics.update_record, (record, ))
else:
self.pool.apply_async(
self.analytics.insert_record, (record, ))
gevent.sleep(0) # force switch
except:
msg = 'AWS billing for environments {0}, month {1} failed'
msg = msg.format(data['envs_ids'], dtime.month)
LOG.exception(msg)
finally:
dtime = helper.next_month(dtime)
except:
msg = 'AWS billing for environments {0} failed'
msg = msg.format(data['envs_ids'])
LOG.exception(msg)
def main():
global app
app = SzrUpdService()
try:
app.load_config()
app.configure()
app.run()
except exceptions.AlreadyRunningError:
LOG.info(helper.exc_info(where=False))
except (SystemExit, KeyboardInterrupt):
pass
except:
LOG.exception('Oops')
def main():
global app
app = LoadStatistics()
try:
app.load_config()
app.configure()
app.run()
except exceptions.AlreadyRunningError:
LOG.info(helper.exc_info(where=False))
except (SystemExit, KeyboardInterrupt):
pass
except:
LOG.exception('Oops')
def _do_iteration(self):
try:
return self.do_iteration()
except SystemExit:
if sys.exc_info()[1].args[0] != 0:
raise
except exceptions.QuitError:
return
except KeyboardInterrupt:
raise
except exceptions.NothingToDoError:
time.sleep(self.nothing_to_do_sleep)
except:
if logging.getLevelName(self.args['--verbosity']) < logging.ERROR:
LOG.exception(helper.exc_info(where=False))
raise
def do_iteration(self):
try:
self.process_poller_billing()
except:
msg = 'Unable to process poller_sessions table, reason: {0}'
msg = msg.format(helper.exc_info(where=False))
LOG.exception(msg)
try:
self.process_aws_billing()
except:
msg = 'Unable to process AWS billing information, reason: {0}'
msg = msg.format(helper.exc_info(where=False))
LOG.exception(msg)
if self.args['--recalculate']:
sys.exit(0)
def run(self):
try:
self.configure()
t = self._serve_forever()
time.sleep(5)
# change permissions
if self.config['group']:
helper.set_gid(self.config['group'])
if self.config['user']:
helper.set_uid(self.config['user'])
LOG.info('Plotter started')
t.join()
except:
LOG.exception(helper.exc_info())
def process_aws_account(self, data, dtime_from, dtime_to):
try:
# Iterate over months from dtime_from to dtime_to
dtime = dtime_from
while dtime.month <= dtime_to.month:
try:
csv_zip_file = self.download_aws_billing_file(data['cred'],
data['bucket_name'],
date=dtime.date())
if csv_zip_file is None:
continue
with zipfile.ZipFile(csv_zip_file, 'r') as f:
f.extract(f.infolist()[0], self.tmp_dir)
csv_file = os.path.join(self.tmp_dir,
os.path.basename(csv_zip_file.strip('.zip')))
for rows in self.csv_reader(csv_file, dtime_from=dtime):
records = self.get_aws_records(rows)
records = [rec for rec in records if int(rec['env_id']) in data['envs_ids']]
if records:
with self._lock:
min_records_dtime = min([record['dtime'] for record in records])
if self.aws_billing_dtime_from:
self.aws_billing_dtime_from = min(self.aws_billing_dtime_from,
min_records_dtime)
else:
self.aws_billing_dtime_from = min_records_dtime
for record in records:
self.pool.wait()
if self.args['--recalculate']:
self.pool.apply_async(self.analytics.update_record, (record,))
else:
self.pool.apply_async(self.analytics.insert_record, (record,))
gevent.sleep(0) # force switch
except:
msg = 'AWS billing for environments {0}, month {1} failed'
msg = msg.format(data['envs_ids'], dtime.month)
LOG.exception(msg)
finally:
dtime = helper.next_month(dtime)
except:
msg = 'AWS billing for environments {0} failed'
msg = msg.format(data['envs_ids'])
LOG.exception(msg)
def get_szr_ver_from_repo(self, devel_branch=None, force=False):
out = {}
if devel_branch:
if 'devel_repos' not in self.scalr_config['scalarizr_update']:
return out
if not force:
try:
return self.get_szr_ver_from_repo.im_func.devel_cache[
devel_branch]
except AttributeError:
self.get_szr_ver_from_repo.im_func.devel_cache = {}
except KeyError:
pass
norm_branch = devel_branch.replace('/', '-').replace('_', '-')
repos = self.scalr_config['scalarizr_update']['devel_repos']
else:
if not force:
try:
return self.get_szr_ver_from_repo.im_func.cache
except AttributeError:
pass
norm_branch = None
repos = self.scalr_config['scalarizr_update']['repos']
for repo_type, repo in repos.iteritems():
for k, func in {
'deb': self.ver_from_deb_repo,
'rpm': self.ver_from_rpm_repo,
'win': self.ver_from_win_repo
}.iteritems():
try:
data = func(repo, branch=norm_branch)
if data:
out.update(data)
out.setdefault(repo_type, {})[k] = data.values()[0]
except:
msg = '{0} repository {1} failed'.format(k, repo_type)
LOG.exception(msg)
if devel_branch:
self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out
else:
self.get_szr_ver_from_repo.im_func.cache = out
return out
def do_iteration(self):
try:
self.process_poller_billing()
except:
msg = 'Unable to process poller_sessions table, reason: {0}'
msg = msg.format(helper.exc_info(where=False))
LOG.exception(msg)
try:
self.process_aws_billing()
except:
msg = 'Unable to process AWS billing information, reason: {0}'
msg = msg.format(helper.exc_info(where=False))
LOG.exception(msg)
if self.args['--recalculate']:
sys.exit(0)
def get_szr_ver_from_repo(self, devel_branch=None, force=False):
out = {}
if devel_branch:
if "devel_repos" not in self.scalr_config["scalarizr_update"]:
return out
if not force:
try:
return self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch]
except AttributeError:
self.get_szr_ver_from_repo.im_func.devel_cache = {}
except KeyError:
pass
norm_branch = devel_branch.replace("/", "-").replace("_", "-")
repos = self.scalr_config["scalarizr_update"]["devel_repos"]
else:
if not force:
try:
return self.get_szr_ver_from_repo.im_func.cache
except AttributeError:
pass
norm_branch = None
repos = self.scalr_config["scalarizr_update"]["repos"]
for repo_type, repo in repos.iteritems():
for k, func in {
"deb": self.ver_from_deb_repo,
"rpm": self.ver_from_rpm_repo,
"win": self.ver_from_win_repo,
}.iteritems():
try:
data = func(repo, branch=norm_branch)
if data:
out.update(data)
out.setdefault(repo_type, {})[k] = data.values()[0]
except:
msg = "{0} repository {1} failed".format(k, repo_type)
LOG.exception(msg)
if devel_branch:
self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out
else:
self.get_szr_ver_from_repo.im_func.cache = out
return out
def get_szr_ver_from_repo(self, devel_branch=None, force=False):
out = {}
if devel_branch:
if 'devel_repos' not in self.scalr_config['scalarizr_update']:
return out
if not force:
try:
return self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch]
except AttributeError:
self.get_szr_ver_from_repo.im_func.devel_cache = {}
except KeyError:
pass
norm_branch = devel_branch.replace('/', '-').replace('_', '-')
repos = self.scalr_config['scalarizr_update']['devel_repos']
else:
if not force:
try:
return self.get_szr_ver_from_repo.im_func.cache
except AttributeError:
pass
norm_branch = None
repos = self.scalr_config['scalarizr_update']['repos']
for repo_type, repo in repos.iteritems():
for k, func in {
'deb': self.ver_from_deb_repo,
'rpm': self.ver_from_rpm_repo,
'win': self.ver_from_win_repo}.iteritems():
try:
data = func(repo, branch=norm_branch)
if data:
out.update(data)
out.setdefault(repo_type, {})[k] = data.values()[0]
except:
msg = '{0} repository {1} failed'.format(k, repo_type)
LOG.exception(msg)
if devel_branch:
self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out
else:
self.get_szr_ver_from_repo.im_func.cache = out
return out
def run(self):
try:
self.configure()
t = self._serve_forever()
while not t.is_alive():
time.sleep(0.5)
# wait before change permissions to allow cherrypy read certificates
time.sleep(2)
# change permissions
if self.config['group']:
helper.set_gid(self.config['group'])
if self.config['user']:
helper.set_uid(self.config['user'])
LOG.info('Plotter started')
t.join()
except:
LOG.exception(helper.exc_info())
def run(self):
try:
self.configure()
t = self._serve_forever()
while not t.is_alive():
time.sleep(0.5)
# wait before change permissions to allow cherrypy read certificates
time.sleep(2)
# change permissions
if self.config['group']:
helper.set_gid(self.config['group'])
if self.config['user']:
helper.set_uid(self.config['user'])
LOG.info('Plotter started')
t.join()
except:
LOG.exception(helper.exc_info())
def _handle_exception(e, msg):
if isinstance(e, boto.exception.EC2ResponseError) and e.status in (401, 403):
LOG.warning(msg)
elif isinstance(e, (libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
oauth2client.client.AccessTokenRefreshError,
gevent.timeout.Timeout,
socket.timeout,
socket.gaierror)):
LOG.warning(msg)
elif isinstance(e, socket.error) and e.errno in (110, 111, 113):
LOG.warning(msg)
elif isinstance(e, googleapiclient.errors.HttpError) and e.resp['status'] in ('403'):
LOG.warning(msg)
elif isinstance(e, ssl.SSLError):
LOG.warning(msg)
elif isinstance(e, greenlet.GreenletExit):
pass
elif 'userDisabled' in str(e):
LOG.warning(msg)
else:
LOG.exception(msg)
def wrapper(*args, **kwds):
try:
return f(*args, **kwds)
except:
LOG.exception('Exception')
raise
def db_update(sorted_data, envs_ids, cred):
platform = cred.platform
for env_id in envs_ids:
for region_data in sorted_data:
try:
sid = uuid.uuid4()
if platform == 'ec2' and 'account_id' in cred:
cloud_account = cryptotool.decrypt_scalr(app.crypto_key, cred['account_id'])
else:
cloud_account = None
if analytics.url_key_map[platform]:
url = urlparse.urlparse(cryptotool.decrypt_scalr(
app.crypto_key, cred[analytics.url_key_map[platform]]).rstrip('/'))
url = '%s%s' % (url.netloc, url.path)
else:
url = ''
query = (
"SELECT client_id "
"FROM client_environments "
"WHERE id={env_id}"
).format(env_id=env_id)
results = app.scalr_db.execute(query, retries=1)
account_id = results[0]['client_id']
query = (
"INSERT IGNORE INTO poller_sessions "
"(sid, account_id, env_id, dtime, platform, url, cloud_location, cloud_account) "
"VALUES "
"(UNHEX('{sid}'), {account_id}, {env_id}, '{dtime}', '{platform}', '{url}',"
"'{cloud_location}', '{cloud_account}')"
).format(
sid=sid.hex, account_id=account_id, env_id=env_id,
dtime=time.strftime(
"%Y-%m-%d %H:%M:%S", time.gmtime(region_data['timestamp'])),
platform=platform, url=url, cloud_location=region_data['region'],
cloud_account=cloud_account
)
app.analytics_db.execute(query, retries=1)
# managed
for managed in region_data['managed']:
if managed['env_id'] != env_id:
continue
query = (
"INSERT IGNORE INTO managed "
"(sid, server_id, instance_type, os) VALUES "
"(UNHEX('{sid}'), UNHEX('{server_id}'), '{instance_type}', {os})"
).format(
sid=sid.hex,
server_id=uuid.UUID(managed['server_id']).hex,
instance_type=managed['instance_type'],
os=managed['os'])
LOG.debug(query)
app.analytics_db.execute(query, retries=1)
## not_managed
#if region_data['not_managed']:
# base_query = (
# "INSERT IGNORE INTO notmanaged "
# "(sid, instance_id, instance_type, os) VALUES %s")
# values_template = "(UNHEX('{sid}'), '{instance_id}', '{instance_type}', {os})"
# i, chunk_size = 0, 20
# while True:
# chunk_not_managed = region_data['not_managed'][
# i * chunk_size:(i + 1) * chunk_size]
# if not chunk_not_managed:
# break
# query = base_query % ','.join(
# [
# values_template.format(
# sid=sid.hex,
# instance_id=not_managed['instance_id'],
# instance_type=not_managed['instance_type'],
# os=not_managed['os']
# )
# for not_managed in chunk_not_managed
# ]
# )
# app.analytics_db.execute(query, retries=1)
# i += 1
except:
msg = 'Database update failed, reason: {0}'
msg = msg.format(helper.exc_info())
LOG.exception(msg)
def do_iteration(self):
try:
dtime_from, dtime_to = self._get_processing_dtime()
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
if self.config['recalculate']:
self.recalculate(date, hour)
else:
self.calculate(date, hour)
except KeyboardInterrupt:
raise
except:
LOG.error(helper.exc_info())
dtime_cur += datetime.timedelta(seconds=3600)
self._pool.join()
if not self.config['recalculate']:
return
# recalculate daily tables
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date = dtime_cur.date()
msg = "Recalculate daily tables for date {0}".format(date)
LOG.info(msg)
try:
self.analytics.recalculate_usage_d(date, self.config['platform'])
except:
msg = "Recalculate usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
try:
self.analytics.recalculate_nm_usage_d(date, self.config['platform'])
except:
msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
dtime_cur += datetime.timedelta(days=1)
# recalculate quarters tables
quarters_calendar = self.analytics.get_quarters_calendar()
start_year = quarters_calendar.year_for_date(dtime_from.date())
start_quarter = quarters_calendar.quarter_for_date(dtime_from.date())
end_year = quarters_calendar.year_for_date(dtime_to.date())
end_quarter = quarters_calendar.quarter_for_date(dtime_to.date())
tmp = []
cur_year = start_year
while cur_year < end_year:
for quarter in range(start_quarter, 5):
tmp.append((cur_year, quarter))
start_quarter = 1
cur_year += 1
for quarter in range(start_quarter, end_quarter + 1):
tmp.append((end_year, quarter))
for year, quarter in tmp:
try:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1}"
msg = msg.format(year, quarter)
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(year, quarter)
except:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}"
msg = msg.format(year, quarter, helper.exc_info())
LOG.warning(msg)
except:
if self.config['recalculate']:
LOG.exception(helper.exc_info())
sys.exit(1)
else:
raise
# quit from iteration loop
raise exceptions.QuitError()
def process_poller_billing(self):
dtime_from, dtime_to = self.get_poller_billing_interval()
LOG.info('Poller billing interval: {0} - {1}'.format(dtime_from, dtime_to))
# process poller_session table
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
msg = "Process poller data, date {0}, hour {1}".format(date, hour)
LOG.info(msg)
if self.args['--recalculate']:
platform = self.config['platform']
generator = self.analytics.get_records(date, hour, platform)
else:
generator = self.analytics.get_servers(date, hour)
for records in generator:
LOG.debug('Records for processing: %s' % len(records))
prices = self.analytics.get_prices(records)
for record in records:
cost = self.analytics.get_cost_from_prices(record, prices) or 0
self.pool.wait()
if self.args['--recalculate']:
record['cost'] = float(cost) * int(record['num'])
self.pool.apply_async(self.analytics.update_record, (record,))
else:
record['cost'] = cost
record['num'] = 1.0
record['cost_distr_type'] = 1
self.pool.apply_async(self.analytics.insert_record, (record,))
gevent.sleep(0) # force switch
self.pool.join()
except:
msg = "Unable to process date {0}, hour {1}".format(date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
# fill farm_usage_d
dtime_cur = dtime_from
msg = 'Poller fill_farm_usage_d interval: {0} - {1}'
LOG.info(msg.format(dtime_cur, dtime_to))
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
self.analytics.fill_farm_usage_d(date, hour)
except:
msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
if not self.args['--recalculate']:
return
# recalculate daily tables
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date = dtime_cur.date()
msg = "Recalculate daily tables for date {0}".format(date)
LOG.debug(msg)
try:
self.analytics.recalculate_usage_d(date, self.config['platform'])
except:
msg = "Recalculate usage_d table for date {0} failed, error: {1}".format(
date, helper.exc_info())
LOG.warning(msg)
dtime_cur += datetime.timedelta(days=1)
# recalculate quarters tables
quarters_calendar = self.analytics.get_quarters_calendar()
start_year = quarters_calendar.year_for_date(dtime_from.date())
start_quarter = quarters_calendar.quarter_for_date(dtime_from.date())
end_year = quarters_calendar.year_for_date(dtime_to.date())
end_quarter = quarters_calendar.quarter_for_date(dtime_to.date())
tmp = []
cur_year = start_year
while cur_year < end_year:
for quarter in range(start_quarter, 5):
tmp.append((cur_year, quarter))
start_quarter = 1
cur_year += 1
for quarter in range(start_quarter, end_quarter + 1):
tmp.append((end_year, quarter))
for year, quarter in tmp:
try:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1}"
msg = msg.format(year, quarter)
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(year, quarter)
except:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed"
msg = msg.format(year, quarter, helper.exc_info())
LOG.exception(msg)
def do_iteration(self):
try:
dtime_from, dtime_to = self._get_processing_dtime()
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
if self.config['recalculate']:
self.recalculate(date, hour)
else:
self.calculate(date, hour)
except KeyboardInterrupt:
raise
except:
LOG.error(helper.exc_info())
dtime_cur += datetime.timedelta(seconds=3600)
self._pool.join()
if not self.config['recalculate']:
return
# recalculate daily tables
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date = dtime_cur.date()
msg = "Recalculate daily tables for date {0}".format(date)
LOG.info(msg)
try:
self.analytics.recalculate_usage_d(date,
self.config['platform'])
except:
msg = "Recalculate usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
try:
self.analytics.recalculate_nm_usage_d(
date, self.config['platform'])
except:
msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
dtime_cur += datetime.timedelta(days=1)
# recalculate quarters tables
quarters_calendar = self.analytics.get_quarters_calendar()
start_year = quarters_calendar.year_for_date(dtime_from.date())
start_quarter = quarters_calendar.quarter_for_date(
dtime_from.date())
end_year = quarters_calendar.year_for_date(dtime_to.date())
end_quarter = quarters_calendar.quarter_for_date(dtime_to.date())
tmp = []
cur_year = start_year
while cur_year < end_year:
for quarter in range(start_quarter, 5):
tmp.append((cur_year, quarter))
start_quarter = 1
cur_year += 1
for quarter in range(start_quarter, end_quarter + 1):
tmp.append((end_year, quarter))
for year, quarter in tmp:
try:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1}"
msg = msg.format(year, quarter)
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(year, quarter)
except:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}"
msg = msg.format(year, quarter, helper.exc_info())
LOG.warning(msg)
except:
if self.config['recalculate']:
LOG.exception(helper.exc_info())
sys.exit(1)
else:
raise
# quit from iteration loop
raise exceptions.QuitError()
def process_poller_billing(self):
dtime_from, dtime_to = self.get_poller_billing_interval()
LOG.info('Poller billing interval: {0} - {1}'.format(
dtime_from, dtime_to))
# process poller_session table
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
msg = "Process poller data, date {0}, hour {1}".format(
date, hour)
LOG.info(msg)
if self.args['--recalculate']:
platform = self.config['platform']
generator = self.analytics.get_records(
date, hour, platform)
else:
generator = self.analytics.get_servers(date, hour)
for records in generator:
LOG.debug('Records for processing: %s' % len(records))
prices = self.analytics.get_prices(records)
for record in records:
cost = self.analytics.get_cost_from_prices(
record, prices) or 0
self.pool.wait()
if self.args['--recalculate']:
record['cost'] = float(cost) * int(record['num'])
self.pool.apply_async(self.analytics.update_record,
(record, ))
else:
record['cost'] = cost
record['num'] = 1.0
record['cost_distr_type'] = 1
self.pool.apply_async(self.analytics.insert_record,
(record, ))
gevent.sleep(0) # force switch
self.pool.join()
except:
msg = "Unable to process date {0}, hour {1}".format(date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
# fill farm_usage_d
dtime_cur = dtime_from
msg = 'Poller fill_farm_usage_d interval: {0} - {1}'
LOG.info(msg.format(dtime_cur, dtime_to))
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
self.analytics.fill_farm_usage_d(date, hour)
except:
msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(
date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
if not self.args['--recalculate']:
return
# recalculate daily tables
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date = dtime_cur.date()
msg = "Recalculate daily tables for date {0}".format(date)
LOG.debug(msg)
try:
self.analytics.recalculate_usage_d(date,
self.config['platform'])
except:
msg = "Recalculate usage_d table for date {0} failed, error: {1}".format(
date, helper.exc_info())
LOG.warning(msg)
dtime_cur += datetime.timedelta(days=1)
# recalculate quarters tables
quarters_calendar = self.analytics.get_quarters_calendar()
start_year = quarters_calendar.year_for_date(dtime_from.date())
start_quarter = quarters_calendar.quarter_for_date(dtime_from.date())
end_year = quarters_calendar.year_for_date(dtime_to.date())
end_quarter = quarters_calendar.quarter_for_date(dtime_to.date())
tmp = []
cur_year = start_year
while cur_year < end_year:
for quarter in range(start_quarter, 5):
tmp.append((cur_year, quarter))
start_quarter = 1
cur_year += 1
for quarter in range(start_quarter, end_quarter + 1):
tmp.append((end_year, quarter))
for year, quarter in tmp:
try:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1}"
msg = msg.format(year, quarter)
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(year, quarter)
except:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed"
msg = msg.format(year, quarter, helper.exc_info())
LOG.exception(msg)
def wrapper(*args, **kwds):
try:
return f(*args, **kwds)
except:
LOG.exception('Exception')
raise
def db_update(sorted_data, envs_ids, cred):
platform = cred.platform
for env_id in envs_ids:
for region_data in sorted_data:
try:
sid = uuid.uuid4()
if platform == 'ec2' and 'account_id' in cred:
cloud_account = cryptotool.decrypt_scalr(
app.crypto_key, cred['account_id'])
else:
cloud_account = None
if analytics.url_key_map[platform]:
url = urlparse.urlparse(
cryptotool.decrypt_scalr(
app.crypto_key,
cred[analytics.url_key_map[platform]]).rstrip('/'))
url = '%s%s' % (url.netloc, url.path)
else:
url = ''
query = ("SELECT client_id "
"FROM client_environments "
"WHERE id={env_id}").format(env_id=env_id)
results = app.scalr_db.execute(query, retries=1)
account_id = results[0]['client_id']
query = (
"INSERT IGNORE INTO poller_sessions "
"(sid, account_id, env_id, dtime, platform, url, cloud_location, cloud_account) "
"VALUES "
"(UNHEX('{sid}'), {account_id}, {env_id}, '{dtime}', '{platform}', '{url}',"
"'{cloud_location}', '{cloud_account}')").format(
sid=sid.hex,
account_id=account_id,
env_id=env_id,
dtime=time.strftime(
"%Y-%m-%d %H:%M:%S",
time.gmtime(region_data['timestamp'])),
platform=platform,
url=url,
cloud_location=region_data['region'],
cloud_account=cloud_account)
app.analytics_db.execute(query, retries=1)
# managed
for managed in region_data['managed']:
if managed['env_id'] != env_id:
continue
query = (
"INSERT IGNORE INTO managed "
"(sid, server_id, instance_type, os) VALUES "
"(UNHEX('{sid}'), UNHEX('{server_id}'), '{instance_type}', {os})"
).format(sid=sid.hex,
server_id=uuid.UUID(managed['server_id']).hex,
instance_type=managed['instance_type'],
os=managed['os'])
LOG.debug(query)
app.analytics_db.execute(query, retries=1)
## not_managed
#if region_data['not_managed']:
# base_query = (
# "INSERT IGNORE INTO notmanaged "
# "(sid, instance_id, instance_type, os) VALUES %s")
# values_template = "(UNHEX('{sid}'), '{instance_id}', '{instance_type}', {os})"
# i, chunk_size = 0, 20
# while True:
# chunk_not_managed = region_data['not_managed'][
# i * chunk_size:(i + 1) * chunk_size]
# if not chunk_not_managed:
# break
# query = base_query % ','.join(
# [
# values_template.format(
# sid=sid.hex,
# instance_id=not_managed['instance_id'],
# instance_type=not_managed['instance_type'],
# os=not_managed['os']
# )
# for not_managed in chunk_not_managed
# ]
# )
# app.analytics_db.execute(query, retries=1)
# i += 1
except:
msg = 'Database update failed, reason: {0}'
msg = msg.format(helper.exc_info())
LOG.exception(msg)
