def execute(self, query, retries=0, retry_timeout=10):
while True:
try:
if self._autocommit or not self._connection:
self._local.connection = self._connection_pool.get(
timeout=10)
self._local.connection.autocommit(self._autocommit)
self._local.cursor = self._connection.cursor()
try:
start_time = time.time()
self._local.cursor.execute(query)
end_time = time.time()
if end_time - start_time > 1:
LOG.debug('Query too slow: %s\n%s...' %
(end_time - start_time, query[:150]))
results = self._local.cursor.fetchall()
if results is not None:
results = tuple(results)
return results
finally:
if self._autocommit:
self._local.cursor.close()
self._connection_pool.put(self._local.connection)
self._local.connection = None
self._local.cursor = None
except (pymysql.err.OperationalError, pymysql.err.InternalError,
socket.timeout):
if not retries:
raise
retries -= 1
time.sleep(retry_timeout)
def execute(self, query, retries=0, retry_timeout=10):
while True:
try:
if self._autocommit or not self._connection:
self._local.connection = self._connection_pool.get(timeout=10)
self._local.connection.autocommit(self._autocommit)
self._local.cursor = self._connection.cursor()
try:
start_time = time.time()
self._local.cursor.execute(query)
end_time = time.time()
if end_time - start_time > 1:
LOG.debug('Query too slow: %s\n%s...' %
(end_time - start_time, query[:150]))
results = self._local.cursor.fetchall()
if results is not None:
results = tuple(results)
return results
finally:
if self._autocommit:
self._local.cursor.close()
self._connection_pool.put(self._local.connection)
self._local.connection = None
self._local.cursor = None
except (pymysql.err.OperationalError, pymysql.err.InternalError, socket.timeout):
if not retries:
raise
retries -= 1
time.sleep(retry_timeout)
def create_pid_file(pid_file):
pid = str(os.getpid())
msg = "Creating pid file: %s" % pid_file
LOG.debug(msg)
if not os.path.exists(os.path.dirname(pid_file)):
os.makedirs(os.path.dirname(pid_file), mode=0o755)
file(pid_file, 'w+').write('%s\n' % pid)
def download_aws_billing_file(self, cred, bucket_name, date=None):
if date is None:
date = datetime.datetime.utcnow().date()
conn = get_s3_conn(cred)
bucket = conn.get_bucket(bucket_name)
account_id = cryptotool.decrypt_scalr(app.crypto_key,
cred['account_id'])
file_name = get_aws_csv_file_name(account_id, date)
key = bucket.get_key(file_name)
if not key:
msg = "AWS detailed billing CSV file {0} wasn't found in bucket {1}"
msg = msg.format(file_name, bucket_name)
if datetime.datetime.utcnow().day == 1:
LOG.warning(msg)
return None
else:
raise Exception(msg)
last_modified_dt = datetime.datetime.strptime(
key.last_modified, self.last_modified_format)
update_interval = self.config['interval']
utcnow = datetime.datetime.utcnow()
delta = datetime.timedelta(seconds=update_interval)
condition1 = utcnow > last_modified_dt and utcnow < last_modified_dt + delta
condition2 = ((utcnow - last_modified_dt).seconds / 3600) % 8 == 0
if condition1 or condition2:
local_file_path = os.path.join(self.tmp_dir, file_name)
LOG.debug('Downloading {0}'.format(file_name))
key.get_contents_to_filename(local_file_path)
return local_file_path
else:
return None
def _is_server_for_update(self, server, status):
repo_url = status["repo_url"]
devel_branch = server.get("user-data.scm_branch", None)
ver_info = self.get_szr_ver_from_repo(devel_branch=devel_branch)
try:
szr_ver_repo = ver_info[repo_url]
except KeyError:
pkg_type = helper.pkg_type_by_name(status["dist"].split()[0])
szr_ver_repo = ver_info[status["repository"]][pkg_type]
if parse_version(server["scalarizr.version"]) >= parse_version(szr_ver_repo):
return False
if "in-progress" in status["state"]:
# skip in-progress server
return False
if status["executed_at"]:
last_update_dt = datetime.datetime.strptime(status["executed_at"], "%a %d %b %Y %H:%M:%S %Z")
last_update_dt = last_update_dt.replace(minute=0, second=0, microsecond=0)
utcnow_dt = datetime.datetime.utcnow()
utcnow_dt = utcnow_dt.replace(minute=0, second=0, microsecond=0)
if last_update_dt == utcnow_dt and status["state"] == "error":
# skip failed server
LOG.debug("Skip server: {0}, reason: server in error state".format(server["server_id"]))
return False
return True
def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
LOG.debug("Daemonize")
# first fork
pid = os.fork()
if pid > 0:
sys.exit(0)
os.chdir('/')
os.setsid()
os.umask(0)
# second fork
pid = os.fork()
if pid > 0:
sys.exit(0)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = file(stdin, 'r')
so = file(stdout, "a+")
se = file(stderr, "a+", 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
def _serve_forever(self):
LOG.debug('Starting plotter')
try:
cherrypy.quickstart(self, '/', {'/': {}})
except:
LOG.error(helper.exc_info())
thread.interrupt_main()
def _is_server_for_update(self, server, status):
repo_url = status['repo_url']
devel_branch = server.get('user-data.scm_branch', None)
ver_info = self.get_szr_ver_from_repo(devel_branch=devel_branch)
try:
szr_ver_repo = ver_info[repo_url]
except KeyError:
pkg_type = helper.pkg_type_by_name(status['dist'].split()[0])
szr_ver_repo = ver_info[status['repository']][pkg_type]
if parse_version(server['scalarizr.version']) >= parse_version(szr_ver_repo):
return False
if 'in-progress' in status['state']:
# skip in-progress server
return False
if status['executed_at']:
last_update_dt = datetime.datetime.strptime(
status['executed_at'], '%a %d %b %Y %H:%M:%S %Z')
last_update_dt = last_update_dt.replace(minute=0, second=0, microsecond=0)
utcnow_dt = datetime.datetime.utcnow()
utcnow_dt = utcnow_dt.replace(minute=0, second=0, microsecond=0)
if last_update_dt == utcnow_dt and status['state'] == 'error':
# skip failed server
LOG.debug(
'Skip server: {0}, reason: server in error state'.format(server['server_id']))
return False
return True
def download_aws_billing_file(self, cred, bucket_name, date=None):
if date is None:
date = datetime.datetime.utcnow().date()
conn = get_s3_conn(cred)
bucket = conn.get_bucket(bucket_name)
account_id = cryptotool.decrypt_scalr(app.crypto_key, cred['account_id'])
file_name = get_aws_csv_file_name(account_id, date)
key = bucket.get_key(file_name)
if not key:
msg = "AWS detailed billing CSV file {0} wasn't found in bucket {1}"
msg = msg.format(file_name, bucket_name)
if datetime.datetime.utcnow().day == 1:
LOG.warning(msg)
return None
else:
raise Exception(msg)
last_modified_dt = datetime.datetime.strptime(key.last_modified, self.last_modified_format)
update_interval = self.config['interval']
utcnow = datetime.datetime.utcnow()
delta = datetime.timedelta(seconds=update_interval)
condition1 = utcnow > last_modified_dt and utcnow < last_modified_dt + delta
condition2 = ((utcnow - last_modified_dt).seconds / 3600) % 8 == 0
if condition1 or condition2:
local_file_path = os.path.join(self.tmp_dir, file_name)
LOG.debug('Downloading {0}'.format(file_name))
key.get_contents_to_filename(local_file_path)
return local_file_path
else:
return None
def do_iteration(self):
for envs in self.analytics.load_envs():
msg = "Processing environments: {}".format([env['id'] for env in envs])
LOG.debug(msg)
try:
self.analytics.load_env_credentials(envs)
unique = {}
for env in envs:
try:
credentials = self.analytics.get_credentials([env])
for cred in credentials:
if cred.platform == 'ec2' and env.get('ec2.detailed_billing.enabled', '0') == '1':
continue
unique.setdefault(cred.unique, {'envs_ids': [], 'cred': cred})
unique[cred.unique]['envs_ids'].append(env['id'])
except:
msg = 'Processing environment: {} failed'.format(env['id'])
LOG.exception(msg)
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(process_credential,
args=(data['cred'],),
kwds={'envs_ids': data['envs_ids']})
gevent.sleep(0) # force switch
except:
msg = 'Processing environments: {} failed'.format([env['id'] for env in envs])
LOG.exception(msg)
self.pool.join()
def _ec2_region(region, cred):
try:
access_key = cryptotool.decrypt_scalr(app.crypto_key, cred["access_key"])
secret_key = cryptotool.decrypt_scalr(app.crypto_key, cred["secret_key"])
kwds = {"aws_access_key_id": access_key, "aws_secret_access_key": secret_key}
proxy_settings = app.proxy_settings.get(cred.platform, {})
kwds["proxy"] = proxy_settings.get("host")
kwds["proxy_port"] = proxy_settings.get("port")
kwds["proxy_user"] = proxy_settings.get("user")
kwds["proxy_pass"] = proxy_settings.get("pass")
msg = "List nodes for platform: 'ec2', region: '{}', envs_ids: {}"
msg = msg.format(region, cred.envs_ids)
LOG.debug(msg)
conn = boto.ec2.connect_to_region(region, **kwds)
cloud_nodes = _ec2_get_only_instances(conn)
timestamp = int(time.time())
nodes = list()
for cloud_node in cloud_nodes:
node = {
"instance_id": cloud_node.id,
"instance_type": cloud_node.instance_type,
"os": cloud_node.platform if cloud_node.platform else "linux",
}
nodes.append(node)
return {"region": region, "timestamp": timestamp, "nodes": nodes} if nodes else dict()
except:
e = sys.exc_info()[1]
msg = "platform: '{platform}', region: '{region}', envs_ids: {envs_ids}. Reason: {error}"
msg = msg.format(
platform=cred.platform, region=region, envs_ids=cred.envs_ids, error=helper.exc_info(where=False)
)
_handle_exception(e, msg)
def __call__(self):
self.change_permissions()
while True:
try:
self.iteration_timestamp = time.time()
self.before_iteration()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
except:
self.on_iteration_error()
raise
finally:
if not g.ready():
g.kill()
self.after_iteration()
iteration_time = time.time() - self.iteration_timestamp
msg = 'End iteration: {0:.1f} seconds'.format(iteration_time)
LOG.debug(msg)
except:
LOG.exception('Iteration failed')
time.sleep(self.error_sleep)
finally:
if self.config['interval']:
next_iteration_time = self.iteration_timestamp + self.config['interval']
time.sleep(next_iteration_time - time.time())
def _serve_forever(self):
LOG.debug('Starting plotter')
try:
cherrypy.quickstart(self, '/', {'/': {}})
except:
LOG.error(helper.exc_info())
thread.interrupt_main()
def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
LOG.debug("Daemonize")
# first fork
pid = os.fork()
if pid > 0:
sys.exit(0)
os.chdir('/')
os.setsid()
os.umask(0)
# second fork
pid = os.fork()
if pid > 0:
sys.exit(0)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = file(stdin, 'r')
so = file(stdout, "a+")
se = file(stderr, "a+", 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
def _handle_exception(e, msg):
if isinstance(e,
boto.exception.EC2ResponseError) and e.status in (401, 403):
LOG.warning(msg)
elif isinstance(e,
(libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
oauth2client.client.AccessTokenRefreshError,
gevent.timeout.Timeout, socket.timeout, socket.gaierror)):
LOG.warning(msg)
elif isinstance(e, socket.error) and e.errno in (110, 111, 113):
LOG.warning(msg)
elif isinstance(e,
googleapiclient.errors.HttpError) and e.resp['status'] in (
'403', ):
LOG.warning(msg)
elif isinstance(e, ssl.SSLError):
LOG.warning(msg)
elif isinstance(e, greenlet.GreenletExit):
pass
elif 'userDisabled' in str(e):
LOG.warning(msg)
elif isinstance(e, exceptions.MissingCredentialsError):
LOG.debug(msg)
else:
LOG.exception(msg)
def _handle_exception(e, msg):
if isinstance(e, boto.exception.EC2ResponseError) and e.status in (401, 403):
LOG.warning(msg)
elif isinstance(
e,
(
libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
libcloud.common.exceptions.BaseHTTPError,
oauth2client.client.AccessTokenRefreshError,
gevent.timeout.Timeout,
socket.timeout,
socket.gaierror,
),
):
LOG.warning(msg)
elif isinstance(e, socket.error):
LOG.warning(msg)
elif isinstance(e, googleapiclient.errors.HttpError) and e.resp["status"] in ("403",):
LOG.warning(msg)
elif isinstance(e, ssl.SSLError):
LOG.warning(msg)
elif isinstance(e, greenlet.GreenletExit):
pass
elif "userDisabled" in str(e):
LOG.warning(msg)
elif isinstance(e, exceptions.MissingCredentialsError):
LOG.debug(msg)
else:
LOG.exception(msg)
def do_iteration(self):
for envs in self.analytics.load_envs():
msg = "Processing environments: {}".format([env["id"] for env in envs])
LOG.debug(msg)
try:
self.analytics.load_env_credentials(envs)
unique = {}
for env in envs:
try:
credentials = self.analytics.get_credentials([env])
for cred in credentials:
if cred.platform == "ec2" and env.get("ec2.detailed_billing.enabled", "0") == "1":
continue
unique.setdefault(cred.unique, {"envs_ids": [], "cred": cred})
unique[cred.unique]["envs_ids"].append(env["id"])
except:
msg = "Processing environment: {} failed".format(env["id"])
LOG.exception(msg)
for data in unique.values():
while len(self.pool) > self.config["pool_size"] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(process_credential, args=(data["cred"],), kwds={"envs_ids": data["envs_ids"]})
gevent.sleep(0) # force switch
except:
msg = "Processing environments: {} failed".format([env["id"] for env in envs])
LOG.exception(msg)
self.pool.join()
def _is_server_for_update(self, server, status):
repo_url = status['repo_url']
devel_branch = server.get('user-data.scm_branch', None)
ver_info = self.get_szr_ver_from_repo(devel_branch=devel_branch)
try:
szr_ver_repo = ver_info[repo_url]
except KeyError:
pkg_type = helper.pkg_type_by_name(status['dist'].split()[0])
szr_ver_repo = ver_info[status['repository']][pkg_type]
if parse_version(
server['scalarizr.version']) >= parse_version(szr_ver_repo):
return False
if 'in-progress' in status['state']:
# skip in-progress server
return False
if status['executed_at']:
last_update_dt = datetime.datetime.strptime(
status['executed_at'], '%a %d %b %Y %H:%M:%S %Z')
last_update_dt = last_update_dt.replace(minute=0,
second=0,
microsecond=0)
utcnow_dt = datetime.datetime.utcnow()
utcnow_dt = utcnow_dt.replace(minute=0, second=0, microsecond=0)
if last_update_dt == utcnow_dt and status['state'] == 'error':
# skip failed server
LOG.debug(
'Skip server: {0}, reason: server in error state'.format(
server['server_id']))
return False
return True
def _stop(self):
LOG.debug(self._stopping_msg)
try:
if not os.path.exists(self.config['pid_file']):
msg = "Can't stop, pid file %s doesn't exist\n" % self.config[
'pid_file']
sys.stderr.write(helper.colorize(helper.Color.FAIL, msg))
return
with file(self.config['pid_file'], 'r') as pf:
pid = int(pf.read().strip())
for ps in psutil.process_iter():
if ps.name() == self.name[0:15]:
# TODO
# SIGINT
helper.kill_children(pid)
helper.kill(pid)
break
else:
msg = "Process with name {0} doesn't exists".format(self.name)
raise Exception(msg)
LOG.info('Stopped')
helper.delete_file(self.config['pid_file'])
except:
msg = "Can't stop, reason: {error}".format(error=helper.exc_info())
raise Exception(msg)
def create_pid_file(pid_file):
pid = str(os.getpid())
msg = "Creating pid file: %s" % pid_file
LOG.debug(msg)
if not os.path.exists(os.path.dirname(pid_file)):
os.makedirs(os.path.dirname(pid_file), mode=0o755)
file(pid_file, 'w+').write('%s\n' % pid)
def __call__(self):
self.change_permissions()
while True:
try:
self.iteration_timestamp = time.time()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
except gevent.Timeout:
raise exceptions.IterationTimeoutError()
finally:
if not g.ready():
g.kill()
except (SystemExit, KeyboardInterrupt):
raise
except exceptions.NothingToDoError:
time_to_sleep = self.nothing_to_do_sleep
except exceptions.QuitError:
sys.exit(0)
except:
LOG.error('Iteration failed, reason: {0}'.format(helper.exc_info()))
self.on_iteration_error()
time_to_sleep = self.error_sleep
else:
time_to_sleep = 0.1
LOG.debug('End iteration: {0:.1f} seconds'.format(time.time() - self.iteration_timestamp))
if self.config['interval']:
time_to_sleep = self.iteration_timestamp + self.config['interval'] - time.time()
time.sleep(time_to_sleep)
def __call__(self):
self.change_permissions()
while True:
try:
self.iteration_timestamp = time.time()
self.before_iteration()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
except:
self.on_iteration_error()
raise
finally:
if not g.ready():
g.kill()
self.after_iteration()
except:
try:
helper.handle_error(message='Iteration failed')
except (SystemExit, KeyboardInterrupt):
return
except:
pass
time.sleep(self.error_sleep)
finally:
iteration_time = time.time() - self.iteration_timestamp
msg = 'End iteration: {0:.1f} seconds'.format(iteration_time)
LOG.debug(msg)
if self.config['interval']:
next_iteration_time = self.iteration_timestamp + self.config['interval']
sleep_time = next_iteration_time - time.time()
if sleep_time:
time.sleep(sleep_time)
def handle_error(message=None, level='exception'):
c, e, t = sys.exc_info()
if message:
message = message.rstrip().rstrip('.') + '. Reason: {}'.format(
exc_info())
else:
message = exc_info()
if isinstance(e, (
KeyboardInterrupt,
GeneratorExit,
greenlet_mod.GreenletExit,
gevent.Timeout,
)):
LOG.debug(message)
raise
if isinstance(e, SystemExit) and sys.exc_info()[1].args[0] == 0:
raise
logging_map = {
'debug': LOG.debug,
'info': LOG.info,
'warning': LOG.warning,
'error': LOG.error,
'critical': LOG.critical,
'exception': LOG.exception,
}
#if isinstance(e, pymysql.err.Error):
# logging_map[min(level, 'error')](message)
#else:
# logging_map[level](message)
logging_map[level](message)
def handle_error(message=None, level='exception'):
c, e, t = sys.exc_info()
if message:
message = message.rstrip().rstrip('.') + '. Reason: {}'.format(exc_info())
else:
message = exc_info()
if isinstance(e, (
KeyboardInterrupt,
GeneratorExit,
greenlet_mod.GreenletExit,
gevent.Timeout,
)
):
LOG.debug(message)
raise
if isinstance(e, SystemExit) and sys.exc_info()[1].args[0] == 0:
raise
logging_map = {
'debug': LOG.debug,
'info': LOG.info,
'warning': LOG.warning,
'error': LOG.error,
'critical': LOG.critical,
'exception': LOG.exception,
}
if isinstance(e, pymysql.err.Error):
logging_map[min(level, 'error')](message)
else:
logging_map[level](message)
def __call__(self):
self.change_permissions()
while True:
try:
self.iteration_timestamp = time.time()
self.before_iteration()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
except:
self.on_iteration_error()
raise
finally:
if not g.ready():
g.kill()
self.after_iteration()
except:
try:
helper.handle_error(message='Iteration failed')
except (SystemExit, KeyboardInterrupt):
return
except:
pass
time.sleep(self.error_sleep)
finally:
iteration_time = time.time() - self.iteration_timestamp
msg = 'End iteration: {0:.1f} seconds'.format(iteration_time)
LOG.debug(msg)
if self.config['interval']:
next_iteration_time = self.iteration_timestamp + self.config[
'interval']
sleep_time = next_iteration_time - time.time()
if sleep_time:
time.sleep(sleep_time)
def wrapper(*args, **kwds):
start_time = time.time()
try:
return f(*args, **kwds)
finally:
end_time = time.time()
msg = 'TIMEIT %s.%s: %s' % (f.__module__, f.__name__, end_time - start_time)
LOG.debug(msg)
def delete_file(file_path):
msg = "Deleting file: %s" % file_path
LOG.debug(msg)
if os.path.exists(file_path):
try:
os.remove(file_path)
except:
LOG.warning(exc_info())
def delete_file(file_path):
msg = "Deleting file: %s" % file_path
LOG.debug(msg)
if os.path.exists(file_path):
try:
os.remove(file_path)
except:
handle_error()
def delete_file(file_path):
msg = "Deleting file: %s" % file_path
LOG.debug(msg)
if os.path.exists(file_path):
try:
os.remove(file_path)
except:
LOG.warning(exc_info())
def delete_file(file_path):
msg = "Deleting file: %s" % file_path
LOG.debug(msg)
if os.path.exists(file_path):
try:
os.remove(file_path)
except:
handle_error()
def execute(self, query, retries=0, retry_timeout=10):
while True:
try:
if self._autocommit or not self._connection:
self._local.connection = self._get_connection_from_pool(
timeout=30)
self._local.connection.autocommit(self._autocommit)
self._local.cursor = self._connection.cursor()
start_time = time.time()
if len(query) > 2000:
msg = '%s...' % query[:2000]
else:
msg = query
LOG.debug(msg)
try:
self._local.cursor.execute(query)
results = self._local.cursor.fetchall()
finally:
end_time = time.time()
try:
if self._autocommit:
self._connection_pool.put(self._local.connection)
self._local.cursor.close()
self._local.connection = None
self._local.cursor = None
except:
msg = 'MySQL finalize connection error'
helper.handle_error(message=msg, level='error')
if end_time - start_time > 5:
LOG.debug('Query too slow: %s\n%s...' %
(end_time - start_time, query[:250]))
if results is not None:
results = tuple(results)
else:
results = tuple()
return results
except exceptions.TimeoutError as e:
LOG.warning(e)
except (pymysql.err.InternalError, pymysql.err.OperationalError,
socket.timeout) as e:
if isinstance(e,
pymysql.err.InternalError) and e.args[0] == 1213:
LOG.warning('MySQL 1213 error, retry')
time.sleep(random.randint(0, 20) / 100.0)
continue
if isinstance(
e, pymysql.err.OperationalError) and e.args[0] == 2013:
LOG.warning('MySQL 2013 error during query: %s' %
query[0:150])
if self._local.connection:
self._connection_pool.remove(self._local.connection)
self._local.connection.close()
self._local.connection = None
if not retries:
raise
retries -= 1
time.sleep(retry_timeout)
def wrapper(*args, **kwds):
start_time = time.time()
try:
return f(*args, **kwds)
finally:
end_time = time.time()
msg = 'TIMEIT %s.%s: %s' % (f.__module__, f.__name__,
end_time - start_time)
LOG.debug(msg)
def process_aws_billing(self):
if self.args['--recalculate']:
return
dtime_from, dtime_to = self.get_aws_billing_interval()
msg = 'AWS billing interval: {0} - {1}'
msg = msg.format(dtime_from, dtime_to)
LOG.debug(msg)
with self._lock:
if not self.aws_billing_dtime_from:
self.aws_billing_dtime_from = dtime_from
else:
self.aws_billing_dtime_from = min(self.aws_billing_dtime_from, dtime_from)
for envs in self.analytics.load_envs():
unique = {}
for env in envs:
if env.get('ec2.detailed_billing.enabled', '0') != '1':
continue
bucket_name = env['ec2.detailed_billing.bucket']
creds = self.analytics.get_creds([env])
cred = next(cred for cred in creds if cred.platform == 'ec2')
unique.setdefault(cred.unique,
{'envs_ids': [], 'cred': cred, 'bucket_name': bucket_name})
unique[cred.unique]['envs_ids'].append(env['id'])
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(self.process_aws_account, args=(data, dtime_from, dtime_to))
self.pool.join()
if not self.aws_billing_dtime_from:
return
dtime_from = self.aws_billing_dtime_from
if self.config['dtime_to']:
dtime_to = self.config['dtime_to']
else:
dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
dtime_to = dtime_hour_ago.replace(minute=59, second=59, microsecond=999999)
# fill farm_usage_d
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
self.analytics.fill_farm_usage_d(date, hour)
except:
msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour)
LOG.exception(msg)
dtime_cur += datetime.timedelta(hours=1)
def insert_not_managed_server(self, server):
try:
LOG.debug('Insert not managed server: %s' % server)
record = server.copy()
record['date'] = server['dtime'].date()
record['num'] = 1
if server['cost'] is None:
record['cost'] = 0
with self._nm_usage_lock:
server_exists = self._not_managed_server_exists(record)
subject_id = self._find_subject_id(record)
self.analytics_db.autocommit(False)
try:
if subject_id:
record['subject_id'] = uuid.UUID(subject_id).hex
else:
record['subject_id'] = uuid.uuid4().hex
nm_subjects_h = NM_subjects_h(record)
self.analytics_db.execute(nm_subjects_h.insert_query(),
retries=1)
if not server_exists:
nm_usage_h = NM_usage_h(record)
record['usage_id'] = nm_usage_h['usage_id']
self.analytics_db.execute(nm_usage_h.insert_query(),
retries=1)
nm_usage_subjects_h = NM_usage_subjects_h(record)
self.analytics_db.execute(
nm_usage_subjects_h.insert_query(), retries=1)
nm_usage_servers_h = NM_usage_servers_h(record)
self.analytics_db.execute(
nm_usage_servers_h.insert_query(), retries=1)
# insert nm_usage_d table
nm_usage_d = NM_usage_d(record)
self.analytics_db.execute(nm_usage_d.insert_query(),
retries=1)
self.analytics_db.commit()
except:
self.analytics_db.rollback()
raise
finally:
self.analytics_db.autocommit(True)
except:
msg = "Unable to insert not managed server: {server}, reason: {error}"
msg = msg.format(server=server, error=helper.exc_info())
LOG.error(msg)
raise
def _start(self):
if helper.check_pid(self.config['pid_file']):
raise exceptions.AlreadyRunningError(self.config['pid_file'])
LOG.debug('Starting')
if self.args['--daemon']:
helper.daemonize()
helper.create_pid_file(self.config['pid_file'])
atexit.register(helper.delete_file, self.config['pid_file'])
LOG.info('Started')
helper.set_proc_name(self.name)
self()
LOG.info('Stopped')
def _encrypt(self, server_id, crypto_key, data, headers=None):
assert server_id, 'server_id'
assert crypto_key, 'scalarizr.key'
assert data, 'data to encrypt'
crypto_algo = dict(name="des_ede3_cbc", key_size=24, iv_size=8)
data = cryptotool.encrypt_scalarizr(crypto_algo, data, cryptotool.decrypt_key(crypto_key))
headers = headers or dict()
headers['X-Signature'], headers['Date'] = cryptotool.sign(data, crypto_key)
headers['X-Server-Id'] = server_id
msg = "Server: {0}, key: {1} ... {2}".format(server_id, crypto_key[0:5], crypto_key[-5:])
LOG.debug(msg)
return data, headers
def execute(self, query, retries=0, retry_timeout=10):
while True:
try:
if self._autocommit or not self._connection:
self._local.connection = self._get_connection_from_pool(timeout=30)
self._local.connection.autocommit(self._autocommit)
self._local.cursor = self._connection.cursor()
start_time = time.time()
if len(query) > 2000:
msg = '%s...' % query[:2000]
else:
msg = query
LOG.debug(msg)
try:
self._local.cursor.execute(query)
results = self._local.cursor.fetchall()
finally:
end_time = time.time()
try:
if self._autocommit:
self._connection_pool.put(self._local.connection)
self._local.cursor.close()
self._local.connection = None
self._local.cursor = None
except:
msg = 'MySQL finalize connection error'
helper.handle_error(message=msg, level='error')
if end_time - start_time > 5:
LOG.debug('Query too slow: %s\n%s...' % (end_time - start_time, query[:250]))
if results is not None:
results = tuple(results)
else:
results = tuple()
return results
except exceptions.TimeoutError as e:
LOG.warning(e)
except (pymysql.err.InternalError, pymysql.err.OperationalError, socket.timeout) as e:
if isinstance(e, pymysql.err.InternalError) and e.args[0] == 1213:
LOG.warning('MySQL 1213 error, retry')
time.sleep(random.randint(0, 20) / 100.0)
continue
if isinstance(e, pymysql.err.OperationalError) and e.args[0] == 2013:
LOG.warning('MySQL 2013 error during query: %s' % query[0:150])
if self._local.connection:
self._connection_pool.remove(self._local.connection)
self._local.connection.close()
self._local.connection = None
if not retries:
raise
retries -= 1
time.sleep(retry_timeout)
def _start(self):
if helper.check_pid(self.config['pid_file']):
raise exceptions.AlreadyRunningError(self.config['pid_file'])
LOG.debug(self._starting_msg)
if self.args['--daemon']:
helper.daemonize()
helper.create_pid_file(self.config['pid_file'])
atexit.register(helper.delete_file, self.config['pid_file'])
helper.set_proc_name(self.name)
self.start_dtime = datetime.datetime.utcnow()
LOG.info('Started')
self()
LOG.info('Stopped')
def get_usage(self, subscription_id, access_token, dtime_from, dtime_to, resolution='Hourly'):
url = self.usage_url.format(subscription_id=subscription_id)
step = datetime.timedelta(hours=12)
reported_dtime_from = dtime_from
if resolution == 'Hourly':
reported_dtime_to = min(reported_dtime_from + step, dtime_to)
elif resolution == 'Daily':
reported_dtime_to = dtime_to.replace(hour=0)
if reported_dtime_from == reported_dtime_to:
return
while reported_dtime_to <= dtime_to:
msg = 'Request Azure billing for subscription {}, reported dtime: {} - {}'
msg = msg.format(subscription_id, reported_dtime_from, reported_dtime_to)
LOG.debug(msg)
params = {
'api-version': self.api_version,
'reportedStartTime': reported_dtime_from.strftime('%Y-%m-%dT%H:%S:%M+00:00'),
'reportedEndTime': reported_dtime_to.strftime('%Y-%m-%dT%H:%M:%S+00:00'),
'aggregationGranularity': resolution,
'showDetails': 'true',
}
headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer %s' % access_token,
}
session = requests.Session()
next_link = requests.Request(method='get', url=url, params=params).prepare().url
while next_link:
resp = session.get(next_link, headers=headers)
resp.raise_for_status()
jsoned_resp = resp.json()
assert 'value' in jsoned_resp, resp.text
rows = []
for row in jsoned_resp['value']:
properties = row['properties']
if 'instanceData' not in properties:
# skip old format
continue
if properties['subscriptionId'] != subscription_id:
continue
row['reported_date'] = reported_dtime_from.date()
rows.append(row)
if rows:
yield rows
next_link = jsoned_resp.get('nextLink')
if reported_dtime_to == dtime_to or resolution == 'Daily':
break
reported_dtime_from = reported_dtime_to
reported_dtime_to = min(reported_dtime_to + step, dtime_to)
def update_server(self, server):
try:
szr_upd_client = self._get_szr_upd_client(server)
timeout = self.config["instances_connection_timeout"]
msg = "Trying to update server: {0}, version: {1}".format(server["server_id"], server["scalarizr.version"])
LOG.debug(msg)
try:
result_id = szr_upd_client.update(async=True, timeout=timeout)
except:
msg = "Unable to update, reason: {0}".format(helper.exc_info())
raise Exception(msg)
LOG.debug("Server: {0}, result: {1}".format(server["server_id"], result_id))
except:
msg = "Server failed: {0}, reason: {1}".format(server["server_id"], helper.exc_info())
LOG.warning(msg)
def update_server(self, server):
try:
szr_upd_client = self._get_szr_upd_client(server)
timeout = self.config['instances_connection_timeout']
msg = "Trying to update server: {0}, version: {1}".format(
server['server_id'], server['scalarizr.version'])
LOG.debug(msg)
try:
result_id = szr_upd_client.update(async=True, timeout=timeout)
except:
msg = 'Unable to update, reason: {0}'.format(helper.exc_info())
raise Exception(msg)
LOG.debug("Server: {0}, result: {1}".format(server['server_id'], result_id))
except:
msg = "Server failed: {0}, reason: {1}".format(server['server_id'], helper.exc_info())
LOG.warning(msg)
def _encrypt(self, server_id, crypto_key, data, headers=None):
assert server_id, 'server_id'
assert crypto_key, 'scalarizr.key'
assert data, 'data to encrypt'
crypto_algo = dict(name="des_ede3_cbc", key_size=24, iv_size=8)
data = cryptotool.encrypt_scalarizr(crypto_algo, data,
cryptotool.decrypt_key(crypto_key))
headers = headers or dict()
headers['X-Signature'], headers['Date'] = cryptotool.sign(
data, crypto_key)
headers['X-Server-Id'] = server_id
msg = "Server: {0}, key: {1} ... {2}".format(server_id,
crypto_key[0:5],
crypto_key[-5:])
LOG.debug(msg)
return data, headers
def __call__(self):
self.change_permissions()
while True:
self.iteration_timestamp = time.time()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
finally:
if not g.ready():
g.kill()
LOG.debug('End iteration: {0:.1f} seconds'.format(
time.time() - self.iteration_timestamp))
if self.config['interval']:
next_iteration_time = self.iteration_timestamp + self.config[
'interval']
time.sleep(next_iteration_time - time.time())
def _set_next_update_dt(self, servers):
for server in servers:
next_update_dt = str(self._scheduled_on(server['schedule']))
if next_update_dt != server['scheduled_on']:
query = ("""INSERT INTO farm_role_settings """
"""(farm_roleid, name, value) """
"""VALUES ({0}, 'scheduled_on', '{1}') """
"""ON DUPLICATE KEY UPDATE value='{1}'""").format(
server['farm_roleid'], next_update_dt)
msg = "Set next update datetime for server: {0} to: {1}"
msg = msg.format(server['server_id'], next_update_dt)
LOG.debug(msg)
try:
self._db.execute(query, retries=1)
except:
msg = 'Unable to update next update datetime for server: {0}, reason: {1}'
msg = msg.format(server['server_id'], helper.exc_info())
LOG.warning(msg)
def _ec2_region(region, cred):
try:
access_key = cryptotool.decrypt_scalr(app.crypto_key,
cred['access_key'])
secret_key = cryptotool.decrypt_scalr(app.crypto_key,
cred['secret_key'])
kwds = {
'aws_access_key_id': access_key,
'aws_secret_access_key': secret_key
}
proxy_settings = app.proxy_settings.get(cred.platform, {})
kwds['proxy'] = proxy_settings.get('host')
kwds['proxy_port'] = proxy_settings.get('port')
kwds['proxy_user'] = proxy_settings.get('user')
kwds['proxy_pass'] = proxy_settings.get('pass')
msg = "List nodes for platform: 'ec2', region: '{}', envs_ids: {}"
msg = msg.format(region, cred.envs_ids)
LOG.debug(msg)
conn = boto.ec2.connect_to_region(region, **kwds)
cloud_nodes = _ec2_get_only_instances(conn)
timestamp = int(time.time())
nodes = list()
for cloud_node in cloud_nodes:
node = {
'instance_id': cloud_node.id,
'instance_type': cloud_node.instance_type,
'os': cloud_node.platform if cloud_node.platform else 'linux'
}
nodes.append(node)
return {
'region': region,
'timestamp': timestamp,
'nodes': nodes
} if nodes else dict()
except:
e = sys.exc_info()[1]
msg = "platform: '{platform}', region: '{region}', envs_ids: {envs_ids}. Reason: {error}"
msg = msg.format(platform=cred.platform,
region=region,
envs_ids=cred.envs_ids,
error=helper.exc_info(where=False))
_handle_exception(e, msg)
def process_message(self, message, server):
try:
try:
request = self.make_request(message, server)
except:
message['status'] = 3
msg = "Make request failed, reason: {error}".format(
error=helper.exc_info())
raise Exception(msg)
if not request['url']:
message['status'] = 3
msg = "Wrong request: {request}".format(request=request)
raise Exception(msg)
msg = "Send message: {message_id}, request: {request}"
msg = msg.format(message_id=message['messageid'],
request={
'url': request['url'],
'headers': request['headers']
})
LOG.debug(msg)
r = requests.post(
request['url'],
data=request['data'],
headers=request['headers'],
timeout=self.config['instances_connection_timeout'])
if r.status_code != 201:
msg = "Bad response code: {code}".format(code=r.status_code)
raise Exception(msg)
message['status'] = 1
msg = "Delivery Ok, message: {message_id}"
msg = msg.format(message_id=message['messageid'])
LOG.debug(msg)
except:
if message['status'] == 0 and int(message['handle_attempts']) >= 2:
message['status'] = 3
msg = "Delivery failed, message: {message_id}, server: {server}, reason: {error}"
server['scalarizr.key'] = '******'
msg = msg.format(message_id=message['messageid'],
server=server,
error=helper.exc_info())
LOG.warning(msg)
self.update(message)
def clean(self):
for directory in os.listdir(self.config['rrd_dir']):
for farm_id in os.listdir('%s/%s' % (self.config['rrd_dir'], directory)):
try:
if self._is_farm_exists({'id': int(farm_id)}):
continue
dir_to_delete = os.path.join(self.config['rrd_dir'], directory, farm_id)
LOG.debug('Delete farm {0}: {1}'.format(farm_id, dir_to_delete))
if self.args['--test']:
continue
shutil.rmtree(dir_to_delete, ignore_errors=True)
except KeyboardInterrupt:
raise
except pymysql.err.Error as e:
if e.args[0] == KeyboardInterrupt:
raise KeyboardInterrupt
LOG.warning(helper.exc_info())
except:
LOG.warning(helper.exc_info())
def _set_next_update_dt(self, servers):
for server in servers:
next_update_dt = str(self._scheduled_on(server['schedule']))
if next_update_dt != server['scheduled_on']:
query = (
"""INSERT INTO farm_role_settings """
"""(farm_roleid, name, value) """
"""VALUES ({0}, 'scheduled_on', '{1}') """
"""ON DUPLICATE KEY UPDATE value='{1}'"""
).format(server['farm_roleid'], next_update_dt)
msg = "Set next update datetime for server: {0} to: {1}"
msg = msg.format(server['server_id'], next_update_dt)
LOG.debug(msg)
try:
self._db.execute(query, retries=1)
except:
msg = 'Unable to update next update datetime for server: {0}, reason: {1}'
msg = msg.format(server['server_id'], helper.exc_info())
LOG.warning(msg)
def process_credential(cred, envs_ids=None):
if envs_ids is None:
envs_ids = [cred.envs_ids]
try:
analytics.Credentials.test(cred, cred.platform)
cloud_data = eval(cred.platform)(cred)
msg = "platform: '{}', envs_ids: {}, cloud data: {}"
msg = msg.format(cred.platform, envs_ids, cloud_data)
LOG.debug(msg)
if cloud_data:
sorted_data = sort_nodes(cloud_data, cred, envs_ids)
sorted_data_update(sorted_data)
db_update(sorted_data, envs_ids, cred)
except:
e = sys.exc_info()[1]
msg = "platform: '{platform}', environments: {envs}. Reason: {error}"
msg = msg.format(platform=cred.platform, envs=envs_ids, error=helper.exc_info(where=False))
_handle_exception(e, msg)
def _ec2_region(region, cred):
try:
access_key = cryptotool.decrypt_scalr(app.crypto_key, cred['access_key'])
secret_key = cryptotool.decrypt_scalr(app.crypto_key, cred['secret_key'])
kwds = {
'aws_access_key_id': access_key,
'aws_secret_access_key': secret_key
}
proxy_settings = app.proxy_settings[cred.platform]
kwds['proxy'] = proxy_settings.get('host', None)
kwds['proxy_port'] = proxy_settings.get('port', None)
kwds['proxy_user'] = proxy_settings.get('user', None)
kwds['proxy_pass'] = proxy_settings.get('pass', None)
msg = "List nodes for platform: 'ec2', region: '{}', envs_ids: {}"
msg = msg.format(region, cred.envs_ids)
LOG.debug(msg)
conn = boto.ec2.connect_to_region(region, **kwds)
cloud_nodes = _ec2_get_only_instances(conn)
timestamp = int(time.time())
nodes = list()
for cloud_node in cloud_nodes:
node = {
'instance_id': cloud_node.id,
'instance_type': cloud_node.instance_type,
'os': cloud_node.platform if cloud_node.platform else 'linux'
}
nodes.append(node)
return {
'region': region,
'timestamp': timestamp,
'nodes': nodes
} if nodes else dict()
except:
e = sys.exc_info()[1]
msg = "platform: '{platform}', region: '{region}', envs_ids: {envs_ids}. Reason: {error}"
msg = msg.format(platform=cred.platform, region=region, envs_ids=cred.envs_ids,
error=helper.exc_info(where=False))
_handle_exception(e, msg)
def process_message(self, message, server):
try:
try:
request = self.make_request(message, server)
except:
message['status'] = 3
msg = "Make request failed, reason: {error}".format(error=helper.exc_info())
raise Exception(msg)
if not request['url']:
message['status'] = 3
msg = "Wrong request: {request}".format(request=request)
raise Exception(msg)
msg = "Send message: {message_id}, request: {request}"
msg = msg.format(
message_id=message['messageid'],
request={'url': request['url'], 'headers': request['headers']})
LOG.debug(msg)
r = requests.post(
request['url'],
data=request['data'],
headers=request['headers'],
timeout=self.config['instances_connection_timeout'])
if r.status_code != 201:
msg = "Bad response code: {code}".format(code=r.status_code)
raise Exception(msg)
message['status'] = 1
msg = "Delivery Ok, message: {message_id}"
msg = msg.format(message_id=message['messageid'])
LOG.debug(msg)
except:
if message['status'] == 0 and int(message['handle_attempts']) >= 2:
message['status'] = 3
msg = "Delivery failed, message: {message_id}, server: {server}, reason: {error}"
server['scalarizr.key'] = '******'
msg = msg.format(
message_id=message['messageid'], server=server, error=helper.exc_info())
LOG.warning(msg)
self.update(message)
def call(cmd, input=None, **kwds):
"""
>>> call("echo 'hello'")
("'hello'\\n", '')
>>> call("echo 'hello'", shell=True)
('hello\\n', '')
"""
if 'stdout' not in kwds:
kwds.update({'stdout': subprocess.PIPE})
if 'stderr' not in kwds:
kwds.update({'stderr': subprocess.PIPE})
if 'stdin' not in kwds:
kwds.update({'stdin': subprocess.PIPE})
msg = "Call '%s'" % cmd
LOG.debug(msg)
if 'shell' in kwds and kwds['shell']:
p = subprocess.Popen(cmd, **kwds)
else:
p = subprocess.Popen(cmd.split(), **kwds)
stdout, stderr = p.communicate(input=input)
return stdout, stderr, p.returncode
def __call__(self):
self._stop = False
while not self._stop:
g = None
try:
self.start_time = time.time()
LOG.debug('Start periodical task ({})'.format(self.task_name))
self.iteration_number += 1
self.task.task_info = {
'period': self.period,
'timeout': self.timeout,
'start_time': self.start_time,
'iteration_number': self.iteration_number,
}
if callable(self.before):
msg = 'Periodical task ({}) call before ({})'
msg = msg.format(self.task_name, self.before.__name__)
LOG.debug(msg)
self.before(self)
g = gevent.spawn(self.task)
g.get(timeout=self.timeout)
except:
if g and not g.ready():
g.kill()
try:
if callable(self.on_error):
self.on_error(self)
except:
msg = 'Periodical task ({}) on error ({}) failed'
msg = msg.format(self.task_name, self.on_error.__name__)
handle_error(message=msg)
msg = 'Periodical task ({}) error: {}'
msg = msg.format(self.task_name, exc_info(where=False))
handle_error(message=msg)
finally:
try:
if callable(self.after):
try:
msg = 'Periodical task ({}) call after ({})'
msg = msg.format(self.task_name,
self.after.__name__)
LOG.debug(msg)
self.after(self)
except:
msg = 'After ({0}) failed'.format(
self.after.__name__)
handle_error(message=msg)
self.end_time = time.time()
task_time = self.end_time - self.start_time
msg = 'End task ({0}): {1:.1f} seconds'
msg = msg.format(self.task_name, task_time)
LOG.info(msg)
if self.period:
next_time = self.start_time + self.period
while time.time() < next_time and not self._stop:
time.sleep(0.5)
except:
msg = 'Task ({}) finally failed'.format(self.task_name)
handle_error(message=msg)
def clean(self):
for directory in os.listdir(self.config['rrd_dir']):
for farm_id in os.listdir('%s/%s' %
(self.config['rrd_dir'], directory)):
try:
if self._is_farm_exists({'id': int(farm_id)}):
continue
dir_to_delete = os.path.join(self.config['rrd_dir'],
directory, farm_id)
LOG.debug('Delete farm {0}: {1}'.format(
farm_id, dir_to_delete))
if self.args['--test']:
continue
shutil.rmtree(dir_to_delete, ignore_errors=True)
except KeyboardInterrupt:
raise
except pymysql.err.Error as e:
if e.args[0] == KeyboardInterrupt:
raise KeyboardInterrupt
LOG.warning(helper.exc_info())
except:
LOG.warning(helper.exc_info())
def get_cost_from_prices(self, server, prices):
account_id = server['account_id']
platform = server['platform']
url = server['url']
cloud_location = server['cloud_location']
instance_type = server['instance_type']
os = server['os']
platform_url = '%s;%s' % (platform, url)
try:
if account_id in prices:
cost = prices[account_id][platform_url][cloud_location][instance_type][os]
else:
cost = prices[0][platform_url][cloud_location][instance_type][os]
except KeyError:
cost = None
msg = (
"Unable to get cost for account_id: {0}, platform: {1}, url: {2}, "
"cloud_location: {3}, instance_type: {4}, os: {5}. Use 0.0, reason: {6}"
).format(account_id, platform, url, cloud_location,
instance_type, os, helper.exc_info())
LOG.debug(msg)
return cost
def call(cmd, input=None, **kwds):
"""
>>> call("echo 'hello'")
("'hello'\\n", '')
>>> call("echo 'hello'", shell=True)
('hello\\n', '')
"""
if 'stdout' not in kwds:
kwds.update({'stdout': subprocess.PIPE})
if 'stderr' not in kwds:
kwds.update({'stderr': subprocess.PIPE})
if 'stdin' not in kwds:
kwds.update({'stdin': subprocess.PIPE})
msg = "Call '%s'" % cmd
LOG.debug(msg)
if 'shell' in kwds and kwds['shell']:
p = subprocess.Popen(cmd, **kwds)
else:
p = subprocess.Popen(cmd.split(), **kwds)
stdout, stderr = p.communicate(input=input)
return stdout, stderr, p.returncode
def __call__(self):
try:
dtime_from, dtime_to = self.get_billing_interval()
quarters_calendar = self.analytics.get_quarters_calendar()
quarter_number = quarters_calendar.quarter_for_date(dtime_from.date())
quarter_year = quarters_calendar.year_for_date(dtime_from.date())
quarter_start_dtime, quarter_end_dtime = quarters_calendar.dtime_for_quarter(
quarter_number, year=quarter_year)
if quarter_start_dtime < dtime_from:
quarter_number, quarter_year = quarters_calendar.next_quarter(quarter_number, quarter_year)
quarter_start_dtime, quarter_end_dtime = quarters_calendar.dtime_for_quarter(
quarter_number, year=quarter_year)
while quarter_start_dtime < dtime_to:
msg = 'Recalculate {} quarter ({} - {}) for year {}'
msg = msg.format(quarter_number, quarter_start_dtime, quarter_end_dtime, quarter_year)
LOG.info(msg)
self.config['dtime_from'] = quarter_start_dtime
self.config['dtime_to'] = min(quarter_end_dtime, dtime_to)
super(RecalculateAWSBilling, self).__call__()
self.fill_farm_usage_d(force=True)
msg = 'Recalculate quarterly_budget'
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(quarter_year, quarter_number)
quarter_number, quarter_year = quarters_calendar.next_quarter(quarter_number, quarter_year)
quarter_start_dtime, quarter_end_dtime = quarters_calendar.dtime_for_quarter(
quarter_number, year=quarter_year)
except:
self.pool.kill()
helper.handle_error(message='Recalculate AWS billing failed')
raise
def process_credential(cred, envs_ids=None):
if envs_ids is None:
envs_ids = [cred.envs_ids]
try:
analytics.Credentials.test(cred, cred.platform)
cloud_data = eval(cred.platform)(cred)
msg = "platform: '{}', envs_ids: {}, cloud data: {}"
msg = msg.format(cred.platform, envs_ids, cloud_data)
LOG.debug(msg)
if cloud_data:
sorted_data = sort_nodes(cloud_data, cred, envs_ids)
sorted_data_update(sorted_data)
db_update(sorted_data, envs_ids, cred)
except:
e = sys.exc_info()[1]
msg = "platform: '{platform}', environments: {envs}. Reason: {error}"
msg = msg.format(platform=cred.platform,
envs=envs_ids,
error=helper.exc_info(where=False))
_handle_exception(e, msg)
def do_iteration(self):
for envs in self.analytics.load_envs():
msg = "Processing environments: {}".format(
[env['id'] for env in envs])
LOG.debug(msg)
try:
self.analytics.load_env_credentials(envs)
unique = {}
for env in envs:
try:
credentials = self.analytics.get_credentials([env])
for cred in credentials:
if cred.platform == 'ec2' and env.get(
'ec2.detailed_billing.enabled',
'0') == '1':
continue
unique.setdefault(cred.unique, {
'envs_ids': [],
'cred': cred
})
unique[cred.unique]['envs_ids'].append(env['id'])
except:
msg = 'Processing environment: {} failed'.format(
env['id'])
LOG.exception(msg)
for data in unique.values():
while len(self.pool) > self.config['pool_size'] * 5 / 10:
gevent.sleep(0.1)
self.pool.apply_async(process_credential,
args=(data['cred'], ),
kwds={'envs_ids': data['envs_ids']})
gevent.sleep(0) # force switch
except:
msg = 'Processing environments: {} failed'.format(
[env['id'] for env in envs])
LOG.exception(msg)
self.pool.join()
