def __call__(self):
self.change_permissions()
while True:
try:
self.iteration_timestamp = time.time()
g = self._do_iteration()
try:
g.get(timeout=self.iteration_timeout)
except gevent.Timeout:
raise exceptions.IterationTimeoutError()
finally:
if not g.ready():
g.kill()
except (SystemExit, KeyboardInterrupt):
raise
except exceptions.NothingToDoError:
time_to_sleep = self.nothing_to_do_sleep
except exceptions.QuitError:
sys.exit(0)
except:
LOG.error('Iteration failed, reason: {0}'.format(helper.exc_info()))
self.on_iteration_error()
time_to_sleep = self.error_sleep
else:
time_to_sleep = 0.1
LOG.debug('End iteration: {0:.1f} seconds'.format(time.time() - self.iteration_timestamp))
if self.config['interval']:
time_to_sleep = self.iteration_timestamp + self.config['interval'] - time.time()
time.sleep(time_to_sleep)
def get_servers(self):
for servers in self._get_servers():
prop = ['scalarizr.api_port', 'scalarizr.key']
self._db.load_server_properties(servers, prop)
for server in servers:
if 'scalarizr.api_port' not in server:
server['scalarizr.api_port'] = 8010
if 'scalarizr.key' not in server:
server['scalarizr.key'] = None
self._db.load_vpc_settings(servers)
out = []
for server in servers:
try:
if server['os_type'] == 'linux':
exclude = ['snum']
elif server['os_type'] == 'windows':
exclude = ['la', 'io', 'snum']
else:
msg = "Wrong os type for server: '%s'" % server['server_id']
raise Exception(msg)
metrics = [m for m in self.config['metrics'] if m not in exclude]
server['metrics'] = metrics
out.append(server)
except:
LOG.error(helper.exc_info())
continue
yield out
def get_servers(self, limit=500):
for servers in self._get_servers(limit=limit):
prop = ['scalarizr.api_port', 'scalarizr.key']
self._db.load_server_properties(servers, prop)
for server in servers:
if 'scalarizr.api_port' not in server:
server['scalarizr.api_port'] = 8010
if 'scalarizr.key' not in server:
server['scalarizr.key'] = None
self._db.load_vpc_settings(servers)
out = []
for server in servers:
try:
if server['os_type'] == 'linux':
exclude = ['snum']
elif server['os_type'] == 'windows':
exclude = ['la', 'io', 'snum']
else:
msg = "Wrong os type for server: '%s'" % server['server_id']
raise Exception(msg)
metrics = [m for m in self.config['metrics'] if m not in exclude]
server['metrics'] = metrics
out.append(server)
except:
LOG.error(helper.exc_info())
continue
yield out
def _serve_forever(self):
LOG.debug('Starting plotter')
try:
cherrypy.quickstart(self, '/', {'/': {}})
except:
LOG.error(helper.exc_info())
thread.interrupt_main()
def _handle_webhook_exception(self, webhook):
exc = sys.exc_info()[1]
if isinstance(exc, (
requests.exceptions.Timeout,
requests.exceptions.ProxyError,
requests.exceptions.ConnectionError)):
if webhook['handle_attempts'] < webhook['attempts']:
webhook['status'] = 0
else:
webhook['status'] = 2
webhook['error_msg'] = str(sys.exc_info()[0].__name__)
msg = "Unable to process webhook: {0}, reason: {1}"
msg = msg.format(webhook['history_id'], helper.exc_info())
LOG.warning(msg)
elif isinstance(exc, (
requests.exceptions.RequestException,
requests.packages.urllib3.exceptions.HTTPError,
requests.packages.urllib3.exceptions.HTTPWarning)):
webhook['status'] = 2
webhook['error_msg'] = str(sys.exc_info()[0].__name__)
msg = "Unable to process webhook: {0}, reason: {1}"
msg = msg.format(webhook['history_id'], helper.exc_info())
LOG.warning(msg)
else:
webhook['status'] = 2
webhook['error_msg'] = 'Internal error'
msg = "Unable to process webhook: {0}, reason: {1}"
msg = msg.format(webhook['history_id'], helper.exc_info())
LOG.error(msg)
def _handle_webhook_exception(self, webhook):
exc = sys.exc_info()[1]
if isinstance(
exc,
(requests.exceptions.Timeout, requests.exceptions.ProxyError,
requests.exceptions.ConnectionError)):
if webhook['handle_attempts'] < webhook['attempts']:
webhook['status'] = 0
else:
webhook['status'] = 2
webhook['error_msg'] = str(sys.exc_info()[0].__name__)
msg = "Unable to process webhook: {0}, reason: {1}"
msg = msg.format(webhook['history_id'], helper.exc_info())
LOG.warning(msg)
elif isinstance(exc,
(requests.exceptions.RequestException,
requests.packages.urllib3.exceptions.HTTPError,
requests.packages.urllib3.exceptions.HTTPWarning)):
webhook['status'] = 2
webhook['error_msg'] = str(sys.exc_info()[0].__name__)
msg = "Unable to process webhook: {0}, reason: {1}"
msg = msg.format(webhook['history_id'], helper.exc_info())
LOG.warning(msg)
else:
webhook['status'] = 2
webhook['error_msg'] = 'Internal error'
msg = "Unable to process webhook: {0}, reason: {1}"
msg = msg.format(webhook['history_id'], helper.exc_info())
LOG.error(msg)
def _average(self, results, ra=None, fa=None, rs=None, fs=None):
ra = ra or dict()
fa = fa or dict()
rs = rs or dict()
fs = fs or dict()
for result in results:
try:
r_key, f_key = self._get_rf_keys(result)
if 'snum' in self.config['metrics']:
rs.setdefault(r_key, {'snum': {'s_running': 0}})
fs.setdefault(f_key, {'snum': {'s_running': 0}})
rs[r_key]['snum']['s_running'] += 1
fs[f_key]['snum']['s_running'] += 1
if not result['data']:
continue
for metrics_group_name, metrics_data in result['data'].iteritems():
if not metrics_data or metrics_group_name == 'io':
continue
for metric_name, value in metrics_data.iteritems():
try:
ra.setdefault(r_key, {})
ra[r_key].setdefault(metrics_group_name, {})
ra[r_key][metrics_group_name].setdefault(metric_name, Average())
fa.setdefault(f_key, {})
fa[f_key].setdefault(metrics_group_name, {})
fa[f_key][metrics_group_name].setdefault(metric_name, Average())
ra[r_key][metrics_group_name][metric_name] += value
fa[f_key][metrics_group_name][metric_name] += value
except:
LOG.error(helper.exc_info())
except:
LOG.error(helper.exc_info())
return ra, fa, rs, fs
def _serve_forever(self):
LOG.debug('Starting plotter')
try:
cherrypy.quickstart(self, '/', {'/': {}})
except:
LOG.error(helper.exc_info())
thread.interrupt_main()
def _average(self, results, ra=None, fa=None, rs=None, fs=None):
ra = ra or dict()
fa = fa or dict()
rs = rs or dict()
fs = fs or dict()
for result in results:
try:
r_key, f_key = self._get_rf_keys(result)
if 'snum' in self.config['metrics']:
rs.setdefault(r_key, {'snum': {'s_running': 0}})
fs.setdefault(f_key, {'snum': {'s_running': 0}})
rs[r_key]['snum']['s_running'] += 1
fs[f_key]['snum']['s_running'] += 1
if not result['data']:
continue
for metrics_group_name, metrics_data in result['data'].iteritems():
if not metrics_data or metrics_group_name == 'io':
continue
for metric_name, value in metrics_data.iteritems():
try:
ra.setdefault(r_key, {})
ra[r_key].setdefault(metrics_group_name, {})
ra[r_key][metrics_group_name].setdefault(metric_name, Average())
fa.setdefault(f_key, {})
fa[f_key].setdefault(metrics_group_name, {})
fa[f_key][metrics_group_name].setdefault(metric_name, Average())
ra[r_key][metrics_group_name][metric_name] += value
fa[f_key][metrics_group_name][metric_name] += value
except:
LOG.error(helper.exc_info())
except:
LOG.error(helper.exc_info())
return ra, fa, rs, fs
def get_proxy_port(self):
for szlr in self.szlrs:
#if (szlr['uuid'] == uuid) and\
# (szlr['client_port'] == self.ctl_port):
if szlr['client_port'] == str(self.ctl_port):
return szlr['proxy_port']
LOG.error("==================get_proxy_port failed! --Chen Leji==================")
return None
def update_nm_usage_h(self, record):
try:
query = NM_usage_h(record).update_query()
self.analytics_db.execute(query, retries=1)
except:
msg = 'Unable to update nm_usage_h record: {record}, reason: {error}'
msg = msg.format(record=record, error=helper.exc_info())
LOG.error(msg)
raise
def _set_usage_cost(self, records):
prices = self.analytics.get_prices(records)
for record in records:
cost = self.analytics.get_cost_from_prices(record, prices) or 0
try:
record['cost'] = float(cost) * int(record['num'])
except:
msg = 'Unable to update usage cost, reason: {error}'
msg = msg.format(error=helper.exc_info())
LOG.error(msg)
def _set_usage_cost(self, records):
prices = self.analytics.get_prices(records)
for record in records:
cost = self.analytics.get_cost_from_prices(record, prices) or 0
try:
record['cost'] = float(cost) * int(record['num'])
except:
msg = 'Unable to update usage cost, reason: {error}'
msg = msg.format(error=helper.exc_info())
LOG.error(msg)
def insert_not_managed_server(self, server):
try:
LOG.debug('Insert not managed server: %s' % server)
record = server.copy()
record['date'] = server['dtime'].date()
record['num'] = 1
if server['cost'] is None:
record['cost'] = 0
with self._nm_usage_lock:
server_exists = self._not_managed_server_exists(record)
subject_id = self._find_subject_id(record)
self.analytics_db.autocommit(False)
try:
if subject_id:
record['subject_id'] = uuid.UUID(subject_id).hex
else:
record['subject_id'] = uuid.uuid4().hex
nm_subjects_h = NM_subjects_h(record)
self.analytics_db.execute(nm_subjects_h.insert_query(),
retries=1)
if not server_exists:
nm_usage_h = NM_usage_h(record)
record['usage_id'] = nm_usage_h['usage_id']
self.analytics_db.execute(nm_usage_h.insert_query(),
retries=1)
nm_usage_subjects_h = NM_usage_subjects_h(record)
self.analytics_db.execute(
nm_usage_subjects_h.insert_query(), retries=1)
nm_usage_servers_h = NM_usage_servers_h(record)
self.analytics_db.execute(
nm_usage_servers_h.insert_query(), retries=1)
# insert nm_usage_d table
nm_usage_d = NM_usage_d(record)
self.analytics_db.execute(nm_usage_d.insert_query(),
retries=1)
self.analytics_db.commit()
except:
self.analytics_db.rollback()
raise
finally:
self.analytics_db.autocommit(True)
except:
msg = "Unable to insert not managed server: {server}, reason: {error}"
msg = msg.format(server=server, error=helper.exc_info())
LOG.error(msg)
raise
def _handle_exception(e, msg):
if type(e) == boto.exception.EC2ResponseError and e.status in [401, 403]:
LOG.warning(msg)
elif type(e) in [
libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
gevent.timeout.Timeout, socket.timeout, socket.gaierror
]:
LOG.warning(msg)
else:
LOG.error(msg)
def __call__(self):
poller_ps, plotter_ps = None, None
if self.args['--plotter']:
plotter = Plotter(self.config)
plotter_ps = plotter.run_in_process()
time.sleep(5)
if not plotter_ps.is_alive():
LOG.critical('Failed to start CherryPy web server')
sys.exit(1)
self.change_permissions()
if self.args['--poller']:
poller = Poller(self.config, self.scalr_config)
while True:
start_time = time.time()
try:
LOG.info('Start poller iteration')
rrdcached_sock_file = self.config['rrd'][
'rrdcached_sock_path']
if not os.path.exists(rrdcached_sock_file):
raise Exception('rrdcached process is not running')
poller_ps = poller.run_in_process()
poller_ps.join(self.config['interval'] * 2)
if poller_ps.is_alive():
LOG.error('Poller iteration timeout. Terminating')
try:
poller_ps.terminate()
except:
msg = 'Unable to terminate, reason: {error}'.format(
error=helper.exc_info())
raise Exception(msg)
LOG.info('Poller iteration time: %.2f' %
(time.time() - start_time))
except KeyboardInterrupt:
raise
except:
msg = 'Poller iteration failed, reason: {error}'.format(
error=helper.exc_info())
LOG.error(msg)
finally:
sleep_time = start_time + self.config[
'interval'] - time.time() - 0.1
if sleep_time > 0:
time.sleep(sleep_time)
if plotter_ps:
plotter_ps.join()
def _do_iteration(self):
try:
self.do_iteration()
except exceptions.QuitError:
sys.exit(0)
except (SystemExit, KeyboardInterrupt):
raise
except exceptions.NothingToDoError:
time.sleep(self.nothing_to_do_sleep)
except:
LOG.error('Iteration failed, reason: {0}'.format(
helper.exc_info()))
self.on_iteration_error()
time.sleep(self.error_sleep)
def process_credential(cred, envs_ids=None):
if envs_ids is None:
envs_ids = [cred.env_id]
try:
cloud_data = eval(cred.platform)(cred)
if cloud_data:
sorted_data = sort_nodes(cloud_data, cred, envs_ids)
sorted_data_update(sorted_data)
db_update(sorted_data, envs_ids, cred)
except:
msg = 'platform: {platform}, environments: {envs}, reason: {error}'
msg = msg.format(platform=cred.platform, envs=envs_ids, error=helper.exc_info())
LOG.error(msg)
def __call__(self):
poller_ps, plotter_ps = None, None
if self.args['--plotter']:
plotter = Plotter(self.config)
plotter_ps = plotter.run_in_process()
time.sleep(5)
if not plotter_ps.is_alive():
LOG.critical('Failed to start CherryPy web server')
sys.exit(1)
self.change_permissions()
if self.args['--poller']:
poller = Poller(self.config, self.scalr_config)
while True:
start_time = time.time()
try:
LOG.info('Start poller iteration')
rrdcached_sock_file = self.config['rrd']['rrdcached_sock_path']
if not os.path.exists(rrdcached_sock_file):
raise Exception('rrdcached process is not running')
poller_ps = poller.run_in_process()
poller_ps.join(self.config['interval'] * 2)
if poller_ps.is_alive():
LOG.error('Poller iteration timeout. Terminating')
try:
poller_ps.terminate()
except:
msg = 'Unable to terminate, reason: {error}'.format(
error=helper.exc_info())
raise Exception(msg)
LOG.info('Poller iteration time: %.2f' % (time.time() - start_time))
except KeyboardInterrupt:
raise
except:
msg = 'Poller iteration failed, reason: {error}'.format(
error=helper.exc_info())
LOG.error(msg)
finally:
sleep_time = start_time + self.config['interval'] - time.time() - 0.1
if sleep_time > 0:
time.sleep(sleep_time)
if plotter_ps:
plotter_ps.join()
def do_iteration(self):
servers = self.get_servers_for_update()
for server in servers:
try:
self._pool.wait()
self._pool.apply_async(self.update_server, (server,))
gevent.sleep(0) # force switch
except:
LOG.warning(helper.exc_info())
self._pool.join()
try:
self.update_scalr_repo_data()
except:
msg = "Unable to update scalr.settings table, reason: {0}".format(helper.exc_info())
LOG.error(msg)
def process_credential(cred, envs_ids=None):
if envs_ids is None:
envs_ids = [cred.env_id]
try:
cloud_data = eval(cred.platform)(cred)
if cloud_data:
sorted_data = sort_nodes(cloud_data, cred, envs_ids)
sorted_data_update(sorted_data)
db_update(sorted_data, envs_ids, cred)
except:
msg = 'platform: {platform}, environments: {envs}, reason: {error}'
msg = msg.format(platform=cred.platform,
envs=envs_ids,
error=helper.exc_info())
LOG.error(msg)
def do_iteration(self):
servers = self.get_servers_for_update()
for server in servers:
try:
self._pool.wait()
self._pool.apply_async(self.update_server, (server, ))
gevent.sleep(0) # force switch
except:
LOG.warning(helper.exc_info())
self._pool.join()
try:
self.update_scalr_repo_data()
except:
msg = 'Unable to update scalr.settings table, reason: {0}'.format(
helper.exc_info())
LOG.error(msg)
def load_statistics(self, **kwds):
result = {'success': True}
try:
self._check_request(kwds)
farm = self._get_farm(kwds['farmId'])
if not farm or farm['status'] != 1:
msg = 'Statistics are not available for terminated farms'
raise Plotter.FarmTerminatedError(msg)
if farm['hash'] != kwds['hash']:
msg = 'Farm hash mismatch'
raise Plotter.PlotterError(msg)
tz = self._get_tz(kwds['farmId'])
if tz:
os.environ['TZ'] = tz
metrics = kwds['metrics'].strip().split(',')
for metric in metrics:
try:
metric = metric.strip()
if metric not in ['cpu', 'la', 'mem', 'net', 'io', 'snum']:
msg = "Unsupported metric '%s'" % metric
raise Plotter.PlotterError(msg)
url = self._plot(kwds, tz, metric)
result.setdefault('metric', dict())
result['metric'][metric] = {'success': True, 'img': url}
except Plotter.PlotterError as e:
result.setdefault('metric', dict())
result['metric'][metric] = {
'success': False,
'msg': str(e)
}
except Plotter.PlotterError as e:
result['success'] = False
result['msg'] = str(e)
except:
result['success'] = False
result['msg'] = 'Internal error. Unable to load statistics'
LOG.error(helper.exc_info())
cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
cherrypy.response.headers['Access-Control-Max-Age'] = 300
if 'Access-Control-Request-Headers' in cherrypy.request.headers:
cherrypy.response.headers['Access-Control-Allow-Headers'] = \
cherrypy.request.headers['Access-Control-Request-Headers']
return result
def _handle_exception(e, msg):
if type(e) == boto.exception.EC2ResponseError and e.status in [401, 403]:
LOG.warning(msg)
elif type(e) in [
libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
gevent.timeout.Timeout,
socket.timeout,
socket.gaierror]:
LOG.warning(msg)
elif type(e) == socket.error and e.errno in [111, 113]:
LOG.warning(msg)
elif type(e) == googleapiclient.errors.HttpError and e.resp['status'] in ['403']:
LOG.warning(msg)
else:
LOG.error(msg)
def _handle_exception(e, msg):
if type(e) == boto.exception.EC2ResponseError and e.status in [401, 403]:
LOG.warning(msg)
elif type(e) in [
libcloud.common.types.InvalidCredsError,
libcloud.common.types.LibcloudError,
libcloud.common.types.MalformedResponseError,
gevent.timeout.Timeout, socket.timeout, socket.gaierror
]:
LOG.warning(msg)
elif type(e) == socket.error and e.errno in [111, 113]:
LOG.warning(msg)
elif type(e) == googleapiclient.errors.HttpError and e.resp['status'] in [
'403'
]:
LOG.warning(msg)
else:
LOG.error(msg)
def process_credential(cred, envs_ids=None):
if envs_ids is None:
envs_ids = [cred.env_id]
try:
for k, v in cred.iteritems():
if k in cred.scheme[cred.platform]:
cred[k] = cryptotool.decrypt_scalr(app.crypto_key, v)
cloud_data = eval(cred.platform)(cred)
if cloud_data:
sorted_data = sort_nodes(cloud_data, cred, envs_ids)
sorted_data_update(sorted_data)
db_update(sorted_data, envs_ids, cred)
except:
msg = 'platform: {platform}, environments: {envs}, reason: {error}'
msg = msg.format(platform=cred.platform,
envs=envs_ids,
error=helper.exc_info())
LOG.error(msg)
def load_statistics(self, **kwds):
result = {'success': True}
try:
self._check_request(kwds)
farm = self._get_farm(kwds['farmId'])
if not farm or farm['status'] != 1:
msg = 'Statistics are not available for terminated farms'
raise Plotter.FarmTerminatedError(msg)
if farm['hash'] != kwds['hash']:
msg = 'Farm hash mismatch'
raise Plotter.PlotterError(msg)
tz = self._get_tz(kwds['farmId'])
if tz:
os.environ['TZ'] = tz
metrics = kwds['metrics'].strip().split(',')
for metric in metrics:
try:
metric = metric.strip()
if metric not in ['cpu', 'la', 'mem', 'net', 'io', 'snum']:
msg = "Unsupported metric '%s'" % metric
raise Plotter.PlotterError(msg)
url = self._plot(kwds, tz, metric)
result.setdefault('metric', dict())
result['metric'][metric] = {'success': True, 'img': url}
except Plotter.PlotterError as e:
result.setdefault('metric', dict())
result['metric'][metric] = {'success': False, 'msg': str(e)}
except Plotter.PlotterError as e:
result['success'] = False
result['msg'] = str(e)
except:
result['success'] = False
result['msg'] = 'Internal error. Unable to load statistics'
LOG.error(helper.exc_info())
cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
cherrypy.response.headers['Access-Control-Max-Age'] = 300
if 'Access-Control-Request-Headers' in cherrypy.request.headers:
cherrypy.response.headers['Access-Control-Allow-Headers'] = \
cherrypy.request.headers['Access-Control-Request-Headers']
return result
def run(self):
srv_pool = multiprocessing.pool.ThreadPool(self.config['pool_size'])
rrd_pool = multiprocessing.pool.ThreadPool(10)
try:
rrd_sock = self.config['rrd']['rrdcached_sock_path']
ra, fa, rs, fs = dict(), dict(), dict(), dict()
for servers in self.get_servers():
results = srv_pool.map(self._process_server, servers)
for result in results:
if result['data']:
file_dir = os.path.join(
self.config['rrd']['dir'],
helper.x1x2(result['farm_id']),
'%s' % result['farm_id'],
'INSTANCE_%s_%s' % (result['farm_role_id'], result['index']))
rrd_pool.apply_async(
rrd.write,
args=(file_dir, result['data'],),
kwds={'sock_path': rrd_sock})
ra, fa, rs, fs = self._average(results, ra=ra, fa=fa, rs=rs, fs=fs)
for k, v in ra.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
for k, v in fa.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
if 'snum' in self.config['metrics']:
for k, v in rs.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
for k, v in fs.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
except:
LOG.error(helper.exc_info())
finally:
srv_pool.close()
rrd_pool.close()
srv_pool.join()
rrd_pool.join()
def run(self):
srv_pool = multiprocessing.pool.ThreadPool(self.config['pool_size'])
rrd_pool = multiprocessing.pool.ThreadPool(10)
try:
rrd_sock = self.config['rrd']['rrdcached_sock_path']
ra, fa, rs, fs = dict(), dict(), dict(), dict()
for servers in self.get_servers():
results = srv_pool.map(self._process_server, servers)
for result in results:
if result['data']:
file_dir = os.path.join(
self.config['rrd']['dir'],
helper.x1x2(result['farm_id']),
'%s' % result['farm_id'],
'INSTANCE_%s_%s' % (result['farm_roleid'], result['index']))
rrd_pool.apply_async(
rrd.write,
args=(file_dir, result['data'],),
kwds={'sock_path': rrd_sock})
ra, fa, rs, fs = self._average(results, ra=ra, fa=fa, rs=rs, fs=fs)
for k, v in ra.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
for k, v in fa.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
if 'snum' in self.config['metrics']:
for k, v in rs.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
for k, v in fs.iteritems():
rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock})
except:
LOG.error(helper.exc_info())
finally:
srv_pool.close()
rrd_pool.close()
srv_pool.join()
rrd_pool.join()
def _load_managed_servers_properties(self, servers):
names = [
'farm.created_by_id',
'farm.project_id',
'env.cc_id',
'role.id',
]
names += self.server_id_map.values()
self.scalr_db.load_server_properties(servers, names)
for server in servers:
for k in self.server_id_map.values():
if k not in server:
continue
server['instance_id'] = server.pop(k)
break
try:
server['user'] = server.pop('farm.created_by_id')
except:
server['user'] = None
try:
server['project_id'] = server.pop('farm.project_id')
except:
server['project_id'] = None
try:
server['cc_id'] = server.pop('env.cc_id')
except:
server['cc_id'] = None
try:
server['role_id'] = server.pop('role.id')
except:
server['role_id'] = None
# cc_id
# first attempt
if not server['cc_id']:
if server['project_id']:
query = (
"SELECT cc_id "
"FROM projects "
"WHERE project_id = UNHEX('{project_id}')").format(
project_id=server['project_id'])
result = self.scalr_db.execute(query, retries=1)
if result and result[0]['cc_id']:
server['cc_id'] = result[0]['cc_id']
# cc_id
# second attempt
if not server['cc_id']:
environment = {'id': server['env_id']}
self.scalr_db.load_client_environment_properties([environment],
['cc_id'])
if 'cc_id' in environment:
server['cc_id'] = environment['cc_id']
if not server['cc_id']:
msg = "Unable to load cc_id for server: {0}".format(server)
LOG.error(msg)
# servers table
server_ids = list(
set(_['server_id'] for _ in servers if _['server_id']))
if not server_ids:
return
query = ("SELECT server_id, farm_id, farm_roleid as farm_role_id "
"FROM servers "
"WHERE server_id IN ({server_id})").format(
server_id=str(server_ids)[1:-1])
results = self.scalr_db.execute(query, retries=1)
existing_server_ids = [_['server_id'] for _ in results]
missing_server_ids = [
_ for _ in server_ids if _ not in existing_server_ids
]
# servers_history
if missing_server_ids:
query = ("SELECT server_id, farm_id, farm_roleid as farm_role_id, "
"cloud_server_id as instance_id "
"FROM servers_history "
"WHERE server_id IN ({server_id})").format(
server_id=str(missing_server_ids)[1:-1])
results += self.scalr_db.execute(query, retries=1)
tmp = dict()
for result in results:
for k, v in result.iteritems():
if k == 'server_id':
continue
tmp.setdefault(result['server_id'], {}).update({k: v})
for server in servers:
if server['server_id'] not in tmp:
continue
for k, v in tmp[server['server_id']].iteritems():
server[k] = v
def insert_managed_server(self, server):
try:
LOG.debug('Insert managed server: %s' % server)
record = server.copy()
if server['platform'] == 'gce':
record['instance_id'] = server['server_id']
record['date'] = server['dtime'].date()
record['num'] = 1
if server['cost'] is None:
record['cost'] = 0
if 'user' not in record:
record['user'] = None
value_id_map = {6: record['user']}
self.analytics_db.autocommit(False)
try:
# insert in usage_h table
usage_h = Usage_h(record)
self.analytics_db.execute(usage_h.insert_query(), retries=1)
record['usage_id'] = usage_h['usage_id']
# insert in usage_servers_h table
usage_servers_h = Usage_servers_h(record)
self.analytics_db.execute(usage_servers_h.insert_query(),
retries=1)
for tag_id, value_id in value_id_map.iteritems():
record['tag_id'] = tag_id
record['value_id'] = value_id
# insert account_tag_values table
account_tag_values = Account_tag_values(record)
self.analytics_db.execute(
account_tag_values.insert_query(), retries=1)
# insert usage_h_tags table
usage_h_tags = Usage_h_tags(record)
self.analytics_db.execute(usage_h_tags.insert_query(),
retries=1)
# insert usage_d table
usage_d = Usage_d(record)
self.analytics_db.execute(usage_d.insert_query(), retries=1)
# insert quarterly_budget
record['year'] = record['date'].year
record['quarter'] = self.get_quarters_calendar(
).quarter_for_date(record['date'])
record['cumulativespend'] = record['cost']
record['spentondate'] = server['dtime']
if record['cc_id']:
record['subject_type'] = 1
record['subject_id'] = record['cc_id']
quarterly_budget = Quarterly_budget(record)
self.analytics_db.execute(quarterly_budget.insert_query(),
retries=1)
if record['project_id']:
record['subject_type'] = 2
record['subject_id'] = record['project_id']
quarterly_budget = Quarterly_budget(record)
self.analytics_db.execute(quarterly_budget.insert_query(),
retries=1)
self.analytics_db.commit()
except:
self.analytics_db.rollback()
raise
finally:
self.analytics_db.autocommit(True)
except:
msg = "Unable to insert managed server: {server}, reason: {error}"
msg = msg.format(server=server, error=helper.exc_info())
LOG.error(msg)
raise
def get_szr_ver_from_repo(self, devel_branch=None, force=False):
out = {}
if devel_branch:
if 'devel_repos' not in self.scalr_config['scalarizr_update']:
return out
if not force:
try:
return self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch]
except AttributeError:
self.get_szr_ver_from_repo.im_func.devel_cache = {}
except KeyError:
pass
norm_branch = devel_branch.replace('/', '-').replace('_', '-')
repos = self.scalr_config['scalarizr_update']['devel_repos']
else:
if not force:
try:
return self.get_szr_ver_from_repo.im_func.cache
except AttributeError:
pass
repos = self.scalr_config['scalarizr_update']['repos']
for repo_type, repo in repos.iteritems():
try:
# deb
try:
deb_repo_url_template = repo['deb_repo_url']
if deb_repo_url_template:
deb_repo_url_template = deb_repo_url_template.strip()
if devel_branch:
deb_repo_url_template = deb_repo_url_template % norm_branch
try:
# try old style deb repo
deb_repo_url = '/'.join(deb_repo_url_template.split())
r = requests.get('%s/Packages' % deb_repo_url)
r.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response.status_code != 404:
raise
# try new style deb repo
deb_repo_url = '%s/dists/%s/%s/binary-amd64' % tuple(deb_repo_url_template.split())
r = requests.get('%s/Packages' % deb_repo_url)
r.raise_for_status()
out[deb_repo_url_template] = re.findall(
'Package: scalarizr\n.*?Version:([ A-Za-z0-9.]*)-?.*\n.*?\n',
r.text,
re.DOTALL)[0].strip()
except:
msg = 'Deb repository {0} failed, reason: {1}'
msg = msg.format(repo_type, helper.exc_info())
LOG.error(msg)
# rpm
try:
rpm_repo_url_template = repo['rpm_repo_url']
if rpm_repo_url_template:
rpm_repo_url_template = rpm_repo_url_template.strip()
if devel_branch:
rpm_repo_url_template = rpm_repo_url_template % norm_branch
for release in ['5', '6']:
rpm_repo_url = rpm_repo_url_template.replace('$releasever', release)
rpm_repo_url = rpm_repo_url.replace('$basearch', 'x86_64')
r = requests.get('%s/repodata/primary.xml.gz' % rpm_repo_url)
r.raise_for_status()
s = StringIO.StringIO(r.content)
f = gzip.GzipFile(fileobj=s, mode='r')
f.seek(0)
xml = minidom.parse(f)
try:
out[rpm_repo_url_template] = re.findall(
'<package type="rpm"><name>scalarizr-base</name>.*?ver="([ A-Za-z0-9.]*)-?.*".*?</package>\n',
xml.toxml(),
re.DOTALL)[0].strip()
except:
out[rpm_repo_url_template] = re.findall(
'<package type="rpm"><name>scalarizr</name>.*?ver="([ A-Za-z0-9.]*)-?.*".*?</package>\n',
xml.toxml(),
re.DOTALL)[0].strip()
except:
msg = 'RPM repository {0} failed, reason: {1}'
msg = msg.format(repo_type, helper.exc_info())
LOG.error(msg)
# win
try:
win_repo_url_template = repo['win_repo_url']
if win_repo_url_template:
win_repo_url_template = win_repo_url_template.strip()
if devel_branch:
win_repo_url = win_repo_url_template % norm_branch
win_repo_url = win_repo_url_template
r = requests.get('%s/x86_64/index' % win_repo_url)
r.raise_for_status()
out[win_repo_url_template] = re.findall(
'scalarizr *scalarizr_(.*).x86_64.exe*\n',
r.text,
re.DOTALL)[0].split('-')[0]
except:
msg = 'Win repository {0} failed, reason: {1}'
msg = msg.format(repo_type, helper.exc_info())
LOG.error(msg)
except:
msg = 'Repository {0} failed, reason: {1}'
msg = msg.format(repo_type, helper.exc_info())
LOG.error(msg)
if devel_branch:
self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out
else:
self.get_szr_ver_from_repo.im_func.cache = out
return out
def do_iteration(self):
try:
dtime_from, dtime_to = self._get_processing_dtime()
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
if self.config['recalculate']:
self.recalculate(date, hour)
else:
self.calculate(date, hour)
except KeyboardInterrupt:
raise
except:
LOG.error(helper.exc_info())
dtime_cur += datetime.timedelta(seconds=3600)
self._pool.join()
if not self.config['recalculate']:
return
# recalculate daily tables
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date = dtime_cur.date()
msg = "Recalculate daily tables for date {0}".format(date)
LOG.info(msg)
try:
self.analytics.recalculate_usage_d(date, self.config['platform'])
except:
msg = "Recalculate usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
try:
self.analytics.recalculate_nm_usage_d(date, self.config['platform'])
except:
msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
dtime_cur += datetime.timedelta(days=1)
# recalculate quarters tables
quarters_calendar = self.analytics.get_quarters_calendar()
start_year = quarters_calendar.year_for_date(dtime_from.date())
start_quarter = quarters_calendar.quarter_for_date(dtime_from.date())
end_year = quarters_calendar.year_for_date(dtime_to.date())
end_quarter = quarters_calendar.quarter_for_date(dtime_to.date())
tmp = []
cur_year = start_year
while cur_year < end_year:
for quarter in range(start_quarter, 5):
tmp.append((cur_year, quarter))
start_quarter = 1
cur_year += 1
for quarter in range(start_quarter, end_quarter + 1):
tmp.append((end_year, quarter))
for year, quarter in tmp:
try:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1}"
msg = msg.format(year, quarter)
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(year, quarter)
except:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}"
msg = msg.format(year, quarter, helper.exc_info())
LOG.warning(msg)
except:
if self.config['recalculate']:
LOG.exception(helper.exc_info())
sys.exit(1)
else:
raise
# quit from iteration loop
raise exceptions.QuitError()
def download_billing_file(self, env, date=None, force=False):
date = date or datetime.datetime.utcnow().date()
bucket_name = env['ec2.detailed_billing.bucket']
if env.get('ec2.detailed_billing.payer_account'):
envs = self.analytics.load_aws_accounts_ids_envs([env['ec2.detailed_billing.payer_account']])
self.analytics.load_env_credentials(envs, platform='ec2')
for e in envs:
if e['client_id'] == env['client_id']:
credentials_env = e
break
else:
msg = 'Can not found AWS credentials for PayerAccount {}'
msg = msg.format(env['ec2.detailed_billing.payer_account'])
raise Exception(msg)
else:
credentials_env = env.copy()
kwds = {
'aws_access_key_id': cryptotool.decrypt_scalr(self.config['crypto_key'],
credentials_env['ec2.access_key']),
'aws_secret_access_key': cryptotool.decrypt_scalr(self.config['crypto_key'],
credentials_env['ec2.secret_key']),
'proxy': self.config['aws_proxy'].get('host'),
'proxy_port': self.config['aws_proxy'].get('port'),
'proxy_user': self.config['aws_proxy'].get('user'),
'proxy_pass': self.config['aws_proxy'].get('pass'),
}
default_region_map = {
'regular': 'us-east-1',
'gov-cloud': 'us-gov-west-1',
'cn-cloud': 'cn-north-1',
}
default_region = default_region_map[env.get('account_type', 'regular')]
region = env.get('ec2.detailed_billing.region', default_region)
conn = boto.s3.connect_to_region(region, **kwds)
bucket = conn.get_bucket(bucket_name)
default_file_name = get_default_aws_csv_file_name(credentials_env['ec2.account_id'], date)
file_name_tmplate = get_aws_csv_file_name_tmplate(credentials_env['ec2.account_id'], date)
files_in_bucket = [key.name for key in bucket.list()
if re.match(file_name_tmplate, key.name)]
if not files_in_bucket:
utcnow = datetime.datetime.utcnow()
if date.month == utcnow.month and utcnow.day < 2:
return None
else:
msg = "Not found any valid files({}, {}) in bucket '{}'"
msg = msg.format(default_file_name, file_name_tmplate, bucket_name)
raise exceptions.FileNotFoundError(msg)
if default_file_name not in files_in_bucket:
file_name = files_in_bucket[0] # use first valid file
msg = "Default AWS detailed billing statistics file '{}' not found in bucket '{}', available {}, use '{}'"
msg = msg.format(default_file_name, bucket_name, files_in_bucket, file_name)
LOG.warning(msg)
else:
file_name = default_file_name
try:
key = bucket.get_key(file_name)
last_modified_dt = datetime.datetime.strptime(key.last_modified, self.last_modified_format)
utcnow = datetime.datetime.utcnow()
seconds_from_last_modified_dt = (utcnow - last_modified_dt).seconds
if hasattr(self, 'task_info') and self.task_info.get('period') > 300:
delta = datetime.timedelta(seconds=self.task_info['period'] - 90)
else:
delta = datetime.timedelta(seconds=210)
condition1 = utcnow > last_modified_dt and utcnow < last_modified_dt + delta
condition2 = seconds_from_last_modified_dt > 3600
condition3 = (seconds_from_last_modified_dt / 3600) % 24 == 0
if force or condition1 or (condition2 and condition3):
with self._downloading_lock:
self.downloading_locks.setdefault(file_name, gevent.lock.RLock())
with self.downloading_locks[file_name]:
csv_zip_file = os.path.join(self.cache_dir, file_name)
csv_file = csv_zip_file.rstrip('.zip')
if os.path.exists(csv_file):
msg = "'{}' already exists in cache directory, use it"
msg = msg.format(os.path.basename(csv_file))
LOG.debug(msg)
return csv_file
while key.size * 3 > helper.get_free_space(self.cache_dir):
LOG.error('Disk is full, waiting 60 sec')
gevent.sleep(60)
LOG.debug("Downloading '{}' for environment {}".format(file_name, env['id']))
attempts = 2
downloading_start_time = time.time()
while True:
try:
key.get_contents_to_filename(csv_zip_file)
assert os.path.isfile(csv_zip_file), os.listdir(self.cache_dir)
downloading_end_time = time.time()
break
except:
attempts -= 1
if not attempts:
raise
downloading_time = downloading_end_time - downloading_start_time
msg = "Downloading '{0}' done in {1:.1f} seconds".format(file_name, downloading_time)
LOG.info(msg)
LOG.debug('Unzipping to {}'.format(csv_file))
while os.path.getsize(csv_zip_file) * 1 > helper.get_free_space(self.cache_dir):
LOG.error('Disk is full, waiting 60 sec')
gevent.sleep(60)
with zipfile.ZipFile(csv_zip_file, 'r') as f:
f.extract(f.infolist()[0], self.cache_dir)
os.remove(csv_zip_file)
return csv_file
else:
msg = "Skipping AWS billing file '{}' for environment {}"
msg = msg.format(file_name, env['id'])
LOG.debug(msg)
return None
except:
msg = "File '{}', bucket '{}', reason: {}"
msg = msg.format(file_name, bucket_name, helper.exc_info())
raise Exception, Exception(msg), sys.exc_info()[2]
def do_iteration(self):
try:
dtime_from, dtime_to = self._get_processing_dtime()
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date, hour = dtime_cur.date(), dtime_cur.hour
try:
if self.config['recalculate']:
self.recalculate(date, hour)
else:
self.calculate(date, hour)
except KeyboardInterrupt:
raise
except:
LOG.error(helper.exc_info())
dtime_cur += datetime.timedelta(seconds=3600)
self._pool.join()
if not self.config['recalculate']:
return
# recalculate daily tables
dtime_cur = dtime_from
while dtime_cur <= dtime_to:
date = dtime_cur.date()
msg = "Recalculate daily tables for date {0}".format(date)
LOG.info(msg)
try:
self.analytics.recalculate_usage_d(date,
self.config['platform'])
except:
msg = "Recalculate usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
try:
self.analytics.recalculate_nm_usage_d(
date, self.config['platform'])
except:
msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}"
msg = msg.format(date, helper.exc_info())
LOG.warning(msg)
dtime_cur += datetime.timedelta(days=1)
# recalculate quarters tables
quarters_calendar = self.analytics.get_quarters_calendar()
start_year = quarters_calendar.year_for_date(dtime_from.date())
start_quarter = quarters_calendar.quarter_for_date(
dtime_from.date())
end_year = quarters_calendar.year_for_date(dtime_to.date())
end_quarter = quarters_calendar.quarter_for_date(dtime_to.date())
tmp = []
cur_year = start_year
while cur_year < end_year:
for quarter in range(start_quarter, 5):
tmp.append((cur_year, quarter))
start_quarter = 1
cur_year += 1
for quarter in range(start_quarter, end_quarter + 1):
tmp.append((end_year, quarter))
for year, quarter in tmp:
try:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1}"
msg = msg.format(year, quarter)
LOG.debug(msg)
self.analytics.recalculate_quarterly_budget(year, quarter)
except:
msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}"
msg = msg.format(year, quarter, helper.exc_info())
LOG.warning(msg)
except:
if self.config['recalculate']:
LOG.exception(helper.exc_info())
sys.exit(1)
else:
raise
# quit from iteration loop
raise exceptions.QuitError()
