def __call__(self): self.change_permissions() while True: try: self.iteration_timestamp = time.time() g = self._do_iteration() try: g.get(timeout=self.iteration_timeout) except gevent.Timeout: raise exceptions.IterationTimeoutError() finally: if not g.ready(): g.kill() except (SystemExit, KeyboardInterrupt): raise except exceptions.NothingToDoError: time_to_sleep = self.nothing_to_do_sleep except exceptions.QuitError: sys.exit(0) except: LOG.error('Iteration failed, reason: {0}'.format(helper.exc_info())) self.on_iteration_error() time_to_sleep = self.error_sleep else: time_to_sleep = 0.1 LOG.debug('End iteration: {0:.1f} seconds'.format(time.time() - self.iteration_timestamp)) if self.config['interval']: time_to_sleep = self.iteration_timestamp + self.config['interval'] - time.time() time.sleep(time_to_sleep)
def get_servers(self): for servers in self._get_servers(): prop = ['scalarizr.api_port', 'scalarizr.key'] self._db.load_server_properties(servers, prop) for server in servers: if 'scalarizr.api_port' not in server: server['scalarizr.api_port'] = 8010 if 'scalarizr.key' not in server: server['scalarizr.key'] = None self._db.load_vpc_settings(servers) out = [] for server in servers: try: if server['os_type'] == 'linux': exclude = ['snum'] elif server['os_type'] == 'windows': exclude = ['la', 'io', 'snum'] else: msg = "Wrong os type for server: '%s'" % server['server_id'] raise Exception(msg) metrics = [m for m in self.config['metrics'] if m not in exclude] server['metrics'] = metrics out.append(server) except: LOG.error(helper.exc_info()) continue yield out
def get_servers(self, limit=500): for servers in self._get_servers(limit=limit): prop = ['scalarizr.api_port', 'scalarizr.key'] self._db.load_server_properties(servers, prop) for server in servers: if 'scalarizr.api_port' not in server: server['scalarizr.api_port'] = 8010 if 'scalarizr.key' not in server: server['scalarizr.key'] = None self._db.load_vpc_settings(servers) out = [] for server in servers: try: if server['os_type'] == 'linux': exclude = ['snum'] elif server['os_type'] == 'windows': exclude = ['la', 'io', 'snum'] else: msg = "Wrong os type for server: '%s'" % server['server_id'] raise Exception(msg) metrics = [m for m in self.config['metrics'] if m not in exclude] server['metrics'] = metrics out.append(server) except: LOG.error(helper.exc_info()) continue yield out
def _serve_forever(self): LOG.debug('Starting plotter') try: cherrypy.quickstart(self, '/', {'/': {}}) except: LOG.error(helper.exc_info()) thread.interrupt_main()
def _handle_webhook_exception(self, webhook): exc = sys.exc_info()[1] if isinstance(exc, ( requests.exceptions.Timeout, requests.exceptions.ProxyError, requests.exceptions.ConnectionError)): if webhook['handle_attempts'] < webhook['attempts']: webhook['status'] = 0 else: webhook['status'] = 2 webhook['error_msg'] = str(sys.exc_info()[0].__name__) msg = "Unable to process webhook: {0}, reason: {1}" msg = msg.format(webhook['history_id'], helper.exc_info()) LOG.warning(msg) elif isinstance(exc, ( requests.exceptions.RequestException, requests.packages.urllib3.exceptions.HTTPError, requests.packages.urllib3.exceptions.HTTPWarning)): webhook['status'] = 2 webhook['error_msg'] = str(sys.exc_info()[0].__name__) msg = "Unable to process webhook: {0}, reason: {1}" msg = msg.format(webhook['history_id'], helper.exc_info()) LOG.warning(msg) else: webhook['status'] = 2 webhook['error_msg'] = 'Internal error' msg = "Unable to process webhook: {0}, reason: {1}" msg = msg.format(webhook['history_id'], helper.exc_info()) LOG.error(msg)
def _handle_webhook_exception(self, webhook): exc = sys.exc_info()[1] if isinstance( exc, (requests.exceptions.Timeout, requests.exceptions.ProxyError, requests.exceptions.ConnectionError)): if webhook['handle_attempts'] < webhook['attempts']: webhook['status'] = 0 else: webhook['status'] = 2 webhook['error_msg'] = str(sys.exc_info()[0].__name__) msg = "Unable to process webhook: {0}, reason: {1}" msg = msg.format(webhook['history_id'], helper.exc_info()) LOG.warning(msg) elif isinstance(exc, (requests.exceptions.RequestException, requests.packages.urllib3.exceptions.HTTPError, requests.packages.urllib3.exceptions.HTTPWarning)): webhook['status'] = 2 webhook['error_msg'] = str(sys.exc_info()[0].__name__) msg = "Unable to process webhook: {0}, reason: {1}" msg = msg.format(webhook['history_id'], helper.exc_info()) LOG.warning(msg) else: webhook['status'] = 2 webhook['error_msg'] = 'Internal error' msg = "Unable to process webhook: {0}, reason: {1}" msg = msg.format(webhook['history_id'], helper.exc_info()) LOG.error(msg)
def _average(self, results, ra=None, fa=None, rs=None, fs=None): ra = ra or dict() fa = fa or dict() rs = rs or dict() fs = fs or dict() for result in results: try: r_key, f_key = self._get_rf_keys(result) if 'snum' in self.config['metrics']: rs.setdefault(r_key, {'snum': {'s_running': 0}}) fs.setdefault(f_key, {'snum': {'s_running': 0}}) rs[r_key]['snum']['s_running'] += 1 fs[f_key]['snum']['s_running'] += 1 if not result['data']: continue for metrics_group_name, metrics_data in result['data'].iteritems(): if not metrics_data or metrics_group_name == 'io': continue for metric_name, value in metrics_data.iteritems(): try: ra.setdefault(r_key, {}) ra[r_key].setdefault(metrics_group_name, {}) ra[r_key][metrics_group_name].setdefault(metric_name, Average()) fa.setdefault(f_key, {}) fa[f_key].setdefault(metrics_group_name, {}) fa[f_key][metrics_group_name].setdefault(metric_name, Average()) ra[r_key][metrics_group_name][metric_name] += value fa[f_key][metrics_group_name][metric_name] += value except: LOG.error(helper.exc_info()) except: LOG.error(helper.exc_info()) return ra, fa, rs, fs
def get_proxy_port(self): for szlr in self.szlrs: #if (szlr['uuid'] == uuid) and\ # (szlr['client_port'] == self.ctl_port): if szlr['client_port'] == str(self.ctl_port): return szlr['proxy_port'] LOG.error("==================get_proxy_port failed! --Chen Leji==================") return None
def update_nm_usage_h(self, record): try: query = NM_usage_h(record).update_query() self.analytics_db.execute(query, retries=1) except: msg = 'Unable to update nm_usage_h record: {record}, reason: {error}' msg = msg.format(record=record, error=helper.exc_info()) LOG.error(msg) raise
def _set_usage_cost(self, records): prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices(record, prices) or 0 try: record['cost'] = float(cost) * int(record['num']) except: msg = 'Unable to update usage cost, reason: {error}' msg = msg.format(error=helper.exc_info()) LOG.error(msg)
def insert_not_managed_server(self, server): try: LOG.debug('Insert not managed server: %s' % server) record = server.copy() record['date'] = server['dtime'].date() record['num'] = 1 if server['cost'] is None: record['cost'] = 0 with self._nm_usage_lock: server_exists = self._not_managed_server_exists(record) subject_id = self._find_subject_id(record) self.analytics_db.autocommit(False) try: if subject_id: record['subject_id'] = uuid.UUID(subject_id).hex else: record['subject_id'] = uuid.uuid4().hex nm_subjects_h = NM_subjects_h(record) self.analytics_db.execute(nm_subjects_h.insert_query(), retries=1) if not server_exists: nm_usage_h = NM_usage_h(record) record['usage_id'] = nm_usage_h['usage_id'] self.analytics_db.execute(nm_usage_h.insert_query(), retries=1) nm_usage_subjects_h = NM_usage_subjects_h(record) self.analytics_db.execute( nm_usage_subjects_h.insert_query(), retries=1) nm_usage_servers_h = NM_usage_servers_h(record) self.analytics_db.execute( nm_usage_servers_h.insert_query(), retries=1) # insert nm_usage_d table nm_usage_d = NM_usage_d(record) self.analytics_db.execute(nm_usage_d.insert_query(), retries=1) self.analytics_db.commit() except: self.analytics_db.rollback() raise finally: self.analytics_db.autocommit(True) except: msg = "Unable to insert not managed server: {server}, reason: {error}" msg = msg.format(server=server, error=helper.exc_info()) LOG.error(msg) raise
def _handle_exception(e, msg): if type(e) == boto.exception.EC2ResponseError and e.status in [401, 403]: LOG.warning(msg) elif type(e) in [ libcloud.common.types.InvalidCredsError, libcloud.common.types.LibcloudError, libcloud.common.types.MalformedResponseError, gevent.timeout.Timeout, socket.timeout, socket.gaierror ]: LOG.warning(msg) else: LOG.error(msg)
def __call__(self): poller_ps, plotter_ps = None, None if self.args['--plotter']: plotter = Plotter(self.config) plotter_ps = plotter.run_in_process() time.sleep(5) if not plotter_ps.is_alive(): LOG.critical('Failed to start CherryPy web server') sys.exit(1) self.change_permissions() if self.args['--poller']: poller = Poller(self.config, self.scalr_config) while True: start_time = time.time() try: LOG.info('Start poller iteration') rrdcached_sock_file = self.config['rrd'][ 'rrdcached_sock_path'] if not os.path.exists(rrdcached_sock_file): raise Exception('rrdcached process is not running') poller_ps = poller.run_in_process() poller_ps.join(self.config['interval'] * 2) if poller_ps.is_alive(): LOG.error('Poller iteration timeout. Terminating') try: poller_ps.terminate() except: msg = 'Unable to terminate, reason: {error}'.format( error=helper.exc_info()) raise Exception(msg) LOG.info('Poller iteration time: %.2f' % (time.time() - start_time)) except KeyboardInterrupt: raise except: msg = 'Poller iteration failed, reason: {error}'.format( error=helper.exc_info()) LOG.error(msg) finally: sleep_time = start_time + self.config[ 'interval'] - time.time() - 0.1 if sleep_time > 0: time.sleep(sleep_time) if plotter_ps: plotter_ps.join()
def _do_iteration(self): try: self.do_iteration() except exceptions.QuitError: sys.exit(0) except (SystemExit, KeyboardInterrupt): raise except exceptions.NothingToDoError: time.sleep(self.nothing_to_do_sleep) except: LOG.error('Iteration failed, reason: {0}'.format( helper.exc_info())) self.on_iteration_error() time.sleep(self.error_sleep)
def process_credential(cred, envs_ids=None): if envs_ids is None: envs_ids = [cred.env_id] try: cloud_data = eval(cred.platform)(cred) if cloud_data: sorted_data = sort_nodes(cloud_data, cred, envs_ids) sorted_data_update(sorted_data) db_update(sorted_data, envs_ids, cred) except: msg = 'platform: {platform}, environments: {envs}, reason: {error}' msg = msg.format(platform=cred.platform, envs=envs_ids, error=helper.exc_info()) LOG.error(msg)
def __call__(self): poller_ps, plotter_ps = None, None if self.args['--plotter']: plotter = Plotter(self.config) plotter_ps = plotter.run_in_process() time.sleep(5) if not plotter_ps.is_alive(): LOG.critical('Failed to start CherryPy web server') sys.exit(1) self.change_permissions() if self.args['--poller']: poller = Poller(self.config, self.scalr_config) while True: start_time = time.time() try: LOG.info('Start poller iteration') rrdcached_sock_file = self.config['rrd']['rrdcached_sock_path'] if not os.path.exists(rrdcached_sock_file): raise Exception('rrdcached process is not running') poller_ps = poller.run_in_process() poller_ps.join(self.config['interval'] * 2) if poller_ps.is_alive(): LOG.error('Poller iteration timeout. Terminating') try: poller_ps.terminate() except: msg = 'Unable to terminate, reason: {error}'.format( error=helper.exc_info()) raise Exception(msg) LOG.info('Poller iteration time: %.2f' % (time.time() - start_time)) except KeyboardInterrupt: raise except: msg = 'Poller iteration failed, reason: {error}'.format( error=helper.exc_info()) LOG.error(msg) finally: sleep_time = start_time + self.config['interval'] - time.time() - 0.1 if sleep_time > 0: time.sleep(sleep_time) if plotter_ps: plotter_ps.join()
def do_iteration(self): servers = self.get_servers_for_update() for server in servers: try: self._pool.wait() self._pool.apply_async(self.update_server, (server,)) gevent.sleep(0) # force switch except: LOG.warning(helper.exc_info()) self._pool.join() try: self.update_scalr_repo_data() except: msg = "Unable to update scalr.settings table, reason: {0}".format(helper.exc_info()) LOG.error(msg)
def do_iteration(self): servers = self.get_servers_for_update() for server in servers: try: self._pool.wait() self._pool.apply_async(self.update_server, (server, )) gevent.sleep(0) # force switch except: LOG.warning(helper.exc_info()) self._pool.join() try: self.update_scalr_repo_data() except: msg = 'Unable to update scalr.settings table, reason: {0}'.format( helper.exc_info()) LOG.error(msg)
def load_statistics(self, **kwds): result = {'success': True} try: self._check_request(kwds) farm = self._get_farm(kwds['farmId']) if not farm or farm['status'] != 1: msg = 'Statistics are not available for terminated farms' raise Plotter.FarmTerminatedError(msg) if farm['hash'] != kwds['hash']: msg = 'Farm hash mismatch' raise Plotter.PlotterError(msg) tz = self._get_tz(kwds['farmId']) if tz: os.environ['TZ'] = tz metrics = kwds['metrics'].strip().split(',') for metric in metrics: try: metric = metric.strip() if metric not in ['cpu', 'la', 'mem', 'net', 'io', 'snum']: msg = "Unsupported metric '%s'" % metric raise Plotter.PlotterError(msg) url = self._plot(kwds, tz, metric) result.setdefault('metric', dict()) result['metric'][metric] = {'success': True, 'img': url} except Plotter.PlotterError as e: result.setdefault('metric', dict()) result['metric'][metric] = { 'success': False, 'msg': str(e) } except Plotter.PlotterError as e: result['success'] = False result['msg'] = str(e) except: result['success'] = False result['msg'] = 'Internal error. Unable to load statistics' LOG.error(helper.exc_info()) cherrypy.response.headers['Access-Control-Allow-Origin'] = '*' cherrypy.response.headers['Access-Control-Max-Age'] = 300 if 'Access-Control-Request-Headers' in cherrypy.request.headers: cherrypy.response.headers['Access-Control-Allow-Headers'] = \ cherrypy.request.headers['Access-Control-Request-Headers'] return result
def _handle_exception(e, msg): if type(e) == boto.exception.EC2ResponseError and e.status in [401, 403]: LOG.warning(msg) elif type(e) in [ libcloud.common.types.InvalidCredsError, libcloud.common.types.LibcloudError, libcloud.common.types.MalformedResponseError, gevent.timeout.Timeout, socket.timeout, socket.gaierror]: LOG.warning(msg) elif type(e) == socket.error and e.errno in [111, 113]: LOG.warning(msg) elif type(e) == googleapiclient.errors.HttpError and e.resp['status'] in ['403']: LOG.warning(msg) else: LOG.error(msg)
def _handle_exception(e, msg): if type(e) == boto.exception.EC2ResponseError and e.status in [401, 403]: LOG.warning(msg) elif type(e) in [ libcloud.common.types.InvalidCredsError, libcloud.common.types.LibcloudError, libcloud.common.types.MalformedResponseError, gevent.timeout.Timeout, socket.timeout, socket.gaierror ]: LOG.warning(msg) elif type(e) == socket.error and e.errno in [111, 113]: LOG.warning(msg) elif type(e) == googleapiclient.errors.HttpError and e.resp['status'] in [ '403' ]: LOG.warning(msg) else: LOG.error(msg)
def process_credential(cred, envs_ids=None): if envs_ids is None: envs_ids = [cred.env_id] try: for k, v in cred.iteritems(): if k in cred.scheme[cred.platform]: cred[k] = cryptotool.decrypt_scalr(app.crypto_key, v) cloud_data = eval(cred.platform)(cred) if cloud_data: sorted_data = sort_nodes(cloud_data, cred, envs_ids) sorted_data_update(sorted_data) db_update(sorted_data, envs_ids, cred) except: msg = 'platform: {platform}, environments: {envs}, reason: {error}' msg = msg.format(platform=cred.platform, envs=envs_ids, error=helper.exc_info()) LOG.error(msg)
def load_statistics(self, **kwds): result = {'success': True} try: self._check_request(kwds) farm = self._get_farm(kwds['farmId']) if not farm or farm['status'] != 1: msg = 'Statistics are not available for terminated farms' raise Plotter.FarmTerminatedError(msg) if farm['hash'] != kwds['hash']: msg = 'Farm hash mismatch' raise Plotter.PlotterError(msg) tz = self._get_tz(kwds['farmId']) if tz: os.environ['TZ'] = tz metrics = kwds['metrics'].strip().split(',') for metric in metrics: try: metric = metric.strip() if metric not in ['cpu', 'la', 'mem', 'net', 'io', 'snum']: msg = "Unsupported metric '%s'" % metric raise Plotter.PlotterError(msg) url = self._plot(kwds, tz, metric) result.setdefault('metric', dict()) result['metric'][metric] = {'success': True, 'img': url} except Plotter.PlotterError as e: result.setdefault('metric', dict()) result['metric'][metric] = {'success': False, 'msg': str(e)} except Plotter.PlotterError as e: result['success'] = False result['msg'] = str(e) except: result['success'] = False result['msg'] = 'Internal error. Unable to load statistics' LOG.error(helper.exc_info()) cherrypy.response.headers['Access-Control-Allow-Origin'] = '*' cherrypy.response.headers['Access-Control-Max-Age'] = 300 if 'Access-Control-Request-Headers' in cherrypy.request.headers: cherrypy.response.headers['Access-Control-Allow-Headers'] = \ cherrypy.request.headers['Access-Control-Request-Headers'] return result
def run(self): srv_pool = multiprocessing.pool.ThreadPool(self.config['pool_size']) rrd_pool = multiprocessing.pool.ThreadPool(10) try: rrd_sock = self.config['rrd']['rrdcached_sock_path'] ra, fa, rs, fs = dict(), dict(), dict(), dict() for servers in self.get_servers(): results = srv_pool.map(self._process_server, servers) for result in results: if result['data']: file_dir = os.path.join( self.config['rrd']['dir'], helper.x1x2(result['farm_id']), '%s' % result['farm_id'], 'INSTANCE_%s_%s' % (result['farm_role_id'], result['index'])) rrd_pool.apply_async( rrd.write, args=(file_dir, result['data'],), kwds={'sock_path': rrd_sock}) ra, fa, rs, fs = self._average(results, ra=ra, fa=fa, rs=rs, fs=fs) for k, v in ra.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) for k, v in fa.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) if 'snum' in self.config['metrics']: for k, v in rs.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) for k, v in fs.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) except: LOG.error(helper.exc_info()) finally: srv_pool.close() rrd_pool.close() srv_pool.join() rrd_pool.join()
def run(self): srv_pool = multiprocessing.pool.ThreadPool(self.config['pool_size']) rrd_pool = multiprocessing.pool.ThreadPool(10) try: rrd_sock = self.config['rrd']['rrdcached_sock_path'] ra, fa, rs, fs = dict(), dict(), dict(), dict() for servers in self.get_servers(): results = srv_pool.map(self._process_server, servers) for result in results: if result['data']: file_dir = os.path.join( self.config['rrd']['dir'], helper.x1x2(result['farm_id']), '%s' % result['farm_id'], 'INSTANCE_%s_%s' % (result['farm_roleid'], result['index'])) rrd_pool.apply_async( rrd.write, args=(file_dir, result['data'],), kwds={'sock_path': rrd_sock}) ra, fa, rs, fs = self._average(results, ra=ra, fa=fa, rs=rs, fs=fs) for k, v in ra.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) for k, v in fa.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) if 'snum' in self.config['metrics']: for k, v in rs.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) for k, v in fs.iteritems(): rrd_pool.apply_async(rrd.write, args=(k, v,), kwds={'sock_path': rrd_sock}) except: LOG.error(helper.exc_info()) finally: srv_pool.close() rrd_pool.close() srv_pool.join() rrd_pool.join()
def _load_managed_servers_properties(self, servers): names = [ 'farm.created_by_id', 'farm.project_id', 'env.cc_id', 'role.id', ] names += self.server_id_map.values() self.scalr_db.load_server_properties(servers, names) for server in servers: for k in self.server_id_map.values(): if k not in server: continue server['instance_id'] = server.pop(k) break try: server['user'] = server.pop('farm.created_by_id') except: server['user'] = None try: server['project_id'] = server.pop('farm.project_id') except: server['project_id'] = None try: server['cc_id'] = server.pop('env.cc_id') except: server['cc_id'] = None try: server['role_id'] = server.pop('role.id') except: server['role_id'] = None # cc_id # first attempt if not server['cc_id']: if server['project_id']: query = ( "SELECT cc_id " "FROM projects " "WHERE project_id = UNHEX('{project_id}')").format( project_id=server['project_id']) result = self.scalr_db.execute(query, retries=1) if result and result[0]['cc_id']: server['cc_id'] = result[0]['cc_id'] # cc_id # second attempt if not server['cc_id']: environment = {'id': server['env_id']} self.scalr_db.load_client_environment_properties([environment], ['cc_id']) if 'cc_id' in environment: server['cc_id'] = environment['cc_id'] if not server['cc_id']: msg = "Unable to load cc_id for server: {0}".format(server) LOG.error(msg) # servers table server_ids = list( set(_['server_id'] for _ in servers if _['server_id'])) if not server_ids: return query = ("SELECT server_id, farm_id, farm_roleid as farm_role_id " "FROM servers " "WHERE server_id IN ({server_id})").format( server_id=str(server_ids)[1:-1]) results = self.scalr_db.execute(query, retries=1) existing_server_ids = [_['server_id'] for _ in results] missing_server_ids = [ _ for _ in server_ids if _ not in existing_server_ids ] # servers_history if missing_server_ids: query = ("SELECT server_id, farm_id, farm_roleid as farm_role_id, " "cloud_server_id as instance_id " "FROM servers_history " "WHERE server_id IN ({server_id})").format( server_id=str(missing_server_ids)[1:-1]) results += self.scalr_db.execute(query, retries=1) tmp = dict() for result in results: for k, v in result.iteritems(): if k == 'server_id': continue tmp.setdefault(result['server_id'], {}).update({k: v}) for server in servers: if server['server_id'] not in tmp: continue for k, v in tmp[server['server_id']].iteritems(): server[k] = v
def insert_managed_server(self, server): try: LOG.debug('Insert managed server: %s' % server) record = server.copy() if server['platform'] == 'gce': record['instance_id'] = server['server_id'] record['date'] = server['dtime'].date() record['num'] = 1 if server['cost'] is None: record['cost'] = 0 if 'user' not in record: record['user'] = None value_id_map = {6: record['user']} self.analytics_db.autocommit(False) try: # insert in usage_h table usage_h = Usage_h(record) self.analytics_db.execute(usage_h.insert_query(), retries=1) record['usage_id'] = usage_h['usage_id'] # insert in usage_servers_h table usage_servers_h = Usage_servers_h(record) self.analytics_db.execute(usage_servers_h.insert_query(), retries=1) for tag_id, value_id in value_id_map.iteritems(): record['tag_id'] = tag_id record['value_id'] = value_id # insert account_tag_values table account_tag_values = Account_tag_values(record) self.analytics_db.execute( account_tag_values.insert_query(), retries=1) # insert usage_h_tags table usage_h_tags = Usage_h_tags(record) self.analytics_db.execute(usage_h_tags.insert_query(), retries=1) # insert usage_d table usage_d = Usage_d(record) self.analytics_db.execute(usage_d.insert_query(), retries=1) # insert quarterly_budget record['year'] = record['date'].year record['quarter'] = self.get_quarters_calendar( ).quarter_for_date(record['date']) record['cumulativespend'] = record['cost'] record['spentondate'] = server['dtime'] if record['cc_id']: record['subject_type'] = 1 record['subject_id'] = record['cc_id'] quarterly_budget = Quarterly_budget(record) self.analytics_db.execute(quarterly_budget.insert_query(), retries=1) if record['project_id']: record['subject_type'] = 2 record['subject_id'] = record['project_id'] quarterly_budget = Quarterly_budget(record) self.analytics_db.execute(quarterly_budget.insert_query(), retries=1) self.analytics_db.commit() except: self.analytics_db.rollback() raise finally: self.analytics_db.autocommit(True) except: msg = "Unable to insert managed server: {server}, reason: {error}" msg = msg.format(server=server, error=helper.exc_info()) LOG.error(msg) raise
def get_szr_ver_from_repo(self, devel_branch=None, force=False): out = {} if devel_branch: if 'devel_repos' not in self.scalr_config['scalarizr_update']: return out if not force: try: return self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] except AttributeError: self.get_szr_ver_from_repo.im_func.devel_cache = {} except KeyError: pass norm_branch = devel_branch.replace('/', '-').replace('_', '-') repos = self.scalr_config['scalarizr_update']['devel_repos'] else: if not force: try: return self.get_szr_ver_from_repo.im_func.cache except AttributeError: pass repos = self.scalr_config['scalarizr_update']['repos'] for repo_type, repo in repos.iteritems(): try: # deb try: deb_repo_url_template = repo['deb_repo_url'] if deb_repo_url_template: deb_repo_url_template = deb_repo_url_template.strip() if devel_branch: deb_repo_url_template = deb_repo_url_template % norm_branch try: # try old style deb repo deb_repo_url = '/'.join(deb_repo_url_template.split()) r = requests.get('%s/Packages' % deb_repo_url) r.raise_for_status() except requests.exceptions.HTTPError as e: if e.response.status_code != 404: raise # try new style deb repo deb_repo_url = '%s/dists/%s/%s/binary-amd64' % tuple(deb_repo_url_template.split()) r = requests.get('%s/Packages' % deb_repo_url) r.raise_for_status() out[deb_repo_url_template] = re.findall( 'Package: scalarizr\n.*?Version:([ A-Za-z0-9.]*)-?.*\n.*?\n', r.text, re.DOTALL)[0].strip() except: msg = 'Deb repository {0} failed, reason: {1}' msg = msg.format(repo_type, helper.exc_info()) LOG.error(msg) # rpm try: rpm_repo_url_template = repo['rpm_repo_url'] if rpm_repo_url_template: rpm_repo_url_template = rpm_repo_url_template.strip() if devel_branch: rpm_repo_url_template = rpm_repo_url_template % norm_branch for release in ['5', '6']: rpm_repo_url = rpm_repo_url_template.replace('$releasever', release) rpm_repo_url = rpm_repo_url.replace('$basearch', 'x86_64') r = requests.get('%s/repodata/primary.xml.gz' % rpm_repo_url) r.raise_for_status() s = StringIO.StringIO(r.content) f = gzip.GzipFile(fileobj=s, mode='r') f.seek(0) xml = minidom.parse(f) try: out[rpm_repo_url_template] = re.findall( '<package type="rpm"><name>scalarizr-base</name>.*?ver="([ A-Za-z0-9.]*)-?.*".*?</package>\n', xml.toxml(), re.DOTALL)[0].strip() except: out[rpm_repo_url_template] = re.findall( '<package type="rpm"><name>scalarizr</name>.*?ver="([ A-Za-z0-9.]*)-?.*".*?</package>\n', xml.toxml(), re.DOTALL)[0].strip() except: msg = 'RPM repository {0} failed, reason: {1}' msg = msg.format(repo_type, helper.exc_info()) LOG.error(msg) # win try: win_repo_url_template = repo['win_repo_url'] if win_repo_url_template: win_repo_url_template = win_repo_url_template.strip() if devel_branch: win_repo_url = win_repo_url_template % norm_branch win_repo_url = win_repo_url_template r = requests.get('%s/x86_64/index' % win_repo_url) r.raise_for_status() out[win_repo_url_template] = re.findall( 'scalarizr *scalarizr_(.*).x86_64.exe*\n', r.text, re.DOTALL)[0].split('-')[0] except: msg = 'Win repository {0} failed, reason: {1}' msg = msg.format(repo_type, helper.exc_info()) LOG.error(msg) except: msg = 'Repository {0} failed, reason: {1}' msg = msg.format(repo_type, helper.exc_info()) LOG.error(msg) if devel_branch: self.get_szr_ver_from_repo.im_func.devel_cache[devel_branch] = out else: self.get_szr_ver_from_repo.im_func.cache = out return out
def do_iteration(self): try: dtime_from, dtime_to = self._get_processing_dtime() dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: if self.config['recalculate']: self.recalculate(date, hour) else: self.calculate(date, hour) except KeyboardInterrupt: raise except: LOG.error(helper.exc_info()) dtime_cur += datetime.timedelta(seconds=3600) self._pool.join() if not self.config['recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.info(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) try: self.analytics.recalculate_nm_usage_d(date, self.config['platform']) except: msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}" msg = msg.format(year, quarter, helper.exc_info()) LOG.warning(msg) except: if self.config['recalculate']: LOG.exception(helper.exc_info()) sys.exit(1) else: raise # quit from iteration loop raise exceptions.QuitError()
def download_billing_file(self, env, date=None, force=False): date = date or datetime.datetime.utcnow().date() bucket_name = env['ec2.detailed_billing.bucket'] if env.get('ec2.detailed_billing.payer_account'): envs = self.analytics.load_aws_accounts_ids_envs([env['ec2.detailed_billing.payer_account']]) self.analytics.load_env_credentials(envs, platform='ec2') for e in envs: if e['client_id'] == env['client_id']: credentials_env = e break else: msg = 'Can not found AWS credentials for PayerAccount {}' msg = msg.format(env['ec2.detailed_billing.payer_account']) raise Exception(msg) else: credentials_env = env.copy() kwds = { 'aws_access_key_id': cryptotool.decrypt_scalr(self.config['crypto_key'], credentials_env['ec2.access_key']), 'aws_secret_access_key': cryptotool.decrypt_scalr(self.config['crypto_key'], credentials_env['ec2.secret_key']), 'proxy': self.config['aws_proxy'].get('host'), 'proxy_port': self.config['aws_proxy'].get('port'), 'proxy_user': self.config['aws_proxy'].get('user'), 'proxy_pass': self.config['aws_proxy'].get('pass'), } default_region_map = { 'regular': 'us-east-1', 'gov-cloud': 'us-gov-west-1', 'cn-cloud': 'cn-north-1', } default_region = default_region_map[env.get('account_type', 'regular')] region = env.get('ec2.detailed_billing.region', default_region) conn = boto.s3.connect_to_region(region, **kwds) bucket = conn.get_bucket(bucket_name) default_file_name = get_default_aws_csv_file_name(credentials_env['ec2.account_id'], date) file_name_tmplate = get_aws_csv_file_name_tmplate(credentials_env['ec2.account_id'], date) files_in_bucket = [key.name for key in bucket.list() if re.match(file_name_tmplate, key.name)] if not files_in_bucket: utcnow = datetime.datetime.utcnow() if date.month == utcnow.month and utcnow.day < 2: return None else: msg = "Not found any valid files({}, {}) in bucket '{}'" msg = msg.format(default_file_name, file_name_tmplate, bucket_name) raise exceptions.FileNotFoundError(msg) if default_file_name not in files_in_bucket: file_name = files_in_bucket[0] # use first valid file msg = "Default AWS detailed billing statistics file '{}' not found in bucket '{}', available {}, use '{}'" msg = msg.format(default_file_name, bucket_name, files_in_bucket, file_name) LOG.warning(msg) else: file_name = default_file_name try: key = bucket.get_key(file_name) last_modified_dt = datetime.datetime.strptime(key.last_modified, self.last_modified_format) utcnow = datetime.datetime.utcnow() seconds_from_last_modified_dt = (utcnow - last_modified_dt).seconds if hasattr(self, 'task_info') and self.task_info.get('period') > 300: delta = datetime.timedelta(seconds=self.task_info['period'] - 90) else: delta = datetime.timedelta(seconds=210) condition1 = utcnow > last_modified_dt and utcnow < last_modified_dt + delta condition2 = seconds_from_last_modified_dt > 3600 condition3 = (seconds_from_last_modified_dt / 3600) % 24 == 0 if force or condition1 or (condition2 and condition3): with self._downloading_lock: self.downloading_locks.setdefault(file_name, gevent.lock.RLock()) with self.downloading_locks[file_name]: csv_zip_file = os.path.join(self.cache_dir, file_name) csv_file = csv_zip_file.rstrip('.zip') if os.path.exists(csv_file): msg = "'{}' already exists in cache directory, use it" msg = msg.format(os.path.basename(csv_file)) LOG.debug(msg) return csv_file while key.size * 3 > helper.get_free_space(self.cache_dir): LOG.error('Disk is full, waiting 60 sec') gevent.sleep(60) LOG.debug("Downloading '{}' for environment {}".format(file_name, env['id'])) attempts = 2 downloading_start_time = time.time() while True: try: key.get_contents_to_filename(csv_zip_file) assert os.path.isfile(csv_zip_file), os.listdir(self.cache_dir) downloading_end_time = time.time() break except: attempts -= 1 if not attempts: raise downloading_time = downloading_end_time - downloading_start_time msg = "Downloading '{0}' done in {1:.1f} seconds".format(file_name, downloading_time) LOG.info(msg) LOG.debug('Unzipping to {}'.format(csv_file)) while os.path.getsize(csv_zip_file) * 1 > helper.get_free_space(self.cache_dir): LOG.error('Disk is full, waiting 60 sec') gevent.sleep(60) with zipfile.ZipFile(csv_zip_file, 'r') as f: f.extract(f.infolist()[0], self.cache_dir) os.remove(csv_zip_file) return csv_file else: msg = "Skipping AWS billing file '{}' for environment {}" msg = msg.format(file_name, env['id']) LOG.debug(msg) return None except: msg = "File '{}', bucket '{}', reason: {}" msg = msg.format(file_name, bucket_name, helper.exc_info()) raise Exception, Exception(msg), sys.exc_info()[2]
def do_iteration(self): try: dtime_from, dtime_to = self._get_processing_dtime() dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: if self.config['recalculate']: self.recalculate(date, hour) else: self.calculate(date, hour) except KeyboardInterrupt: raise except: LOG.error(helper.exc_info()) dtime_cur += datetime.timedelta(seconds=3600) self._pool.join() if not self.config['recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.info(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) try: self.analytics.recalculate_nm_usage_d( date, self.config['platform']) except: msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date( dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}" msg = msg.format(year, quarter, helper.exc_info()) LOG.warning(msg) except: if self.config['recalculate']: LOG.exception(helper.exc_info()) sys.exit(1) else: raise # quit from iteration loop raise exceptions.QuitError()