def _stop(self): LOG.debug(self._stopping_msg) try: if not os.path.exists(self.config['pid_file']): msg = "Can't stop, pid file %s doesn't exist\n" % self.config[ 'pid_file'] sys.stderr.write(helper.colorize(helper.Color.FAIL, msg)) return with file(self.config['pid_file'], 'r') as pf: pid = int(pf.read().strip()) for ps in psutil.process_iter(): if ps.name() == self.name[0:15]: # TODO # SIGINT helper.kill_children(pid) helper.kill(pid) break else: msg = "Process with name {0} doesn't exists".format(self.name) raise Exception(msg) LOG.info('Stopped') helper.delete_file(self.config['pid_file']) except: msg = "Can't stop, reason: {error}".format(error=helper.exc_info()) raise Exception(msg)
def _get_szr_conn_info(server, port, instances_connection_policy): ip = { 'public': server['remote_ip'], 'local': server['local_ip'], 'auto': server['remote_ip'] if server['remote_ip'] else server['local_ip'], }[instances_connection_policy] headers = {} if server['platform'] == 'ec2' and 'ec2.vpc.id' in server and 'router.vpc.ip' in server: if server['remote_ip']: ip = server['remote_ip'] else: headers.update({ 'X-Receiver-Host': server['local_ip'], 'X-Receiver-Port': port, }) ip = server['router.vpc.ip'] port = 80 # Start - Added by Chen Leji if not server['remote_ip']: LOG.info("=============Apply FLOATINGIP_PROXY Patch=============") ip = "localhost" proxy = floatingip_proxy.szrProxy(port, server['server_id']) port = proxy.get_proxy_port() return ip, port, headers
def process_aws_billing(self): if self.args['--recalculate']: return dtime_from, dtime_to = self.get_aws_billing_interval() msg = 'AWS billing interval: {0} - {1}' msg = msg.format(dtime_from, dtime_to) LOG.info(msg) for envs in self.analytics.load_envs(): unique = {} for env in envs: if env.get('ec2.detailed_billing.enabled', '0') != '1': continue bucket_name = env['ec2.detailed_billing.bucket'] creds = self.analytics.get_creds([env]) cred = next(cred for cred in creds if cred.platform == 'ec2') unique.setdefault(cred.unique, { 'envs_ids': [], 'cred': cred, 'bucket_name': bucket_name }) unique[cred.unique]['envs_ids'].append(env['id']) for data in unique.values(): while len(self.pool) > self.config['pool_size'] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(self.process_aws_account, args=(data, dtime_from, dtime_to)) self.pool.join() if not self.aws_billing_dtime_from: return dtime_from = self.aws_billing_dtime_from if self.config['dtime_to']: dtime_to = self.config['dtime_to'] else: dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta( hours=1) dtime_to = dtime_hour_ago.replace(minute=59, second=59, microsecond=999999) # fill farm_usage_d dtime_cur = dtime_from msg = 'AWS fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour, platform='ec2') except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format( date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1)
def __call__(self): self._stop = False while not self._stop: g = None try: self.start_time = time.time() LOG.debug('Start periodical task ({})'.format(self.task_name)) self.iteration_number += 1 self.task.task_info = { 'period': self.period, 'timeout': self.timeout, 'start_time': self.start_time, 'iteration_number': self.iteration_number, } if callable(self.before): msg = 'Periodical task ({}) call before ({})' msg = msg.format(self.task_name, self.before.__name__) LOG.debug(msg) self.before(self) g = gevent.spawn(self.task) g.get(timeout=self.timeout) except: if g and not g.ready(): g.kill() try: if callable(self.on_error): self.on_error(self) except: msg = 'Periodical task ({}) on error ({}) failed' msg = msg.format(self.task_name, self.on_error.__name__) handle_error(message=msg) msg = 'Periodical task ({}) error: {}' msg = msg.format(self.task_name, exc_info(where=False)) handle_error(message=msg) finally: try: if callable(self.after): try: msg = 'Periodical task ({}) call after ({})' msg = msg.format(self.task_name, self.after.__name__) LOG.debug(msg) self.after(self) except: msg = 'After ({0}) failed'.format( self.after.__name__) handle_error(message=msg) self.end_time = time.time() task_time = self.end_time - self.start_time msg = 'End task ({0}): {1:.1f} seconds' msg = msg.format(self.task_name, task_time) LOG.info(msg) if self.period: next_time = self.start_time + self.period while time.time() < next_time and not self._stop: time.sleep(0.5) except: msg = 'Task ({}) finally failed'.format(self.task_name) handle_error(message=msg)
def __call__(self): self._stop = False while not self._stop: g = None try: self.start_time = time.time() LOG.debug('Start periodical task ({})'.format(self.task_name)) self.iteration_number += 1 self.task.task_info = { 'period': self.period, 'timeout': self.timeout, 'start_time': self.start_time, 'iteration_number': self.iteration_number, } if callable(self.before): msg = 'Periodical task ({}) call before ({})' msg = msg.format(self.task_name, self.before.__name__) LOG.debug(msg) self.before(self) g = gevent.spawn(self.task) g.get(timeout=self.timeout) except: if g and not g.ready(): g.kill() try: if callable(self.on_error): self.on_error(self) except: msg = 'Periodical task ({}) on error ({}) failed' msg = msg.format(self.task_name, self.on_error.__name__) handle_error(message=msg) msg = 'Periodical task ({}) error: {}' msg = msg.format(self.task_name, exc_info(where=False)) handle_error(message=msg) finally: try: if callable(self.after): try: msg = 'Periodical task ({}) call after ({})' msg = msg.format(self.task_name, self.after.__name__) LOG.debug(msg) self.after(self) except: msg = 'After ({0}) failed'.format(self.after.__name__) handle_error(message=msg) self.end_time = time.time() task_time = self.end_time - self.start_time msg = 'End task ({0}): {1:.1f} seconds' msg = msg.format(self.task_name, task_time) LOG.info(msg) if self.period: next_time = self.start_time + self.period while time.time() < next_time and not self._stop: time.sleep(0.5) except: msg = 'Task ({}) finally failed'.format(self.task_name) handle_error(message=msg)
def main(): app = AnalyticsProcessing() try: app.load_config() app.configure() app.run() except exceptions.AlreadyRunningError: LOG.info(helper.exc_info()) except (SystemExit, KeyboardInterrupt): pass except: LOG.exception('Oops')
def _start(self): if helper.check_pid(self.config['pid_file']): raise exceptions.AlreadyRunningError(self.config['pid_file']) LOG.debug('Starting') if self.args['--daemon']: helper.daemonize() helper.create_pid_file(self.config['pid_file']) atexit.register(helper.delete_file, self.config['pid_file']) LOG.info('Started') helper.set_proc_name(self.name) self() LOG.info('Stopped')
def __call__(self): poller_ps, plotter_ps = None, None if self.args['--plotter']: plotter = Plotter(self.config) plotter_ps = plotter.run_in_process() time.sleep(5) if not plotter_ps.is_alive(): LOG.critical('Failed to start CherryPy web server') sys.exit(1) self.change_permissions() if self.args['--poller']: poller = Poller(self.config, self.scalr_config) while True: start_time = time.time() try: LOG.info('Start poller iteration') rrdcached_sock_file = self.config['rrd'][ 'rrdcached_sock_path'] if not os.path.exists(rrdcached_sock_file): raise Exception('rrdcached process is not running') poller_ps = poller.run_in_process() poller_ps.join(self.config['interval'] * 2) if poller_ps.is_alive(): LOG.error('Poller iteration timeout. Terminating') try: poller_ps.terminate() except: msg = 'Unable to terminate, reason: {error}'.format( error=helper.exc_info()) raise Exception(msg) LOG.info('Poller iteration time: %.2f' % (time.time() - start_time)) except KeyboardInterrupt: raise except: msg = 'Poller iteration failed, reason: {error}'.format( error=helper.exc_info()) LOG.error(msg) finally: sleep_time = start_time + self.config[ 'interval'] - time.time() - 0.1 if sleep_time > 0: time.sleep(sleep_time) if plotter_ps: plotter_ps.join()
def main(): global app app = SzrUpdService() try: app.load_config() app.configure() app.run() except exceptions.AlreadyRunningError: LOG.info(helper.exc_info(where=False)) except (SystemExit, KeyboardInterrupt): pass except: LOG.exception('Oops')
def _start(self): if helper.check_pid(self.config['pid_file']): raise exceptions.AlreadyRunningError(self.config['pid_file']) LOG.debug(self._starting_msg) if self.args['--daemon']: helper.daemonize() helper.create_pid_file(self.config['pid_file']) atexit.register(helper.delete_file, self.config['pid_file']) helper.set_proc_name(self.name) self.start_dtime = datetime.datetime.utcnow() LOG.info('Started') self() LOG.info('Stopped')
def main(): global app app = LoadStatistics() try: app.load_config() app.configure() app.run() except exceptions.AlreadyRunningError: LOG.info(helper.exc_info(where=False)) except (SystemExit, KeyboardInterrupt): pass except: LOG.exception('Oops')
def process_aws_billing(self): if self.args['--recalculate']: return dtime_from, dtime_to = self.get_aws_billing_interval() msg = 'AWS billing interval: {0} - {1}' msg = msg.format(dtime_from, dtime_to) LOG.info(msg) for envs in self.analytics.load_envs(): unique = {} for env in envs: if env.get('ec2.detailed_billing.enabled', '0') != '1': continue bucket_name = env['ec2.detailed_billing.bucket'] creds = self.analytics.get_creds([env]) cred = next(cred for cred in creds if cred.platform == 'ec2') unique.setdefault(cred.unique, {'envs_ids': [], 'cred': cred, 'bucket_name': bucket_name}) unique[cred.unique]['envs_ids'].append(env['id']) for data in unique.values(): while len(self.pool) > self.config['pool_size'] * 5 / 10: gevent.sleep(0.1) self.pool.apply_async(self.process_aws_account, args=(data, dtime_from, dtime_to)) self.pool.join() if not self.aws_billing_dtime_from: return dtime_from = self.aws_billing_dtime_from if self.config['dtime_to']: dtime_to = self.config['dtime_to'] else: dtime_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=1) dtime_to = dtime_hour_ago.replace(minute=59, second=59, microsecond=999999) # fill farm_usage_d dtime_cur = dtime_from msg = 'AWS fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour, platform='ec2') except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1)
def __call__(self): poller_ps, plotter_ps = None, None if self.args['--plotter']: plotter = Plotter(self.config) plotter_ps = plotter.run_in_process() time.sleep(5) if not plotter_ps.is_alive(): LOG.critical('Failed to start CherryPy web server') sys.exit(1) self.change_permissions() if self.args['--poller']: poller = Poller(self.config, self.scalr_config) while True: start_time = time.time() try: LOG.info('Start poller iteration') rrdcached_sock_file = self.config['rrd']['rrdcached_sock_path'] if not os.path.exists(rrdcached_sock_file): raise Exception('rrdcached process is not running') poller_ps = poller.run_in_process() poller_ps.join(self.config['interval'] * 2) if poller_ps.is_alive(): LOG.error('Poller iteration timeout. Terminating') try: poller_ps.terminate() except: msg = 'Unable to terminate, reason: {error}'.format( error=helper.exc_info()) raise Exception(msg) LOG.info('Poller iteration time: %.2f' % (time.time() - start_time)) except KeyboardInterrupt: raise except: msg = 'Poller iteration failed, reason: {error}'.format( error=helper.exc_info()) LOG.error(msg) finally: sleep_time = start_time + self.config['interval'] - time.time() - 0.1 if sleep_time > 0: time.sleep(sleep_time) if plotter_ps: plotter_ps.join()
def _start(self): if helper.check_pid(self.config['pid_file']): raise exceptions.AlreadyRunningError(self.config['pid_file']) LOG.debug(self._starting_msg) LOG.debug('Log file: {}'.format(self.config['log_file'])) LOG.debug('Pid file: {}'.format(self.config['pid_file'])) if self.args['--daemon']: helper.daemonize() helper.create_pid_file(self.config['pid_file']) atexit.register(helper.delete_file, self.config['pid_file']) helper.set_proc_name(self.name) self.start_dtime = datetime.datetime.utcnow() LOG.info('Started') self() LOG.info('Stopped')
def __call__(self): try: dtime_from, dtime_to = self.get_billing_interval() msg = 'AWS billing interval: {} - {}'.format(dtime_from, dtime_to) LOG.info(msg) self._create_cache_dir() aws_accounts_ids = self.analytics.load_aws_accounts_ids() for chunk in helper.chunks(aws_accounts_ids, 100): envs = self.analytics.load_aws_accounts_ids_envs(chunk) envs = [env for env in envs if env.get('ec2.is_enabled', '0') == '1'] self.analytics.load_env_credentials(envs, platform='ec2') envs = [env for env in envs if env.get('ec2.detailed_billing.enabled', '0') == '1' and env.get('ec2.detailed_billing.payer_account') in (None, '')] if not envs: continue self._wait_pool() self.pool.apply_async(self.process_envs, args=(envs, dtime_from, dtime_to)) aws_payers_accounts = self.analytics.load_aws_payers_accounts() for chunk in helper.chunks(aws_payers_accounts, 100): envs = self.analytics.load_aws_payers_accounts_envs(chunk) envs = [env for env in envs if env.get('ec2.is_enabled', '0') == '1'] self.analytics.load_env_credentials(envs, platform='ec2') envs = [env for env in envs if env.get('ec2.detailed_billing.enabled', '0') == '1'] if not envs: continue self._wait_pool() self.pool.apply_async(self.process_envs, args=(envs, dtime_from, dtime_to)) self.pool.join() except: self.pool.kill() helper.handle_error(message='AWS billing failed') raise finally: self.downloading_locks = {} try: self._remove_cache_dir() except: msg = 'Unable to remove cache dir {}' msg = msg.format(self.cache_dir) helper.handle_error(message=msg, level='error')
def run(self): try: self.configure() t = self._serve_forever() time.sleep(5) # change permissions if self.config['group']: helper.set_gid(self.config['group']) if self.config['user']: helper.set_uid(self.config['user']) LOG.info('Plotter started') t.join() except: LOG.exception(helper.exc_info())
def __call__(self): try: dtime_from, dtime_to = self.get_billing_interval() msg = 'Azure billing interval: {} - {}'.format(dtime_from, dtime_to) LOG.info(msg) azure_subscriptions_ids = self.analytics.load_azure_subscriptions_ids() for chunk in helper.chunks(azure_subscriptions_ids, 100): envs = self.analytics.load_azure_subscriptions_ids_envs(chunk) self.analytics.load_env_credentials(envs, platform='azure') if not envs: continue self._wait_pool() self.pool.apply_async(self.process_envs, args=(envs, dtime_from, dtime_to)) self.pool.join() except: self.pool.kill() helper.handle_error(message='Azure billing failed') raise
def run(self): try: self.configure() t = self._serve_forever() while not t.is_alive(): time.sleep(0.5) # wait before change permissions to allow cherrypy read certificates time.sleep(2) # change permissions if self.config['group']: helper.set_gid(self.config['group']) if self.config['user']: helper.set_uid(self.config['user']) LOG.info('Plotter started') t.join() except: LOG.exception(helper.exc_info())
def recalculate(self, date, hour): try: msg = "Recalculate hourly tables for date {0}, hour {1}".format(date, hour) LOG.info(msg) for usage_h_records, nm_usage_h_records in itertools.izip_longest( self.analytics.get_usage_h_records(date, hour, self.config['platform']), self.analytics.get_nm_usage_h_records(date, hour, self.config['platform'])): usage_h_records = usage_h_records or [] LOG.info('usage_h records for recalculating: %s' % len(usage_h_records)) nm_usage_h_records = nm_usage_h_records or [] LOG.info('nm_usage_h records for recalculating: %s' % len(nm_usage_h_records)) self._set_usage_cost(usage_h_records + nm_usage_h_records) for record in usage_h_records: self._pool.wait() self._pool.apply_async(self.analytics.update_usage_h, (record,)) gevent.sleep(0) # force switch for record in nm_usage_h_records: self._pool.wait() self._pool.apply_async(self.analytics.update_nm_usage_h, (record,)) gevent.sleep(0) # force switch self._pool.join() self.analytics.fill_farm_usage_d(date, hour, platform=self.config['platform']) except: msg = "Unable to recalculate date {date}, hour {hour}, reason: {error}".format( date=date, hour=hour, error=helper.exc_info()) raise Exception(msg)
def calculate(self, date, hour): try: msg = "Calculate date {0}, hour {1}".format(date, hour) LOG.info(msg) for managed_servers, not_managed_servers in itertools.izip_longest( self.analytics.get_managed_servers(date, hour), self.analytics.get_not_managed_servers(date, hour)): managed_servers = managed_servers or [] LOG.info('Managed servers for processing: %s' % len(managed_servers)) not_managed_servers = not_managed_servers or [] LOG.info('Not managed servers for processing: %s' % len(not_managed_servers)) self._set_servers_cost(managed_servers + not_managed_servers) for server in managed_servers: self._pool.wait() self._pool.apply_async(self.analytics.insert_managed_server, (server,)) gevent.sleep(0) # force switch for server in not_managed_servers: self._pool.wait() self._pool.apply_async(self.analytics.insert_not_managed_server, (server,)) gevent.sleep(0) # force switch self._pool.join() self.analytics.fill_farm_usage_d(date, hour) except: msg = "Unable to calculate date {date}, hour {hour}, reason: {error}".format( date=date, hour=hour, error=helper.exc_info()) raise Exception(msg)
def calculate(self, date, hour): try: msg = "Calculate date {0}, hour {1}".format(date, hour) LOG.info(msg) for managed_servers, not_managed_servers in itertools.izip_longest( self.analytics.get_managed_servers(date, hour), self.analytics.get_not_managed_servers(date, hour)): managed_servers = managed_servers or [] LOG.info('Managed servers for processing: %s' % len(managed_servers)) not_managed_servers = not_managed_servers or [] LOG.info('Not managed servers for processing: %s' % len(not_managed_servers)) self._set_servers_cost(managed_servers + not_managed_servers) for server in managed_servers: self._pool.wait() self._pool.apply_async( self.analytics.insert_managed_server, (server, )) gevent.sleep(0) # force switch for server in not_managed_servers: self._pool.wait() self._pool.apply_async( self.analytics.insert_not_managed_server, (server, )) gevent.sleep(0) # force switch self._pool.join() #self.analytics.fill_farm_usage_d(date, hour) except: msg = "Unable to calculate date {date}, hour {hour}, reason: {error}".format( date=date, hour=hour, error=helper.exc_info()) raise Exception(msg)
def __call__(self): try: dtime_from, dtime_to = self.get_billing_interval() quarters_calendar = self.analytics.get_quarters_calendar() quarter_number = quarters_calendar.quarter_for_date(dtime_from.date()) quarter_year = quarters_calendar.year_for_date(dtime_from.date()) quarter_start_dtime, quarter_end_dtime = quarters_calendar.dtime_for_quarter( quarter_number, year=quarter_year) if quarter_start_dtime < dtime_from: quarter_number, quarter_year = quarters_calendar.next_quarter(quarter_number, quarter_year) quarter_start_dtime, quarter_end_dtime = quarters_calendar.dtime_for_quarter( quarter_number, year=quarter_year) while quarter_start_dtime < dtime_to: msg = 'Recalculate {} quarter ({} - {}) for year {}' msg = msg.format(quarter_number, quarter_start_dtime, quarter_end_dtime, quarter_year) LOG.info(msg) self.config['dtime_from'] = quarter_start_dtime self.config['dtime_to'] = min(quarter_end_dtime, dtime_to) super(RecalculateAWSBilling, self).__call__() self.fill_farm_usage_d(force=True) msg = 'Recalculate quarterly_budget' LOG.debug(msg) self.analytics.recalculate_quarterly_budget(quarter_year, quarter_number) quarter_number, quarter_year = quarters_calendar.next_quarter(quarter_number, quarter_year) quarter_start_dtime, quarter_end_dtime = quarters_calendar.dtime_for_quarter( quarter_number, year=quarter_year) except: self.pool.kill() helper.handle_error(message='Recalculate AWS billing failed') raise
def _stop(self): LOG.debug(self._stopping_msg) try: if not os.path.exists(self.config['pid_file']): msg = "Can't stop, pid file %s doesn't exist\n" % self.config['pid_file'] sys.stderr.write(helper.colorize(helper.Color.FAIL, msg)) return with file(self.config['pid_file'], 'r') as pf: pid = int(pf.read().strip()) for ps in psutil.process_iter(): if ps.name() == self.name[0:15]: # TODO # SIGINT helper.kill_children(pid) helper.kill(pid) break else: msg = "Process with name {0} doesn't exists".format(self.name) raise Exception(msg) LOG.info('Stopped') helper.delete_file(self.config['pid_file']) except: msg = "Can't stop, reason: {error}".format(error=helper.exc_info()) raise Exception(msg)
def __call__(self): try: dtime_from, dtime_to = self.get_billing_interval() LOG.info('Scalr Poller billing interval: {} - {}'.format(dtime_from, dtime_to)) dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour for platform in self.config['platform']: try: msg = "Process Scalr Poller data, date {}, hour {}, platform '{}'" msg = msg.format(date, hour, platform) LOG.debug(msg) for records in self.analytics.get_poller_servers(date, hour, platform=platform): LOG.debug('Scalr Poller records for processing: {}'.format(len(records))) prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices(record, prices) or 0 record['cost'] = cost record['num'] = 1.0 record['cost_distr_type'] = 1 for chunk in helper.chunks(records, insert_chunk_size): self.pool.wait() self.pool.apply_async(self.analytics.insert_records, (chunk,), {'callback': self.on_insert_records}) gevent.sleep(0) # force switch except: msg = "Scalr Poller billing unable to process date {}, hour {}, platform '{}'" msg = msg.format(date, hour, platform) helper.handle_error(message=msg) self.pool.join() dtime_cur += datetime.timedelta(hours=1) except: self.pool.kill() helper.handle_error(message='Scalr Poller billing failed') raise
def recalculate(self, date, hour): try: msg = "Recalculate hourly tables for date {0}, hour {1}".format( date, hour) LOG.info(msg) for usage_h_records, nm_usage_h_records in itertools.izip_longest( self.analytics.get_usage_h_records( date, hour, self.config['platform']), self.analytics.get_nm_usage_h_records( date, hour, self.config['platform'])): usage_h_records = usage_h_records or [] LOG.info('usage_h records for recalculating: %s' % len(usage_h_records)) nm_usage_h_records = nm_usage_h_records or [] LOG.info('nm_usage_h records for recalculating: %s' % len(nm_usage_h_records)) self._set_usage_cost(usage_h_records + nm_usage_h_records) for record in usage_h_records: self._pool.wait() self._pool.apply_async(self.analytics.update_usage_h, (record, )) gevent.sleep(0) # force switch for record in nm_usage_h_records: self._pool.wait() self._pool.apply_async(self.analytics.update_nm_usage_h, (record, )) gevent.sleep(0) # force switch self._pool.join() self.analytics.fill_farm_usage_d(date, hour, platform=self.config['platform']) except: msg = "Unable to recalculate date {date}, hour {hour}, reason: {error}".format( date=date, hour=hour, error=helper.exc_info()) raise Exception(msg)
def do_iteration(self): try: dtime_from, dtime_to = self._get_processing_dtime() dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: if self.config['recalculate']: self.recalculate(date, hour) else: self.calculate(date, hour) except KeyboardInterrupt: raise except: LOG.error(helper.exc_info()) dtime_cur += datetime.timedelta(seconds=3600) self._pool.join() if not self.config['recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.info(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) try: self.analytics.recalculate_nm_usage_d(date, self.config['platform']) except: msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}" msg = msg.format(year, quarter, helper.exc_info()) LOG.warning(msg) except: if self.config['recalculate']: LOG.exception(helper.exc_info()) sys.exit(1) else: raise # quit from iteration loop raise exceptions.QuitError()
def do_iteration(self): try: dtime_from, dtime_to = self._get_processing_dtime() dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: if self.config['recalculate']: self.recalculate(date, hour) else: self.calculate(date, hour) except KeyboardInterrupt: raise except: LOG.error(helper.exc_info()) dtime_cur += datetime.timedelta(seconds=3600) self._pool.join() if not self.config['recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.info(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) try: self.analytics.recalculate_nm_usage_d( date, self.config['platform']) except: msg = "Recalculate nm_usage_d table for date {0} failed, reason: {1}" msg = msg.format(date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date( dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed, reason: {2}" msg = msg.format(year, quarter, helper.exc_info()) LOG.warning(msg) except: if self.config['recalculate']: LOG.exception(helper.exc_info()) sys.exit(1) else: raise # quit from iteration loop raise exceptions.QuitError()
def process_poller_billing(self): dtime_from, dtime_to = self.get_poller_billing_interval() LOG.info('Poller billing interval: {0} - {1}'.format(dtime_from, dtime_to)) # process poller_session table dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: msg = "Process poller data, date {0}, hour {1}".format(date, hour) LOG.info(msg) if self.args['--recalculate']: platform = self.config['platform'] generator = self.analytics.get_records(date, hour, platform) else: generator = self.analytics.get_servers(date, hour) for records in generator: LOG.debug('Records for processing: %s' % len(records)) prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices(record, prices) or 0 self.pool.wait() if self.args['--recalculate']: record['cost'] = float(cost) * int(record['num']) self.pool.apply_async(self.analytics.update_record, (record,)) else: record['cost'] = cost record['num'] = 1.0 record['cost_distr_type'] = 1 self.pool.apply_async(self.analytics.insert_record, (record,)) gevent.sleep(0) # force switch self.pool.join() except: msg = "Unable to process date {0}, hour {1}".format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) # fill farm_usage_d dtime_cur = dtime_from msg = 'Poller fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour) except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) if not self.args['--recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.debug(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, error: {1}".format( date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed" msg = msg.format(year, quarter, helper.exc_info()) LOG.exception(msg)
def __call__(self): try: dtime_from, dtime_to = self.get_billing_interval() LOG.info('Scalr Poller billing recalculate interval: {} - {}'.format(dtime_from, dtime_to)) # process poller_session table dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour for platform in self.config['platform']: try: msg = "Recalculate Scalr Poller data, date {}, hour {}, platform '{}'" msg = msg.format(date, hour, platform) LOG.debug(msg) for records in self.analytics.get_records(date, hour, platform): LOG.debug('Scalr Poller records to recalculate: {}'.format(len(records))) prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices(record, prices) or 0 self.pool.wait() record['cost'] = float(cost) * int(record['num']) self.pool.apply_async(self.analytics.update_record, (record,), {'callback': self.on_insert_record}) gevent.sleep(0) # force switch except: msg = "Scalr Poller billing unable to recalculate date {}, hour {}, platform '{}'" msg = msg.format(date, hour, platform) helper.handle_error(message=msg) self.pool.join() dtime_cur += datetime.timedelta(hours=1) # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() for platform in self.config['platform']: try: msg = "Recalculate daily tables for date {}, platform '{}'" msg = msg.format(date, platform) LOG.debug(msg) self.analytics.recalculate_usage_d(date, platform) except: msg = "Recalculate usage_d table for date {}, platform '{}' failed" msg = msg.format(date, platform) helper.handle_error(message=msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {}, quarter {}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {}, quarter {} failed" msg = msg.format(year, quarter, helper.exc_info(where=False)) helper.handle_error(message=msg) except: self.pool.kill() helper.handle_error(message='Recalculate Scalr Poller billing failde')
def delete_data(self, csv_file, envs, period): envs_ids = list(set(int(env['id']) for env in envs)) dtime_from, dtime_to = period msg = 'Deleting AWS detailed billing data for environments: {}, period: {} - {}' msg = msg.format(envs_ids, dtime_from, dtime_to) LOG.info(msg) with self.analytics.lock: self.analytics.analytics_db.autocommit(False) try: # aws_billing_records for rows in self.csv_reader(csv_file, envs, dtime_from=dtime_from, dtime_to=dtime_to): records_ids = [row['RecordId'] for row in rows] for chunk in helper.chunks(records_ids, 1000): if chunk: query = ( "DELETE FROM aws_billing_records " "WHERE record_id IN ({record_id})" ).format(record_id=str(chunk)[1:-1]) self.analytics.analytics_db.execute(query) _dtime_from = dtime_from step_days = 15 while _dtime_from < dtime_to: _dtime_to = min(_dtime_from + datetime.timedelta(days=step_days), dtime_to) # usage_servers_h, usage_h query = ( "DELETE uh, us " "FROM usage_h uh " "LEFT JOIN usage_servers_h us ON uh.usage_id=us.usage_id " "WHERE uh.platform='ec2' " "AND uh.dtime BETWEEN '{dtime_from}' AND '{dtime_to}' " "AND uh.env_id IN ({env_id})" ).format(env_id=str(envs_ids)[1:-1], dtime_from=_dtime_from, dtime_to=_dtime_to) self.analytics.analytics_db.execute(query) # usage_d query = ( "DELETE FROM usage_d " "WHERE platform='ec2' " "AND date BETWEEN '{date_from}' AND '{date_to}' " "AND env_id IN ({env_id})" ).format(env_id=str(envs_ids)[1:-1], date_from=_dtime_from.date(), date_to=_dtime_to.date()) self.analytics.analytics_db.execute(query) # farm_usage_d query = ( "DELETE FROM farm_usage_d " "WHERE platform='ec2' " "AND date BETWEEN '{date_from}' AND '{date_to}' " "AND env_id IN ({env_id})" ).format(env_id=str(envs_ids)[1:-1], date_from=_dtime_from.date(), date_to=_dtime_to.date()) self.analytics.analytics_db.execute(query) _dtime_from += datetime.timedelta(days=step_days) self.analytics.analytics_db.commit() except: self.analytics.analytics_db.rollback() raise finally: self.analytics.analytics_db.autocommit(True)
def do_iteration(self): global debug_rate_counter global debug_rate_timestamp debug_rate_time = time.time() - debug_rate_timestamp rate = round(debug_rate_counter / debug_rate_time, 2) LOG.info('Average rate: %s, %s' % (rate, rate * 60)) debug_rate_counter = 0 debug_rate_timestamp = time.time() while len(self._processing_messages) > self._max_processing_messages: LOG.warning('Reached the limit of simultaneously processed messages') time.sleep(1) messages = self.get_messages() messages = [m for m in messages if m['message_id'] not in self._processing_messages] num, idx = int(self.config['workers']), int(self.config['index']) def filter_messages(message): if message.get('server_id') and message.get('farm_id'): return int(message['farm_id']) % num == idx - 1 else: return idx == 1 if num > 1: messages = filter(filter_messages, messages) if not messages: time.sleep(self.nothing_todo_sleep) return self.load_servers_data(messages) server_statuses = [ 'Running', 'Initializing', 'Importing', 'Temporary', 'Pending terminate', 'Pending suspend', ] for message in messages: try: self._processing_messages.add(message['message_id']) if message.get('server_id') is None or \ message['server_status'] not in server_statuses or ( message['server_status'] in ('Pending terminate', 'Pending suspend') and int(message['handle_attempts']) >= 1): msg = ( "Server {message_server_id} doesn't exist or not in right status, " "set message {message_id} status to 3").format(**message) LOG.warning(msg) message['status'] = 3 self._pool.wait() self._pool.apply_async(self.update, (message,)) else: self._pool.wait() self._pool.apply_async(self.process_message, (message,)) except: msg = "Unable to process message: {message_id}, reason: {error}" msg = msg.format(message_id=message['message_id'], error=helper.exc_info()) LOG.warning(msg) LOG.info('Messages still in processing: %s' % len(self._processing_messages))
def download_billing_file(self, env, date=None, force=False): date = date or datetime.datetime.utcnow().date() bucket_name = env['ec2.detailed_billing.bucket'] if env.get('ec2.detailed_billing.payer_account'): envs = self.analytics.load_aws_accounts_ids_envs([env['ec2.detailed_billing.payer_account']]) self.analytics.load_env_credentials(envs, platform='ec2') for e in envs: if e['client_id'] == env['client_id']: credentials_env = e break else: msg = 'Can not found AWS credentials for PayerAccount {}' msg = msg.format(env['ec2.detailed_billing.payer_account']) raise Exception(msg) else: credentials_env = env.copy() kwds = { 'aws_access_key_id': cryptotool.decrypt_scalr(self.config['crypto_key'], credentials_env['ec2.access_key']), 'aws_secret_access_key': cryptotool.decrypt_scalr(self.config['crypto_key'], credentials_env['ec2.secret_key']), 'proxy': self.config['aws_proxy'].get('host'), 'proxy_port': self.config['aws_proxy'].get('port'), 'proxy_user': self.config['aws_proxy'].get('user'), 'proxy_pass': self.config['aws_proxy'].get('pass'), } default_region_map = { 'regular': 'us-east-1', 'gov-cloud': 'us-gov-west-1', 'cn-cloud': 'cn-north-1', } default_region = default_region_map[env.get('account_type', 'regular')] region = env.get('ec2.detailed_billing.region', default_region) conn = boto.s3.connect_to_region(region, **kwds) bucket = conn.get_bucket(bucket_name) default_file_name = get_default_aws_csv_file_name(credentials_env['ec2.account_id'], date) file_name_tmplate = get_aws_csv_file_name_tmplate(credentials_env['ec2.account_id'], date) files_in_bucket = [key.name for key in bucket.list() if re.match(file_name_tmplate, key.name)] if not files_in_bucket: utcnow = datetime.datetime.utcnow() if date.month == utcnow.month and utcnow.day < 2: return None else: msg = "Not found any valid files({}, {}) in bucket '{}'" msg = msg.format(default_file_name, file_name_tmplate, bucket_name) raise exceptions.FileNotFoundError(msg) if default_file_name not in files_in_bucket: file_name = files_in_bucket[0] # use first valid file msg = "Default AWS detailed billing statistics file '{}' not found in bucket '{}', available {}, use '{}'" msg = msg.format(default_file_name, bucket_name, files_in_bucket, file_name) LOG.warning(msg) else: file_name = default_file_name try: key = bucket.get_key(file_name) last_modified_dt = datetime.datetime.strptime(key.last_modified, self.last_modified_format) utcnow = datetime.datetime.utcnow() seconds_from_last_modified_dt = (utcnow - last_modified_dt).seconds if hasattr(self, 'task_info') and self.task_info.get('period') > 300: delta = datetime.timedelta(seconds=self.task_info['period'] - 90) else: delta = datetime.timedelta(seconds=210) condition1 = utcnow > last_modified_dt and utcnow < last_modified_dt + delta condition2 = seconds_from_last_modified_dt > 3600 condition3 = (seconds_from_last_modified_dt / 3600) % 24 == 0 if force or condition1 or (condition2 and condition3): with self._downloading_lock: self.downloading_locks.setdefault(file_name, gevent.lock.RLock()) with self.downloading_locks[file_name]: csv_zip_file = os.path.join(self.cache_dir, file_name) csv_file = csv_zip_file.rstrip('.zip') if os.path.exists(csv_file): msg = "'{}' already exists in cache directory, use it" msg = msg.format(os.path.basename(csv_file)) LOG.debug(msg) return csv_file while key.size * 3 > helper.get_free_space(self.cache_dir): LOG.error('Disk is full, waiting 60 sec') gevent.sleep(60) LOG.debug("Downloading '{}' for environment {}".format(file_name, env['id'])) attempts = 2 downloading_start_time = time.time() while True: try: key.get_contents_to_filename(csv_zip_file) assert os.path.isfile(csv_zip_file), os.listdir(self.cache_dir) downloading_end_time = time.time() break except: attempts -= 1 if not attempts: raise downloading_time = downloading_end_time - downloading_start_time msg = "Downloading '{0}' done in {1:.1f} seconds".format(file_name, downloading_time) LOG.info(msg) LOG.debug('Unzipping to {}'.format(csv_file)) while os.path.getsize(csv_zip_file) * 1 > helper.get_free_space(self.cache_dir): LOG.error('Disk is full, waiting 60 sec') gevent.sleep(60) with zipfile.ZipFile(csv_zip_file, 'r') as f: f.extract(f.infolist()[0], self.cache_dir) os.remove(csv_zip_file) return csv_file else: msg = "Skipping AWS billing file '{}' for environment {}" msg = msg.format(file_name, env['id']) LOG.debug(msg) return None except: msg = "File '{}', bucket '{}', reason: {}" msg = msg.format(file_name, bucket_name, helper.exc_info()) raise Exception, Exception(msg), sys.exc_info()[2]
def process_poller_billing(self): dtime_from, dtime_to = self.get_poller_billing_interval() LOG.info('Poller billing interval: {0} - {1}'.format( dtime_from, dtime_to)) # process poller_session table dtime_cur = dtime_from while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: msg = "Process poller data, date {0}, hour {1}".format( date, hour) LOG.info(msg) if self.args['--recalculate']: platform = self.config['platform'] generator = self.analytics.get_records( date, hour, platform) else: generator = self.analytics.get_servers(date, hour) for records in generator: LOG.debug('Records for processing: %s' % len(records)) prices = self.analytics.get_prices(records) for record in records: cost = self.analytics.get_cost_from_prices( record, prices) or 0 self.pool.wait() if self.args['--recalculate']: record['cost'] = float(cost) * int(record['num']) self.pool.apply_async(self.analytics.update_record, (record, )) else: record['cost'] = cost record['num'] = 1.0 record['cost_distr_type'] = 1 self.pool.apply_async(self.analytics.insert_record, (record, )) gevent.sleep(0) # force switch self.pool.join() except: msg = "Unable to process date {0}, hour {1}".format(date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) # fill farm_usage_d dtime_cur = dtime_from msg = 'Poller fill_farm_usage_d interval: {0} - {1}' LOG.info(msg.format(dtime_cur, dtime_to)) while dtime_cur <= dtime_to: date, hour = dtime_cur.date(), dtime_cur.hour try: self.analytics.fill_farm_usage_d(date, hour) except: msg = 'Unable to fill farm_usage_d table for date {0}, hour {1}'.format( date, hour) LOG.exception(msg) dtime_cur += datetime.timedelta(hours=1) if not self.args['--recalculate']: return # recalculate daily tables dtime_cur = dtime_from while dtime_cur <= dtime_to: date = dtime_cur.date() msg = "Recalculate daily tables for date {0}".format(date) LOG.debug(msg) try: self.analytics.recalculate_usage_d(date, self.config['platform']) except: msg = "Recalculate usage_d table for date {0} failed, error: {1}".format( date, helper.exc_info()) LOG.warning(msg) dtime_cur += datetime.timedelta(days=1) # recalculate quarters tables quarters_calendar = self.analytics.get_quarters_calendar() start_year = quarters_calendar.year_for_date(dtime_from.date()) start_quarter = quarters_calendar.quarter_for_date(dtime_from.date()) end_year = quarters_calendar.year_for_date(dtime_to.date()) end_quarter = quarters_calendar.quarter_for_date(dtime_to.date()) tmp = [] cur_year = start_year while cur_year < end_year: for quarter in range(start_quarter, 5): tmp.append((cur_year, quarter)) start_quarter = 1 cur_year += 1 for quarter in range(start_quarter, end_quarter + 1): tmp.append((end_year, quarter)) for year, quarter in tmp: try: msg = "Recalculate quarterly_budget table for year {0}, quarter {1}" msg = msg.format(year, quarter) LOG.debug(msg) self.analytics.recalculate_quarterly_budget(year, quarter) except: msg = "Recalculate quarterly_budget table for year {0}, quarter {1} failed" msg = msg.format(year, quarter, helper.exc_info()) LOG.exception(msg)