class ServiceWatcher(object): def __init__(self, conf, service, **kwargs): self.conf = conf self.running = False for k in ['host', 'port', 'type']: if k not in service: raise Exception('Missing field "%s" in service configuration' % k) self.name = '%s|%s|%s' % \ (service['type'], service['host'], service['port']) self.service = service self.rise = int_value(self._load_item_config('rise'), 1) self.fall = int_value(self._load_item_config('fall'), 1) self.check_interval = float_value( self._load_item_config('check_interval'), 1) self.deregister_on_exit = true_value( self._load_item_config('deregister_on_exit', False)) self.logger = get_logger(self.conf) self.pool_manager = get_pool_manager() self.cs = ConscienceClient(self.conf, pool_manager=self.pool_manager, logger=self.logger) # FIXME: explain that self.client = ProxyClient(self.conf, pool_manager=self.pool_manager, no_ns_in_url=True, logger=self.logger) self.last_status = False self.status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {} } if self.service.get('slots', None): self.service_definition['tags']['tag.slots'] = \ ','.join(self.service['slots']) for name, tag in (('location', 'tag.loc'), ('service_id', 'tag.service_id'), ('tls', 'tag.tls')): if self.service.get(name): self.service_definition['tags'][tag] = \ self.service[name] self.service_checks = list() self.service_stats = list() self.init_checkers(service) self.init_stats(service) def _load_item_config(self, item, default=None): return self.service.get(item, self.conf.get(item)) or default def start(self): self.logger.info('watcher "%s" starting', self.name) self.running = True self.watch() self.running = False def stop(self): self.logger.info('watcher "%s" stopping', self.name) if self.deregister_on_exit: self.logger.info('watcher "%s" deregister service', self.name) try: self.status = False self.last_status = False self.register() except Exception as e: self.logger.warn('Failed to register service: %s', e) self.running = False def check(self): """Perform the registered checks on the service until any of them fails of the end of the list is reached.""" self.status = True for service_check in (x for x in self.service_checks if self.running): if not service_check.service_status(): self.status = False return def get_stats(self): """Update service definition with all configured stats""" if not self.status: return collectors = (x for x in self.service_stats if self.running) try: for stat in collectors: stats = stat.get_stats() self.service_definition['tags'].update(stats) except Exception as ex: self.logger.debug("get_stats error: %s, skipping %s", ex, [type(col).__name__ for col in collectors] or "<nothing>") self.status = False def register(self): # only accept a final zero/down-registration when exiting if not self.running and self.status: return # Alert when the status changes if self.status != self.last_status: if self.status: self.logger.info('service "%s" is now up', self.name) else: self.logger.warn('service "%s" is now down', self.name) self.last_status = self.status # Use a boolean so we can easily convert it to a number in conscience self.service_definition['tags']['tag.up'] = self.status try: self.cs.register(self.service_definition, retries=False) except OioException as rqe: self.logger.warn("Failed to register service %s: %s", self.service_definition["addr"], rqe) def watch(self): try: while self.running: self.check() self.get_stats() self.register() sleep(self.check_interval) except Exception as e: self.logger.warn('ERROR in watcher "%s"', e) self.failed = True raise e finally: self.logger.info('watcher "%s" stopped', self.name) def init_checkers(self, service): for check in service['checks']: check['host'] = check.get('host') or service['host'] check['port'] = check.get('port') or service['port'] check['name'] = check.get('name') or "%s|%s|%s" % \ (check['type'], check['host'], check['port']) check['rise'] = check.get('rise') or self.rise check['fall'] = check.get('fall') or self.fall check['type'] = check.get('type') or 'unknown' service_check_class = CHECKERS_MODULES.get(check['type']) if not service_check_class: raise Exception( 'Invalid check type "%s", valid types: %s' % (check['type'], ', '.join(CHECKERS_MODULES.keys()))) service_check = service_check_class(self, check, self.logger) self.service_checks.append(service_check) def init_stats(self, service): """Initialize service stat fetchers""" self.service_stats[:] = [] for stat in service['stats']: stat.setdefault('host', service['host']) stat.setdefault('port', service['port']) stat.setdefault('path', "") service_stat_class = STATS_MODULES.get(stat['type'], None) if not service_stat_class: raise Exception( 'Invalid stat type "%s", valid types: %s' % (stat['type'], ', '.join(STATS_MODULES.keys()))) service_stat = service_stat_class(self, stat, self.logger) self.service_stats.append(service_stat)
class ServiceWatcher(object): def __init__(self, conf, service, **kwargs): self.conf = conf self.running = False for k in ['host', 'port', 'type']: if k not in service: raise Exception( 'Missing field "%s" in service configuration' % k) self.name = '%s|%s' % \ (service['host'], service['port']) self.check_interval = float_value(conf.get('check_interval'), 1) self.service = service self.rise = int_value(conf.get('rise'), 1) self.fall = int_value(conf.get('fall'), 1) self.logger = get_logger(self.conf) self.cs = ConscienceClient(self.conf) self.client = Client(self.conf) self.last_status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {}} if self.service.get('location', None): self.service_definition['tags']['tag.loc'] = \ self.service['location'] self.service_checks = list() self.service_stats = list() self.init_checkers(service) self.init_stats(service) def start(self): self.logger.info('watcher "%s" starting', self.name) self.running = True self.watch() def stop(self): self.logger.info('watcher "%s" stopping', self.name) self.running = False def check(self): status = True for service_check in self.service_checks: if not service_check.service_status(): status = False if status != self.last_status: if status: self.logger.info('service "%s" is now up', self.name) else: self.logger.warn('service "%s" is now down', self.name) self.last_status = status def get_stats(self): """Update service definition with all configured stats""" for stat in self.service_stats: stats = stat.get_stats() self.logger.debug("Stat fetcher '%s' returned %s", str(stat), str(stats)) self.service_definition['tags'].update(stats) def register(self): # Use a boolean so we can easily convert it to a number in conscience self.service_definition['tags']['tag.up'] = self.last_status self.cs.register( self.service['type'], self.service_definition) def watch(self): try: while self.running: self.check() self.get_stats() self.register() sleep(self.check_interval) except Exception as e: self.logger.warn('ERROR in watcher "%s"', e) self.failed = True raise e finally: self.logger.info('watcher "%s" stopped', self.name) def init_checkers(self, service): for check in service['checks']: check['host'] = check.get('host') or service['host'] check['port'] = check.get('port') or service['port'] check['name'] = check.get('name') or "%s|%s|%s" % \ (check['type'], check['host'], check['port']) check['rise'] = check.get('rise') or self.rise check['fall'] = check.get('fall') or self.fall check['type'] = check.get('type') or 'unknown' service_check_class = CHECKERS_MODULES.get(check['type']) if not service_check_class: raise Exception( 'Invalid check type "%s", valid types: %s' % (check['type'], ', '.join(CHECKERS_MODULES.keys()))) service_check = service_check_class(self, check, self.logger) self.service_checks.append(service_check) def init_stats(self, service): """Initialize service stat fetchers""" self.service_stats[:] = [] for stat in service['stats']: stat.setdefault('host', service['host']) stat.setdefault('port', service['port']) stat.setdefault('path', "") service_stat_class = STATS_MODULES.get(stat['type'], None) if not service_stat_class: raise Exception( 'Invalid stat type "%s", valid types: %s' % (stat['type'], ', '.join(STATS_MODULES.keys()))) service_stat = service_stat_class(self, stat, self.logger) self.service_stats.append(service_stat)
class ServiceWatcher(object): def __init__(self, conf, service, **kwargs): self.conf = conf for k in ['host', 'port', 'type']: if k not in service: raise Exception('Missing field "%s" in service configuration' % k) self.name = '%s|%s' % \ (service['host'], service['port']) self.check_interval = float_value(conf.get('check_interval'), 1) self.service = service self.rise = int_value(conf.get('rise'), 1) self.fall = int_value(conf.get('fall'), 1) self.logger = get_logger(self.conf) self.cs = ConscienceClient(self.conf) self.init_checkers(service) self.last_status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {} } def start(self): self.logger.info('watcher "%s" starting', self.name) self.running = True self.watch() def stop(self): self.logger.info('watcher "%s" stopping', self.name) self.running = False def check(self): status = True for service_check in self.service_checks: if not service_check.service_status(): status = False if status != self.last_status: if status: self.logger.info('service "%s" is now up', self.name) else: self.logger.warn('service "%s" is now down', self.name) self.last_status = status def register(self): tag_up = 'true' if self.last_status else 'false' self.service_definition['tags']['tag.up'] = tag_up self.cs.register(self.service['type'], self.service_definition) def watch(self): try: while self.running: self.check() self.register() sleep(self.check_interval) except Exception as e: self.logger.warn('ERROR in watcher "%s"', e) self.failed = True raise e finally: self.logger.info('watcher "%s" stopped', self.name) def init_checkers(self, service): self.service_checks = [] for check in service['checks']: check['host'] = check.get('host') or service['host'] check['port'] = check.get('port') or service['port'] check['name'] = check.get('name') or "%s|%s|%s" % \ (check['type'], check['host'], check['port']) check['rise'] = check.get('rise') or self.rise check['fall'] = check.get('fall') or self.fall check['type'] = check.get('type') or 'unknown' service_check_class = CHECKERS_MODULES.get(check['type']) if not service_check_class: raise Exception( 'Invalid check type "%s", valid types: %s' % (check['type'], ', '.join(self.checkers.keys()))) service_check = service_check_class(check, self.logger) self.service_checks.append(service_check)
class ServiceWatcher(object): def __init__(self, conf, service, **kwargs): self.conf = conf self.running = False for k in ['host', 'port', 'type']: if k not in service: raise Exception('Missing field "%s" in service configuration' % k) self.name = '%s|%s' % \ (service['host'], service['port']) self.check_interval = float_value(conf.get('check_interval'), 1) self.service = service self.rise = int_value(conf.get('rise'), 1) self.fall = int_value(conf.get('fall'), 1) self.logger = get_logger(self.conf) self.cs = ConscienceClient(self.conf) self.client = Client(self.conf) self.last_status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {} } if self.service.get('location', None): self.service_definition['tags']['tag.loc'] = \ self.service['location'] self.service_checks = list() self.service_stats = list() self.init_checkers(service) self.init_stats(service) def start(self): self.logger.info('watcher "%s" starting', self.name) self.running = True self.watch() def stop(self): self.logger.info('watcher "%s" stopping', self.name) self.running = False def check(self): status = True for service_check in self.service_checks: if not service_check.service_status(): status = False if status != self.last_status: if status: self.logger.info('service "%s" is now up', self.name) else: self.logger.warn('service "%s" is now down', self.name) self.last_status = status def get_stats(self): """Update service definition with all configured stats""" for stat in self.service_stats: stats = stat.get_stats() self.logger.debug("Stat fetcher '%s' returned %s", str(stat), str(stats)) self.service_definition['tags'].update(stats) def register(self): # Use a boolean so we can easily convert it to a number in conscience self.service_definition['tags']['tag.up'] = self.last_status self.cs.register(self.service['type'], self.service_definition) def watch(self): try: while self.running: self.check() self.get_stats() self.register() sleep(self.check_interval) except Exception as e: self.logger.warn('ERROR in watcher "%s"', e) self.failed = True raise e finally: self.logger.info('watcher "%s" stopped', self.name) def init_checkers(self, service): for check in service['checks']: check['host'] = check.get('host') or service['host'] check['port'] = check.get('port') or service['port'] check['name'] = check.get('name') or "%s|%s|%s" % \ (check['type'], check['host'], check['port']) check['rise'] = check.get('rise') or self.rise check['fall'] = check.get('fall') or self.fall check['type'] = check.get('type') or 'unknown' service_check_class = CHECKERS_MODULES.get(check['type']) if not service_check_class: raise Exception( 'Invalid check type "%s", valid types: %s' % (check['type'], ', '.join(CHECKERS_MODULES.keys()))) service_check = service_check_class(self, check, self.logger) self.service_checks.append(service_check) def init_stats(self, service): """Initialize service stat fetchers""" self.service_stats[:] = [] for stat in service['stats']: stat.setdefault('host', service['host']) stat.setdefault('port', service['port']) stat.setdefault('path', "") service_stat_class = STATS_MODULES.get(stat['type'], None) if not service_stat_class: raise Exception( 'Invalid stat type "%s", valid types: %s' % (stat['type'], ', '.join(STATS_MODULES.keys()))) service_stat = service_stat_class(self, stat, self.logger) self.service_stats.append(service_stat)
class ServiceWatcher(object): def __init__(self, conf, service, **kwargs): self.conf = conf for k in ['host', 'port', 'type']: if k not in service: raise Exception( 'Missing field "%s" in service configuration' % k) self.name = '%s|%s' % \ (service['host'], service['port']) self.check_interval = float_value(conf.get('check_interval'), 1) self.service = service self.rise = int_value(conf.get('rise'), 1) self.fall = int_value(conf.get('fall'), 1) self.logger = get_logger(self.conf) self.cs = ConscienceClient(self.conf) self.init_checkers(service) self.last_status = False self.failed = False self.service_definition = { 'ns': self.conf['namespace'], 'type': self.service['type'], 'addr': '%s:%s' % (self.service['host'], self.service['port']), 'score': 0, 'tags': {}} def start(self): self.logger.info('watcher "%s" starting', self.name) self.running = True self.watch() def stop(self): self.logger.info('watcher "%s" stopping', self.name) self.running = False def check(self): status = True for service_check in self.service_checks: if not service_check.service_status(): status = False if status != self.last_status: if status: self.logger.info('service "%s" is now up', self.name) else: self.logger.warn('service "%s" is now down', self.name) self.last_status = status def register(self): tag_up = 'true' if self.last_status else 'false' self.service_definition['tags']['tag.up'] = tag_up self.cs.register( self.service['type'], self.service_definition) def watch(self): try: while self.running: self.check() self.register() sleep(self.check_interval) except Exception as e: self.logger.warn('ERROR in watcher "%s"', e) self.failed = True raise e finally: self.logger.info('watcher "%s" stopped', self.name) def init_checkers(self, service): self.service_checks = [] for check in service['checks']: check['host'] = check.get('host') or service['host'] check['port'] = check.get('port') or service['port'] check['name'] = check.get('name') or "%s|%s|%s" % \ (check['type'], check['host'], check['port']) check['rise'] = check.get('rise') or self.rise check['fall'] = check.get('fall') or self.fall check['type'] = check.get('type') or 'unknown' service_check_class = CHECKERS_MODULES.get(check['type']) if not service_check_class: raise Exception( 'Invalid check type "%s", valid types: %s' % (check['type'], ', '.join(self.checkers.keys()))) service_check = service_check_class(check, self.logger) self.service_checks.append(service_check)