class ServiceWatcher(object):
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception('Missing field "%s" in service configuration' %
k)
self.name = '%s|%s|%s' % \
(service['type'], service['host'], service['port'])
self.service = service
self.rise = int_value(self._load_item_config('rise'), 1)
self.fall = int_value(self._load_item_config('fall'), 1)
self.check_interval = float_value(
self._load_item_config('check_interval'), 1)
self.deregister_on_exit = true_value(
self._load_item_config('deregister_on_exit', False))
self.logger = get_logger(self.conf)
self.pool_manager = get_pool_manager()
self.cs = ConscienceClient(self.conf,
pool_manager=self.pool_manager,
logger=self.logger)
# FIXME: explain that
self.client = ProxyClient(self.conf,
pool_manager=self.pool_manager,
no_ns_in_url=True,
logger=self.logger)
self.last_status = False
self.status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}
}
if self.service.get('slots', None):
self.service_definition['tags']['tag.slots'] = \
','.join(self.service['slots'])
for name, tag in (('location', 'tag.loc'),
('service_id', 'tag.service_id'), ('tls',
'tag.tls')):
if self.service.get(name):
self.service_definition['tags'][tag] = \
self.service[name]
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
def _load_item_config(self, item, default=None):
return self.service.get(item, self.conf.get(item)) or default
def start(self):
self.logger.info('watcher "%s" starting', self.name)
self.running = True
self.watch()
self.running = False
def stop(self):
self.logger.info('watcher "%s" stopping', self.name)
if self.deregister_on_exit:
self.logger.info('watcher "%s" deregister service', self.name)
try:
self.status = False
self.last_status = False
self.register()
except Exception as e:
self.logger.warn('Failed to register service: %s', e)
self.running = False
def check(self):
"""Perform the registered checks on the service until any of
them fails of the end of the list is reached."""
self.status = True
for service_check in (x for x in self.service_checks if self.running):
if not service_check.service_status():
self.status = False
return
def get_stats(self):
"""Update service definition with all configured stats"""
if not self.status:
return
collectors = (x for x in self.service_stats if self.running)
try:
for stat in collectors:
stats = stat.get_stats()
self.service_definition['tags'].update(stats)
except Exception as ex:
self.logger.debug("get_stats error: %s, skipping %s", ex,
[type(col).__name__ for col in collectors]
or "<nothing>")
self.status = False
def register(self):
# only accept a final zero/down-registration when exiting
if not self.running and self.status:
return
# Alert when the status changes
if self.status != self.last_status:
if self.status:
self.logger.info('service "%s" is now up', self.name)
else:
self.logger.warn('service "%s" is now down', self.name)
self.last_status = self.status
# Use a boolean so we can easily convert it to a number in conscience
self.service_definition['tags']['tag.up'] = self.status
try:
self.cs.register(self.service_definition, retries=False)
except OioException as rqe:
self.logger.warn("Failed to register service %s: %s",
self.service_definition["addr"], rqe)
def watch(self):
try:
while self.running:
self.check()
self.get_stats()
self.register()
sleep(self.check_interval)
except Exception as e:
self.logger.warn('ERROR in watcher "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('watcher "%s" stopped', self.name)
def init_checkers(self, service):
for check in service['checks']:
check['host'] = check.get('host') or service['host']
check['port'] = check.get('port') or service['port']
check['name'] = check.get('name') or "%s|%s|%s" % \
(check['type'], check['host'], check['port'])
check['rise'] = check.get('rise') or self.rise
check['fall'] = check.get('fall') or self.fall
check['type'] = check.get('type') or 'unknown'
service_check_class = CHECKERS_MODULES.get(check['type'])
if not service_check_class:
raise Exception(
'Invalid check type "%s", valid types: %s' %
(check['type'], ', '.join(CHECKERS_MODULES.keys())))
service_check = service_check_class(self, check, self.logger)
self.service_checks.append(service_check)
def init_stats(self, service):
"""Initialize service stat fetchers"""
self.service_stats[:] = []
for stat in service['stats']:
stat.setdefault('host', service['host'])
stat.setdefault('port', service['port'])
stat.setdefault('path', "")
service_stat_class = STATS_MODULES.get(stat['type'], None)
if not service_stat_class:
raise Exception(
'Invalid stat type "%s", valid types: %s' %
(stat['type'], ', '.join(STATS_MODULES.keys())))
service_stat = service_stat_class(self, stat, self.logger)
self.service_stats.append(service_stat)
class ServiceWatcher(object):
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception(
'Missing field "%s" in service configuration' % k)
self.name = '%s|%s' % \
(service['host'], service['port'])
self.check_interval = float_value(conf.get('check_interval'), 1)
self.service = service
self.rise = int_value(conf.get('rise'), 1)
self.fall = int_value(conf.get('fall'), 1)
self.logger = get_logger(self.conf)
self.cs = ConscienceClient(self.conf)
self.client = Client(self.conf)
self.last_status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}}
if self.service.get('location', None):
self.service_definition['tags']['tag.loc'] = \
self.service['location']
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
def start(self):
self.logger.info('watcher "%s" starting', self.name)
self.running = True
self.watch()
def stop(self):
self.logger.info('watcher "%s" stopping', self.name)
self.running = False
def check(self):
status = True
for service_check in self.service_checks:
if not service_check.service_status():
status = False
if status != self.last_status:
if status:
self.logger.info('service "%s" is now up', self.name)
else:
self.logger.warn('service "%s" is now down', self.name)
self.last_status = status
def get_stats(self):
"""Update service definition with all configured stats"""
for stat in self.service_stats:
stats = stat.get_stats()
self.logger.debug("Stat fetcher '%s' returned %s",
str(stat), str(stats))
self.service_definition['tags'].update(stats)
def register(self):
# Use a boolean so we can easily convert it to a number in conscience
self.service_definition['tags']['tag.up'] = self.last_status
self.cs.register(
self.service['type'], self.service_definition)
def watch(self):
try:
while self.running:
self.check()
self.get_stats()
self.register()
sleep(self.check_interval)
except Exception as e:
self.logger.warn('ERROR in watcher "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('watcher "%s" stopped', self.name)
def init_checkers(self, service):
for check in service['checks']:
check['host'] = check.get('host') or service['host']
check['port'] = check.get('port') or service['port']
check['name'] = check.get('name') or "%s|%s|%s" % \
(check['type'], check['host'], check['port'])
check['rise'] = check.get('rise') or self.rise
check['fall'] = check.get('fall') or self.fall
check['type'] = check.get('type') or 'unknown'
service_check_class = CHECKERS_MODULES.get(check['type'])
if not service_check_class:
raise Exception(
'Invalid check type "%s", valid types: %s' %
(check['type'], ', '.join(CHECKERS_MODULES.keys())))
service_check = service_check_class(self, check, self.logger)
self.service_checks.append(service_check)
def init_stats(self, service):
"""Initialize service stat fetchers"""
self.service_stats[:] = []
for stat in service['stats']:
stat.setdefault('host', service['host'])
stat.setdefault('port', service['port'])
stat.setdefault('path', "")
service_stat_class = STATS_MODULES.get(stat['type'], None)
if not service_stat_class:
raise Exception(
'Invalid stat type "%s", valid types: %s' %
(stat['type'], ', '.join(STATS_MODULES.keys())))
service_stat = service_stat_class(self, stat, self.logger)
self.service_stats.append(service_stat)
class ServiceWatcher(object):
def __init__(self, conf, service, **kwargs):
self.conf = conf
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception('Missing field "%s" in service configuration' %
k)
self.name = '%s|%s' % \
(service['host'], service['port'])
self.check_interval = float_value(conf.get('check_interval'), 1)
self.service = service
self.rise = int_value(conf.get('rise'), 1)
self.fall = int_value(conf.get('fall'), 1)
self.logger = get_logger(self.conf)
self.cs = ConscienceClient(self.conf)
self.init_checkers(service)
self.last_status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}
}
def start(self):
self.logger.info('watcher "%s" starting', self.name)
self.running = True
self.watch()
def stop(self):
self.logger.info('watcher "%s" stopping', self.name)
self.running = False
def check(self):
status = True
for service_check in self.service_checks:
if not service_check.service_status():
status = False
if status != self.last_status:
if status:
self.logger.info('service "%s" is now up', self.name)
else:
self.logger.warn('service "%s" is now down', self.name)
self.last_status = status
def register(self):
tag_up = 'true' if self.last_status else 'false'
self.service_definition['tags']['tag.up'] = tag_up
self.cs.register(self.service['type'], self.service_definition)
def watch(self):
try:
while self.running:
self.check()
self.register()
sleep(self.check_interval)
except Exception as e:
self.logger.warn('ERROR in watcher "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('watcher "%s" stopped', self.name)
def init_checkers(self, service):
self.service_checks = []
for check in service['checks']:
check['host'] = check.get('host') or service['host']
check['port'] = check.get('port') or service['port']
check['name'] = check.get('name') or "%s|%s|%s" % \
(check['type'], check['host'], check['port'])
check['rise'] = check.get('rise') or self.rise
check['fall'] = check.get('fall') or self.fall
check['type'] = check.get('type') or 'unknown'
service_check_class = CHECKERS_MODULES.get(check['type'])
if not service_check_class:
raise Exception(
'Invalid check type "%s", valid types: %s' %
(check['type'], ', '.join(self.checkers.keys())))
service_check = service_check_class(check, self.logger)
self.service_checks.append(service_check)
class ServiceWatcher(object):
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception('Missing field "%s" in service configuration' %
k)
self.name = '%s|%s' % \
(service['host'], service['port'])
self.check_interval = float_value(conf.get('check_interval'), 1)
self.service = service
self.rise = int_value(conf.get('rise'), 1)
self.fall = int_value(conf.get('fall'), 1)
self.logger = get_logger(self.conf)
self.cs = ConscienceClient(self.conf)
self.client = Client(self.conf)
self.last_status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}
}
if self.service.get('location', None):
self.service_definition['tags']['tag.loc'] = \
self.service['location']
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
def start(self):
self.logger.info('watcher "%s" starting', self.name)
self.running = True
self.watch()
def stop(self):
self.logger.info('watcher "%s" stopping', self.name)
self.running = False
def check(self):
status = True
for service_check in self.service_checks:
if not service_check.service_status():
status = False
if status != self.last_status:
if status:
self.logger.info('service "%s" is now up', self.name)
else:
self.logger.warn('service "%s" is now down', self.name)
self.last_status = status
def get_stats(self):
"""Update service definition with all configured stats"""
for stat in self.service_stats:
stats = stat.get_stats()
self.logger.debug("Stat fetcher '%s' returned %s", str(stat),
str(stats))
self.service_definition['tags'].update(stats)
def register(self):
# Use a boolean so we can easily convert it to a number in conscience
self.service_definition['tags']['tag.up'] = self.last_status
self.cs.register(self.service['type'], self.service_definition)
def watch(self):
try:
while self.running:
self.check()
self.get_stats()
self.register()
sleep(self.check_interval)
except Exception as e:
self.logger.warn('ERROR in watcher "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('watcher "%s" stopped', self.name)
def init_checkers(self, service):
for check in service['checks']:
check['host'] = check.get('host') or service['host']
check['port'] = check.get('port') or service['port']
check['name'] = check.get('name') or "%s|%s|%s" % \
(check['type'], check['host'], check['port'])
check['rise'] = check.get('rise') or self.rise
check['fall'] = check.get('fall') or self.fall
check['type'] = check.get('type') or 'unknown'
service_check_class = CHECKERS_MODULES.get(check['type'])
if not service_check_class:
raise Exception(
'Invalid check type "%s", valid types: %s' %
(check['type'], ', '.join(CHECKERS_MODULES.keys())))
service_check = service_check_class(self, check, self.logger)
self.service_checks.append(service_check)
def init_stats(self, service):
"""Initialize service stat fetchers"""
self.service_stats[:] = []
for stat in service['stats']:
stat.setdefault('host', service['host'])
stat.setdefault('port', service['port'])
stat.setdefault('path', "")
service_stat_class = STATS_MODULES.get(stat['type'], None)
if not service_stat_class:
raise Exception(
'Invalid stat type "%s", valid types: %s' %
(stat['type'], ', '.join(STATS_MODULES.keys())))
service_stat = service_stat_class(self, stat, self.logger)
self.service_stats.append(service_stat)
class ServiceWatcher(object):
def __init__(self, conf, service, **kwargs):
self.conf = conf
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception(
'Missing field "%s" in service configuration' % k)
self.name = '%s|%s' % \
(service['host'], service['port'])
self.check_interval = float_value(conf.get('check_interval'), 1)
self.service = service
self.rise = int_value(conf.get('rise'), 1)
self.fall = int_value(conf.get('fall'), 1)
self.logger = get_logger(self.conf)
self.cs = ConscienceClient(self.conf)
self.init_checkers(service)
self.last_status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}}
def start(self):
self.logger.info('watcher "%s" starting', self.name)
self.running = True
self.watch()
def stop(self):
self.logger.info('watcher "%s" stopping', self.name)
self.running = False
def check(self):
status = True
for service_check in self.service_checks:
if not service_check.service_status():
status = False
if status != self.last_status:
if status:
self.logger.info('service "%s" is now up', self.name)
else:
self.logger.warn('service "%s" is now down', self.name)
self.last_status = status
def register(self):
tag_up = 'true' if self.last_status else 'false'
self.service_definition['tags']['tag.up'] = tag_up
self.cs.register(
self.service['type'], self.service_definition)
def watch(self):
try:
while self.running:
self.check()
self.register()
sleep(self.check_interval)
except Exception as e:
self.logger.warn('ERROR in watcher "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('watcher "%s" stopped', self.name)
def init_checkers(self, service):
self.service_checks = []
for check in service['checks']:
check['host'] = check.get('host') or service['host']
check['port'] = check.get('port') or service['port']
check['name'] = check.get('name') or "%s|%s|%s" % \
(check['type'], check['host'], check['port'])
check['rise'] = check.get('rise') or self.rise
check['fall'] = check.get('fall') or self.fall
check['type'] = check.get('type') or 'unknown'
service_check_class = CHECKERS_MODULES.get(check['type'])
if not service_check_class:
raise Exception(
'Invalid check type "%s", valid types: %s' %
(check['type'], ', '.join(self.checkers.keys())))
service_check = service_check_class(check, self.logger)
self.service_checks.append(service_check)