def keep_alive(service):
''' Ping the registry on an interval to show good health. '''
# the default tt is 10 sec
ttl = service.ttl if hasattr(service, 'ttl') else 10
# register the service with consul
register_service(service)
# add a ttl check for the service in case we die
consulSession.agent.check.register(
name=service.consul_name,
check=Check.ttl(str(ttl) + 's'),
)
# the keep alive check
def run_check():
# continuously run
while True:
# sleep every 2 seconds
time.sleep(2)
# tell the agent that we are passing the ttl check
consulSession.agent.check.ttl_pass(service.consul_name,
'Agent alive and reachable.')
# create a thread that will run the consumer
thread = threading.Thread(target=run_check)
# start the thread
thread.start()
def keep_alive(service):
''' Ping the registry on an interval to show good health. '''
# the default tt is 10 sec
ttl = service.ttl if hasattr(service, 'ttl') else 10
# register the service with consul
register_service(service)
# add a ttl check for the service in case we die
consulSession.agent.check.register(
name = service.consul_name,
check = Check.ttl(str(ttl) + 's'),
)
# the keep alive check
def run_check():
# continuously run
while True:
# sleep every 2 seconds
time.sleep(2)
# tell the agent that we are passing the ttl check
consulSession.agent.check.ttl_pass(service.consul_name, 'Agent alive and reachable.')
# create a thread that will run the consumer
thread = threading.Thread(target=run_check)
# start the thread
thread.start()
def register(self):
logger.info(
'Registering %s on Consul as %s with %s:%s',
self._service_name,
self._service_id,
self._advertise_address,
self._advertise_port,
)
try:
ttl_check = Check.ttl(self._ttl_interval)
registered = self._client.agent.service.register(
self._service_name,
service_id=self._service_id,
address=self._advertise_address,
port=self._advertise_port,
check=ttl_check,
tags=self._tags,
)
if not registered:
raise RegistererError(
'{} registration on Consul failed'.format(
self._service_name))
except (ConnectionError, ConsulException) as e:
raise RegistererError(str(e))
def register(self, name, register_type=MicroRegistry.TYPES_KV,
value=None, **options):
registry_name = register_type + ":" + name
try:
if registry_name not in self.registries:
registry = {
'name': name,
'type': register_type,
'value': value,
'options': options,
'registered': False
}
self.registries[registry_name] = registry
if register_type == MicroRegistry.TYPES_KV:
yield from self.kv.put(name, value, **options)
if register_type == MicroRegistry.TYPES_SERVICE:
service_id = options.pop('service_id', name)
check_ttl = options.pop('check_ttl', None)
if check_ttl:
options['check'] = Check.ttl(check_ttl)
ttl = None
if 'ttl_ping' in options:
ttl = options.pop('ttl_ping')
yield from self.agent.service.register(name,
service_id=service_id)
if ttl:
self.health_tasks[service_id] = self._loop.create_task(
self._health_ttl_ping(service_id, ttl))
self.registries[registry_name]['registered'] = True
return True
except Exception:
logger.critical('failed to register with consul')
self.registries[registry_name]['registered'] = False
return False
def test_get_failed_cluster_checks(consul_service, consul1, consul2, consul3,
consul4):
consul_service.register(consul1, "service1")
consul_service.register(consul2, "service1")
consul_service.register(consul3, "service1",
check=Check.ttl("1ms")) # failing
consul_service.register(consul4, "service2")
consul_service.register(consul3, "service2")
consul_service.register(consul3, "service3")
consul_service.register(consul2, "service4",
check=Check.ttl("1ms")) # failing
time.sleep(0.01)
assert set(get_failed_cluster_checks(consul1, ["service1", "service2", "service3", "service4"]).keys()) \
== {"service:service1", "service:service4"}
def register(self):
"""
Register service in consul cluster.
:return: None
"""
logger.debug("registering service")
with ignore_connection_errors():
self.session.agent.service.register(
name=self.service_name,
service_id=self.service_id,
port=self.port,
tags=self.tags,
# format the ttl into XXXs format
check=Check.ttl("%ss" % self.ttl))
def test_reboot_fails_with_failing_checks(run_cli, consul_cluster,
forward_consul_port, default_config,
reboot_task, mock_subprocess_run,
mocker):
consul_cluster[0].agent.service.register("A", tags=["rebootmgr"])
consul_cluster[1].agent.service.register("A",
tags=["rebootmgr"],
check=Check.ttl("1ms")) # Failing
time.sleep(0.01)
mocker.patch("time.sleep")
mock_subprocess_run(["shutdown", "-r", "+1"])
result = run_cli(rebootmgr, ["-v"])
assert result.exit_code == 2
def register(self):
"""
Register service in consul cluster.
:return: None
"""
logger.debug("registering service")
with ignore_connection_errors():
self.session.agent.service.register(
name=self.service_name,
service_id=self.service_id,
address=self.address,
port=self.port,
tags=self.tags,
# format the ttl into XXXs format
check=Check.ttl("%ss" % self.ttl)
)
def register_consul(self, port):
self.consul_service.register(
name=ConsulConfig.service_name,
service_id=ConsulConfig.service_id,
address=ConsulConfig.service_host,
port=port,
token=ConsulConfig.token,
)
self.consul_check.register(
name=f"service '{ConsulConfig.service_name}' check",
check=Check.ttl("10000000s"),
check_id=ConsulConfig.check_id,
service_id=ConsulConfig.service_id,
token=ConsulConfig.token,
)
self.consul_check.ttl_pass(ConsulConfig.check_id)
def ttl(cls, service, params):
"""
Consul TTL Check
You can also register a TTL check with Consul.
Keep in mind, this means Consul will expect a regular heartbeat ping to its API to keep the service
marked healthy.
SERVICE_CHECK_TTL=30s
"""
# https://github.com/cablehead/python-consul/blob/53eb41c4760b983aec878ef73e72c11e0af501bb/consul/base.py#L103
# https://github.com/gliderlabs/registrator/blob/master/docs/user/backends.md#consul-ttl-check
ttl = cls._value(params, 'ttl')
if ttl:
# Set check to be marked as critical after *ttl* (e.g. "10s") unless the
# check
ret = Check.ttl(ttl)
return cls._post_process(ret, params)
return None
def test_post_reboot_consul_checks_failing(run_cli, consul_cluster,
forward_consul_port, default_config,
reboot_in_progress, reboot_task,
mocker):
"""
Test if we fail if consul checks are failing after reboot.
"""
consul_cluster[0].agent.service.register("A", tags=["rebootmgr"])
consul_cluster[1].agent.service.register("A",
tags=["rebootmgr"],
check=Check.ttl("1ms")) # Failing
time.sleep(0.01)
mocker.patch("time.sleep")
mocked_run = mocker.patch("subprocess.run")
result = run_cli(rebootmgr, ["-v"])
mocked_run.assert_not_called()
assert result.exit_code == EXIT_CONSUL_CHECKS_FAILED
def test_post_reboot_wait_until_healthy(run_cli, consul_cluster,
forward_consul_port, default_config,
reboot_in_progress, reboot_task,
mocker):
"""
Test if we wait until consul checks are passing after reboot.
"""
consul_cluster[0].agent.service.register("A", tags=["rebootmgr"])
consul_cluster[1].agent.service.register("A",
tags=["rebootmgr"],
check=Check.ttl("1000s"))
consul_cluster[1].agent.check.ttl_fail("service:A")
sleep_counter = 2
def fake_sleep(seconds):
"""
While we're waiting for consul checks to start passing,
we sleep 120 seconds at a time.
Count how often this happens, and after a few times, we
will set the failing check to passing.
We ignore sleep requests for different amounts of time.
"""
nonlocal sleep_counter
if seconds == WAIT_UNTIL_HEALTHY_SLEEP_TIME:
sleep_counter -= 1
if sleep_counter <= 0:
consul_cluster[1].agent.check.ttl_pass("service:A")
mocker.patch("time.sleep", new=fake_sleep)
mocked_run = mocker.patch("subprocess.run")
result = run_cli(rebootmgr, ["-v", "--post-reboot-wait-until-healthy"])
mocked_run.assert_not_called()
assert sleep_counter == 0
assert result.exit_code == 0
def register(self):
logger.info('Registering service {} with id {}'.format(self.name, self.id))
self.client.agent.service.register(name=self.name, service_id=self.id, check=Check.ttl('10s'))
self.client.agent.check.ttl_pass(check_id=self.check_id)
self.session = self.client.session.create(behavior='delete', checks=[self.check_id])