async def _monitor_loop(self): """async Monitor loop to monitor registered services""" # check health of all micro-services every N seconds round_cnt = 0 check_count = {} # dict to hold current count of current status. # In case of ok and running status, count will always be 1. # In case of of non running statuses, count shows since when this status is set. while True: round_cnt += 1 self._logger.debug( "Starting next round#{} of service monitoring, sleep/i:{} ping/t:{} max/a:{}" .format(round_cnt, self._sleep_interval, self._ping_timeout, self._max_attempts)) for service_record in ServiceRegistry.all(): if service_record._id not in check_count: check_count.update({service_record._id: 1}) # Try ping if service status is either running or doubtful (i.e. give service a chance to recover) if service_record._status not in [ ServiceRecord.Status.Running, ServiceRecord.Status.Unresponsive, ServiceRecord.Status.Failed ]: continue self._logger.debug("Service: {} Status: {}".format( service_record._name, service_record._status)) if service_record._status == ServiceRecord.Status.Failed: if self._restart_failed == "auto": if service_record._id not in self.restarted_services: self.restarted_services.append(service_record._id) asyncio.ensure_future( self.restart_service(service_record)) continue try: url = "{}://{}:{}/fledge/service/ping".format( service_record._protocol, service_record._address, service_record._management_port) async with aiohttp.ClientSession() as session: async with session.get( url, timeout=self._ping_timeout) as resp: text = await resp.text() res = json.loads(text) if res["uptime"] is None: raise ValueError('res.uptime is None') except (asyncio.TimeoutError, aiohttp.client_exceptions.ServerTimeoutError) as ex: service_record._status = ServiceRecord.Status.Unresponsive check_count[service_record._id] += 1 self._logger.info("ServerTimeoutError: %s, %s", str(ex), service_record.__repr__()) except aiohttp.client_exceptions.ClientConnectorError as ex: service_record._status = ServiceRecord.Status.Unresponsive check_count[service_record._id] += 1 self._logger.info("ClientConnectorError: %s, %s", str(ex), service_record.__repr__()) except ValueError as ex: service_record._status = ServiceRecord.Status.Unresponsive check_count[service_record._id] += 1 self._logger.info("Invalid response: %s, %s", str(ex), service_record.__repr__()) except Exception as ex: service_record._status = ServiceRecord.Status.Unresponsive check_count[service_record._id] += 1 self._logger.info("Exception occurred: %s, %s", str(ex), service_record.__repr__()) else: service_record._status = ServiceRecord.Status.Running check_count[service_record._id] = 1 if check_count[service_record._id] > self._max_attempts: ServiceRegistry.mark_as_failed(service_record._id) check_count[service_record._id] = 0 try: audit = AuditLogger(connect.get_storage_async()) await audit.failure('SRVFL', {'name': service_record._name}) except Exception as ex: self._logger.info("Failed to audit service failure %s", str(ex)) await self._sleep(self._sleep_interval)
async def test_get_health(self, mocker, client): # empty service registry resp = await client.get('/fledge/service') assert 200 == resp.status result = await resp.text() json_response = json.loads(result) assert {'services': []} == json_response mocker.patch.object(InterestRegistry, "__init__", return_value=None) mocker.patch.object(InterestRegistry, "get", return_value=list()) with patch.object(ServiceRegistry._logger, 'info') as log_patch_info: # populated service registry ServiceRegistry.register( 'name1', 'Storage', 'address1', 1, 1, 'protocol1') ServiceRegistry.register( 'name2', 'Southbound', 'address2', 2, 2, 'protocol2') s_id_3 = ServiceRegistry.register( 'name3', 'Southbound', 'address3', 3, 3, 'protocol3') s_id_4 = ServiceRegistry.register( 'name4', 'Southbound', 'address4', 4, 4, 'protocol4') ServiceRegistry.unregister(s_id_3) ServiceRegistry.mark_as_failed(s_id_4) resp = await client.get('/fledge/service') assert 200 == resp.status result = await resp.text() json_response = json.loads(result) assert json_response == { 'services': [ { 'type': 'Storage', 'service_port': 1, 'address': 'address1', 'protocol': 'protocol1', 'status': 'running', 'name': 'name1', 'management_port': 1 }, { 'type': 'Southbound', 'service_port': 2, 'address': 'address2', 'protocol': 'protocol2', 'status': 'running', 'name': 'name2', 'management_port': 2 }, { 'type': 'Southbound', 'service_port': 3, 'address': 'address3', 'protocol': 'protocol3', 'status': 'shutdown', 'name': 'name3', 'management_port': 3 }, { 'type': 'Southbound', 'service_port': 4, 'address': 'address4', 'protocol': 'protocol4', 'status': 'failed', 'name': 'name4', 'management_port': 4 } ] } assert 6 == log_patch_info.call_count