async def test_unregister(self, mocker):
# register a service
with patch.object(Service._logger, 'info') as log_info:
idx = Service.register("StorageService2", "Storage", "127.0.0.1", 8888, 1888)
assert str(uuid.UUID(idx, version=4)) == idx
assert 1 == log_info.call_count
arg, kwarg = log_info.call_args
assert arg[0].startswith('Registered service instance id=')
assert arg[0].endswith(': <StorageService2, type=Storage, protocol=http, address=127.0.0.1, service port=8888,'
' management port=1888, status=1>')
mocker.patch.object(InterestRegistry, '__init__', return_value=None)
mocker.patch.object(InterestRegistry, 'get', return_value=list())
# deregister the same
with patch.object(Service._logger, 'info') as log_info2:
t = Service.unregister(idx)
assert idx == t
assert 1 == log_info2.call_count
args, kwargs = log_info2.call_args
assert args[0].startswith('Stopped service instance id=')
assert args[0].endswith(': <StorageService2, type=Storage, protocol=http, address=127.0.0.1, '
'service port=8888, management port=1888, status=2>')
s = Service.get(idx)
assert s[0]._status == 2 # Unregistered
async def test_get(self):
with patch.object(Service._logger, 'info') as log_info:
s = Service.register("StorageService", "Storage", "localhost", 8881, 1888)
c = Service.register("CoreService", "Core", "localhost", 7771, 1777)
d = Service.register("SouthService", "Southbound", "127.0.0.1", 9991, 1999, "https")
assert 3 == log_info.call_count
_service = Service.get()
assert 3 == len(_service)
assert s == _service[0]._id
assert "StorageService" == _service[0]._name
assert "Storage" == _service[0]._type
assert "localhost" == _service[0]._address
assert 8881 == int(_service[0]._port)
assert 1888 == int(_service[0]._management_port)
# validates default set to HTTP
assert "http" == _service[0]._protocol
assert c == _service[1]._id
assert "CoreService" == _service[1]._name
assert "Core" == _service[1]._type
assert "localhost" == _service[1]._address
assert 7771 == int(_service[1]._port)
assert 1777 == int(_service[1]._management_port)
# validates default set to HTTP
assert "http" == _service[1]._protocol
assert d == _service[2]._id
assert "SouthService" == _service[2]._name
assert "Southbound" == _service[2]._type
assert "127.0.0.1" == _service[2]._address
assert 9991 == int(_service[2]._port)
assert 1999 == int(_service[2]._management_port)
assert "https" == _service[2]._protocol
def test_register_with_bad_management_port(self):
"""raise NonNumericPortError"""
with pytest.raises(Exception) as excinfo:
with patch.object(ServiceRegistry._logger, 'info') as log_info:
ServiceRegistry.register("B name", "Storage", "127.0.0.1", 1234, "m01", 'http')
assert 0 == len(ServiceRegistry._registry)
assert 0 == log_info.call_count
assert excinfo.type is NonNumericPortError
def test_unregister_non_existing_service_record(self):
"""raise DoesNotExist"""
with pytest.raises(Exception) as excinfo:
with patch.object(ServiceRegistry._logger, 'info') as log_info:
ServiceRegistry.unregister("blah")
assert 0 == len(ServiceRegistry._registry)
assert 0 == log_info.call_count
assert excinfo.type is DoesNotExist
async def delete_service(request):
""" Delete an existing service
:Example:
curl -X DELETE http://localhost:8081/fledge/service/<svc name>
"""
try:
svc = request.match_info.get('service_name', None)
storage = connect.get_storage_async()
result = await get_schedule(storage, svc)
if result['count'] == 0:
return web.HTTPNotFound(
reason='{} service does not exist.'.format(svc))
config_mgr = ConfigurationManager(storage)
# In case of notification service, if notifications exists, then deletion is not allowed
if 'notification' in result['rows'][0]['process_name']:
notf_children = await config_mgr.get_category_child(
category_name="Notifications")
children = [x['key'] for x in notf_children]
if len(notf_children) > 0:
return web.HTTPBadRequest(
reason=
'Notification service `{}` can not be deleted, as {} notification instances exist.'
.format(svc, children))
# First disable the schedule
svc_schedule = result['rows'][0]
sch_id = uuid.UUID(svc_schedule['id'])
if svc_schedule['enabled'].lower() == 't':
await server.Server.scheduler.disable_schedule(sch_id)
# return control to event loop
await asyncio.sleep(1)
# Delete all configuration for the service name
await config_mgr.delete_category_and_children_recursively(svc)
# Remove from registry as it has been already shutdown via disable_schedule() and since
# we intend to delete the schedule also, there is no use of its Service registry entry
try:
services = ServiceRegistry.get(name=svc)
ServiceRegistry.remove_from_registry(services[0]._id)
except service_registry_exceptions.DoesNotExist:
pass
# Delete schedule
await server.Server.scheduler.delete_schedule(sch_id)
except Exception as ex:
raise web.HTTPInternalServerError(reason=str(ex))
else:
return web.json_response(
{'result': 'Service {} deleted successfully.'.format(svc)})
def test_get_storage(self):
with patch.object(ServiceRegistry._logger, 'info') as log_info:
ServiceRegistry.register("Fledge Storage", "Storage", "127.0.0.1",
37449, 37843)
storage_client = connect.get_storage_async()
assert isinstance(storage_client, StorageClientAsync)
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(
': <Fledge Storage, type=Storage, protocol=http, address=127.0.0.1, service port=37449,'
' management port=37843, status=1>')
async def test_get_fail(self):
with pytest.raises(DoesNotExist) as excinfo:
with patch.object(Service._logger, 'info') as log_info:
Service.register("StorageService", "Storage", "127.0.0.1", 8888, 9999)
Service.get('incorrect_id')
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(
': <StorageService1, type=Storage, protocol=http, address=127.0.0.1, service port=8888,'
' management port=9999, status=1>')
assert str(excinfo).endswith('DoesNotExist')
async def test_duplicate_address_and_mgt_port_registration(self):
with patch.object(Service._logger, 'info') as log_info:
idx1 = Service.register("StorageService1", "Storage", "127.0.0.1", 9999, 1999)
assert str(uuid.UUID(idx1, version=4)) == idx1
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(': <StorageService1, type=Storage, protocol=http, address=127.0.0.1, service port=9999,'
' management port=1999, status=1>')
with pytest.raises(AlreadyExistsWithTheSameAddressAndManagementPort) as excinfo:
Service.register("StorageService2", "Storage", "127.0.0.1", 9998, 1999)
assert str(excinfo).endswith('AlreadyExistsWithTheSameAddressAndManagementPort')
def test_exception_when_non_fledge_storage(self, mock_logger):
with patch.object(ServiceRegistry._logger, 'info') as log_info:
ServiceRegistry.register("foo", "Storage", "127.0.0.1", 1, 2)
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(
': <foo, type=Storage, protocol=http, address=127.0.0.1, service port=1, '
'management port=2, status=1>')
with pytest.raises(DoesNotExist) as excinfo:
connect.get_storage_async()
assert str(excinfo).endswith('DoesNotExist')
mock_logger.exception.assert_called_once_with('')
async def get_plugin(request):
""" GET lists of rule plugins and delivery plugins
:Example:
curl -X GET http://localhost:8081/fledge/notification/plugin
"""
try:
notification_service = ServiceRegistry.get(
s_type=ServiceRecord.Type.Notification.name)
_address, _port = notification_service[
0]._address, notification_service[0]._port
except service_registry_exceptions.DoesNotExist:
raise web.HTTPNotFound(reason="No Notification service available.")
try:
url = 'http://{}:{}/notification/rules'.format(_address, _port)
rule_plugins = json.loads(await _hit_get_url(url))
url = 'http://{}:{}/notification/delivery'.format(_address, _port)
delivery_plugins = json.loads(await _hit_get_url(url))
except Exception as ex:
raise web.HTTPInternalServerError(reason=ex)
else:
return web.json_response({
'rules': rule_plugins,
'delivery': delivery_plugins
})
def test_register_with_same_address_and_mgt_port(self):
"""raise AlreadyExistsWithTheSameAddressAndManagementPort"""
with patch.object(ServiceRegistry._logger, 'info') as log_info1:
ServiceRegistry.register("A name", "Storage", "127.0.0.1", 1, 1234, 'http')
assert 1 == len(ServiceRegistry._registry)
assert 1 == log_info1.call_count
args, kwargs = log_info1.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(': <A name, type=Storage, protocol=http, address=127.0.0.1, service port=1,'
' management port=1234, status=1>')
with pytest.raises(Exception) as excinfo:
with patch.object(ServiceRegistry._logger, 'info') as log_info2:
ServiceRegistry.register("B name", "Storage", "127.0.0.1", 2, 1234, 'http')
assert 1 == len(ServiceRegistry._registry)
assert 0 == log_info2.call_count
assert excinfo.type is AlreadyExistsWithTheSameAddressAndManagementPort
async def test_register(self):
with patch.object(Service._logger, 'info') as log_info:
idx = Service.register("StorageService1", "Storage", "127.0.0.1", 9999, 1999)
assert str(uuid.UUID(idx, version=4)) == idx
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(': <StorageService1, type=Storage, protocol=http, address=127.0.0.1, service port=9999,'
' management port=1999, status=1>')
async def test__monitor_good_uptime(self):
async def async_mock(return_value):
return return_value
# used to mock client session context manager
class AsyncSessionContextManagerMock(MagicMock):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def __aenter__(self):
client_response_mock = MagicMock(spec=aiohttp.ClientResponse)
# mock response (good)
client_response_mock.text.side_effect = [
async_mock('{"uptime": "bla"}')]
return client_response_mock
async def __aexit__(self, *args):
return None
# as monitor loop is as infinite loop, this exception is thrown when we need to exit the loop
class TestMonitorException(Exception):
pass
# register a service
with patch.object(ServiceRegistry._logger, 'info') as log_info:
s_id_1 = ServiceRegistry.register(
'sname1', 'Storage', 'saddress1', 1, 1, 'protocol1')
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(': <sname1, type=Storage, protocol=protocol1, address=saddress1, service port=1, '
'management port=1, status=1>')
monitor = Monitor()
monitor._sleep_interval = Monitor._DEFAULT_SLEEP_INTERVAL
monitor._max_attempts = Monitor._DEFAULT_MAX_ATTEMPTS
# throw the TestMonitorException when sleep is called (end of infinite loop)
with patch.object(Monitor, '_sleep', side_effect=TestMonitorException()):
with patch.object(aiohttp.ClientSession, 'get', return_value=AsyncSessionContextManagerMock()):
with pytest.raises(Exception) as excinfo:
await monitor._monitor_loop()
assert excinfo.type is TestMonitorException
# service is good, so it should remain in the service registry
assert len(ServiceRegistry.get(idx=s_id_1)) is 1
assert ServiceRegistry.get(idx=s_id_1)[0]._status is ServiceRecord.Status.Running
def services_health_litmus_test():
all_svc_status = [
ServiceRecord.Status(int(service_record._status)).name.upper()
for service_record in ServiceRegistry.all()
]
if 'FAILED' in all_svc_status:
return 'red'
elif 'UNRESPONSIVE' in all_svc_status:
return 'amber'
return 'green'
def test_register_with_service_port_none(self):
with patch.object(ServiceRegistry._logger, 'info') as log_info:
s_id = ServiceRegistry.register("A name", "Southbound", "127.0.0.1", None, 4321, 'http')
assert 36 == len(s_id) # uuid version 4 len
assert 1 == len(ServiceRegistry._registry)
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(': <A name, type=Southbound, protocol=http, address=127.0.0.1, service port=None,'
' management port=4321, status=1>')
async def test_run_no_interests_in_cat(self):
storage_client_mock = MagicMock(spec=StorageClientAsync)
cfg_mgr = ConfigurationManager(storage_client_mock)
with patch.object(ServiceRegistry._logger, 'info') as log_info:
s_id_1 = ServiceRegistry.register('sname1', 'Storage', 'saddress1',
1, 1, 'http')
s_id_2 = ServiceRegistry.register('sname2', 'Southbound',
'saddress2', 2, 2, 'http')
s_id_3 = ServiceRegistry.register('sname3', 'Southbound',
'saddress3', 3, 3, 'http')
assert 3 == log_info.call_count
i_reg = InterestRegistry(cfg_mgr)
i_reg.register(s_id_1, 'catname2')
i_reg.register(s_id_2, 'catname2')
i_reg.register(s_id_3, 'catname3')
# used to mock client session context manager
async def async_mock(return_value):
return return_value
class AsyncSessionContextManagerMock(MagicMock):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def __aenter__(self):
client_response_mock = MagicMock(spec=aiohttp.ClientResponse)
client_response_mock.text.side_effect = [async_mock(None)]
status_mock = Mock()
status_mock.side_effect = [200]
client_response_mock.status = status_mock()
return client_response_mock
async def __aexit__(self, *args):
return None
with patch.object(ConfigurationManager,
'get_category_all_items') as cm_get_patch:
with patch.object(aiohttp.ClientSession, 'post') as post_patch:
await cb.run('catname1')
post_patch.assert_not_called()
cm_get_patch.assert_not_called()
def get_readings_async():
""" Storage Object """
try:
services = ServiceRegistry.get(name="Fledge Storage")
storage_svc = services[0]
_readings = ReadingsStorageClientAsync(core_mgt_host=None, core_mgt_port=None,
svc=storage_svc)
# _logger.info(type(_storage))
except Exception as ex:
_logger.exception(str(ex))
raise
return _readings
async def test_run_general_exception(self):
storage_client_mock = MagicMock(spec=StorageClientAsync)
cfg_mgr = ConfigurationManager(storage_client_mock)
with patch.object(ServiceRegistry._logger, 'info') as log_info:
s_id_1 = ServiceRegistry.register('sname1', 'Storage', 'saddress1',
1, 1, 'http')
assert 1 == log_info.call_count
args, kwargs = log_info.call_args
assert args[0].startswith('Registered service instance id=')
assert args[0].endswith(
': <sname1, type=Storage, protocol=http, address=saddress1, service port=1, management port=1, status=1>'
)
i_reg = InterestRegistry(cfg_mgr)
i_reg.register(s_id_1, 'catname1')
# used to mock client session context manager
async def async_mock(return_value):
return return_value
class AsyncSessionContextManagerMock(MagicMock):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def __aenter__(self):
raise Exception
async def __aexit__(self, *args):
return None
with patch.object(ConfigurationManager,
'get_category_all_items',
return_value=async_mock(None)) as cm_get_patch:
with patch.object(aiohttp.ClientSession,
'post',
return_value=AsyncSessionContextManagerMock()
) as post_patch:
with patch.object(cb._LOGGER, 'exception') as exception_patch:
await cb.run('catname1')
exception_patch.assert_called_once_with(
'Unable to notify microservice with uuid %s due to exception: %s',
s_id_1, '')
post_patch.assert_has_calls([
call('http://saddress1:1/fledge/change',
data='{"category": "catname1", "items": null}',
headers={'content-type': 'application/json'})
])
cm_get_patch.assert_called_once_with('catname1')
def get_service_records():
sr_list = list()
for service_record in ServiceRegistry.all():
sr_list.append(
{
'name': service_record._name,
'type': service_record._type,
'address': service_record._address,
'management_port': service_record._management_port,
'service_port': service_record._port,
'protocol': service_record._protocol,
'status': ServiceRecord.Status(int(service_record._status)).name.lower()
})
recs = {'services': sr_list}
return recs
def test_unregister(self, mocker):
mocker.patch.object(InterestRegistry, '__init__', return_value=None)
mocker.patch.object(InterestRegistry, 'get', return_value=list())
with patch.object(ServiceRegistry._logger, 'info') as log_info1:
reg_id = ServiceRegistry.register("A name", "Storage", "127.0.0.1", 1234, 4321, 'http')
assert 1 == len(ServiceRegistry._registry)
assert 1 == log_info1.call_count
arg, kwarg = log_info1.call_args
assert arg[0].startswith('Registered service instance id=')
assert arg[0].endswith(': <A name, type=Storage, protocol=http, address=127.0.0.1, service port=1234,'
' management port=4321, status=1>')
with patch.object(ServiceRegistry._logger, 'info') as log_info2:
s_id = ServiceRegistry.unregister(reg_id)
assert 36 == len(s_id) # uuid version 4 len
assert 1 == len(ServiceRegistry._registry)
s = ServiceRegistry.get(idx=s_id)
assert s[0]._status == 2
assert 1 == log_info2.call_count
args, kwargs = log_info2.call_args
assert args[0].startswith('Stopped service instance id=')
assert args[0].endswith(': <A name, type=Storage, protocol=http, address=127.0.0.1, service port=1234,'
' management port=4321, status=2>')
async def test__monitor_exceed_attempts(self, mocker):
class AsyncSessionContextManagerMock(MagicMock):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def __aenter__(self):
# mock response (error- exception)
raise Exception("test")
async def __aexit__(self, *args):
return None
# as monitor loop is as infinite loop, this exception is thrown when we need to exit the loop
class TestMonitorException(Exception):
pass
# register a service
s_id_1 = ServiceRegistry.register(
'sname1', 'Storage', 'saddress1', 1, 1, 'protocol1')
monitor = Monitor()
monitor._sleep_interval = Monitor._DEFAULT_SLEEP_INTERVAL
monitor._max_attempts = Monitor._DEFAULT_MAX_ATTEMPTS
sleep_side_effect_list = list()
# _MAX_ATTEMPTS is 15
# throw exception on the 16th time sleep is called - the first 15 sleeps are used during retries
for i in range(0, 15):
sleep_side_effect_list.append(asyncio.sleep(.01))
sleep_side_effect_list.append(TestMonitorException())
with patch.object(Monitor, '_sleep', side_effect=sleep_side_effect_list):
with patch.object(aiohttp.ClientSession, 'get', return_value=AsyncSessionContextManagerMock()):
with pytest.raises(Exception) as excinfo:
await monitor._monitor_loop()
assert excinfo.type in [TestMonitorException, TypeError]
assert ServiceRegistry.get(idx=s_id_1)[0]._status is ServiceRecord.Status.Failed
async def run(category_name):
""" Callback run by configuration category to notify changes to interested microservices
Note: this method is async as needed
Args:
configuration_name (str): name of category that was changed
"""
# get all interest records regarding category_name
cfg_mgr = ConfigurationManager()
interest_registry = InterestRegistry(cfg_mgr)
try:
interest_records = interest_registry.get(category_name=category_name)
except interest_registry_exceptions.DoesNotExist:
return
category_value = await cfg_mgr.get_category_all_items(category_name)
payload = {"category" : category_name, "items" : category_value}
headers = {'content-type': 'application/json'}
# for each microservice interested in category_name, notify change
for i in interest_records:
# get microservice management server info of microservice through service registry
try:
service_record = ServiceRegistry.get(idx=i._microservice_uuid)[0]
except service_registry_exceptions.DoesNotExist:
_LOGGER.exception("Unable to notify microservice with uuid %s as it is not found in the service registry", i._microservice_uuid)
continue
url = "{}://{}:{}/fledge/change".format(service_record._protocol, service_record._address, service_record._management_port)
async with aiohttp.ClientSession() as session:
try:
async with session.post(url, data=json.dumps(payload, sort_keys=True), headers=headers) as resp:
result = await resp.text()
status_code = resp.status
if status_code in range(400, 500):
_LOGGER.error("Bad request error code: %d, reason: %s", status_code, resp.reason)
if status_code in range(500, 600):
_LOGGER.error("Server error code: %d, reason: %s", status_code, resp.reason)
except Exception as ex:
_LOGGER.exception("Unable to notify microservice with uuid %s due to exception: %s", i._microservice_uuid, str(ex))
continue
async def delete_notification(request):
""" Delete an existing notification
:Example:
curl -X DELETE http://localhost:8081/fledge/notification/<notification_name>
"""
try:
notification_service = ServiceRegistry.get(
s_type=ServiceRecord.Type.Notification.name)
_address, _port = notification_service[
0]._address, notification_service[0]._port
except service_registry_exceptions.DoesNotExist:
raise web.HTTPNotFound(reason="No Notification service available.")
try:
notif = request.match_info.get('notification_name', None)
if notif is None:
raise ValueError("Notification name is required for deletion.")
# Stop & remove notification
url = 'http://{}:{}/notification/{}'.format(_address, _port,
urllib.parse.quote(notif))
notification = json.loads(await _hit_delete_url(url))
# Removes the child categories for the rule and delivery plugins, Removes the category for the notification itself
storage = connect.get_storage_async()
config_mgr = ConfigurationManager(storage)
await config_mgr.delete_category_and_children_recursively(notif)
audit = AuditLogger(storage)
await audit.information('NTFDL', {"name": notif})
except ValueError as ex:
raise web.HTTPBadRequest(reason=str(ex))
except Exception as ex:
raise web.HTTPInternalServerError(reason=str(ex))
else:
return web.json_response(
{'result': 'Notification {} deleted successfully.'.format(notif)})
async def _services_with_assets(storage_client, south_services):
sr_list = list()
try:
try:
services_from_registry = ServiceRegistry.get(s_type="Southbound")
except DoesNotExist:
services_from_registry = []
def is_svc_in_service_registry(name):
return next(
(svc for svc in services_from_registry if svc._name == name),
None)
installed_plugins = _get_installed_plugins()
for s_record in services_from_registry:
plugin, assets = await _get_tracked_plugin_assets_and_readings(
storage_client, s_record._name)
plugin_version = ''
for p in installed_plugins:
if p["name"] == plugin:
plugin_version = p["version"]
break
sr_list.append({
'name':
s_record._name,
'address':
s_record._address,
'management_port':
s_record._management_port,
'service_port':
s_record._port,
'protocol':
s_record._protocol,
'status':
ServiceRecord.Status(int(s_record._status)).name.lower(),
'assets':
assets,
'plugin': {
'name': plugin,
'version': plugin_version
},
'schedule_enabled':
await _get_schedule_status(storage_client, s_record._name)
})
for s_name in south_services:
south_svc = is_svc_in_service_registry(s_name)
if not south_svc:
plugin, assets = await _get_tracked_plugin_assets_and_readings(
storage_client, s_name)
plugin_version = ''
for p in installed_plugins:
if p["name"] == plugin:
plugin_version = p["version"]
break
sr_list.append({
'name':
s_name,
'address':
'',
'management_port':
'',
'service_port':
'',
'protocol':
'',
'status':
'',
'assets':
assets,
'plugin': {
'name': plugin,
'version': plugin_version
},
'schedule_enabled':
await _get_schedule_status(storage_client, s_name)
})
except:
raise
else:
return sr_list
async def test_get_health(self, mocker, client):
# empty service registry
resp = await client.get('/fledge/service')
assert 200 == resp.status
result = await resp.text()
json_response = json.loads(result)
assert {'services': []} == json_response
mocker.patch.object(InterestRegistry, "__init__", return_value=None)
mocker.patch.object(InterestRegistry, "get", return_value=list())
with patch.object(ServiceRegistry._logger, 'info') as log_patch_info:
# populated service registry
ServiceRegistry.register(
'name1', 'Storage', 'address1', 1, 1, 'protocol1')
ServiceRegistry.register(
'name2', 'Southbound', 'address2', 2, 2, 'protocol2')
s_id_3 = ServiceRegistry.register(
'name3', 'Southbound', 'address3', 3, 3, 'protocol3')
s_id_4 = ServiceRegistry.register(
'name4', 'Southbound', 'address4', 4, 4, 'protocol4')
ServiceRegistry.unregister(s_id_3)
ServiceRegistry.mark_as_failed(s_id_4)
resp = await client.get('/fledge/service')
assert 200 == resp.status
result = await resp.text()
json_response = json.loads(result)
assert json_response == {
'services': [
{
'type': 'Storage',
'service_port': 1,
'address': 'address1',
'protocol': 'protocol1',
'status': 'running',
'name': 'name1',
'management_port': 1
},
{
'type': 'Southbound',
'service_port': 2,
'address': 'address2',
'protocol': 'protocol2',
'status': 'running',
'name': 'name2',
'management_port': 2
},
{
'type': 'Southbound',
'service_port': 3,
'address': 'address3',
'protocol': 'protocol3',
'status': 'shutdown',
'name': 'name3',
'management_port': 3
},
{
'type': 'Southbound',
'service_port': 4,
'address': 'address4',
'protocol': 'protocol4',
'status': 'failed',
'name': 'name4',
'management_port': 4
}
]
}
assert 6 == log_patch_info.call_count
async def test_register_invalid_mgt_port(self):
with pytest.raises(NonNumericPortError) as excinfo:
Service.register("StorageService2", "Core", "127.0.0.1", 8888, "199a")
assert str(excinfo).endswith('NonNumericPortError')
async def test_register_wrong_type(self):
with pytest.raises(ServiceRecord.InvalidServiceType) as excinfo:
Service.register("StorageService1", "WrongType", "127.0.0.1", 9999, 1999)
assert str(excinfo).endswith('InvalidServiceType')
async def _monitor_loop(self):
"""async Monitor loop to monitor registered services"""
# check health of all micro-services every N seconds
round_cnt = 0
check_count = {} # dict to hold current count of current status.
# In case of ok and running status, count will always be 1.
# In case of of non running statuses, count shows since when this status is set.
while True:
round_cnt += 1
self._logger.debug(
"Starting next round#{} of service monitoring, sleep/i:{} ping/t:{} max/a:{}"
.format(round_cnt, self._sleep_interval, self._ping_timeout,
self._max_attempts))
for service_record in ServiceRegistry.all():
if service_record._id not in check_count:
check_count.update({service_record._id: 1})
# Try ping if service status is either running or doubtful (i.e. give service a chance to recover)
if service_record._status not in [
ServiceRecord.Status.Running,
ServiceRecord.Status.Unresponsive,
ServiceRecord.Status.Failed
]:
continue
self._logger.debug("Service: {} Status: {}".format(
service_record._name, service_record._status))
if service_record._status == ServiceRecord.Status.Failed:
if self._restart_failed == "auto":
if service_record._id not in self.restarted_services:
self.restarted_services.append(service_record._id)
asyncio.ensure_future(
self.restart_service(service_record))
continue
try:
url = "{}://{}:{}/fledge/service/ping".format(
service_record._protocol, service_record._address,
service_record._management_port)
async with aiohttp.ClientSession() as session:
async with session.get(
url, timeout=self._ping_timeout) as resp:
text = await resp.text()
res = json.loads(text)
if res["uptime"] is None:
raise ValueError('res.uptime is None')
except (asyncio.TimeoutError,
aiohttp.client_exceptions.ServerTimeoutError) as ex:
service_record._status = ServiceRecord.Status.Unresponsive
check_count[service_record._id] += 1
self._logger.info("ServerTimeoutError: %s, %s", str(ex),
service_record.__repr__())
except aiohttp.client_exceptions.ClientConnectorError as ex:
service_record._status = ServiceRecord.Status.Unresponsive
check_count[service_record._id] += 1
self._logger.info("ClientConnectorError: %s, %s", str(ex),
service_record.__repr__())
except ValueError as ex:
service_record._status = ServiceRecord.Status.Unresponsive
check_count[service_record._id] += 1
self._logger.info("Invalid response: %s, %s", str(ex),
service_record.__repr__())
except Exception as ex:
service_record._status = ServiceRecord.Status.Unresponsive
check_count[service_record._id] += 1
self._logger.info("Exception occurred: %s, %s", str(ex),
service_record.__repr__())
else:
service_record._status = ServiceRecord.Status.Running
check_count[service_record._id] = 1
if check_count[service_record._id] > self._max_attempts:
ServiceRegistry.mark_as_failed(service_record._id)
check_count[service_record._id] = 0
try:
audit = AuditLogger(connect.get_storage_async())
await audit.failure('SRVFL',
{'name': service_record._name})
except Exception as ex:
self._logger.info("Failed to audit service failure %s",
str(ex))
await self._sleep(self._sleep_interval)
async def test_run_good(self):
storage_client_mock = MagicMock(spec=StorageClientAsync)
cfg_mgr = ConfigurationManager(storage_client_mock)
with patch.object(ServiceRegistry._logger, 'info') as log_info:
s_id_1 = ServiceRegistry.register('sname1', 'Storage', 'saddress1',
1, 1, 'http')
s_id_2 = ServiceRegistry.register('sname2', 'Southbound',
'saddress2', 2, 2, 'http')
s_id_3 = ServiceRegistry.register('sname3', 'Southbound',
'saddress3', 3, 3, 'http')
assert 3 == log_info.call_count
i_reg = InterestRegistry(cfg_mgr)
i_reg.register(s_id_1, 'catname1')
i_reg.register(s_id_1, 'catname2')
i_reg.register(s_id_2, 'catname1')
i_reg.register(s_id_2, 'catname2')
i_reg.register(s_id_3, 'catname3')
# used to mock client session context manager
async def async_mock(return_value):
return return_value
class AsyncSessionContextManagerMock(MagicMock):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def __aenter__(self):
client_response_mock = MagicMock(spec=aiohttp.ClientResponse)
client_response_mock.text.side_effect = [async_mock(None)]
status_mock = Mock()
status_mock.side_effect = [200]
client_response_mock.status = status_mock()
return client_response_mock
async def __aexit__(self, *args):
return None
with patch.object(ConfigurationManager,
'get_category_all_items',
return_value=async_mock(None)) as cm_get_patch:
with patch.object(aiohttp.ClientSession,
'post',
return_value=AsyncSessionContextManagerMock()
) as post_patch:
await cb.run('catname1')
post_patch.assert_has_calls([
call('http://saddress1:1/fledge/change',
data='{"category": "catname1", "items": null}',
headers={'content-type': 'application/json'}),
call('http://saddress2:2/fledge/change',
data='{"category": "catname1", "items": null}',
headers={'content-type': 'application/json'})
])
cm_get_patch.assert_called_once_with('catname1')
with patch.object(ConfigurationManager,
'get_category_all_items',
return_value=async_mock(None)) as cm_get_patch:
with patch.object(aiohttp.ClientSession,
'post',
return_value=AsyncSessionContextManagerMock()
) as post_patch:
await cb.run('catname2')
post_patch.assert_has_calls([
call('http://saddress1:1/fledge/change',
data='{"category": "catname2", "items": null}',
headers={'content-type': 'application/json'}),
call('http://saddress2:2/fledge/change',
data='{"category": "catname2", "items": null}',
headers={'content-type': 'application/json'})
])
cm_get_patch.assert_called_once_with('catname2')
with patch.object(ConfigurationManager,
'get_category_all_items',
return_value=async_mock(None)) as cm_get_patch:
with patch.object(aiohttp.ClientSession,
'post',
return_value=AsyncSessionContextManagerMock()
) as post_patch:
await cb.run('catname3')
post_patch.assert_called_once_with(
'http://saddress3:3/fledge/change',
data='{"category": "catname3", "items": null}',
headers={'content-type': 'application/json'})
cm_get_patch.assert_called_once_with('catname3')
async def put_notification(request):
"""
Update an existing notification
:Example:
curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"description":"Test Notification modified"}'
curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"rule": "threshold", "channel": "email"}'
curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"notification_type": "one shot", "enabled": false}'
curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"enabled": false}'
curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"description":"Test Notification", "rule": "threshold", "channel": "email", "notification_type": "one shot", "enabled": false, "rule_config": {}, "delivery_config": {}}'
"""
try:
notification_service = ServiceRegistry.get(
s_type=ServiceRecord.Type.Notification.name)
_address, _port = notification_service[
0]._address, notification_service[0]._port
except service_registry_exceptions.DoesNotExist:
raise web.HTTPNotFound(reason="No Notification service available.")
try:
notif = request.match_info.get('notification_name', None)
if notif is None:
raise ValueError("Notification name is required for updation.")
# TODO: Stop notification before update
data = await request.json()
if not isinstance(data, dict):
raise ValueError('Data payload must be a valid JSON')
description = data.get('description', None)
rule = data.get('rule', None)
channel = data.get('channel', None)
notification_type = data.get('notification_type', None)
enabled = data.get('enabled', None)
rule_config = data.get('rule_config', {})
delivery_config = data.get('delivery_config', {})
if utils.check_reserved(notif) is False:
raise ValueError('Invalid notification instance name.')
if rule is not None and utils.check_reserved(rule) is False:
raise ValueError('Invalid rule property in payload.')
if channel is not None and utils.check_reserved(channel) is False:
raise ValueError('Invalid channel property in payload.')
if notification_type is not None and notification_type not in NOTIFICATION_TYPE:
raise ValueError('Invalid notification_type property in payload.')
if enabled is not None:
if enabled not in ['true', 'false', True, False]:
raise ValueError(
'Only "true", "false", true, false are allowed for value of enabled.'
)
is_enabled = "true" if (
(type(enabled) is str and enabled.lower() in ['true']) or
((type(enabled) is bool and enabled is True))) else "false"
storage = connect.get_storage_async()
config_mgr = ConfigurationManager(storage)
current_config = await config_mgr._read_category_val(notif)
if current_config is None:
raise NotFoundError(
'No {} notification instance found'.format(notif))
rule_changed = True if rule is not None and rule != current_config[
'rule']['value'] else False
channel_changed = True if channel is not None and channel != current_config[
'channel']['value'] else False
try:
# Get default config for rule and channel plugins
url = str(request.url)
url_parts = url.split("/fledge/notification")
url = '{}/fledge/notification/plugin'.format(url_parts[0])
try:
# When authentication is mandatory we need to pass token in request header
auth_token = request.token
except AttributeError:
auth_token = None
list_plugins = json.loads(await _hit_get_url(url, auth_token))
search_rule = rule if rule_changed else current_config['rule'][
'value']
r = list(
filter(lambda rules: rules['name'] == search_rule,
list_plugins['rules']))
if len(r) == 0:
raise KeyError
rule_plugin_config = r[0]['config']
search_channel = channel if channel_changed else current_config[
'channel']['value']
c = list(
filter(lambda channels: channels['name'] == search_channel,
list_plugins['delivery']))
if len(c) == 0:
raise KeyError
delivery_plugin_config = c[0]['config']
except KeyError:
raise ValueError(
"Invalid rule plugin:{} and/or delivery plugin:{} supplied.".
format(rule, channel))
# Verify if rule_config contains valid keys
if rule_config != {}:
for k, v in rule_config.items():
if k not in rule_plugin_config:
raise ValueError("Invalid key:{} in rule plugin:{}".format(
k, rule_plugin_config))
# Verify if delivery_config contains valid keys
if delivery_config != {}:
for k, v in delivery_config.items():
if k not in delivery_plugin_config:
raise ValueError(
"Invalid key:{} in delivery plugin:{}".format(
k, delivery_plugin_config))
if rule_changed: # A new rule has been supplied
category_desc = rule_plugin_config['plugin']['description']
category_name = "rule{}".format(notif)
await config_mgr.create_category(
category_name=category_name,
category_description=category_desc,
category_value=rule_plugin_config,
keep_original_items=False)
if channel_changed: # A new delivery has been supplied
category_desc = delivery_plugin_config['plugin']['description']
category_name = "delivery{}".format(notif)
await config_mgr.create_category(
category_name=category_name,
category_description=category_desc,
category_value=delivery_plugin_config,
keep_original_items=False)
notification_config = {}
if description is not None:
notification_config.update({"description": description})
if rule is not None:
notification_config.update({"rule": rule})
if channel is not None:
notification_config.update({"channel": channel})
if notification_type is not None:
notification_config.update(
{"notification_type": notification_type})
if enabled is not None:
notification_config.update({"enable": is_enabled})
await _update_configurations(config_mgr, notif, notification_config,
rule_config, delivery_config)
except ValueError as e:
raise web.HTTPBadRequest(reason=str(e))
except NotFoundError as e:
raise web.HTTPNotFound(reason=str(e))
except Exception as ex:
raise web.HTTPInternalServerError(reason=str(ex))
else:
# TODO: Start notification after update
return web.json_response(
{'result': "Notification {} updated successfully".format(notif)})
