async def test_get_fail(self): with pytest.raises(DoesNotExist) as excinfo: with patch.object(Service._logger, 'info') as log_info: Service.register("StorageService", "Storage", "127.0.0.1", 8888, 9999) Service.get('incorrect_id') assert 1 == log_info.call_count args, kwargs = log_info.call_args assert args[0].startswith('Registered service instance id=') assert args[0].endswith( ': <StorageService1, type=Storage, protocol=http, address=127.0.0.1, service port=8888,' ' management port=9999, status=1>') assert str(excinfo).endswith('DoesNotExist')
async def test_unregister(self, mocker): # register a service with patch.object(Service._logger, 'info') as log_info: idx = Service.register("StorageService2", "Storage", "127.0.0.1", 8888, 1888) assert str(uuid.UUID(idx, version=4)) == idx assert 1 == log_info.call_count arg, kwarg = log_info.call_args assert arg[0].startswith('Registered service instance id=') assert arg[0].endswith(': <StorageService2, type=Storage, protocol=http, address=127.0.0.1, service port=8888,' ' management port=1888, status=1>') mocker.patch.object(InterestRegistry, '__init__', return_value=None) mocker.patch.object(InterestRegistry, 'get', return_value=list()) # deregister the same with patch.object(Service._logger, 'info') as log_info2: t = Service.unregister(idx) assert idx == t assert 1 == log_info2.call_count args, kwargs = log_info2.call_args assert args[0].startswith('Stopped service instance id=') assert args[0].endswith(': <StorageService2, type=Storage, protocol=http, address=127.0.0.1, ' 'service port=8888, management port=1888, status=2>') s = Service.get(idx) assert s[0]._status == 2 # Unregistered
async def test_get(self): with patch.object(Service._logger, 'info') as log_info: s = Service.register("StorageService", "Storage", "localhost", 8881, 1888) c = Service.register("CoreService", "Core", "localhost", 7771, 1777) d = Service.register("SouthService", "Southbound", "127.0.0.1", 9991, 1999, "https") assert 3 == log_info.call_count _service = Service.get() assert 3 == len(_service) assert s == _service[0]._id assert "StorageService" == _service[0]._name assert "Storage" == _service[0]._type assert "localhost" == _service[0]._address assert 8881 == int(_service[0]._port) assert 1888 == int(_service[0]._management_port) # validates default set to HTTP assert "http" == _service[0]._protocol assert c == _service[1]._id assert "CoreService" == _service[1]._name assert "Core" == _service[1]._type assert "localhost" == _service[1]._address assert 7771 == int(_service[1]._port) assert 1777 == int(_service[1]._management_port) # validates default set to HTTP assert "http" == _service[1]._protocol assert d == _service[2]._id assert "SouthService" == _service[2]._name assert "Southbound" == _service[2]._type assert "127.0.0.1" == _service[2]._address assert 9991 == int(_service[2]._port) assert 1999 == int(_service[2]._management_port) assert "https" == _service[2]._protocol
async def get_plugin(request): """ GET lists of rule plugins and delivery plugins :Example: curl -X GET http://localhost:8081/fledge/notification/plugin """ try: notification_service = ServiceRegistry.get( s_type=ServiceRecord.Type.Notification.name) _address, _port = notification_service[ 0]._address, notification_service[0]._port except service_registry_exceptions.DoesNotExist: raise web.HTTPNotFound(reason="No Notification service available.") try: url = 'http://{}:{}/notification/rules'.format(_address, _port) rule_plugins = json.loads(await _hit_get_url(url)) url = 'http://{}:{}/notification/delivery'.format(_address, _port) delivery_plugins = json.loads(await _hit_get_url(url)) except Exception as ex: raise web.HTTPInternalServerError(reason=ex) else: return web.json_response({ 'rules': rule_plugins, 'delivery': delivery_plugins })
async def test__monitor_good_uptime(self): async def async_mock(return_value): return return_value # used to mock client session context manager class AsyncSessionContextManagerMock(MagicMock): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) async def __aenter__(self): client_response_mock = MagicMock(spec=aiohttp.ClientResponse) # mock response (good) client_response_mock.text.side_effect = [ async_mock('{"uptime": "bla"}')] return client_response_mock async def __aexit__(self, *args): return None # as monitor loop is as infinite loop, this exception is thrown when we need to exit the loop class TestMonitorException(Exception): pass # register a service with patch.object(ServiceRegistry._logger, 'info') as log_info: s_id_1 = ServiceRegistry.register( 'sname1', 'Storage', 'saddress1', 1, 1, 'protocol1') assert 1 == log_info.call_count args, kwargs = log_info.call_args assert args[0].startswith('Registered service instance id=') assert args[0].endswith(': <sname1, type=Storage, protocol=protocol1, address=saddress1, service port=1, ' 'management port=1, status=1>') monitor = Monitor() monitor._sleep_interval = Monitor._DEFAULT_SLEEP_INTERVAL monitor._max_attempts = Monitor._DEFAULT_MAX_ATTEMPTS # throw the TestMonitorException when sleep is called (end of infinite loop) with patch.object(Monitor, '_sleep', side_effect=TestMonitorException()): with patch.object(aiohttp.ClientSession, 'get', return_value=AsyncSessionContextManagerMock()): with pytest.raises(Exception) as excinfo: await monitor._monitor_loop() assert excinfo.type is TestMonitorException # service is good, so it should remain in the service registry assert len(ServiceRegistry.get(idx=s_id_1)) is 1 assert ServiceRegistry.get(idx=s_id_1)[0]._status is ServiceRecord.Status.Running
async def delete_service(request): """ Delete an existing service :Example: curl -X DELETE http://localhost:8081/fledge/service/<svc name> """ try: svc = request.match_info.get('service_name', None) storage = connect.get_storage_async() result = await get_schedule(storage, svc) if result['count'] == 0: return web.HTTPNotFound( reason='{} service does not exist.'.format(svc)) config_mgr = ConfigurationManager(storage) # In case of notification service, if notifications exists, then deletion is not allowed if 'notification' in result['rows'][0]['process_name']: notf_children = await config_mgr.get_category_child( category_name="Notifications") children = [x['key'] for x in notf_children] if len(notf_children) > 0: return web.HTTPBadRequest( reason= 'Notification service `{}` can not be deleted, as {} notification instances exist.' .format(svc, children)) # First disable the schedule svc_schedule = result['rows'][0] sch_id = uuid.UUID(svc_schedule['id']) if svc_schedule['enabled'].lower() == 't': await server.Server.scheduler.disable_schedule(sch_id) # return control to event loop await asyncio.sleep(1) # Delete all configuration for the service name await config_mgr.delete_category_and_children_recursively(svc) # Remove from registry as it has been already shutdown via disable_schedule() and since # we intend to delete the schedule also, there is no use of its Service registry entry try: services = ServiceRegistry.get(name=svc) ServiceRegistry.remove_from_registry(services[0]._id) except service_registry_exceptions.DoesNotExist: pass # Delete schedule await server.Server.scheduler.delete_schedule(sch_id) except Exception as ex: raise web.HTTPInternalServerError(reason=str(ex)) else: return web.json_response( {'result': 'Service {} deleted successfully.'.format(svc)})
def get_readings_async(): """ Storage Object """ try: services = ServiceRegistry.get(name="Fledge Storage") storage_svc = services[0] _readings = ReadingsStorageClientAsync(core_mgt_host=None, core_mgt_port=None, svc=storage_svc) # _logger.info(type(_storage)) except Exception as ex: _logger.exception(str(ex)) raise return _readings
async def run(category_name): """ Callback run by configuration category to notify changes to interested microservices Note: this method is async as needed Args: configuration_name (str): name of category that was changed """ # get all interest records regarding category_name cfg_mgr = ConfigurationManager() interest_registry = InterestRegistry(cfg_mgr) try: interest_records = interest_registry.get(category_name=category_name) except interest_registry_exceptions.DoesNotExist: return category_value = await cfg_mgr.get_category_all_items(category_name) payload = {"category" : category_name, "items" : category_value} headers = {'content-type': 'application/json'} # for each microservice interested in category_name, notify change for i in interest_records: # get microservice management server info of microservice through service registry try: service_record = ServiceRegistry.get(idx=i._microservice_uuid)[0] except service_registry_exceptions.DoesNotExist: _LOGGER.exception("Unable to notify microservice with uuid %s as it is not found in the service registry", i._microservice_uuid) continue url = "{}://{}:{}/fledge/change".format(service_record._protocol, service_record._address, service_record._management_port) async with aiohttp.ClientSession() as session: try: async with session.post(url, data=json.dumps(payload, sort_keys=True), headers=headers) as resp: result = await resp.text() status_code = resp.status if status_code in range(400, 500): _LOGGER.error("Bad request error code: %d, reason: %s", status_code, resp.reason) if status_code in range(500, 600): _LOGGER.error("Server error code: %d, reason: %s", status_code, resp.reason) except Exception as ex: _LOGGER.exception("Unable to notify microservice with uuid %s due to exception: %s", i._microservice_uuid, str(ex)) continue
async def delete_notification(request): """ Delete an existing notification :Example: curl -X DELETE http://localhost:8081/fledge/notification/<notification_name> """ try: notification_service = ServiceRegistry.get( s_type=ServiceRecord.Type.Notification.name) _address, _port = notification_service[ 0]._address, notification_service[0]._port except service_registry_exceptions.DoesNotExist: raise web.HTTPNotFound(reason="No Notification service available.") try: notif = request.match_info.get('notification_name', None) if notif is None: raise ValueError("Notification name is required for deletion.") # Stop & remove notification url = 'http://{}:{}/notification/{}'.format(_address, _port, urllib.parse.quote(notif)) notification = json.loads(await _hit_delete_url(url)) # Removes the child categories for the rule and delivery plugins, Removes the category for the notification itself storage = connect.get_storage_async() config_mgr = ConfigurationManager(storage) await config_mgr.delete_category_and_children_recursively(notif) audit = AuditLogger(storage) await audit.information('NTFDL', {"name": notif}) except ValueError as ex: raise web.HTTPBadRequest(reason=str(ex)) except Exception as ex: raise web.HTTPInternalServerError(reason=str(ex)) else: return web.json_response( {'result': 'Notification {} deleted successfully.'.format(notif)})
def test_unregister(self, mocker): mocker.patch.object(InterestRegistry, '__init__', return_value=None) mocker.patch.object(InterestRegistry, 'get', return_value=list()) with patch.object(ServiceRegistry._logger, 'info') as log_info1: reg_id = ServiceRegistry.register("A name", "Storage", "127.0.0.1", 1234, 4321, 'http') assert 1 == len(ServiceRegistry._registry) assert 1 == log_info1.call_count arg, kwarg = log_info1.call_args assert arg[0].startswith('Registered service instance id=') assert arg[0].endswith(': <A name, type=Storage, protocol=http, address=127.0.0.1, service port=1234,' ' management port=4321, status=1>') with patch.object(ServiceRegistry._logger, 'info') as log_info2: s_id = ServiceRegistry.unregister(reg_id) assert 36 == len(s_id) # uuid version 4 len assert 1 == len(ServiceRegistry._registry) s = ServiceRegistry.get(idx=s_id) assert s[0]._status == 2 assert 1 == log_info2.call_count args, kwargs = log_info2.call_args assert args[0].startswith('Stopped service instance id=') assert args[0].endswith(': <A name, type=Storage, protocol=http, address=127.0.0.1, service port=1234,' ' management port=4321, status=2>')
async def test__monitor_exceed_attempts(self, mocker): class AsyncSessionContextManagerMock(MagicMock): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) async def __aenter__(self): # mock response (error- exception) raise Exception("test") async def __aexit__(self, *args): return None # as monitor loop is as infinite loop, this exception is thrown when we need to exit the loop class TestMonitorException(Exception): pass # register a service s_id_1 = ServiceRegistry.register( 'sname1', 'Storage', 'saddress1', 1, 1, 'protocol1') monitor = Monitor() monitor._sleep_interval = Monitor._DEFAULT_SLEEP_INTERVAL monitor._max_attempts = Monitor._DEFAULT_MAX_ATTEMPTS sleep_side_effect_list = list() # _MAX_ATTEMPTS is 15 # throw exception on the 16th time sleep is called - the first 15 sleeps are used during retries for i in range(0, 15): sleep_side_effect_list.append(asyncio.sleep(.01)) sleep_side_effect_list.append(TestMonitorException()) with patch.object(Monitor, '_sleep', side_effect=sleep_side_effect_list): with patch.object(aiohttp.ClientSession, 'get', return_value=AsyncSessionContextManagerMock()): with pytest.raises(Exception) as excinfo: await monitor._monitor_loop() assert excinfo.type in [TestMonitorException, TypeError] assert ServiceRegistry.get(idx=s_id_1)[0]._status is ServiceRecord.Status.Failed
async def _services_with_assets(storage_client, south_services): sr_list = list() try: try: services_from_registry = ServiceRegistry.get(s_type="Southbound") except DoesNotExist: services_from_registry = [] def is_svc_in_service_registry(name): return next( (svc for svc in services_from_registry if svc._name == name), None) installed_plugins = _get_installed_plugins() for s_record in services_from_registry: plugin, assets = await _get_tracked_plugin_assets_and_readings( storage_client, s_record._name) plugin_version = '' for p in installed_plugins: if p["name"] == plugin: plugin_version = p["version"] break sr_list.append({ 'name': s_record._name, 'address': s_record._address, 'management_port': s_record._management_port, 'service_port': s_record._port, 'protocol': s_record._protocol, 'status': ServiceRecord.Status(int(s_record._status)).name.lower(), 'assets': assets, 'plugin': { 'name': plugin, 'version': plugin_version }, 'schedule_enabled': await _get_schedule_status(storage_client, s_record._name) }) for s_name in south_services: south_svc = is_svc_in_service_registry(s_name) if not south_svc: plugin, assets = await _get_tracked_plugin_assets_and_readings( storage_client, s_name) plugin_version = '' for p in installed_plugins: if p["name"] == plugin: plugin_version = p["version"] break sr_list.append({ 'name': s_name, 'address': '', 'management_port': '', 'service_port': '', 'protocol': '', 'status': '', 'assets': assets, 'plugin': { 'name': plugin, 'version': plugin_version }, 'schedule_enabled': await _get_schedule_status(storage_client, s_name) }) except: raise else: return sr_list
async def put_notification(request): """ Update an existing notification :Example: curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"description":"Test Notification modified"}' curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"rule": "threshold", "channel": "email"}' curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"notification_type": "one shot", "enabled": false}' curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"enabled": false}' curl -X PUT http://localhost:8081/fledge/notification/<notification_name> -d '{"description":"Test Notification", "rule": "threshold", "channel": "email", "notification_type": "one shot", "enabled": false, "rule_config": {}, "delivery_config": {}}' """ try: notification_service = ServiceRegistry.get( s_type=ServiceRecord.Type.Notification.name) _address, _port = notification_service[ 0]._address, notification_service[0]._port except service_registry_exceptions.DoesNotExist: raise web.HTTPNotFound(reason="No Notification service available.") try: notif = request.match_info.get('notification_name', None) if notif is None: raise ValueError("Notification name is required for updation.") # TODO: Stop notification before update data = await request.json() if not isinstance(data, dict): raise ValueError('Data payload must be a valid JSON') description = data.get('description', None) rule = data.get('rule', None) channel = data.get('channel', None) notification_type = data.get('notification_type', None) enabled = data.get('enabled', None) rule_config = data.get('rule_config', {}) delivery_config = data.get('delivery_config', {}) if utils.check_reserved(notif) is False: raise ValueError('Invalid notification instance name.') if rule is not None and utils.check_reserved(rule) is False: raise ValueError('Invalid rule property in payload.') if channel is not None and utils.check_reserved(channel) is False: raise ValueError('Invalid channel property in payload.') if notification_type is not None and notification_type not in NOTIFICATION_TYPE: raise ValueError('Invalid notification_type property in payload.') if enabled is not None: if enabled not in ['true', 'false', True, False]: raise ValueError( 'Only "true", "false", true, false are allowed for value of enabled.' ) is_enabled = "true" if ( (type(enabled) is str and enabled.lower() in ['true']) or ((type(enabled) is bool and enabled is True))) else "false" storage = connect.get_storage_async() config_mgr = ConfigurationManager(storage) current_config = await config_mgr._read_category_val(notif) if current_config is None: raise NotFoundError( 'No {} notification instance found'.format(notif)) rule_changed = True if rule is not None and rule != current_config[ 'rule']['value'] else False channel_changed = True if channel is not None and channel != current_config[ 'channel']['value'] else False try: # Get default config for rule and channel plugins url = str(request.url) url_parts = url.split("/fledge/notification") url = '{}/fledge/notification/plugin'.format(url_parts[0]) try: # When authentication is mandatory we need to pass token in request header auth_token = request.token except AttributeError: auth_token = None list_plugins = json.loads(await _hit_get_url(url, auth_token)) search_rule = rule if rule_changed else current_config['rule'][ 'value'] r = list( filter(lambda rules: rules['name'] == search_rule, list_plugins['rules'])) if len(r) == 0: raise KeyError rule_plugin_config = r[0]['config'] search_channel = channel if channel_changed else current_config[ 'channel']['value'] c = list( filter(lambda channels: channels['name'] == search_channel, list_plugins['delivery'])) if len(c) == 0: raise KeyError delivery_plugin_config = c[0]['config'] except KeyError: raise ValueError( "Invalid rule plugin:{} and/or delivery plugin:{} supplied.". format(rule, channel)) # Verify if rule_config contains valid keys if rule_config != {}: for k, v in rule_config.items(): if k not in rule_plugin_config: raise ValueError("Invalid key:{} in rule plugin:{}".format( k, rule_plugin_config)) # Verify if delivery_config contains valid keys if delivery_config != {}: for k, v in delivery_config.items(): if k not in delivery_plugin_config: raise ValueError( "Invalid key:{} in delivery plugin:{}".format( k, delivery_plugin_config)) if rule_changed: # A new rule has been supplied category_desc = rule_plugin_config['plugin']['description'] category_name = "rule{}".format(notif) await config_mgr.create_category( category_name=category_name, category_description=category_desc, category_value=rule_plugin_config, keep_original_items=False) if channel_changed: # A new delivery has been supplied category_desc = delivery_plugin_config['plugin']['description'] category_name = "delivery{}".format(notif) await config_mgr.create_category( category_name=category_name, category_description=category_desc, category_value=delivery_plugin_config, keep_original_items=False) notification_config = {} if description is not None: notification_config.update({"description": description}) if rule is not None: notification_config.update({"rule": rule}) if channel is not None: notification_config.update({"channel": channel}) if notification_type is not None: notification_config.update( {"notification_type": notification_type}) if enabled is not None: notification_config.update({"enable": is_enabled}) await _update_configurations(config_mgr, notif, notification_config, rule_config, delivery_config) except ValueError as e: raise web.HTTPBadRequest(reason=str(e)) except NotFoundError as e: raise web.HTTPNotFound(reason=str(e)) except Exception as ex: raise web.HTTPInternalServerError(reason=str(ex)) else: # TODO: Start notification after update return web.json_response( {'result': "Notification {} updated successfully".format(notif)})
async def post_notification(request): """ Create a new notification to run a specific plugin :Example: curl -X POST http://localhost:8081/fledge/notification -d '{"name": "Test Notification", "description":"Test Notification", "rule": "threshold", "channel": "email", "notification_type": "one shot", "enabled": false}' curl -X POST http://localhost:8081/fledge/notification -d '{"name": "Test Notification", "description":"Test Notification", "rule": "threshold", "channel": "email", "notification_type": "one shot", "enabled": false, "rule_config": {}, "delivery_config": {}}' """ try: notification_service = ServiceRegistry.get( s_type=ServiceRecord.Type.Notification.name) _address, _port = notification_service[ 0]._address, notification_service[0]._port except service_registry_exceptions.DoesNotExist: raise web.HTTPNotFound(reason="No Notification service available.") try: data = await request.json() if not isinstance(data, dict): raise ValueError('Data payload must be a valid JSON') name = data.get('name', None) description = data.get('description', None) rule = data.get('rule', None) channel = data.get('channel', None) notification_type = data.get('notification_type', None) enabled = data.get('enabled', None) rule_config = data.get('rule_config', {}) delivery_config = data.get('delivery_config', {}) if name is None or name.strip() == "": raise ValueError('Missing name property in payload.') if description is None: raise ValueError('Missing description property in payload.') if rule is None: raise ValueError('Missing rule property in payload.') if channel is None: raise ValueError('Missing channel property in payload.') if notification_type is None: raise ValueError('Missing notification_type property in payload.') if utils.check_reserved(name) is False: raise ValueError('Invalid name property in payload.') if utils.check_reserved(rule) is False: raise ValueError('Invalid rule property in payload.') if utils.check_reserved(channel) is False: raise ValueError('Invalid channel property in payload.') if notification_type not in NOTIFICATION_TYPE: raise ValueError('Invalid notification_type property in payload.') if enabled is not None: if enabled not in ['true', 'false', True, False]: raise ValueError( 'Only "true", "false", true, false are allowed for value of enabled.' ) is_enabled = "true" if ( (type(enabled) is str and enabled.lower() in ['true']) or ((type(enabled) is bool and enabled is True))) else "false" storage = connect.get_storage_async() config_mgr = ConfigurationManager(storage) curr_config = await config_mgr.get_category_all_items(name) if curr_config is not None: raise ValueError( "A Category with name {} already exists.".format(name)) try: # Get default config for rule and channel plugins url = '{}/plugin'.format(request.url) try: # When authentication is mandatory we need to pass token in request header auth_token = request.token except AttributeError: auth_token = None list_plugins = json.loads(await _hit_get_url(url, auth_token)) r = list( filter(lambda rules: rules['name'] == rule, list_plugins['rules'])) c = list( filter(lambda channels: channels['name'] == channel, list_plugins['delivery'])) if len(r) == 0 or len(c) == 0: raise KeyError rule_plugin_config = r[0]['config'] delivery_plugin_config = c[0]['config'] except KeyError: raise ValueError( "Invalid rule plugin {} and/or delivery plugin {} supplied.". format(rule, channel)) # Verify if rule_config contains valid keys if rule_config != {}: for k, v in rule_config.items(): if k not in rule_plugin_config: raise ValueError( "Invalid key {} in rule_config {} supplied for plugin {}." .format(k, rule_config, rule)) # Verify if delivery_config contains valid keys if delivery_config != {}: for k, v in delivery_config.items(): if k not in delivery_plugin_config: raise ValueError( "Invalid key {} in delivery_config {} supplied for plugin {}." .format(k, delivery_config, channel)) # First create templates for notification and rule, channel plugins post_url = 'http://{}:{}/notification/{}'.format( _address, _port, urllib.parse.quote(name)) await _hit_post_url(post_url) # Create Notification template post_url = 'http://{}:{}/notification/{}/rule/{}'.format( _address, _port, urllib.parse.quote(name), urllib.parse.quote(rule)) await _hit_post_url(post_url) # Create Notification rule template post_url = 'http://{}:{}/notification/{}/delivery/{}'.format( _address, _port, urllib.parse.quote(name), urllib.parse.quote(channel)) await _hit_post_url(post_url) # Create Notification delivery template # Create configurations notification_config = { "description": description, "rule": rule, "channel": channel, "notification_type": notification_type, "enable": is_enabled, } await _update_configurations(config_mgr, name, notification_config, rule_config, delivery_config) audit = AuditLogger(storage) await audit.information('NTFAD', {"name": name}) except ValueError as ex: raise web.HTTPBadRequest(reason=str(ex)) except Exception as e: raise web.HTTPInternalServerError(reason=str(e)) else: return web.json_response( {'result': "Notification {} created successfully".format(name)})