def test_exception_when_non_foglamp_storage(self, mock_logger): with patch.object(ServiceRegistry._logger, 'info') as log_info: ServiceRegistry.register("foo", "Storage", "127.0.0.1", 1, 2) assert 1 == log_info.call_count args, kwargs = log_info.call_args assert args[0].startswith('Registered service instance id=') assert args[0].endswith(': <foo, type=Storage, protocol=http, address=127.0.0.1, service port=1, ' 'management port=2, status=1>') with pytest.raises(DoesNotExist) as excinfo: connect.get_storage_async() assert str(excinfo).endswith('DoesNotExist') mock_logger.exception.assert_called_once_with('')
async def get_categories(request): """ Args: request: Returns: the list of known categories in the configuration database :Example: curl -sX GET http://localhost:8081/foglamp/category curl -sX GET http://localhost:8081/foglamp/category?root=true curl -sX GET 'http://localhost:8081/foglamp/category?root=true&children=true' """ cf_mgr = ConfigurationManager(connect.get_storage_async()) if 'root' in request.query and request.query['root'].lower() in ['true', 'false']: is_root = True if request.query['root'].lower() == 'true' else False # to get nested categories, if children is true is_children = True if 'children' in request.query and request.query['children'].lower() == 'true' else False if is_children: categories_json = await cf_mgr.get_all_category_names(root=is_root, children=is_children) else: categories = await cf_mgr.get_all_category_names(root=is_root) categories_json = [{"key": c[0], "description": c[1], "displayName": c[2]} for c in categories] else: categories = await cf_mgr.get_all_category_names() categories_json = [{"key": c[0], "description": c[1], "displayName": c[2]} for c in categories] return web.json_response({'categories': categories_json})
async def get_backup_download(request): """ Download back up file by id :Example: wget -O foglamp-backup-1.tar.gz http://localhost:8081/foglamp/backup/1/download """ backup_id = request.match_info.get('backup_id', None) try: backup_id = int(backup_id) backup = Backup(connect.get_storage_async()) backup_json = await backup.get_backup_details(backup_id) # Strip filename from backup path file_name_path = str(backup_json["file_name"]).split('data/backup/') file_name = str(file_name_path[1]) dir_name = _FOGLAMP_DATA + '/backup/' if _FOGLAMP_DATA else _FOGLAMP_ROOT + "/data/backup/" source = dir_name + file_name # Create tar file t = tarfile.open(source + ".tar.gz", "w:gz") t.add(source, arcname=os.path.basename(source)) t.close() # Path of tar.gz file gz_path = Path(source + ".tar.gz") except ValueError: raise web.HTTPBadRequest(reason='Invalid backup id') except exceptions.DoesNotExist: raise web.HTTPNotFound(reason='Backup id {} does not exist'.format(backup_id)) except Exception as ex: raise web.HTTPInternalServerError(reason=(str(ex))) return web.FileResponse(path=gz_path)
async def get_backup_details(request): """ Returns the details of a backup :Example: curl -X GET http://localhost:8081/foglamp/backup/1 """ backup_id = request.match_info.get('backup_id', None) try: backup_id = int(backup_id) backup = Backup(connect.get_storage_async()) backup_json = await backup.get_backup_details(backup_id) resp = { "status": _get_status(int(backup_json["status"])), 'id': backup_json["id"], 'date': backup_json["ts"] } except ValueError: raise web.HTTPBadRequest(reason='Invalid backup id') except exceptions.DoesNotExist: raise web.HTTPNotFound( reason='Backup id {} does not exist'.format(backup_id)) except Exception as ex: raise web.HTTPException(reason=(str(ex))) return web.json_response(resp)
async def validate_token(cls, token): """ check existence and validity of token * exists in user_logins table * its not expired :param token: :return: """ storage_client = connect.get_storage_async() payload = PayloadBuilder().SELECT("token_expiration") \ .ALIAS("return", ("token_expiration", 'token_expiration')) \ .FORMAT("return", ("token_expiration", "YYYY-MM-DD HH24:MI:SS.MS")) \ .WHERE(['token', '=', token]).payload() result = await storage_client.query_tbl_with_payload('user_logins', payload) if len(result['rows']) == 0: raise User.InvalidToken("Token appears to be invalid") r = result['rows'][0] token_expiry = r["token_expiration"] curr_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") fmt = "%Y-%m-%d %H:%M:%S.%f" diff = datetime.strptime(token_expiry, fmt) - datetime.strptime(curr_time, fmt) if diff.seconds < 0: raise User.TokenExpired("The token has expired, login again") # verification of expiry set to false, # as we want to refresh token on each successful request # and extend it to keep session alive user_payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM], options={'verify_exp': False}) return user_payload["uid"]
async def get_scheduled_process(request): """ Returns: a list of all the defined scheduled_processes from scheduled_processes table :Example: curl -X GET http://localhost:8081/foglamp/schedule/process/purge curl -X GET http://localhost:8081/foglamp/schedule/process/purge%2Cbackup%2Crestore curl -X GET http://localhost:8081/foglamp/schedule/process/purge%2Cbackup%2Cstats%20collector """ scheduled_process_names = request.match_info.get('scheduled_process_name', None) scheduled_process_name = scheduled_process_names.split(',') payload = PayloadBuilder().SELECT("name").WHERE( ["name", "in", scheduled_process_name]).payload() _storage = connect.get_storage_async() scheduled_process = await _storage.query_tbl_with_payload( 'scheduled_processes', payload) if len(scheduled_process['rows']) == 0: raise web.HTTPNotFound(reason='No such Scheduled Process: {}.'.format( scheduled_process_name)) if len(scheduled_process['rows']) == 1: retval = scheduled_process['rows'][0].get("name") else: retval = scheduled_process['rows'] return web.json_response(retval)
async def get_filter_pipeline(request: web.Request) -> web.Response: """ GET filter pipeline :Example: curl -X GET http://localhost:8081/foglamp/filter/<user_name>/pipeline """ user_name = request.match_info.get('user_name', None) try: storage = connect.get_storage_async() cf_mgr = ConfigurationManager(storage) # Fetch the filter items: get category items category_info = await cf_mgr.get_category_all_items( category_name=user_name) if category_info is None: raise ValueError("No such '{}' category found.".format(user_name)) filter_value_from_storage = json.loads( category_info['filter']['value']) except KeyError: msg = "No filter pipeline exists for {}".format(user_name) _LOGGER.info(msg) raise web.HTTPNotFound(reason=msg) except StorageServerError as ex: _LOGGER.exception("Get pipeline: %s, caught exception: %s", user_name, str(ex.error)) raise web.HTTPInternalServerError(reason=str(ex.error)) except ValueError as ex: raise web.HTTPNotFound(reason=ex) except Exception as ex: raise web.HTTPInternalServerError(reason=ex) else: return web.json_response({'result': filter_value_from_storage})
async def get_category_item(request): """ Args: request: category_name & config_item are required Returns: the configuration item in the given category. :Example: curl -X GET http://localhost:8081/foglamp/category/PURGE_READ/age """ category_name = request.match_info.get('category_name', None) config_item = request.match_info.get('config_item', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None config_item = urllib.parse.unquote( config_item) if config_item is not None else None cf_mgr = ConfigurationManager(connect.get_storage_async()) category_item = await cf_mgr.get_category_item(category_name, config_item) if category_item is None: raise web.HTTPNotFound( reason="No such Category item found for {}".format(config_item)) return web.json_response(category_item)
async def get_south_services(request): """ Args: request: Returns: list of all south services with tracked assets and readings count :Example: curl -X GET http://localhost:8081/foglamp/south """ if 'cached' in request.query and request.query['cached'].lower( ) == 'false': _get_installed_plugins.cache_clear() storage_client = connect.get_storage_async() cf_mgr = ConfigurationManager(storage_client) try: south_cat = await cf_mgr.get_category_child("South") south_categories = [nc["key"] for nc in south_cat] except: return web.json_response({'services': []}) response = await _services_with_assets(storage_client, south_categories) return web.json_response({'services': response})
async def delete_child_category(request): """ Args: request: category_name, child_category are required Returns: remove the link b/w child category and its parent :Example: curl -X DELETE http://localhost:8081/foglamp/category/{category_name}/children/{child_category} """ category_name = request.match_info.get('category_name', None) child_category = request.match_info.get('child_category', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None cf_mgr = ConfigurationManager(connect.get_storage_async()) try: result = await cf_mgr.delete_child_category(category_name, child_category) except TypeError as ex: raise web.HTTPBadRequest(reason=str(ex)) except ValueError as ex: raise web.HTTPNotFound(reason=str(ex)) return web.json_response({"children": result})
async def delete_category(request): """ Args: request: category_name required Returns: Success message on successful deletion Raises: TypeError/ValueError/Exception on error :Example: curl -X DELETE http://localhost:8081/foglamp/category/{category_name} """ category_name = request.match_info.get('category_name', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None try: cf_mgr = ConfigurationManager(connect.get_storage_async()) await cf_mgr.delete_category_and_children_recursively(category_name) except (ValueError, TypeError) as ex: raise web.HTTPBadRequest(reason=ex) except Exception as ex: raise web.HTTPInternalServerError(reason=ex) else: return web.json_response({ 'result': 'Category {} deleted successfully.'.format(category_name) })
async def create_child_category(request): """ Args: request: category_name is required and JSON object that defines the child category Returns: parent of the children being added :Example: curl -d '{"children": ["coap", "http", "sinusoid"]}' -X POST http://localhost:8081/foglamp/category/south/children """ cf_mgr = ConfigurationManager(connect.get_storage_async()) data = await request.json() if not isinstance(data, dict): raise ValueError('Data payload must be a dictionary') category_name = request.match_info.get('category_name', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None children = data.get('children') try: r = await cf_mgr.create_child_category(category_name, children) except TypeError as ex: raise web.HTTPBadRequest(reason=str(ex)) except ValueError as ex: raise web.HTTPNotFound(reason=str(ex)) return web.json_response(r)
async def get_category(request): """ Args: request: category_name is required Returns: the configuration items in the given category. :Example: curl -X GET http://localhost:8081/foglamp/category/PURGE_READ """ category_name = request.match_info.get('category_name', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None cf_mgr = ConfigurationManager(connect.get_storage_async()) category = await cf_mgr.get_category_all_items(category_name) if category is None: raise web.HTTPNotFound( reason="No such Category found for {}".format(category_name)) try: request.is_core_mgt except AttributeError: category = hide_password(category) return web.json_response(category)
async def all(cls): storage_client = connect.get_storage_async() payload = PayloadBuilder().SELECT("id", "uname", "role_id").WHERE( ['enabled', '=', 't']).payload() result = await storage_client.query_tbl_with_payload( 'users', payload) return result['rows']
async def update_configuration_item_bulk(request): """ Bulk update config items :Example: curl -X PUT -H "Content-Type: application/json" -d '{"config_item_key": "<some value>", "config_item2_key": "<some value>" }' http://localhost:8081/foglamp/category/{category_name} """ category_name = request.match_info.get('category_name', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None try: data = await request.json() if not data: return web.HTTPBadRequest(reason='Nothing to update') cf_mgr = ConfigurationManager(connect.get_storage_async()) await cf_mgr.update_configuration_item_bulk(category_name, data) except (NameError, KeyError) as ex: raise web.HTTPNotFound(reason=ex) except (ValueError, TypeError) as ex: raise web.HTTPBadRequest(reason=ex) except Exception as ex: raise web.HTTPInternalServerError(reason=ex) else: result = await cf_mgr.get_category_all_items(category_name) return web.json_response(result)
async def get_notifications(request): """ GET list of notifications :Example: curl -X GET http://localhost:8081/foglamp/notification """ try: storage = connect.get_storage_async() config_mgr = ConfigurationManager(storage) all_notifications = await config_mgr._read_all_child_category_names("Notifications") notifications = [] for notification in all_notifications: notification_config = await config_mgr._read_category_val(notification['child']) notification = { "name": notification_config['name']['value'], "rule": notification_config['rule']['value'], "channel": notification_config['channel']['value'], "notificationType": notification_config['notification_type']['value'], "enable": notification_config['enable']['value'], } notifications.append(notification) except Exception as ex: raise web.HTTPInternalServerError(reason=ex) else: return web.json_response({'notifications': notifications})
async def _read_config(self): """Reads configuration""" default_config = { "sleep_interval": { "description": "Time in seconds to sleep between health checks. (must be greater than 5)", "type": "integer", "default": str(self._DEFAULT_SLEEP_INTERVAL) }, "ping_timeout": { "description": "Timeout for a response from any given micro-service. (must be greater than 0)", "type": "integer", "default": str(self._DEFAULT_PING_TIMEOUT) }, "max_attempts": { "description": "Maximum number of attempts for finding a heartbeat of service", "type": "integer", "default": str(self._DEFAULT_MAX_ATTEMPTS) }, "restart_failed": { "description": "Restart failed microservice - manual/auto", "type": "string", "default": self._DEFAULT_RESTART_FAILED } } storage_client = connect.get_storage_async() cfg_manager = ConfigurationManager(storage_client) await cfg_manager.create_category('SMNTR', default_config, 'Service Monitor') config = await cfg_manager.get_category_all_items('SMNTR') self._sleep_interval = int(config['sleep_interval']['value']) self._ping_timeout = int(config['ping_timeout']['value']) self._max_attempts = int(config['max_attempts']['value']) self._restart_failed = config['restart_failed']['value']
async def delete(cls, user_id): """ Args: user_id: user id to delete Returns: json response """ # either keep 1 admin user or just reserve id:1 for superuser if int(user_id) == 1: raise ValueError("Super admin user can not be deleted") storage_client = connect.get_storage_async() try: # first delete the active login references await cls.delete_user_tokens(user_id) payload = PayloadBuilder().SET(enabled="f").WHERE(['id', '=', user_id]).AND_WHERE(['enabled', '=', 't']).payload() result = await storage_client.update_tbl("users", payload) except StorageServerError as ex: if ex.error["retryable"]: pass # retry INSERT raise ValueError(ERROR_MSG) return result
async def get_role_id_by_name(cls, name): storage_client = connect.get_storage_async() payload = PayloadBuilder().SELECT("id").WHERE(['name', '=', name]).payload() result = await storage_client.query_tbl_with_payload( 'roles', payload) return result["rows"]
async def certificate_login(cls, username, host): """ Args: username: username host: IP address Returns: uid: User id token: jwt token is_admin: boolean flag """ storage_client = connect.get_storage_async() # get user info on the basis of username payload = PayloadBuilder().SELECT("id", "role_id").WHERE(['uname', '=', username])\ .AND_WHERE(['enabled', '=', 't']).payload() result = await storage_client.query_tbl_with_payload( 'users', payload) if len(result['rows']) == 0: raise User.DoesNotExist('User does not exist') found_user = result['rows'][0] uid, jwt_token, is_admin = await cls._get_new_token( storage_client, found_user, host) return uid, jwt_token, is_admin
async def delete_parent_category(request): """ Args: request: category_name Returns: remove the link b/w parent-child category for the parent :Example: curl -X DELETE http://localhost:8081/foglamp/category/{category_name}/parent """ category_name = request.match_info.get('category_name', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None cf_mgr = ConfigurationManager(connect.get_storage_async()) try: await cf_mgr.delete_parent_category(category_name) except TypeError as ex: raise web.HTTPBadRequest(reason=str(ex)) except ValueError as ex: raise web.HTTPNotFound(reason=str(ex)) return web.json_response({ "message": "Parent-child relationship for the parent-{} is deleted".format( category_name) })
async def delete_configuration_item_value(request): """ Args: request: category_name, config_item are required Returns: set the configuration item value to empty string in the given category :Example: curl -X DELETE http://localhost:8081/foglamp/category/{category_name}/{config_item}/value For {category_name}=>PURGE delete value for {config_item}=>age curl -X DELETE http://localhost:8081/foglamp/category/PURGE_READ/age/value """ category_name = request.match_info.get('category_name', None) config_item = request.match_info.get('config_item', None) # TODO: make it optimized and elegant cf_mgr = ConfigurationManager(connect.get_storage_async()) try: category_item = await cf_mgr.get_category_item(category_name, config_item) if category_item is None: raise ValueError await cf_mgr.set_category_item_value_entry(category_name, config_item, category_item['default']) except ValueError: raise web.HTTPNotFound(reason="No detail found for the category_name: {} and config_item: {}".format(category_name, config_item)) result = await cf_mgr.get_category_item(category_name, config_item) if result is None: raise web.HTTPNotFound(reason="No detail found for the category_name: {} and config_item: {}".format(category_name, config_item)) return web.json_response(result)
async def get_tasks_latest(request): """ Returns: the list of the most recent task execution for each name from tasks table :Example: curl -X GET http://localhost:8081/foglamp/task/latest curl -X GET http://localhost:8081/foglamp/task/latest?name=xxx """ payload = PayloadBuilder().SELECT("id", "process_name", "state", "start_time", "end_time", "reason", "pid", "exit_code")\ .ALIAS("return", ("start_time", 'start_time'), ("end_time", 'end_time'))\ .FORMAT("return", ("start_time", "YYYY-MM-DD HH24:MI:SS.MS"), ("end_time", "YYYY-MM-DD HH24:MI:SS.MS"))\ .ORDER_BY(["process_name", "asc"], ["start_time", "desc"]) if 'name' in request.query and request.query['name'] != '': name = request.query['name'] payload.WHERE(["process_name", "=", name]) try: _storage = connect.get_storage_async() results = await _storage.query_tbl_with_payload( 'tasks', payload.payload()) if len(results['rows']) == 0: raise web.HTTPNotFound(reason="No Tasks found") tasks = [] previous_process = None for row in results['rows']: if previous_process != row['process_name']: tasks.append(row) previous_process = row['process_name'] new_tasks = [] for task in tasks: new_tasks.append({ 'id': str(task['id']), 'name': task['process_name'], 'state': [t.name.capitalize() for t in list(Task.State)][int(task['state']) - 1], 'startTime': str(task['start_time']), 'endTime': str(task['end_time']), 'exitCode': task['exit_code'], 'reason': task['reason'], 'pid': task['pid'] }) return web.json_response({'tasks': new_tasks}) except (ValueError, TaskNotFoundError) as ex: raise web.HTTPNotFound(reason=str(ex))
async def delete_configuration_item_value(request): """ Args: request: category_name, config_item are required Returns: set the configuration item value to empty string in the given category :Example: curl -X DELETE http://localhost:8081/foglamp/category/{category_name}/{config_item}/value For {category_name}=>PURGE delete value for {config_item}=>age curl -X DELETE http://localhost:8081/foglamp/category/PURGE_READ/age/value """ category_name = request.match_info.get('category_name', None) config_item = request.match_info.get('config_item', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None config_item = urllib.parse.unquote( config_item) if config_item is not None else None cf_mgr = ConfigurationManager(connect.get_storage_async()) try: category_item = await cf_mgr.get_category_item(category_name, config_item) if category_item is None: raise ValueError try: is_core_mgt = request.is_core_mgt except AttributeError: if 'readonly' in category_item: if category_item['readonly'] == 'true': raise TypeError( "Delete not allowed for {} item_name as it has readonly attribute set" .format(config_item)) await cf_mgr.set_category_item_value_entry(category_name, config_item, category_item['default']) except ValueError: raise web.HTTPNotFound( reason= "No detail found for the category_name: {} and config_item: {}". format(category_name, config_item)) except TypeError as ex: raise web.HTTPBadRequest(reason=str(ex)) result = await cf_mgr.get_category_item(category_name, config_item) if result is None: raise web.HTTPNotFound( reason= "No detail found for the category_name: {} and config_item: {}". format(category_name, config_item)) return web.json_response(result)
async def is_user_exists(cls, username, password): payload = PayloadBuilder().SELECT("id", "pwd").WHERE(['uname', '=', username]).AND_WHERE(['enabled', '=', 't']).payload() storage_client = connect.get_storage_async() result = await storage_client.query_tbl_with_payload('users', payload) if len(result['rows']) == 0: return None found_user = result['rows'][0] is_valid_pwd = cls.check_password(found_user['pwd'], str(password)) return result['rows'][0]['id'] if is_valid_pwd else None
def test_get_storage(self): with patch.object(ServiceRegistry._logger, 'info') as log_info: ServiceRegistry.register("FogLAMP Storage", "Storage", "127.0.0.1", 37449, 37843) storage_client = connect.get_storage_async() assert isinstance(storage_client, StorageClientAsync) assert 1 == log_info.call_count args, kwargs = log_info.call_args assert args[0].startswith('Registered service instance id=') assert args[0].endswith(': <FogLAMP Storage, type=Storage, protocol=http, address=127.0.0.1, service port=37449,' ' management port=37843, status=1>')
async def delete_token(cls, token): storage_client = connect.get_storage_async() payload = PayloadBuilder().WHERE(['token', '=', token]).payload() try: res = await storage_client.delete_from_tbl("user_logins", payload) except StorageServerError as ex: if not ex.error["retryable"]: pass raise ValueError(ERROR_MSG) return res
async def set_configuration_item(request): """ Args: request: category_name, config_item, {"value" : "<some value>"} are required Returns: set the configuration item value in the given category. :Example: curl -X PUT -H "Content-Type: application/json" -d '{"value": "<some value>" }' http://localhost:8081/foglamp/category/{category_name}/{config_item} For {category_name}=>PURGE update value for {config_item}=>age curl -X PUT -H "Content-Type: application/json" -d '{"value": "24"}' http://localhost:8081/foglamp/category/PURGE_READ/age """ category_name = request.match_info.get('category_name', None) config_item = request.match_info.get('config_item', None) category_name = urllib.parse.unquote( category_name) if category_name is not None else None config_item = urllib.parse.unquote( config_item) if config_item is not None else None data = await request.json() cf_mgr = ConfigurationManager(connect.get_storage_async()) try: value = data['value'] if isinstance(value, dict): pass elif not isinstance(value, str): raise web.HTTPBadRequest( reason='{} should be a string literal, in double quotes'. format(value)) except KeyError: raise web.HTTPBadRequest( reason='Missing required value for {}'.format(config_item)) try: await cf_mgr.set_category_item_value_entry(category_name, config_item, value) except ValueError as ex: raise web.HTTPNotFound(reason=ex) except TypeError as ex: raise web.HTTPBadRequest(reason=ex) result = await cf_mgr.get_category_item(category_name, config_item) if result is None: raise web.HTTPNotFound( reason= "No detail found for the category_name: {} and config_item: {}". format(category_name, config_item)) return web.json_response(result)
async def update(cls, user_id, user_data): """ Args: user_id: logged user id user_data: user dict Returns: updated user info dict """ kwargs = dict() if 'role_id' in user_data: kwargs.update({"role_id": user_data['role_id']}) storage_client = connect.get_storage_async() hashed_pwd = None pwd_history_list = [] if 'password' in user_data: if len(user_data['password']): hashed_pwd = cls.hash_password(user_data['password']) current_datetime = datetime.now() kwargs.update({ "pwd": hashed_pwd, "pwd_last_changed": str(current_datetime) }) # get password history list pwd_history_list = await cls._get_password_history( storage_client, user_id, user_data) try: payload = PayloadBuilder().SET(**kwargs).WHERE( ['id', '=', user_id]).AND_WHERE(['enabled', '=', 't']).payload() result = await storage_client.update_tbl("users", payload) if result['rows_affected']: # FIXME: FOGL-1226 active session delete only in case of role_id and password updation # delete all active sessions await cls.delete_user_tokens(user_id) if 'password' in user_data: # insert pwd history and delete oldest pwd if USED_PASSWORD_HISTORY_COUNT exceeds await cls._insert_pwd_history_with_oldest_pwd_deletion_if_count_exceeds( storage_client, user_id, hashed_pwd, pwd_history_list) return True except StorageServerError as ex: if ex.error["retryable"]: pass # retry UPDATE raise ValueError(ERROR_MSG) except Exception: raise
async def create_backup(request): """ Creates a backup :Example: curl -X POST http://localhost:8081/foglamp/backup """ try: backup = Backup(connect.get_storage_async()) status = await backup.create_backup() except Exception as ex: raise web.HTTPException(reason=str(ex)) return web.json_response({"status": status})