def _pre_forward(operation: Operation) -> Operation: """Called before forwarding an operation""" # Validate that the operation can be forwarded if operation.operation_type not in routable_operations: raise RoutingRequestException( f"Operation type '{operation.operation_type}' can not be forwarded" ) if operation.operation_type == "REQUEST_CREATE": operation.model = ( beer_garden.requests.RequestValidator.instance().validate_request( operation.model)) # Save the request so it'll have an ID and we'll have something to update operation.model = db.create(operation.model) # Clear parent before forwarding so the child doesn't freak out about an # unknown request operation.model.parent = None operation.model.has_parent = False beer_garden.files.forward_file(operation) # Pull out and store the wait event, if it exists wait_event = operation.kwargs.pop("wait_event", None) if wait_event: beer_garden.requests.request_map[operation.model.id] = wait_event return operation
async def _generate_get_response(self, instance_id, start_line, end_line): wait_event = Event() response = await self.client( Operation( operation_type="INSTANCE_LOGS", args=[instance_id], kwargs={ "wait_event": wait_event, "start_line": start_line, "end_line": end_line, }, ), serialize_kwargs={"to_string": False}, ) wait_timeout = float(self.get_argument("timeout", default="15")) if wait_timeout < 0: wait_timeout = None if not await event_wait(wait_event, wait_timeout): raise TimeoutExceededError("Timeout exceeded") response = await self.client( Operation(operation_type="REQUEST_READ", args=[response["id"]]), serialize_kwargs={"to_string": False}, ) if response["status"] == "ERROR": raise RequestProcessingError(response["output"])
async def patch(self): """ --- summary: Initiate administrative actions description: | The body of the request needs to contain a set of instructions detailing the operations to perform. Currently the supported operations are `rescan`: ```JSON [ { "operation": "rescan" } ] ``` * Will remove from the registry and database any currently stopped plugins who's directory has been removed. * Will add and start any new plugin directories. And reloading the plugin logging configuration: ```JSON [ { "operation": "reload", "path": "/config/logging/plugin" } ] ``` parameters: - name: patch in: body required: true description: Instructions for operations schema: $ref: '#/definitions/Patch' responses: 204: description: Operation successfully initiated 50x: $ref: '#/definitions/50xError' tags: - Admin """ self.verify_user_permission_for_object(GARDEN_UPDATE, local_garden()) operations = SchemaParser.parse_patch( self.request.decoded_body, many=True, from_string=True ) for op in operations: if op.operation == "rescan": await self.client(Operation(operation_type="RUNNER_RESCAN")) elif op.operation == "reload": if op.path == "/config/logging/plugin": await self.client(Operation(operation_type="PLUGIN_LOG_RELOAD")) else: raise ModelValidationError(f"Unsupported path '{op.path}'") else: raise ModelValidationError(f"Unsupported operation '{op.operation}'") self.set_status(204)
def forward_file(operation: Operation) -> None: """Send file data before forwarding an operation with a file parameter.""" # HEADS UP - THIS IS PROBABLY BROKEN # import here bypasses circular dependency import beer_garden.router as router for file_id in _find_chunk_params(operation.model.parameters): file = check_chunks(file_id) args = [file.file_name, file.file_size, file.chunk_size] # Make sure we get all of the other data kwargs = _safe_build_object( dict, file, ignore=[ "file_name", "file_size", "chunk_size", ], upsert=True, ) file_op = Operation( operation_type="FILE_CREATE", args=args, kwargs=kwargs, target_garden_name=operation.target_garden_name, source_garden_name=operation.source_garden_name, ) # This should put push the file operations before the current one router.forward_processor.put(file_op) for chunk_id in file.chunks.values(): chunk = check_chunk(chunk_id) c_args = [chunk.file_id, chunk.offset, chunk.data] c_kwargs = _safe_build_object(dict, chunk, ignore=["file_id", "offset", "data"], upsert=True) chunk_op = Operation( operation_type="FILE_CHUNK", args=c_args, kwargs=c_kwargs, target_garden_name=operation.target_garden_name, source_garden_name=operation.source_garden_name, ) # This should put push the file operations before the current one router.forward_processor.put(chunk_op)
def process_wait(request: Request, timeout: float) -> Request: """Helper to process a request and wait for completion using a threading.Event Args: request: Request to create timeout: Timeout used for wait Returns: The completed request """ # We need a better solution for this. Because the Request library is imported # everywhere it causes issues when importing the router at the top because all of # the functions are not initialized. So we either leave this as is, or move the # requests import to the end of all of the files. import beer_garden.router as router req_complete = threading.Event() # Send the request through the router to allow for commands to work across Gardens created_request = router.route( Operation( operation_type="REQUEST_CREATE", model=request, model_type="Request", kwargs={"wait_event": req_complete}, )) if not req_complete.wait(timeout): raise TimeoutError( "Request did not complete before the specified timeout") return db.query_unique(Request, id=created_request.id)
async def get(self, garden_name): """ --- summary: Retrieve a specific Garden parameters: - name: garden_name in: path required: true description: Read specific Garden Information type: string responses: 200: description: Garden with the given garden_name schema: $ref: '#/definitions/Garden' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Garden """ response = await self.client( Operation(operation_type="GARDEN_READ", args=[garden_name]) ) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
def test_listen_create_request(self): """Published the Request over HTTP and verifies of STOMP""" stomp_connection = self.create_stomp_connection() request_model = self.create_request("test_listen_create_request") sample_operation_request = Operation( operation_type="REQUEST_CREATE", model=request_model, model_type="Request", ) listener = MessageListener() stomp_connection.set_listener('', listener) stomp_connection.subscribe(destination='Beer_Garden_Events', id='event_listener', ack='auto', headers={ 'subscription-type': 'MULTICAST', 'durable-subscription-name': 'events' }) self.easy_client.forward(sample_operation_request) time.sleep(10) assert listener.create_event_captured if stomp_connection.is_connected(): stomp_connection.disconnect()
async def get(self): """ --- summary: Get a list of all namespaces known to this garden responses: 200: description: List of Namespaces 50x: $ref: '#/definitions/50xError' tags: - Namespace """ permitted_gardens = self.permissioned_queryset(Garden, GARDEN_READ) permitted_requests = self.permissioned_queryset(Request, REQUEST_READ) permitted_systems = self.permissioned_queryset(System, SYSTEM_READ) response = await self.client( Operation( operation_type="NAMESPACE_READ_ALL", kwargs={ "garden_queryset": permitted_gardens, "system_queryset": permitted_systems, "request_queryset": permitted_requests, }, ) ) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def get(self): """ --- summary: Retrieve all Jobs. responses: 200: description: Successfully retrieved all systems. schema: type: array items: $ref: '#/definitions/Job' 50x: $ref: '#/definitions/50xError' tags: - Jobs """ filter_params = {} for key in self.request.arguments.keys(): if key in JobSchema.get_attribute_names(): filter_params[key] = self.get_query_argument(key) response = await self.client( Operation(operation_type="JOB_READ_ALL", kwargs={"filter_params": filter_params})) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def delete(self, job_id): """ --- summary: Delete a specific Job. description: Will remove a specific job. No further executions will occur. parameters: - name: job_id in: path required: true description: The ID of the Job type: string responses: 204: description: Job has been successfully deleted. 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Jobs """ await self.client(Operation(operation_type="JOB_DELETE", args=[job_id])) self.set_status(204)
async def get(self, job_id): """ --- summary: Retrieve a specific Job parameters: - name: job_id in: path required: true description: The ID of the Job type: string responses: 200: description: Job with the given ID schema: $ref: '#/definitions/Job' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Jobs """ response = await self.client( Operation(operation_type="JOB_READ", args=[job_id])) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def delete(self, garden_name): """ --- summary: Delete a specific Garden parameters: - name: garden_name in: path required: true description: Garden to use type: string responses: 204: description: Garden has been successfully deleted 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Garden """ garden = self.get_or_raise(Garden, GARDEN_DELETE, name=garden_name) await self.client( Operation(operation_type="GARDEN_DELETE", args=[garden.name])) self.set_status(204)
def _pre_forward(operation: Operation) -> Operation: """Called before forwarding an operation""" # Validate that the operation can be forwarded if operation.operation_type not in routable_operations: raise RoutingRequestException( f"Operation type '{operation.operation_type}' can not be forwarded" ) if operation.operation_type == "REQUEST_CREATE": # Save the request so it'll have an ID and we'll have something to update local_request = create_request(operation.model) if operation.model.namespace == config.get("garden.name"): operation.model = local_request else: # When the target is a remote garden, just capture the id. We don't # want to replace the entire model, as we'd lose the base64 encoded file # parameter data. operation.model.id = local_request.id # Clear parent before forwarding so the child doesn't freak out about an # unknown request operation.model.parent = None operation.model.has_parent = False # Pull out and store the wait event, if it exists wait_event = operation.kwargs.pop("wait_event", None) if wait_event: beer_garden.requests.request_map[operation.model.id] = wait_event return operation
async def get(self, system_id, command_name): """ --- summary: Retrieve a specific Command parameters: - name: system_id in: path required: true description: The ID of the System type: string - name: command_name in: path required: true description: The name of the Command type: string responses: 200: description: Command with the given name schema: $ref: '#/definitions/Command' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Commands """ response = await self.client( Operation(operation_type="COMMAND_READ", args=[system_id, command_name])) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def delete(self, instance_id): """ --- summary: Delete a specific Instance parameters: - name: instance_id in: path required: true description: The ID of the Instance type: string responses: 204: description: Instance has been successfully deleted 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Instances """ _ = self.get_or_raise(System, SYSTEM_UPDATE, instances__id=instance_id) await self.client( Operation(operation_type="INSTANCE_DELETE", args=[instance_id])) self.set_status(204)
async def get(self, instance_id): """ --- summary: Retrieve queue information for instance parameters: - name: instance_id in: path required: true description: The instance ID to pull queues for type: string responses: 200: description: List of queue information objects for this instance schema: type: array items: $ref: '#/definitions/Queue' 50x: $ref: '#/definitions/50xError' tags: - Queues """ _ = self.get_or_raise(System, QUEUE_READ, instances__id=instance_id) response = await self.client( Operation(operation_type="QUEUE_READ_INSTANCE", args=[instance_id])) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def delete(self, runner_id): """ --- summary: Delete a runner parameters: - name: runner_id in: path required: true description: The ID of the Runner type: string responses: 200: description: List of runner states schema: $ref: '#/definitions/Runner' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Runners """ response = await self.client( Operation( operation_type="RUNNER_DELETE", kwargs={ "runner_id": runner_id, "remove": True }, )) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def get(self, instance_id): """ --- summary: Retrieve a specific Instance parameters: - name: instance_id in: path required: true description: The ID of the Instance type: string responses: 200: description: Instance with the given ID schema: $ref: '#/definitions/Instance' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Instances """ _ = self.get_or_raise(System, SYSTEM_READ, instances__id=instance_id) response = await self.client( Operation(operation_type="INSTANCE_READ", args=[instance_id])) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
def route(operation: Operation): """Entry point into the routing subsystem Args: operation: The operation to route Returns: """ operation = _pre_route(operation) logger.debug(f"Routing {operation!r}") if not operation.operation_type: raise RoutingRequestException("Missing operation type") if operation.operation_type not in route_functions.keys(): raise RoutingRequestException( f"Unknown operation type '{operation.operation_type}'") # Determine which garden the operation is targeting if not operation.target_garden_name: operation.target_garden_name = _determine_target_garden(operation) if not operation.target_garden_name: raise UnknownGardenException( f"Could not determine the target garden for routing {operation!r}") # If it's targeted at THIS garden, execute if operation.target_garden_name == config.get("garden.name"): return execute_local(operation) else: return initiate_forward(operation)
def _determine_target_garden(operation: Operation) -> str: """Determine the system the operation is targeting""" # Certain operations are ASSUMED to be targeted at the local garden if ("READ" in operation.operation_type or "JOB" in operation.operation_type or "FILE" in operation.operation_type or operation.operation_type in ("PLUGIN_LOG_RELOAD", "SYSTEM_CREATE", "SYSTEM_RESCAN") or "PUBLISH_EVENT" in operation.operation_type or "RUNNER" in operation.operation_type or operation.operation_type in ("PLUGIN_LOG_RELOAD", "SYSTEM_CREATE")): return config.get("garden.name") # Otherwise, each operation needs to be "parsed" if operation.operation_type in ("SYSTEM_RELOAD", "SYSTEM_UPDATE"): return _system_id_lookup(operation.args[0]) if operation.operation_type == "SYSTEM_DELETE": # Force deletes get routed to local garden if operation.kwargs.get("force"): return config.get("garden.name") return _system_id_lookup(operation.args[0]) if "INSTANCE" in operation.operation_type: if "system_id" in operation.kwargs and "instance_name" in operation.kwargs: return _system_id_lookup(operation.kwargs["system_id"]) else: return _instance_id_lookup(operation.args[0]) if operation.operation_type == "REQUEST_CREATE": target_system = System( namespace=operation.model.namespace, name=operation.model.system, version=operation.model.system_version, ) return _system_name_lookup(target_system) if operation.operation_type.startswith("REQUEST"): request = db.query_unique(Request, id=operation.args[0]) operation.kwargs["request"] = request return config.get("garden.name") if "GARDEN" in operation.operation_type: if operation.operation_type == "GARDEN_SYNC": sync_target = operation.kwargs.get("sync_target") if sync_target: return sync_target return config.get("garden.name") if operation.operation_type == "QUEUE_DELETE": # Need to deconstruct the queue name parts = operation.args[0].split(".") version = parts[2].replace("-", ".") return _system_name_lookup( System(namespace=parts[0], name=parts[1], version=version)) raise Exception(f"Bad operation type {operation.operation_type}")
def _pre_route(operation: Operation) -> Operation: """Called before any routing logic is applied""" # If no source garden is defined set it to the local garden if operation.source_garden_name is None: operation.source_garden_name = config.get("garden.name") if operation.operation_type == "REQUEST_CREATE": if operation.model.namespace is None: operation.model.namespace = config.get("garden.name") elif operation.operation_type == "SYSTEM_READ_ALL": if operation.kwargs.get("filter_params", {}).get("namespace") == "": operation.kwargs["filter_params"]["namespace"] = config.get( "garden.name") return operation
async def get(self, request_id): """ --- summary: Retrieve a specific Request parameters: - name: request_id in: path required: true description: The ID of the Request type: string responses: 200: description: Request with the given ID schema: $ref: '#/definitions/Request' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Requests """ _ = self.get_or_raise(Request, REQUEST_READ, id=request_id) response = await self.client( Operation(operation_type="REQUEST_READ", args=[request_id]) ) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def delete(self): """ --- summary: Delete a file parameters: - name: file_id in: query required: true description: The ID of the file type: string responses: 200: description: The file and all of its contents have been removed. schema: $ref: '#/definitions/FileStatus' 400: $ref: '#/definitions/400Error' 50x: $ref: '#/definitions/50xError' tags: - Files """ file_id = self.get_argument("file_id", default=None) if file_id is None: raise ValueError("Cannot delete a file without an id.") response = await self.client( Operation(operation_type="FILE_DELETE", args=[file_id])) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def get(self): """ --- summary: Get plugin logging configuration description: | Will return a Python logging configuration that can be used to configure plugin logging. parameters: - name: local in: query required: false description: Whether to request the local plugin logging configuration type: boolean default: false responses: 200: description: Logging Configuration for system 50x: $ref: '#/definitions/50xError' tags: - Logging """ local = self.get_query_argument("local", None) if local is None: local = False else: local = bool(local.lower() == "true") response = await self.client( Operation(operation_type="PLUGIN_LOG_READ", kwargs={"local": local})) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def get(self): """ --- summary: Get the plugin logging configuration deprecated: true parameters: - name: system_name in: query required: false description: UNUSED type: string responses: 200: description: Logging Configuration for system schema: $ref: '#/definitions/LoggingConfig' 50x: $ref: '#/definitions/50xError' tags: - Deprecated """ response = await self.client( Operation(operation_type="PLUGIN_LOG_READ_LEGACY")) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
def process(body) -> Tuple[str, dict]: """Processes a message body prior to sending We always want to send Operations. So if the given message is an Event we'll wrap it in an Operation. Args: body: the message body to process Returns: Tuple of the serialized message and headers dict """ many = isinstance(body, list) if body.__class__.__name__ == "Event": body = Operation(operation_type="PUBLISH_EVENT", model=body, model_type="Event") model_class = (body[0] if many else body).__class__.__name__ if not isinstance(body, str): body = SchemaParser.serialize(body, to_string=True, many=many) return body, {"model_class": model_class, "many": many}
async def delete(self, queue_name): """ --- summary: Clear a queue by canceling all requests parameters: - name: queue_name in: path required: true description: The name of the queue to clear type: string responses: 204: description: Queue successfully cleared 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Queues """ self.verify_user_permission_for_object(QUEUE_DELETE, local_garden()) await self.client( Operation(operation_type="QUEUE_DELETE", args=[queue_name])) self.set_status(204)
async def post(self): """ --- summary: Exports a list of Jobs from a list of IDs. description: | Jobs will be scheduled from a provided list to run on the intervals set in their trigger arguments. parameters: - name: ids in: body description: A list of the Jobs IDs whose job definitions should be \ exported. Omitting this parameter or providing an empty map will export \ all jobs. schema: $ref: '#/definitions/JobExport' responses: 201: description: A list of jobs has been exported. schema: type: array items: $ref: '#/definitions/JobImport' 400: $ref: '#/definitions/400Error' 50x: $ref: '#/definitions/50xError' tags: - Jobs """ filter_params_dict = {} permitted_objects_filter = self.permitted_objects_filter(Job, JOB_READ) # self.request_body is designed to return a 400 on a completely absent body # but we want to return all jobs if that's the case if len(self.request.body) > 0: decoded_body_as_dict = self.request_body if len(decoded_body_as_dict) > 0: # i.e. it has keys input_schema = JobExportInputSchema() validated_input_data_dict = input_schema.load( decoded_body_as_dict).data filter_params_dict["id__in"] = validated_input_data_dict["ids"] response_objects = await self.client( Operation( operation_type="JOB_READ_ALL", kwargs={ "q_filter": permitted_objects_filter, "filter_params": filter_params_dict, }, ), serialize_kwargs={"return_raw": True}, ) response = SchemaParser.serialize(response_objects, to_string=True, schema_name="JobExportSchema") self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
async def patch(self): """ --- summary: Partially update a Garden description: | The body of the request needs to contain a set of instructions detailing the updates to apply. Currently the only operations are: * sync ```JSON [ { "operation": "" } ] ``` parameters: - name: garden_name in: path required: true description: Garden to use type: string - name: patch in: body required: true description: Instructions for how to update the Garden schema: $ref: '#/definitions/Patch' responses: 200: description: Execute Patch action against Gardens schema: $ref: '#/definitions/Garden' 400: $ref: '#/definitions/400Error' 404: $ref: '#/definitions/404Error' 50x: $ref: '#/definitions/50xError' tags: - Garden """ patch = SchemaParser.parse_patch(self.request.decoded_body, from_string=True) for op in patch: operation = op.operation.lower() if operation == "sync": response = await self.client( Operation( operation_type="GARDEN_SYNC", ) ) else: raise ModelValidationError(f"Unsupported operation '{op.operation}'") self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
def test_publish_create_request(self): """Publish a Request over STOMP and verify it via HTTP.""" stomp_connection = self.create_stomp_connection() request_model = self.create_request("test_publish_create_request") sample_operation_request = Operation( operation_type="REQUEST_CREATE", model=request_model, model_type="Request", target_garden_name="docker", ) listener = MessageListener() stomp_connection.set_listener("", listener) stomp_connection.subscribe( destination="Beer_Garden_Events", id="event_listener", ack="auto", headers={ "subscription-type": "MULTICAST", "durable-subscription-name": "events", }, ) stomp_connection.send( body=SchemaParser.serialize_operation(sample_operation_request, to_string=True), headers={ "model_class": sample_operation_request.__class__.__name__, }, destination="Beer_Garden_Operations", ) time.sleep(10) requests = self.easy_client.find_requests() found_request = False print(len(requests)) for request in requests: print(SchemaParser.serialize_request(request, to_string=True)) if ("generated-by" in request.metadata and request.metadata["generated-by"] == "test_publish_create_request"): found_request = True break assert found_request assert listener.create_event_captured if stomp_connection.is_connected(): stomp_connection.disconnect()