def _pre_forward(operation: Operation) -> Operation:
"""Called before forwarding an operation"""
# Validate that the operation can be forwarded
if operation.operation_type not in routable_operations:
raise RoutingRequestException(
f"Operation type '{operation.operation_type}' can not be forwarded"
)
if operation.operation_type == "REQUEST_CREATE":
operation.model = (
beer_garden.requests.RequestValidator.instance().validate_request(
operation.model))
# Save the request so it'll have an ID and we'll have something to update
operation.model = db.create(operation.model)
# Clear parent before forwarding so the child doesn't freak out about an
# unknown request
operation.model.parent = None
operation.model.has_parent = False
beer_garden.files.forward_file(operation)
# Pull out and store the wait event, if it exists
wait_event = operation.kwargs.pop("wait_event", None)
if wait_event:
beer_garden.requests.request_map[operation.model.id] = wait_event
return operation
async def _generate_get_response(self, instance_id, start_line, end_line):
wait_event = Event()
response = await self.client(
Operation(
operation_type="INSTANCE_LOGS",
args=[instance_id],
kwargs={
"wait_event": wait_event,
"start_line": start_line,
"end_line": end_line,
},
),
serialize_kwargs={"to_string": False},
)
wait_timeout = float(self.get_argument("timeout", default="15"))
if wait_timeout < 0:
wait_timeout = None
if not await event_wait(wait_event, wait_timeout):
raise TimeoutExceededError("Timeout exceeded")
response = await self.client(
Operation(operation_type="REQUEST_READ", args=[response["id"]]),
serialize_kwargs={"to_string": False},
)
if response["status"] == "ERROR":
raise RequestProcessingError(response["output"])
async def patch(self):
"""
---
summary: Initiate administrative actions
description: |
The body of the request needs to contain a set of instructions
detailing the operations to perform.
Currently the supported operations are `rescan`:
```JSON
[
{ "operation": "rescan" }
]
```
* Will remove from the registry and database any currently stopped
plugins who's directory has been removed.
* Will add and start any new plugin directories.
And reloading the plugin logging configuration:
```JSON
[
{
"operation": "reload",
"path": "/config/logging/plugin"
}
]
```
parameters:
- name: patch
in: body
required: true
description: Instructions for operations
schema:
$ref: '#/definitions/Patch'
responses:
204:
description: Operation successfully initiated
50x:
$ref: '#/definitions/50xError'
tags:
- Admin
"""
self.verify_user_permission_for_object(GARDEN_UPDATE, local_garden())
operations = SchemaParser.parse_patch(
self.request.decoded_body, many=True, from_string=True
)
for op in operations:
if op.operation == "rescan":
await self.client(Operation(operation_type="RUNNER_RESCAN"))
elif op.operation == "reload":
if op.path == "/config/logging/plugin":
await self.client(Operation(operation_type="PLUGIN_LOG_RELOAD"))
else:
raise ModelValidationError(f"Unsupported path '{op.path}'")
else:
raise ModelValidationError(f"Unsupported operation '{op.operation}'")
self.set_status(204)
def forward_file(operation: Operation) -> None:
"""Send file data before forwarding an operation with a file parameter."""
# HEADS UP - THIS IS PROBABLY BROKEN
# import here bypasses circular dependency
import beer_garden.router as router
for file_id in _find_chunk_params(operation.model.parameters):
file = check_chunks(file_id)
args = [file.file_name, file.file_size, file.chunk_size]
# Make sure we get all of the other data
kwargs = _safe_build_object(
dict,
file,
ignore=[
"file_name",
"file_size",
"chunk_size",
],
upsert=True,
)
file_op = Operation(
operation_type="FILE_CREATE",
args=args,
kwargs=kwargs,
target_garden_name=operation.target_garden_name,
source_garden_name=operation.source_garden_name,
)
# This should put push the file operations before the current one
router.forward_processor.put(file_op)
for chunk_id in file.chunks.values():
chunk = check_chunk(chunk_id)
c_args = [chunk.file_id, chunk.offset, chunk.data]
c_kwargs = _safe_build_object(dict,
chunk,
ignore=["file_id", "offset", "data"],
upsert=True)
chunk_op = Operation(
operation_type="FILE_CHUNK",
args=c_args,
kwargs=c_kwargs,
target_garden_name=operation.target_garden_name,
source_garden_name=operation.source_garden_name,
)
# This should put push the file operations before the current one
router.forward_processor.put(chunk_op)
def process_wait(request: Request, timeout: float) -> Request:
"""Helper to process a request and wait for completion using a threading.Event
Args:
request: Request to create
timeout: Timeout used for wait
Returns:
The completed request
"""
# We need a better solution for this. Because the Request library is imported
# everywhere it causes issues when importing the router at the top because all of
# the functions are not initialized. So we either leave this as is, or move the
# requests import to the end of all of the files.
import beer_garden.router as router
req_complete = threading.Event()
# Send the request through the router to allow for commands to work across Gardens
created_request = router.route(
Operation(
operation_type="REQUEST_CREATE",
model=request,
model_type="Request",
kwargs={"wait_event": req_complete},
))
if not req_complete.wait(timeout):
raise TimeoutError(
"Request did not complete before the specified timeout")
return db.query_unique(Request, id=created_request.id)
async def get(self, garden_name):
"""
---
summary: Retrieve a specific Garden
parameters:
- name: garden_name
in: path
required: true
description: Read specific Garden Information
type: string
responses:
200:
description: Garden with the given garden_name
schema:
$ref: '#/definitions/Garden'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Garden
"""
response = await self.client(
Operation(operation_type="GARDEN_READ", args=[garden_name])
)
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
def test_listen_create_request(self):
"""Published the Request over HTTP and verifies of STOMP"""
stomp_connection = self.create_stomp_connection()
request_model = self.create_request("test_listen_create_request")
sample_operation_request = Operation(
operation_type="REQUEST_CREATE",
model=request_model,
model_type="Request",
)
listener = MessageListener()
stomp_connection.set_listener('', listener)
stomp_connection.subscribe(destination='Beer_Garden_Events',
id='event_listener',
ack='auto',
headers={
'subscription-type': 'MULTICAST',
'durable-subscription-name': 'events'
})
self.easy_client.forward(sample_operation_request)
time.sleep(10)
assert listener.create_event_captured
if stomp_connection.is_connected():
stomp_connection.disconnect()
async def get(self):
"""
---
summary: Get a list of all namespaces known to this garden
responses:
200:
description: List of Namespaces
50x:
$ref: '#/definitions/50xError'
tags:
- Namespace
"""
permitted_gardens = self.permissioned_queryset(Garden, GARDEN_READ)
permitted_requests = self.permissioned_queryset(Request, REQUEST_READ)
permitted_systems = self.permissioned_queryset(System, SYSTEM_READ)
response = await self.client(
Operation(
operation_type="NAMESPACE_READ_ALL",
kwargs={
"garden_queryset": permitted_gardens,
"system_queryset": permitted_systems,
"request_queryset": permitted_requests,
},
)
)
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def get(self):
"""
---
summary: Retrieve all Jobs.
responses:
200:
description: Successfully retrieved all systems.
schema:
type: array
items:
$ref: '#/definitions/Job'
50x:
$ref: '#/definitions/50xError'
tags:
- Jobs
"""
filter_params = {}
for key in self.request.arguments.keys():
if key in JobSchema.get_attribute_names():
filter_params[key] = self.get_query_argument(key)
response = await self.client(
Operation(operation_type="JOB_READ_ALL",
kwargs={"filter_params": filter_params}))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def delete(self, job_id):
"""
---
summary: Delete a specific Job.
description: Will remove a specific job. No further executions will occur.
parameters:
- name: job_id
in: path
required: true
description: The ID of the Job
type: string
responses:
204:
description: Job has been successfully deleted.
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Jobs
"""
await self.client(Operation(operation_type="JOB_DELETE",
args=[job_id]))
self.set_status(204)
async def get(self, job_id):
"""
---
summary: Retrieve a specific Job
parameters:
- name: job_id
in: path
required: true
description: The ID of the Job
type: string
responses:
200:
description: Job with the given ID
schema:
$ref: '#/definitions/Job'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Jobs
"""
response = await self.client(
Operation(operation_type="JOB_READ", args=[job_id]))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def delete(self, garden_name):
"""
---
summary: Delete a specific Garden
parameters:
- name: garden_name
in: path
required: true
description: Garden to use
type: string
responses:
204:
description: Garden has been successfully deleted
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Garden
"""
garden = self.get_or_raise(Garden, GARDEN_DELETE, name=garden_name)
await self.client(
Operation(operation_type="GARDEN_DELETE", args=[garden.name]))
self.set_status(204)
def _pre_forward(operation: Operation) -> Operation:
"""Called before forwarding an operation"""
# Validate that the operation can be forwarded
if operation.operation_type not in routable_operations:
raise RoutingRequestException(
f"Operation type '{operation.operation_type}' can not be forwarded"
)
if operation.operation_type == "REQUEST_CREATE":
# Save the request so it'll have an ID and we'll have something to update
local_request = create_request(operation.model)
if operation.model.namespace == config.get("garden.name"):
operation.model = local_request
else:
# When the target is a remote garden, just capture the id. We don't
# want to replace the entire model, as we'd lose the base64 encoded file
# parameter data.
operation.model.id = local_request.id
# Clear parent before forwarding so the child doesn't freak out about an
# unknown request
operation.model.parent = None
operation.model.has_parent = False
# Pull out and store the wait event, if it exists
wait_event = operation.kwargs.pop("wait_event", None)
if wait_event:
beer_garden.requests.request_map[operation.model.id] = wait_event
return operation
async def get(self, system_id, command_name):
"""
---
summary: Retrieve a specific Command
parameters:
- name: system_id
in: path
required: true
description: The ID of the System
type: string
- name: command_name
in: path
required: true
description: The name of the Command
type: string
responses:
200:
description: Command with the given name
schema:
$ref: '#/definitions/Command'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Commands
"""
response = await self.client(
Operation(operation_type="COMMAND_READ",
args=[system_id, command_name]))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def delete(self, instance_id):
"""
---
summary: Delete a specific Instance
parameters:
- name: instance_id
in: path
required: true
description: The ID of the Instance
type: string
responses:
204:
description: Instance has been successfully deleted
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Instances
"""
_ = self.get_or_raise(System, SYSTEM_UPDATE, instances__id=instance_id)
await self.client(
Operation(operation_type="INSTANCE_DELETE", args=[instance_id]))
self.set_status(204)
async def get(self, instance_id):
"""
---
summary: Retrieve queue information for instance
parameters:
- name: instance_id
in: path
required: true
description: The instance ID to pull queues for
type: string
responses:
200:
description: List of queue information objects for this instance
schema:
type: array
items:
$ref: '#/definitions/Queue'
50x:
$ref: '#/definitions/50xError'
tags:
- Queues
"""
_ = self.get_or_raise(System, QUEUE_READ, instances__id=instance_id)
response = await self.client(
Operation(operation_type="QUEUE_READ_INSTANCE",
args=[instance_id]))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def delete(self, runner_id):
"""
---
summary: Delete a runner
parameters:
- name: runner_id
in: path
required: true
description: The ID of the Runner
type: string
responses:
200:
description: List of runner states
schema:
$ref: '#/definitions/Runner'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Runners
"""
response = await self.client(
Operation(
operation_type="RUNNER_DELETE",
kwargs={
"runner_id": runner_id,
"remove": True
},
))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def get(self, instance_id):
"""
---
summary: Retrieve a specific Instance
parameters:
- name: instance_id
in: path
required: true
description: The ID of the Instance
type: string
responses:
200:
description: Instance with the given ID
schema:
$ref: '#/definitions/Instance'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Instances
"""
_ = self.get_or_raise(System, SYSTEM_READ, instances__id=instance_id)
response = await self.client(
Operation(operation_type="INSTANCE_READ", args=[instance_id]))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
def route(operation: Operation):
"""Entry point into the routing subsystem
Args:
operation: The operation to route
Returns:
"""
operation = _pre_route(operation)
logger.debug(f"Routing {operation!r}")
if not operation.operation_type:
raise RoutingRequestException("Missing operation type")
if operation.operation_type not in route_functions.keys():
raise RoutingRequestException(
f"Unknown operation type '{operation.operation_type}'")
# Determine which garden the operation is targeting
if not operation.target_garden_name:
operation.target_garden_name = _determine_target_garden(operation)
if not operation.target_garden_name:
raise UnknownGardenException(
f"Could not determine the target garden for routing {operation!r}")
# If it's targeted at THIS garden, execute
if operation.target_garden_name == config.get("garden.name"):
return execute_local(operation)
else:
return initiate_forward(operation)
def _determine_target_garden(operation: Operation) -> str:
"""Determine the system the operation is targeting"""
# Certain operations are ASSUMED to be targeted at the local garden
if ("READ" in operation.operation_type or "JOB" in operation.operation_type
or "FILE" in operation.operation_type or operation.operation_type
in ("PLUGIN_LOG_RELOAD", "SYSTEM_CREATE", "SYSTEM_RESCAN")
or "PUBLISH_EVENT" in operation.operation_type
or "RUNNER" in operation.operation_type or operation.operation_type
in ("PLUGIN_LOG_RELOAD", "SYSTEM_CREATE")):
return config.get("garden.name")
# Otherwise, each operation needs to be "parsed"
if operation.operation_type in ("SYSTEM_RELOAD", "SYSTEM_UPDATE"):
return _system_id_lookup(operation.args[0])
if operation.operation_type == "SYSTEM_DELETE":
# Force deletes get routed to local garden
if operation.kwargs.get("force"):
return config.get("garden.name")
return _system_id_lookup(operation.args[0])
if "INSTANCE" in operation.operation_type:
if "system_id" in operation.kwargs and "instance_name" in operation.kwargs:
return _system_id_lookup(operation.kwargs["system_id"])
else:
return _instance_id_lookup(operation.args[0])
if operation.operation_type == "REQUEST_CREATE":
target_system = System(
namespace=operation.model.namespace,
name=operation.model.system,
version=operation.model.system_version,
)
return _system_name_lookup(target_system)
if operation.operation_type.startswith("REQUEST"):
request = db.query_unique(Request, id=operation.args[0])
operation.kwargs["request"] = request
return config.get("garden.name")
if "GARDEN" in operation.operation_type:
if operation.operation_type == "GARDEN_SYNC":
sync_target = operation.kwargs.get("sync_target")
if sync_target:
return sync_target
return config.get("garden.name")
if operation.operation_type == "QUEUE_DELETE":
# Need to deconstruct the queue name
parts = operation.args[0].split(".")
version = parts[2].replace("-", ".")
return _system_name_lookup(
System(namespace=parts[0], name=parts[1], version=version))
raise Exception(f"Bad operation type {operation.operation_type}")
def _pre_route(operation: Operation) -> Operation:
"""Called before any routing logic is applied"""
# If no source garden is defined set it to the local garden
if operation.source_garden_name is None:
operation.source_garden_name = config.get("garden.name")
if operation.operation_type == "REQUEST_CREATE":
if operation.model.namespace is None:
operation.model.namespace = config.get("garden.name")
elif operation.operation_type == "SYSTEM_READ_ALL":
if operation.kwargs.get("filter_params", {}).get("namespace") == "":
operation.kwargs["filter_params"]["namespace"] = config.get(
"garden.name")
return operation
async def get(self, request_id):
"""
---
summary: Retrieve a specific Request
parameters:
- name: request_id
in: path
required: true
description: The ID of the Request
type: string
responses:
200:
description: Request with the given ID
schema:
$ref: '#/definitions/Request'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Requests
"""
_ = self.get_or_raise(Request, REQUEST_READ, id=request_id)
response = await self.client(
Operation(operation_type="REQUEST_READ", args=[request_id])
)
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def delete(self):
"""
---
summary: Delete a file
parameters:
- name: file_id
in: query
required: true
description: The ID of the file
type: string
responses:
200:
description: The file and all of its contents have been removed.
schema:
$ref: '#/definitions/FileStatus'
400:
$ref: '#/definitions/400Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Files
"""
file_id = self.get_argument("file_id", default=None)
if file_id is None:
raise ValueError("Cannot delete a file without an id.")
response = await self.client(
Operation(operation_type="FILE_DELETE", args=[file_id]))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def get(self):
"""
---
summary: Get plugin logging configuration
description: |
Will return a Python logging configuration that can be used to configure
plugin logging.
parameters:
- name: local
in: query
required: false
description: Whether to request the local plugin logging configuration
type: boolean
default: false
responses:
200:
description: Logging Configuration for system
50x:
$ref: '#/definitions/50xError'
tags:
- Logging
"""
local = self.get_query_argument("local", None)
if local is None:
local = False
else:
local = bool(local.lower() == "true")
response = await self.client(
Operation(operation_type="PLUGIN_LOG_READ",
kwargs={"local": local}))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def get(self):
"""
---
summary: Get the plugin logging configuration
deprecated: true
parameters:
- name: system_name
in: query
required: false
description: UNUSED
type: string
responses:
200:
description: Logging Configuration for system
schema:
$ref: '#/definitions/LoggingConfig'
50x:
$ref: '#/definitions/50xError'
tags:
- Deprecated
"""
response = await self.client(
Operation(operation_type="PLUGIN_LOG_READ_LEGACY"))
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
def process(body) -> Tuple[str, dict]:
"""Processes a message body prior to sending
We always want to send Operations. So if the given message is an Event we'll wrap
it in an Operation.
Args:
body: the message body to process
Returns:
Tuple of the serialized message and headers dict
"""
many = isinstance(body, list)
if body.__class__.__name__ == "Event":
body = Operation(operation_type="PUBLISH_EVENT",
model=body,
model_type="Event")
model_class = (body[0] if many else body).__class__.__name__
if not isinstance(body, str):
body = SchemaParser.serialize(body, to_string=True, many=many)
return body, {"model_class": model_class, "many": many}
async def delete(self, queue_name):
"""
---
summary: Clear a queue by canceling all requests
parameters:
- name: queue_name
in: path
required: true
description: The name of the queue to clear
type: string
responses:
204:
description: Queue successfully cleared
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Queues
"""
self.verify_user_permission_for_object(QUEUE_DELETE, local_garden())
await self.client(
Operation(operation_type="QUEUE_DELETE", args=[queue_name]))
self.set_status(204)
async def post(self):
"""
---
summary: Exports a list of Jobs from a list of IDs.
description: |
Jobs will be scheduled from a provided list to run on the intervals
set in their trigger arguments.
parameters:
- name: ids
in: body
description: A list of the Jobs IDs whose job definitions should be \
exported. Omitting this parameter or providing an empty map will export \
all jobs.
schema:
$ref: '#/definitions/JobExport'
responses:
201:
description: A list of jobs has been exported.
schema:
type: array
items:
$ref: '#/definitions/JobImport'
400:
$ref: '#/definitions/400Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Jobs
"""
filter_params_dict = {}
permitted_objects_filter = self.permitted_objects_filter(Job, JOB_READ)
# self.request_body is designed to return a 400 on a completely absent body
# but we want to return all jobs if that's the case
if len(self.request.body) > 0:
decoded_body_as_dict = self.request_body
if len(decoded_body_as_dict) > 0: # i.e. it has keys
input_schema = JobExportInputSchema()
validated_input_data_dict = input_schema.load(
decoded_body_as_dict).data
filter_params_dict["id__in"] = validated_input_data_dict["ids"]
response_objects = await self.client(
Operation(
operation_type="JOB_READ_ALL",
kwargs={
"q_filter": permitted_objects_filter,
"filter_params": filter_params_dict,
},
),
serialize_kwargs={"return_raw": True},
)
response = SchemaParser.serialize(response_objects,
to_string=True,
schema_name="JobExportSchema")
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
async def patch(self):
"""
---
summary: Partially update a Garden
description: |
The body of the request needs to contain a set of instructions detailing the
updates to apply. Currently the only operations are:
* sync
```JSON
[
{ "operation": "" }
]
```
parameters:
- name: garden_name
in: path
required: true
description: Garden to use
type: string
- name: patch
in: body
required: true
description: Instructions for how to update the Garden
schema:
$ref: '#/definitions/Patch'
responses:
200:
description: Execute Patch action against Gardens
schema:
$ref: '#/definitions/Garden'
400:
$ref: '#/definitions/400Error'
404:
$ref: '#/definitions/404Error'
50x:
$ref: '#/definitions/50xError'
tags:
- Garden
"""
patch = SchemaParser.parse_patch(self.request.decoded_body, from_string=True)
for op in patch:
operation = op.operation.lower()
if operation == "sync":
response = await self.client(
Operation(
operation_type="GARDEN_SYNC",
)
)
else:
raise ModelValidationError(f"Unsupported operation '{op.operation}'")
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
def test_publish_create_request(self):
"""Publish a Request over STOMP and verify it via HTTP."""
stomp_connection = self.create_stomp_connection()
request_model = self.create_request("test_publish_create_request")
sample_operation_request = Operation(
operation_type="REQUEST_CREATE",
model=request_model,
model_type="Request",
target_garden_name="docker",
)
listener = MessageListener()
stomp_connection.set_listener("", listener)
stomp_connection.subscribe(
destination="Beer_Garden_Events",
id="event_listener",
ack="auto",
headers={
"subscription-type": "MULTICAST",
"durable-subscription-name": "events",
},
)
stomp_connection.send(
body=SchemaParser.serialize_operation(sample_operation_request,
to_string=True),
headers={
"model_class": sample_operation_request.__class__.__name__,
},
destination="Beer_Garden_Operations",
)
time.sleep(10)
requests = self.easy_client.find_requests()
found_request = False
print(len(requests))
for request in requests:
print(SchemaParser.serialize_request(request, to_string=True))
if ("generated-by" in request.metadata
and request.metadata["generated-by"]
== "test_publish_create_request"):
found_request = True
break
assert found_request
assert listener.create_event_captured
if stomp_connection.is_connected():
stomp_connection.disconnect()
