def put(request: Request, headers: dict = None, **kwargs) -> None: """Put a Request on a queue If a routing_key is specified in the kwargs, it will be used. If not, system and instance info on the Request will be used, along with the ``is_admin`` kwarg. If the Request has an ID it will be added to the headers as 'request_id'. Args: request: The Request to publish headers: Headers to use when publishing **kwargs: is_admin: Will be passed to get_routing_key Other arguments will be passed to the client publish method Returns: None """ kwargs["headers"] = headers or {} if request.id: kwargs["headers"]["request_id"] = request.id if "routing_key" not in kwargs: kwargs["routing_key"] = get_routing_key( request.namespace, request.system, request.system_version, request.instance_name, is_admin=kwargs.get("is_admin", False), ) clients["pika"].publish(SchemaParser.serialize_request(request), **kwargs)
def post_bg_request(base_url, bg_request): return HTTPRequest( base_url + "/api/v1/requests/", method="POST", headers=RestClient.JSON_HEADERS, body=SchemaParser.serialize_request(bg_request, to_string=True), )
def on_message_callback_complete(self, basic_deliver, future): """Invoked when the future returned by _on_message_callback completes. :param pika.Spec.Basic.Deliver basic_deliver: basic_deliver method :param concurrent.futures.Future future: Completed future :return: None """ delivery_tag = basic_deliver.delivery_tag if not future.exception(): try: self.logger.debug('Acking message %s', delivery_tag) self._channel.basic_ack(delivery_tag) except Exception as ex: self.logger.exception( 'Error acking message %s, about to shut down: %s', delivery_tag, ex) self._panic_event.set() else: real_ex = future.exception() if isinstance(real_ex, RepublishRequestException): try: with BlockingConnection(self._connection_parameters) as c: headers = real_ex.headers headers.update({'request_id': real_ex.request.id}) props = pika.BasicProperties( app_id='beer-garden', content_type='text/plain', headers=headers, priority=1, ) c.channel().basic_publish( exchange=basic_deliver.exchange, properties=props, routing_key=basic_deliver.routing_key, body=SchemaParser.serialize_request( real_ex.request)) self._channel.basic_ack(delivery_tag) except Exception as ex: self.logger.exception( 'Error republishing message %s, about to shut down: %s', delivery_tag, ex) self._panic_event.set() elif isinstance(real_ex, DiscardMessageException): self.logger.info( 'Nacking message %s, not attempting to requeue', delivery_tag) self._channel.basic_nack(delivery_tag, requeue=False) else: # If request processing throws anything else we terminate self.logger.exception( 'Unexpected exception during request %s processing, about ' 'to shut down: %s', delivery_tag, real_ex, exc_info=False) self._panic_event.set()
def test_publish_create_request(self): """Publish a Request over STOMP and verify it via HTTP.""" stomp_connection = self.create_stomp_connection() request_model = self.create_request("test_publish_create_request") sample_operation_request = Operation( operation_type="REQUEST_CREATE", model=request_model, model_type="Request", target_garden_name="docker", ) listener = MessageListener() stomp_connection.set_listener("", listener) stomp_connection.subscribe( destination="Beer_Garden_Events", id="event_listener", ack="auto", headers={ "subscription-type": "MULTICAST", "durable-subscription-name": "events", }, ) stomp_connection.send( body=SchemaParser.serialize_operation(sample_operation_request, to_string=True), headers={ "model_class": sample_operation_request.__class__.__name__, }, destination="Beer_Garden_Operations", ) time.sleep(10) requests = self.easy_client.find_requests() found_request = False print(len(requests)) for request in requests: print(SchemaParser.serialize_request(request, to_string=True)) if ("generated-by" in request.metadata and request.metadata["generated-by"] == "test_publish_create_request"): found_request = True break assert found_request assert listener.create_event_captured if stomp_connection.is_connected(): stomp_connection.disconnect()
def publish_request(self, request, **kwargs): if "headers" not in kwargs: kwargs["headers"] = {} kwargs["headers"]["request_id"] = str(request.id) if "routing_key" not in kwargs: kwargs["routing_key"] = get_routing_key(request.system, request.system_version, request.instance_name) return self.publish(SchemaParser.serialize_request(request), **kwargs)
def test_publish_create_request(self): """Published the Request over STOMP and verifies of HTTP""" stomp_connection = self.create_stomp_connection() request_model = self.create_request("test_publish_create_request") sample_operation_request = Operation( operation_type="REQUEST_CREATE", model=request_model, model_type="Request", ) listener = MessageListener() stomp_connection.set_listener('', listener) stomp_connection.subscribe(destination='Beer_Garden_Events', id='event_listener', ack='auto', headers={ 'subscription-type': 'MULTICAST', 'durable-subscription-name': 'events' }) stomp_connection.send( body=SchemaParser.serialize_operation(sample_operation_request, to_string=True), headers={ "model_class": sample_operation_request.__class__.__name__, }, destination="Beer_Garden_Operations", ) time.sleep(10) requests = self.easy_client.find_requests() found_request = False print(len(requests)) for request in requests: print(SchemaParser.serialize_request(request, to_string=True)) if "generated-by" in request.metadata and request.metadata[ "generated-by"] == "test_publish_create_request": found_request = True break assert found_request assert listener.create_event_captured if stomp_connection.is_connected(): stomp_connection.disconnect()
def create_request(self, request, **kwargs): """Create a new Request Args: request: New request definition **kwargs: Extra request parameters Keyword Args: blocking (bool): Wait for request to complete before returning timeout (int): Maximum seconds to wait for completion Returns: Request: The newly-created Request """ return self.client.post_requests( SchemaParser.serialize_request(request), **kwargs)
async def post(self): """ --- summary: Create a new Request parameters: - name: request in: body description: The Request definition schema: $ref: '#/definitions/Request' - name: blocking in: query required: false description: Flag indicating whether to wait for request completion type: boolean default: false - name: timeout in: query required: false description: Max seconds to wait for request completion. (-1 = wait forever) type: float default: -1 - name: request in: formData required: true description: | For multipart/form-data requests (required when uploading a a file as a parameter) this field acts the same as the request (body) parameter and should be formatted per that field's definition. type: object - name: file_upload in: formData required: false type: file description: | A file to upload for use as input to a "bytes" type request parameter. NOTE: The name of the field in the submitted form data should match the name of the actual "bytes" type field that the command being tasked is expecting. The "file_upload" name is just a stand-in, since the actual expected name will vary for each command. consumes: - application/json - application/x-www-form-urlencoded - multipart/form-data responses: 201: description: A new Request has been created schema: $ref: '#/definitions/Request' headers: Instance-Status: type: string description: | Current status of the Instance that will process the created Request 400: $ref: '#/definitions/400Error' 50x: $ref: '#/definitions/50xError' tags: - Requests """ if self.request.mime_type == "application/json": request_model = self.parser.parse_request( self.request.decoded_body, from_string=True ) elif self.request.mime_type == "application/x-www-form-urlencoded": request_model = self._parse_form_request() elif self.request.mime_type == "multipart/form-data": request_model = self._parse_multipart_form_data() else: raise ModelValidationError("Unsupported or missing content-type header") self.verify_user_permission_for_object(REQUEST_CREATE, request_model) if self.current_user: request_model.requester = self.current_user.username wait_event = None if self.get_argument("blocking", default="").lower() == "true": wait_event = Event() # Also don't publish latency measurements self.request.ignore_latency = True try: created_request = await self.client( Operation( operation_type="REQUEST_CREATE", model=request_model, model_type="Request", kwargs={"wait_event": wait_event}, ), serialize_kwargs={"to_string": False}, ) except UnknownGardenException as ex: req_system = BrewtilsSystem( namespace=request_model.namespace, name=request_model.system, version=request_model.system_version, ) raise ModelValidationError( f"Could not find a garden containing system {req_system}" ) from ex # Wait for the request to complete, if requested if wait_event: wait_timeout = float(self.get_argument("timeout", default="-1")) if wait_timeout < 0: wait_timeout = None if not await event_wait(wait_event, wait_timeout): raise TimeoutExceededError("Timeout exceeded") # Reload to get the completed request response = await self.client( Operation(operation_type="REQUEST_READ", args=[created_request["id"]]) ) else: # We don't want to echo back the base64 encoding of any file parameters remove_bytes_parameter_base64(created_request["parameters"], False) response = SchemaParser.serialize_request(created_request) self.set_status(201) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
def finish_message(self, basic_deliver, future): """Finish processing a message This should be invoked as the final part of message processing. It's responsible for acking / nacking messages back to the broker. The main complexity here depends on whether the request processing future has an exception: - If there is no exception it acks the message - If there is an exception: - If the exception is an instance of DiscardMessageException it nacks the message and does not requeue it - If the exception is an instance of RepublishRequestException it will construct an entirely new BlockingConnection, use that to publish a new message, and then ack the original message - If the exception is not an instance of either the panic_event is set and the consumer will self-destruct Also, if there's ever an error acking a message the panic_event is set and the consumer will self-destruct. Args: basic_deliver: future: Completed future Returns: None """ delivery_tag = basic_deliver.delivery_tag if not future.exception(): try: self.logger.debug("Acking message %s", delivery_tag) self._channel.basic_ack(delivery_tag) except Exception as ex: self.logger.exception( "Error acking message %s, about to shut down: %s", delivery_tag, ex) self._panic_event.set() else: real_ex = future.exception() if isinstance(real_ex, RepublishRequestException): try: with BlockingConnection(self._connection_parameters) as c: headers = real_ex.headers headers.update({"request_id": real_ex.request.id}) props = BasicProperties( app_id="beer-garden", content_type="text/plain", headers=headers, priority=1, delivery_mode=PERSISTENT_DELIVERY_MODE, ) c.channel().basic_publish( exchange=basic_deliver.exchange, properties=props, routing_key=basic_deliver.routing_key, body=SchemaParser.serialize_request( real_ex.request), ) self._channel.basic_ack(delivery_tag) except Exception as ex: self.logger.exception( "Error republishing message %s, about to shut down: %s", delivery_tag, ex, ) self._panic_event.set() elif isinstance(real_ex, DiscardMessageException): self.logger.info( "Nacking message %s, not attempting to requeue", delivery_tag) self._channel.basic_nack(delivery_tag, requeue=False) else: # If request processing throws anything else we terminate self.logger.exception( "Unexpected exception during request %s processing, about " "to shut down: %s", delivery_tag, real_ex, exc_info=False, ) self._panic_event.set()
async def post(self): """ --- summary: Create a new Request parameters: - name: request in: body description: The Request definition schema: $ref: '#/definitions/Request' - name: blocking in: query required: false description: Flag indicating whether to wait for request completion type: boolean default: false - name: timeout in: query required: false description: Max seconds to wait for request completion. (-1 = wait forever) type: float default: -1 consumes: - application/json - application/x-www-form-urlencoded responses: 201: description: A new Request has been created schema: $ref: '#/definitions/Request' headers: Instance-Status: type: string description: | Current status of the Instance that will process the created Request 400: $ref: '#/definitions/400Error' 50x: $ref: '#/definitions/50xError' tags: - Requests """ if self.request.mime_type == "application/json": request_model = self.parser.parse_request( self.request.decoded_body, from_string=True ) elif self.request.mime_type == "application/x-www-form-urlencoded": request_model = self._parse_form_request() else: raise ModelValidationError("Unsupported or missing content-type header") if self.current_user: request_model.requester = self.current_user.username wait_event = None if self.get_argument("blocking", default="").lower() == "true": wait_event = Event() # Also don't publish latency measurements self.request.ignore_latency = True created_request = await self.client( Operation( operation_type="REQUEST_CREATE", model=request_model, model_type="Request", kwargs={"wait_event": wait_event}, ), serialize_kwargs={"to_string": False}, ) # Wait for the request to complete, if requested if wait_event: wait_timeout = float(self.get_argument("timeout", default="-1")) if wait_timeout < 0: wait_timeout = None if not await event_wait(wait_event, wait_timeout): raise TimeoutExceededError("Timeout exceeded") # Reload to get the completed request response = await self.client( Operation(operation_type="REQUEST_READ", args=[created_request["id"]]) ) else: response = SchemaParser.serialize_request(created_request) self.set_status(201) self.set_header("Content-Type", "application/json; charset=UTF-8") self.write(response)
def test_success(self, processor, bg_request): serialized = SchemaParser.serialize_request(bg_request) assert_request_equal(processor._parse(serialized), bg_request)