Esempio n. 1
0
    def handle_error(
        self,
        _error,
        _category=EC_COMMUNICATION,
        _severity=SEV_ERROR,
        _web_socket=None,
        _close_socket=None,
        _message_to_reply_to=None,
    ):
        """

        :param _error: The error message
        :param _category: The category of the error
        :param _severity: Error severity
        :param _web_socket: If the sender was external, the web socket
        :param _close_socket: Close the web socket
        :param _message_to_reply_to: The source message, if set, the error message is sent to the sender
        :return:
        """
        """ Handles and logs errors, decides whether to keep the connection open if an error occurrs


        """
        _error = write_to_log(
            _data=self.log_prefix + "In handler.handle_error error with error :" + str(_error),
            _category=_category,
            _severity=_severity,
        )
        if _web_socket:

            if _message_to_reply_to:
                _web_socket.send_message(reply_with_error_message(self, _message_to_reply_to, _error))

            if _close_socket:
                _web_socket.close(code=PROTOCOL_ERROR, reason=_error)
Esempio n. 2
0
    def handle_message(self, _source_web_socket, _message_data):
        """
        Handle inbound and outbound messages.
        :param _source_web_socket: The web socket of the source, if none, it is outbound
        :param _message_data: The message data
        """

        if _source_web_socket is None:
            # This is an outbound message

            self.outbound_message_count += 1
            # TODO: Handle multiple broker peers(other peers?), be informed trough some kind of messaging.(PROD-25)
            # For now, however, always have one broker.
            _message_data["source"] = self.address

            self.send_to_address(self.broker_address, _message_data)

        else:
            self.inbound_message_count += 1
            # This is an inbound message
            try:
                self.schema_tools.validate(_message_data)
            except Exception as e:
                _error = "An error occurred validating an inbound message:" + str(e)
                write_to_log(_error, _category=EC_COMMUNICATION, _severity=SEV_ERROR)
                # Respond to sender with an error message
                self.send_to_address(self.broker_address, reply_with_error_message(self, _message_data, _error))
            else:
                self.process_handler.forward_message(_message_data)
Esempio n. 3
0
    def handle_message(self, _source_web_socket, _message_data):
        """
        Handle inbound and outbound messages.
        :param _source_web_socket: The web socket of the source, if none, it is outbound
        :param _message_data: The message data
        """

        if _source_web_socket is None:
            # This is an outbound message

            self.outbound_message_count += 1
            # TODO: Handle multiple broker peers(other peers?), be informed trough some kind of messaging.(PROD-25)
            # For now, however, always have one broker.
            _message_data["source"] = self.address

            self.send_to_address(self.broker_address, _message_data)

        else:
            self.inbound_message_count += 1
            # This is an inbound message
            try:
                self.schema_tools.validate(_message_data)
            except Exception as e:
                _error = "An error occurred validating an inbound message:" + str(
                    e)
                write_to_log(_error,
                             _category=EC_COMMUNICATION,
                             _severity=SEV_ERROR)
                # Respond to sender with an error message
                self.send_to_address(
                    self.broker_address,
                    reply_with_error_message(self, _message_data, _error))
            else:
                self.process_handler.forward_message(_message_data)
Esempio n. 4
0
    def handle(self, _message):
        """
        Handles all incoming messages to the process handler on the monitor queue.

        :param _message: A message
        """
        _message_data = _message
        self.write_dbg_info("Worker process got a message:" + str(_message_data))

        if _message_data["schemaRef"] == "ref://bpm.message.bpm.process.start":
            # The message is a process start message,start a process.
            if self.job_running:
                # If the worker has a job currently running, reply with an error message and do nothing
                # TODO: This message should probably be handled in some way by the agent (PROD-28)
                self.send_queue.put([None, reply_with_error_message(self, _message_data,
                                                                    "Worker.handle_message: Cannot start BPM Process, process " + str(
                                                                        self.bpm_process_id) + " already running.")])
            else:
                self.write_dbg_info("Starting BPM process")
                self.start_bpm_process(_message=_message_data)

        elif _message_data["schemaRef"] == "ref://bpm.message.bpm.process.command":
            # The message is a command to the bpm process
            self.write_dbg_info("Got a BPM process control message:" + _message_data["command"])
            if _message_data["command"] == "stop":
                # Told to stop. Stopping BPM process.
                self.terminate_bpm_process(_message_data)

                # TODO: The broker should be able to offer the users the option to retry running the process (PROD-28)

                # TODO: Implement "pause" (PROD-29)

        elif _message_data["schemaRef"] == "ref://bpm.message.worker.process_command":
            # The message is a command to the actual worker process
            if _message_data["command"] == "stop":
                # Told to stop the worker. Stopping process.

                self.write_dbg_info("Told to stop the worker.")
                # First stop the monitor, we don't want any more commands coming in.
                self.monitor.stop()
                self.write_dbg_info("Terminate BPM thread")
                self.terminate_bpm_process(_message_data)

                self.write_dbg_info("Worker monitor stopped.")
                self.send_queue.put([None, log_process_state_message(
                    _changed_by=message_is_none(_message_data, "userId", "No user mentioned"),
                    _state="stopped",
                    _process_id=self.process_id,
                    _reason=message_is_none(_message_data, "reason", "No reason mentioned"))])
                self.write_dbg_info("Reported stopping to process handler")
                self.terminated = True
            else:
                self.write_dbg_info("Unhandled command " + _message_data["command"])

        elif _message_data["schemaRef"] == "ref://of.message":
            #: TODO: Figure out if processes should just ever get general messages, and if so, how to handle?(PROD-30)
            self.message = _message_data
            self.write_dbg_info("Got a message:" + self.message)
Esempio n. 5
0
 def handle_error(self, _error_message, _message):
     """
     A generic function for handling errors
     :param _error: The error message
     """
     write_to_log(
         "An error occured, replying with an error message. Error: " +
         str(_error_message),
         _category=EC_SERVICE,
         _severity=SEV_ERROR)
     self.message_monitor.queue.put(
         [None,
          reply_with_error_message(self, _message, _error_message)])
Esempio n. 6
0
    def received_message(self, message):
        """
        Receives a message from a peer and puts it on the queue

        :param message: The message
        """


        if str(message) != "":
            self.write_dbg_info("Got this message(putting on queue):" + str(message))
            if isinstance(message, TextMessage):
                monitor.queue.put([self, json.loads(str(message))])
            else:
                monitor.queue.put([self, json.loads(message)])
        else:
            self.send_message(reply_with_error_message(_runtime_instance=os.getpid(),
                                                       _error_message="Cannot send empty messages to agent",
                                                       _message={}))
Esempio n. 7
0
    def handle_control(self, _web_socket, _message_data):
        """
        Handle control messages, put them on the correct queue
        :param _web_socket: Yet unused in this category
        :param _message_data: The control message
        """
        if _web_socket.address != self.broker_address:
            self.send_to_address(_message_data["source"],
                     reply_with_error_message(
                         _message_data,
                         "Control messages can only originate from the broker.")
                     )
            return

        self.schema_tools.validate(_message_data)

        if _message_data["schemaRef"] == "ref://bpm.message.worker.process_command":
            self.process_handler.forward_message(_message_data)
        else:
            self.control_queue.put(_message_data)
Esempio n. 8
0
    def handle_control(self, _web_socket, _message_data):
        """
        Handle control messages, put them on the correct queue
        :param _web_socket: Yet unused in this category
        :param _message_data: The control message
        """
        if _web_socket.address != self.broker_address:
            self.send_to_address(
                _message_data["source"],
                reply_with_error_message(
                    _message_data,
                    "Control messages can only originate from the broker."))
            return

        self.schema_tools.validate(_message_data)

        if _message_data[
                "schemaRef"] == "ref://bpm.message.worker.process_command":
            self.process_handler.forward_message(_message_data)
        else:
            self.control_queue.put(_message_data)
Esempio n. 9
0
    def shut_down(self, _user_id):
        """
        Shuts down the worker handler and all jobs.
        :param _user_id:
        :return:
        """
        try:

            """
            TODO: First tell broker and incoming message handle we are shutting down.
            Broker should then remove agent from destinations for messaging

            Move all available workers processes to temporary list, shut them all down
            All queued jobs should be unqueued and be returned to the broker to be run elsewhere
            Pause(means do not run next step all running jobs and send states to broker where they should be queued or
            continued elsewhere

            Wait n seconds for remaining jobs to complete
            Pause all running jobs and save state to broker
            n seconds for remaining and queued jobs to complete


            """
            self.write_dbg_info("Shutting down process handler")

            self.write_dbg_info("Telling BPM processes to stop")
            # Send process control messages to all bpm processes
            for _process_id, _process in self.busy_workers.items():
                _process["queue"].put(bpm_process_control(_destination="",
                                                          _destination_process_id=_process_id,
                                                          _command="stop",
                                                          _reason="Shutting down agent",
                                                          _message_id=0,
                                                          _source="",
                                                          _source_process_id=self.process_id,
                                                          _user_id=_user_id))
                self.write_dbg_info(str(_process_id) + " told to stop.")

            self.write_dbg_info("BPM processes told to shut down, waiting a bit..")

            self.write_dbg_info("Clearing job queue")
            # Loop through job queue, remove all items, tell broker
            while True:
                try:
                    _item = self.job_queue.get_nowait()
                    self.write_dbg_info("Job in queue:" + str(_item))
                    self.message_monitor.queue.put([None,
                                                    reply_with_error_message(self, _item,
                                                                             "Unqueued, agent shutting down.")]
                                                   )
                except queue.Empty:
                    break

            _check_count = 3
            while len(self.busy_workers) > 0 and _check_count > 0:
                self.write_dbg_info("Still running BPM processes, waiting another second before stopping "
                                      "the remaining workers. Times left:" + str(_check_count))
                time.sleep(1)
                _check_count -= 1

            self.write_dbg_info("Telling worker processes to shut down")

            # Tell workers to shut down
            for _process in self.workers.values():
                self.write_dbg_info("Telling worker process " + _process["processId"] + " to stop")
                _process["queue"].put(worker_process_control(_destination="",
                                                             _destination_process_id=_process["processId"],
                                                             _command="stop",
                                                             _reason="Shutting down agent",
                                                             _message_id=0,
                                                             _source="",
                                                             _source_process_id=self.process_id,
                                                             _user_id=_user_id))

            time.sleep(.1)
            # Kill the remaining processes

            for _process in list(self.workers.values()):
                self.write_dbg_info("Killing unresponsive process " + _process["processId"] + " (pid:" + str(
                    _process["pid"]) + ")")

                _process["process"].terminate()
                self.message_monitor.queue.put([None, log_process_state_message(_changed_by=_user_id,
                                                                                _state="killed",
                                                                                _process_id=_process["processId"],
                                                                                _reason="Unresponsive, killed by agent")
                                                ])
                del self.workers[_process["processId"]]
        except Exception as e:
            self.write_dbg_info("Failed to properly shut down, error:" + str(e))
Esempio n. 10
0
def on_process_start(_handler, _message_data):
    print("Control handler mockup, putting error on queue...")
    _handler.message_monitor.queue.put([None, reply_with_error_message(_handler, _message_data, "Mockup error.")])