def on_destination_process_control_message(web_socket, message):
print("\non_destination_process_control_message: " + str(message))
if message['schemaRef'] == "ref://bpm.message.bpm.process.start":
# This is the first start process, respond with process instance
web_socket.context.process_id = message['processId']
web_socket.received_message(json.dumps(
{
"processId": web_socket.context.process_id,
"spawnedBy": web_socket.context.user["_id"],
"spawnedWhen": str(datetime.datetime.utcnow()),
"name": "Test_process_name",
"processDefinitionId": message["processDefinitionId"],
"schemaRef": "ref://bpm.process.bpm"
})
)
for _state in ["running", "paused", "stopped", "killed", "failed", "finished"]:
print("Send :" + _state)
web_socket.received_message(json.dumps(log_process_state_message(_changed_by=web_socket.context.user["_id"],
_state= _state, _reason="testing",
_process_id=web_socket.context.process_id)
))
web_socket.received_message(json.dumps(
{
"destination": "source_peer" ,
"processId": web_socket.context.process_id,
"schemaRef": "ref://bpm.message.bpm.process.result",
"sourceProcessId": web_socket.context.process_id,
"messageId": getNextMessageId(),
"source": "destination_peer",
"globals" : {"context": "context"},
"result" : {"result": "result"}
})
)
def handle_bpm_process_start(self, _message_data):
"""
Handle and act on BPM process start messages.
:param _message_data: The start message
"""
self.write_dbg_info("BPM Process(definition: " + _message_data[
"processDefinitionId"] + "): Looking for a worker.")
_worker = self.acquire_worker(_message_data["processId"])
if _worker:
# Add the message to its queue
self.write_dbg_info("BPM Process " + _message_data["processId"] +
": Putting the message on the workers' queue.")
_worker["queue"].put(_message_data)
else:
# There was no available worker, put the job on the queue
self.job_queue.put(_message_data)
self.message_monitor.queue.put([None,
log_process_state_message(
_changed_by=zero_object_id,
_state="queued",
_process_id=_message_data["processId"],
_reason="Queued until a worker becomes available")]
)
self.write_dbg_info("BPM Process(definition: " + _message_data[
"processDefinitionId"] + "): Queued for execution when a worker becomes available. "
"Max worker count limit reached.")
def handle(self, _message):
"""
Handles all incoming messages to the process handler on the monitor queue.
:param _message: A message
"""
_message_data = _message
self.write_dbg_info("Worker process got a message:" + str(_message_data))
if _message_data["schemaRef"] == "ref://bpm.message.bpm.process.start":
# The message is a process start message,start a process.
if self.job_running:
# If the worker has a job currently running, reply with an error message and do nothing
# TODO: This message should probably be handled in some way by the agent (PROD-28)
self.send_queue.put([None, reply_with_error_message(self, _message_data,
"Worker.handle_message: Cannot start BPM Process, process " + str(
self.bpm_process_id) + " already running.")])
else:
self.write_dbg_info("Starting BPM process")
self.start_bpm_process(_message=_message_data)
elif _message_data["schemaRef"] == "ref://bpm.message.bpm.process.command":
# The message is a command to the bpm process
self.write_dbg_info("Got a BPM process control message:" + _message_data["command"])
if _message_data["command"] == "stop":
# Told to stop. Stopping BPM process.
self.terminate_bpm_process(_message_data)
# TODO: The broker should be able to offer the users the option to retry running the process (PROD-28)
# TODO: Implement "pause" (PROD-29)
elif _message_data["schemaRef"] == "ref://bpm.message.worker.process_command":
# The message is a command to the actual worker process
if _message_data["command"] == "stop":
# Told to stop the worker. Stopping process.
self.write_dbg_info("Told to stop the worker.")
# First stop the monitor, we don't want any more commands coming in.
self.monitor.stop()
self.write_dbg_info("Terminate BPM thread")
self.terminate_bpm_process(_message_data)
self.write_dbg_info("Worker monitor stopped.")
self.send_queue.put([None, log_process_state_message(
_changed_by=message_is_none(_message_data, "userId", "No user mentioned"),
_state="stopped",
_process_id=self.process_id,
_reason=message_is_none(_message_data, "reason", "No reason mentioned"))])
self.write_dbg_info("Reported stopping to process handler")
self.terminated = True
else:
self.write_dbg_info("Unhandled command " + _message_data["command"])
elif _message_data["schemaRef"] == "ref://of.message":
#: TODO: Figure out if processes should just ever get general messages, and if so, how to handle?(PROD-30)
self.message = _message_data
self.write_dbg_info("Got a message:" + self.message)
def report_finished(self):
"""
Report that a BPM process has finished successfully
"""
# Stop tracing
sys.settrace(None)
# Send finished running state to broker.
self.send_queue.put([None, log_process_state_message(
_changed_by=self.bpm_process_thread.user_id,
_state="finished",
_reason="Process finished successfully",
_process_id=self.bpm_process_id)])
def kill_unresponsive_bpm_process(self, _bpm_process_id, _user_id):
"""
Kill a worked process
:param _bpm_process_id: The process id of the BPM process
:param _user_id: The user id of the killer
"""
_worker = self.busy_workers[_bpm_process_id]
_worker["process"].terminate()
del self.busy_workers[_bpm_process_id]
del self.workers[_worker["processId"]]
# Send first a state message for the (logical) BPM process
self.message_monitor.queue.put([None, log_process_state_message(_changed_by=_user_id,
_state="killed",
_process_id=_bpm_process_id,
_reason="Unresponsive, killed.")])
# Then a state message for the actual worker process
self.message_monitor.queue.put([None, log_process_state_message(_changed_by=_user_id,
_state="killed",
_process_id=_worker["processId"],
_reason="Had an unresponsive BPM process")])
self.write_dbg_info(self.log_prefix + "Killed")
def report_error(self, _error, e):
"""
Report an error from inside a BPM process.
:param _error:
:param e:
:return:
"""
# Stop tracing
sys.settrace(None)
# Send failed running state to broker.
self.send_queue.put([None, log_process_state_message(
_changed_by=self.bpm_process_thread.user_id,
_state="failed",
_reason=_error,
_process_id=self.bpm_process_id)])
# Create a return value for the caller
return {"bpm_error":
{
"exception": str(type(e)),
"message": _error
}
}
def report_termination(self, _globals):
"""
Reports termination of a BPM process.
:param _globals: The current globals of the BPM process
"""
# Stop tracing
sys.settrace(None)
self.write_dbg_info("XXXXXXXXXXXX-Script has been stopped: " + self.script_path + ".")
# Send stopped running state to broker.
self.send_queue.put([None, log_process_state_message(
_changed_by=self.bpm_process_thread.user_id,
_state="stopped",
_reason=self.bpm_process_thread.termination_message["reason"],
_process_id=self.bpm_process_id)])
# Create a return value for the caller
return {"bpm_error":
{
"exception": TerminationException,
"message": "The process was stopped, reason:" +
self.bpm_process_thread.termination_message["reason"],
"globals": self.sanitize_result(_globals)
}
}
def stop_broker(_reason, _restart=None):
global web_root, process_id
if _restart:
write_to_log("BROKER WAS TOLD TO RESTART, shutting down orderly",
_category=EC_SERVICE, _severity=SEV_INFO, _process_id=process_id)
else:
write_to_log("BROKER WAS TERMINATED, shutting down orderly",
_category=EC_SERVICE, _severity=SEV_INFO, _process_id=process_id)
write_srvc_dbg("Reason:" + str(_reason))
_exit_status = 0
write_srvc_dbg("Stop the monitor")
try:
of.common.messaging.websocket.monitor.stop()
except Exception as e:
write_to_log("Exception trying to stop monitor:", _category=EC_INVALID)
_exit_status += 1
time.sleep(1)
# TODO: Terminate all child processes.(PROD-146)
try:
database_access.save(log_process_state_message(_changed_by=zero_object_id,
_state="killed",
_process_id=process_id,
_reason="Broker was terminated, reason: \"" +
_reason + "\", shutting down gracefully"),
_user=None)
except Exception as e:
write_to_log("Exception trying to write log item to Mongo DB backend:" + str(e), _category=EC_SERVICE,
_severity=SEV_ERROR)
_exit_status += 1
try:
write_srvc_dbg("Unsubscribing the web socket plugin...")
web_socket_plugin.unsubscribe()
write_srvc_dbg("Stopping the web socket plugin...")
web_socket_plugin.stop()
write_srvc_dbg("Shutting down web server...")
cherrypy.engine.stop()
write_srvc_dbg("Web server shut down...")
except Exception as e:
write_to_log("Exception trying to shut down web server:" + str(e), _category=EC_SERVICE,
_severity=SEV_ERROR)
_exit_status += 4
if _restart:
write_srvc_dbg("Broker was told to restart, so it now starts a new broker instance...")
_broker_process = Process(target=run_broker, name="optimalframework_broker", daemon=False)
_broker_process.start()
_broker_process.join()
write_srvc_dbg("Broker exiting with exit status " + str(_exit_status))
if os.name != "nt":
os._exit(_exit_status)
else:
cherrypy.engine.exit()
return _exit_status
def shut_down(self, _user_id):
"""
Shuts down the worker handler and all jobs.
:param _user_id:
:return:
"""
try:
"""
TODO: First tell broker and incoming message handle we are shutting down.
Broker should then remove agent from destinations for messaging
Move all available workers processes to temporary list, shut them all down
All queued jobs should be unqueued and be returned to the broker to be run elsewhere
Pause(means do not run next step all running jobs and send states to broker where they should be queued or
continued elsewhere
Wait n seconds for remaining jobs to complete
Pause all running jobs and save state to broker
n seconds for remaining and queued jobs to complete
"""
self.write_dbg_info("Shutting down process handler")
self.write_dbg_info("Telling BPM processes to stop")
# Send process control messages to all bpm processes
for _process_id, _process in self.busy_workers.items():
_process["queue"].put(bpm_process_control(_destination="",
_destination_process_id=_process_id,
_command="stop",
_reason="Shutting down agent",
_message_id=0,
_source="",
_source_process_id=self.process_id,
_user_id=_user_id))
self.write_dbg_info(str(_process_id) + " told to stop.")
self.write_dbg_info("BPM processes told to shut down, waiting a bit..")
self.write_dbg_info("Clearing job queue")
# Loop through job queue, remove all items, tell broker
while True:
try:
_item = self.job_queue.get_nowait()
self.write_dbg_info("Job in queue:" + str(_item))
self.message_monitor.queue.put([None,
reply_with_error_message(self, _item,
"Unqueued, agent shutting down.")]
)
except queue.Empty:
break
_check_count = 3
while len(self.busy_workers) > 0 and _check_count > 0:
self.write_dbg_info("Still running BPM processes, waiting another second before stopping "
"the remaining workers. Times left:" + str(_check_count))
time.sleep(1)
_check_count -= 1
self.write_dbg_info("Telling worker processes to shut down")
# Tell workers to shut down
for _process in self.workers.values():
self.write_dbg_info("Telling worker process " + _process["processId"] + " to stop")
_process["queue"].put(worker_process_control(_destination="",
_destination_process_id=_process["processId"],
_command="stop",
_reason="Shutting down agent",
_message_id=0,
_source="",
_source_process_id=self.process_id,
_user_id=_user_id))
time.sleep(.1)
# Kill the remaining processes
for _process in list(self.workers.values()):
self.write_dbg_info("Killing unresponsive process " + _process["processId"] + " (pid:" + str(
_process["pid"]) + ")")
_process["process"].terminate()
self.message_monitor.queue.put([None, log_process_state_message(_changed_by=_user_id,
_state="killed",
_process_id=_process["processId"],
_reason="Unresponsive, killed by agent")
])
del self.workers[_process["processId"]]
except Exception as e:
self.write_dbg_info("Failed to properly shut down, error:" + str(e))
def stop_agent(_reason, _restart=False):
"""
Shuts down the agent
:param _reason: The reason for shutting down
:param _restart: If set, the agent will restart
"""
global _process_queue_manager, _start_pid
# Make sure this is not a child process also calling signal handler
if _start_pid != os.getpid():
write_srvc_dbg("Ignoring child processes' signal call to stop_agent().")
return
if _restart is True:
write_srvc_dbg( "--------------AGENT WAS TOLD TO RESTART------------")
else:
write_srvc_dbg( "--------------AGENT WAS TERMINATED, shutting down orderly------------")
write_srvc_dbg( "Reason:" + str(_reason))
write_srvc_dbg("Process Id: " + str(process_id))
try:
write_srvc_dbg("try and tell the broker about shutting down")
_control_monitor.handler.message_monitor.queue.put([None,
log_process_state_message(_changed_by=zero_object_id,
_state="stopped",
_process_id=process_id,
_reason="Agent stopped at " +
_address)])
# Give some time for it to get there
time.sleep(0.1)
write_srvc_dbg("try and tell the broker about shutting down, done")
except Exception as e:
write_to_log("Tried and tell the broker about shutting down, failed, error:" + str(e),
_category=EC_COMMUNICATION, _severity=SEV_ERROR)
write_srvc_dbg( "Stop the control monitor.")
_control_monitor.stop(_reverse_order=True)
time.sleep(0.4)
write_srvc_dbg("Control monitor stopped.")
_exit_status = 0
_process_queue_manager.shutdown()
write_srvc_dbg("Process queue manager shut down.")
if _restart is True:
write_to_log("Agent was told to restart, so now it starts a new agent instance.", _category=EC_SERVICE, _severity=SEV_INFO )
#set_start_method("spawn", force=True)
_agent_process = Process(target=run_agent, name="optimalbpm_agent", daemon=False)
_agent_process.start()
# On the current process (source) must still exist while the new process runs if its to be run using without
# pOpen. TODO: Investigate if it really is impossible to create standalone(non-child) processes using Process.
write_srvc_dbg("Pid of new instance: " + str(_agent_process.pid))
_agent_process.join()
global _terminated
_terminated = True
write_srvc_dbg("Agent exiting with exit status " + str(_exit_status))
if os.name == "nt":
return _exit_status
else:
os._exit(_exit_status)
def start_agent(_cfg_filename = None):
"""
Starts the agent; Loads settings, connects to database, registers process and starts the web server.
"""
global process_id, _control_monitor, _terminated, _address, _process_queue_manager, _broker_url, \
_username, _password, _peers, _log_to_database_severity, _verify_SSL
_process_id = str(ObjectId())
of.common.logging.callback = log_locally
_terminated = False
# Handle multiprocessing on windows
freeze_support()
write_srvc_dbg("=====start_agent===============================")
try:
if _cfg_filename is None:
_cfg_filename = resolve_config_path()
_settings = JSONXPath(_cfg_filename)
except Exception as e:
write_to_log("Error loading settings: " + str(e), _category=EC_SERVICE, _severity=SEV_FATAL,
_process_id=_process_id)
return
of.common.logging.severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/severityLevel", _default="warning"))
_log_to_database_severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/brokerLevel", _default="warning"))
write_srvc_dbg("===register signal handlers===")
register_signals(stop_agent)
# An _address is completely necessary.
_address = _settings.get("agent/address", _default=None)
if not _address or _address == "":
raise Exception(write_to_log(
"Fatal error: Agent cannot start, missing [agent] _address setting in configuration file.",
_category=EC_SERVICE, _severity=SEV_FATAL))
# An _address is completely necessary.
_verify_SSL = _settings.get("agent/verifySSL", _default=True)
# Gather credentials
_broker_url = _settings.get("agent/brokerUrl", _default="127.0.0.1:8080")
_username = _settings.get("agent/username")
if not _username:
raise Exception(write_to_log("Username must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_password = _settings.get("agent/password")
if not _password:
raise Exception(write_to_log("Password must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_retries = int(_settings.get("agent/connectionRetries", 5))
# Register at the broker
if not register_agent(_retries):
raise Exception(write_to_log("Fatal: The agent failed to register with the broker, tried " + str(
_retries + 1) + " time(s), quitting.", _category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
of.common.logging.callback = log_to_database
_repository_base_folder = _settings.get_path("agent/repositoryFolder",
_default=os.path.join(os.path.dirname(__file__), "repositories"))
write_srvc_dbg("Load schema tool")
try:
# Initiate a schema tools instance for validation other purposes.
_schema_tools = SchemaTools(_json_schema_folders=[of_schema_folder(),
os.path.abspath(os.path.join(script_dir, "..", "schemas", "namespaces"))
],
_uri_handlers={"ref": None})
except Exception as e:
raise Exception(write_to_log("An error occurred while loading schema tools:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
return
write_srvc_dbg("Load schema tool done")
try:
write_srvc_dbg("Initializing monitors")
# Init the monitor for incoming messages
_message_monitor = Monitor(
_handler=AgentWebSocketHandler(_process_id=_process_id,
_peers=_peers,
_schema_tools=_schema_tools,
_address=_address,
_broker_address="broker"))
# The manager for the process queue
_process_queue_manager = multiprocessing.Manager()
# Init the monitor for the worker queue
_worker_monitor = Monitor(
_handler=WorkerSupervisor(_process_id=_process_id,
_message_monitor=_message_monitor,
_repo_base_folder=_repository_base_folder,
_severity=of.common.logging.severity),
_queue=_process_queue_manager.Queue())
# Init the monitor for the agent queue
_control_monitor = Monitor(
_handler=ControlHandler(_process_id=_process_id,
_message_monitor=_message_monitor,
_worker_monitor=_worker_monitor,
_stop_agent=stop_agent
))
# The global variable for handling websockets. TODO: Could this be done without globals? (PROD-33)
of.common.messaging.websocket.monitor = _message_monitor
write_srvc_dbg("Initializing monitors done")
except Exception as e:
raise Exception(write_to_log("Fatal: An error occurred while initiating the monitors and handlers:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
# Try to connect to websocket, quit on failure
if not connect_to_websocket():
os._exit(1)
write_srvc_dbg("Register agent system process")
_control_monitor.handler.message_monitor.queue.put(
[None, store_process_system_document(_process_id=_process_id,
_name="Agent instance(" + _address + ")")])
write_srvc_dbg("Log agent system state")
_control_monitor.handler.message_monitor.queue.put([None,
log_process_state_message(_changed_by=zero_object_id,
_state="running",
_process_id=_process_id,
_reason="Agent starting up at " +
_address)])
# Security check to remind broker if it is unsecured
if not _verify_SSL:
try:
call_api("https://"+ _broker_url + "/status", _data={}, _session_id= _session_id, _verify_SSL=True)
except SSLError as e:
write_to_log("There is a problem with the security certificate:\n" + str(e) + "\n"
"This is a security risk, and and important thing to _address.",
_category=EC_NOTIFICATION, _severity=SEV_WARNING)
except Exception as e:
write_to_log("An error occured while checking status of broker and SSL certificate:" + str(e),
_category=EC_NOTIFICATION, _severity=SEV_ERROR)
write_srvc_dbg("Agent up and running.")
while not _terminated:
time.sleep(0.1)
write_srvc_dbg("Exiting main thread")
def start_bpm_process(self, _message):
"""
Initializes and starts a BPM process based on the content of the message
:param _message: A message_bpm_process_start-message
"""
try:
# Extract caller information
self.bpm_process_id = _message["processId"]
self.bpm_source = str(message_is_none(_message, "source", ""))
# Execution context
_entry_point = message_is_none(_message, "entryPoint", None)
_globals = message_is_none(_message, "globals", {})
# Create the source path to the module
_source_path = os.path.join(os.path.expanduser(self.repo_base_folder), _message["processDefinitionId"])
self.source_path = _source_path
self.write_dbg_info("Source path: " + _source_path)
if _entry_point:
# If there is an entry point, it is a function call
_module_name = _entry_point["moduleName"]
if "functionName" in _entry_point:
# Add repository location to sys.path to be able to import
sys.path.append(self.source_path)
_module = importlib.import_module(_module_name)
_function_name = _entry_point["functionName"]
self.write_dbg_info("In start_bpm_process, initializing: " + str(_message))
# Find function
_function = getattr(_module, _function_name)
else:
_function = None
else:
# If there is no entry point, it is a module
_module_name = "main"
_function = None
self.script_path = os.path.join(_source_path, _module_name + ".py")
# Send an instance message to the broker. This tells the broker that a worker has instantiated a process.
self.send_queue.put(
[None, store_bpm_process_instance_message(_start_message=_message, _worker_process_id=self.process_id)])
self.write_dbg_info("In start_bpm_process, initializing of " + self.script_path + " done.")
except Exception as e:
_error_message = self.log_prefix + "error initiating BPM process. processDefinitionId: " + \
message_is_none(_message, "processDefinitionId", "not set") + \
", process_id: " + message_is_none(_message, "processId", "not set") + \
", Error: " + str(e)
write_to_log(_error_message, _category=EC_NOTIFICATION, _severity=SEV_ERROR)
self.send_queue.put([None, log_process_state_message(
_changed_by=_message["userId"],
_state="failed",
_reason=_error_message,
_process_id=self.bpm_process_id)])
else:
try:
_thread_name = "BPM Process " + self.bpm_process_id + " thread"
# Load data
self.load_data(_source_path=_source_path)
# Add callbacks to scope
_globals.update(
{
"report_error": self.report_error,
"report_result": self.report_result,
"log_progress": self.log_progress,
"log_message": self.log_message,
"pause": self.pause,
"get_data": self.get_data
}
)
# Add the name space modules to the process globals
self.load_namespaces(_source_path=_source_path, _dest_globals=_globals)
if _function:
_function.__globals__.update(_globals)
self.bpm_process_thread = BPMTread(target=self.run_function,
name=_thread_name, args=[_function],
_user_id=_message["userId"],
_start_message=_message)
else:
self.bpm_process_thread = BPMTread(target=self.run_module,
name=_thread_name,
args=[_module_name, _globals],
_user_id=_message["userId"],
_start_message=_message)
# Send running state.
if "reason" in _message:
_reason = _message["reason"]
else:
_reason = "No reason"
self.send_queue.put([None, log_process_state_message(
_changed_by=self.bpm_process_thread.user_id,
_state="running",
_process_id=self.bpm_process_id,
_reason=_reason)])
threading.settrace(self.trace_calls)
self.bpm_process_thread.start()
except Exception as e:
_error_message = self.log_prefix + "error initiating module " + self.script_path + '.run() - ' + str(e) +\
" processDefinitionId: " + message_is_none(_message, "processDefinitionId", "not set") + \
", process_id: " + message_is_none(_message, "processId", "not set") + \
", Error: " + str(e)
write_to_log(_error_message, _category=EC_NOTIFICATION, _severity=SEV_ERROR)
self.send_queue.put([None, log_process_state_message(
_changed_by= _message["userId"],
_state="failed",
_reason=_error_message,
_process_id=self.bpm_process_id)])