def write_dbg_info(self, _data):
"""
Shortcut to writing debug information
:param _message: The message
:param _severity: The severity of the log information, a constant defined in the built-in logging module
"""
write_to_log(self.log_prefix + _data, _category=EC_NOTIFICATION, _severity=SEV_DEBUG, _process_id=self.process_id)
def run_worker_process(_parent_process_id, _process_id, _queue, _send_queue, _repo_base_folder, _severity):
"""
This function is the first thing that is called when the worker process is initialized
:param _parent_process_id: The process_id of the parent process.
:param _process_id: The process id of this worker
:param _queue: The instance of multiprocess queue on which the worker expects process messages to appear.
:param _send_queue: The queue the process uses for external communication
:param _repo_base_folder: The repository base folder.
"""
global send_queue, process_id
of.common.logging.callback = log_to_queue
of.common.logging.severity = _severity
process_id = _process_id
send_queue = _send_queue
write_to_log("Run.Run_worker_process: Started a new worker process.")
# Create a worker handler using the supplied parameters
_worker_handler = WorkerHandler(_process_id=_process_id,
_parent_process_id=_parent_process_id,
_send_queue=_send_queue,
_repo_base_folder=_repo_base_folder)
# Start an monitor for that the inbound queue, specify handler
_worker_monitor = Monitor(_handler=_worker_handler,
_queue=_queue)
# Run until terminated
while not _worker_handler.terminated:
time.sleep(0.1)
write_to_log("Run.Run_worker_process: Exiting worker process.")
def handle(self, _item):
"""
Called when the monitor has gotten a message from a worker process
"""
if _item[0]:
# If the source websocket is set, it is not from a worker process, raise error.
raise Exception("The process handler only handles messages from worker processes.")
else:
# TODO: Should any filtering be done here? (PROD-27)
_message_data = _item[1]
if not isinstance(_message_data, dict):
write_to_log("A worker process sent message data that is not a dict, this might be a an attack.",
_category=EC_BREAKIN, _severity=SEV_ERROR)
raise Exception(self.log_prefix + "A worker process sent message data that is not a dict, this "
"might be a an attack: " + str(_message_data))
if "schemaRef" not in _message_data:
raise Exception(self.log_prefix + "Missing schemaRef: " + str(_message_data))
if _message_data["schemaRef"] == "ref://bpm.message.bpm.process.result":
# A process result message implies that the worker is done and available for new jobs
self.release_worker(_message_data["sourceProcessId"])
elif _message_data["schemaRef"] == "ref://of.log.process_state" and \
_message_data["processId"] in self.workers and \
_message_data["state"] in ["killed"]:
# If a worker is logging that it is being killed, it should be remove from the workers
self.write_dbg_info(self.log_prefix + "Worker " + _message_data["processId"] + " shut down, removing from workers.")
del self.workers[_message_data["processId"]]
self.write_dbg_info("Forwarding " + str(_message_data))
# Pass the message on to the message queue, heading for the last destination
self.message_monitor.queue.put([_item[0], _item[1]])
def handle_message(self, _source_web_socket, _message_data):
"""
Handle inbound and outbound messages.
:param _source_web_socket: The web socket of the source, if none, it is outbound
:param _message_data: The message data
"""
if _source_web_socket is None:
# This is an outbound message
self.outbound_message_count += 1
# TODO: Handle multiple broker peers(other peers?), be informed trough some kind of messaging.(PROD-25)
# For now, however, always have one broker.
_message_data["source"] = self.address
self.send_to_address(self.broker_address, _message_data)
else:
self.inbound_message_count += 1
# This is an inbound message
try:
self.schema_tools.validate(_message_data)
except Exception as e:
_error = "An error occurred validating an inbound message:" + str(
e)
write_to_log(_error,
_category=EC_COMMUNICATION,
_severity=SEV_ERROR)
# Respond to sender with an error message
self.send_to_address(
self.broker_address,
reply_with_error_message(self, _message_data, _error))
else:
self.process_handler.forward_message(_message_data)
def write_to_log(self, **kwargs):
"""
This function provides a way for registered peers to write to the log even if they have no web socket.
:param kwargs:
:return:
"""
_session_id = kwargs["_session_id"]
# Only registered peers get to write to the log, other trying is a probe
if _session_id not in self.peers:
write_to_log("Client that wasn't registered as peer tried to write to log ",
_category=EC_PROBE, _severity=SEV_WARNING)
return None
_message = cherrypy.request.json
write_to_log(_data=_message.get("data"),
_category=category_identifiers.index(_message.get("category")),
_severity=severity_identifiers.index(_message.get("severity")),
_occurred_when=_message.get("occurred_when"),
_address=_message.get("address"),
_process_id=_message.get("process_id"),
_user_id=_message.get("user_id"),
_pid=_message.get("pid"),
_uid=_message.get("uid"),
_node_id=_message.get("node_id")
)
return "{}"
def run_worker_process(_parent_process_id, _process_id, _queue, _send_queue,
_repo_base_folder, _severity):
"""
This function is the first thing that is called when the worker process is initialized
:param _parent_process_id: The process_id of the parent process.
:param _process_id: The process id of this worker
:param _queue: The instance of multiprocess queue on which the worker expects process messages to appear.
:param _send_queue: The queue the process uses for external communication
:param _repo_base_folder: The repository base folder.
"""
global send_queue, process_id
of.common.logging.callback = log_to_queue
of.common.logging.severity = _severity
process_id = _process_id
send_queue = _send_queue
write_to_log("Run.Run_worker_process: Started a new worker process.")
# Create a worker handler using the supplied parameters
_worker_handler = WorkerHandler(_process_id=_process_id,
_parent_process_id=_parent_process_id,
_send_queue=_send_queue,
_repo_base_folder=_repo_base_folder)
# Start an monitor for that the inbound queue, specify handler
_worker_monitor = Monitor(_handler=_worker_handler, _queue=_queue)
# Run until terminated
while not _worker_handler.terminated:
time.sleep(0.1)
write_to_log("Run.Run_worker_process: Exiting worker process.")
def handle_message(self, _source_web_socket, _message_data):
"""
Handle inbound and outbound messages.
:param _source_web_socket: The web socket of the source, if none, it is outbound
:param _message_data: The message data
"""
if _source_web_socket is None:
# This is an outbound message
self.outbound_message_count += 1
# TODO: Handle multiple broker peers(other peers?), be informed trough some kind of messaging.(PROD-25)
# For now, however, always have one broker.
_message_data["source"] = self.address
self.send_to_address(self.broker_address, _message_data)
else:
self.inbound_message_count += 1
# This is an inbound message
try:
self.schema_tools.validate(_message_data)
except Exception as e:
_error = "An error occurred validating an inbound message:" + str(e)
write_to_log(_error, _category=EC_COMMUNICATION, _severity=SEV_ERROR)
# Respond to sender with an error message
self.send_to_address(self.broker_address, reply_with_error_message(self, _message_data, _error))
else:
self.process_handler.forward_message(_message_data)
def agent_control(self, _address, _command, _reason, _user):
"""
Controls an agent's running state
:param _address: The address of the agent to control
:param _command: Can be "stop" or "restart".
:param _user: A user instance
"""
write_to_log("Control.agent_control: Got the command " + str(_command),
_category=EC_SERVICE,
_severity=SEV_DEBUG)
self.send_queue.put([
None,
agent_control(_destination=_address,
_destination_process_id=zero_object_id,
_command=_command,
_reason=_reason,
_user_id=_user["_id"],
_source=self.address,
_source_process_id=self.process_id,
_message_id=1)
])
return {}
def run_module(self, _module_name, _globals):
"""
Run a BPM process module
:param _module_name: Full path to the model.
:param _globals: The globals.
"""
self.write_dbg_info("->>>>>>>>>>>> Running module " + _module_name + ", globals: " + str(_globals))
try:
# Call the run_module, it returns the globals after execution
_new_globals = runpy.run_path(os.path.join(self.source_path, _module_name + ".py"), init_globals=_globals)
except TerminationException as e:
write_to_log("TerminationException running the process:" + str(e),
_category=EC_NOTIFICATION, _severity=SEV_ERROR)
# The process was terminated
_result = self.report_termination(_globals)
_new_globals = _globals
except Exception as e:
# An unhandled error occured running the process
write_to_log("Error running the process:" + str(e),
_category=EC_NOTIFICATION, _severity=SEV_ERROR)
_result = self.report_error(self.log_prefix + "Error in module " + _module_name + ".py:" + str(e), e)
_new_globals = None
else:
# The process ended normally
self.report_finished()
# A module has no return value
_result = None
self.write_dbg_info(self.log_prefix + "<<<<<<<<<<<<- Done calling " + self.script_path + ".")
self.report_result(_globals=_new_globals, _result=_result)
self.job_running = False
def restart_agent(self, _reason):
try:
# Wait 3 seconds and then restart the agent.
time.sleep(3)
self.stop_agent(_reason=_reason, _restart=True)
except Exception as e:
write_to_log("Error terminating self:" + str(e), _category=EC_SERVICE, _severity=SEV_ERROR)
def agent_control(self, **kwargs):
write_to_log("Got an agent control call, command "+ cherrypy.request.json["command"] +
", reason: " + cherrypy.request.json["reason"] +
", address: " + cherrypy.request.json["address"] +
str(cherrypy.request.json))
return self._control.agent_control(cherrypy.request.json["address"],
cherrypy.request.json["command"],
cherrypy.request.json["reason"],
kwargs["_user"])
def error_handler(self, exception):
"""
If an unhandled error occurs, unregister at the broker, and close the socket
"""
write_to_log(self.log_prefix + "An exception handle in the socket, closing. Error: " + str(exception),
_category=EC_COMMUNICATION, _severity=SEV_ERROR)
monitor.handler.unregister_web_socket(self)
# Tell the socket to close
return False
def restart_agent(self, _reason):
try:
# Wait 3 seconds and then restart the agent.
time.sleep(3)
self.stop_agent(_reason=_reason, _restart=True)
except Exception as e:
write_to_log("Error terminating self:" + str(e),
_category=EC_SERVICE,
_severity=SEV_ERROR)
def refresh_static(self, _web_config):
"""
This function regenerates all the static content that is used to initialize the user interface
plugins.
:param _web_config: An instance of the CherryPy web configuration
"""
self.admin_ui_hooks = self.refresh_hooks()
_imports = ""
_systemjs = ""
_admin_menus = []
# has_right(id_right_admin_everything, kwargs["user"])
for _curr_plugin_key, _curr_plugin_info in self.plugins.items():
# Add any plugin configuration for the Admin user interface
if "admin-ui" in _curr_plugin_info:
_curr_ui_def = _curr_plugin_info["admin-ui"]
if "mountpoint" not in _curr_ui_def:
write_to_log("Error loading admin-ui for " + _curr_plugin_key + " no mount point.",
_category=EC_SERVICE, _severity=SEV_ERROR)
continue
_mount_point = _curr_ui_def["mountpoint"]
if _mount_point[0] == "/":
write_to_log(
"Not mounting " + _mount_point + ", cannot mount admin-specific ui under root(root can "
"never depend on admin), use root-ui instead.",
_category=EC_SERVICE, _severity=SEV_ERROR)
continue
# Mount the static content at a mount point somewhere under /admin
_web_config.update({
"/admin/" + _mount_point: {
"tools.staticdir.on": True,
"tools.staticdir.dir": os.path.join(_curr_plugin_info["baseDirectoryName"], _mount_point),
"tools.trailing_slash.on": True
}
})
_systemjs += "System.config({\"packages\": {\"" + _mount_point + "\": {\"defaultExtension\": \"ts\"}}});\n"
# Add menus
if "menus" in _curr_ui_def:
_admin_menus += _curr_ui_def["menus"]
self.admin_systemjs_init = _systemjs
self.admin_menus = _admin_menus
def handle_error(self, _error_message, _message):
"""
A generic function for handling errors
:param _error: The error message
"""
write_to_log(
"An error occured, replying with an error message. Error: " +
str(_error_message),
_category=EC_SERVICE,
_severity=SEV_ERROR)
self.message_monitor.queue.put(
[None,
reply_with_error_message(self, _message, _error_message)])
def closed(self, code, reason=None):
"""
Called when the socket is closed.
:param code: A web socket error code as defined in: http://tools.ietf.org/html/rfc6455#section-7.4.1
:param reason: A string describing the reason for closing the connection
"""
# TODO: Handle the rest of the possible web socket error codes
self.connected = False
if code == ABNORMAL_CLOSE:
write_to_log(self.log_prefix + "The connection to " + self.address + " has been abnormally closed.",
_category=EC_COMMUNICATION, _severity=SEV_ERROR)
self.close(code=code, reason=reason)
else:
self.monitor_message_queue_thread.terminated = True
self.write_dbg_info(self.log_prefix + "Closed, code: " + str(code) + ", reason: " + str(reason))
def __init__(self, _process_id, _address, _database_access):
"""
Initializes the broker web service and includes and initiates the other parts of the API as well
:param _database_access: A DatabaseAccess instance for database connectivity
:param _process_id: The system process id of the broker
:param _address: The peer address of the broker
:param _stop_broker: A callback to a function that shuts down the broker
"""
write_to_log(_process_id=_process_id, _category=EC_SERVICE, _severity=SEV_DEBUG,
_data="Initializing broker REST API.")
self.peers = {}
self.process_id = _process_id
self.address = _address
self.node = CherryPyNode(_database_access=_database_access)
def handle_error(
self,
_error,
_category=EC_COMMUNICATION,
_severity=SEV_ERROR,
_web_socket=None,
_close_socket=None,
_message_to_reply_to=None,
):
"""
:param _error: The error message
:param _category: The category of the error
:param _severity: Error severity
:param _web_socket: If the sender was external, the web socket
:param _close_socket: Close the web socket
:param _message_to_reply_to: The source message, if set, the error message is sent to the sender
:return:
"""
""" Handles and logs errors, decides whether to keep the connection open if an error occurrs
"""
_error = write_to_log(
_data=self.log_prefix + "In handler.handle_error error with error :" + str(_error),
_category=_category,
_severity=_severity,
)
if _web_socket:
if _message_to_reply_to:
_web_socket.send_message(reply_with_error_message(self, _message_to_reply_to, _error))
if _close_socket:
_web_socket.close(code=PROTOCOL_ERROR, reason=_error)
def register_agent(_retries, _connect=False):
global _broker_url, _username, _password, _peers, _session_id, _verify_SSL
_retry_count = _retries + 1
write_srvc_dbg("Register agent session (adress : " + _address + ") at broker(URL: https://" +
_broker_url + ")")
# Register session at the broker
_data = None
while _retry_count > 0:
try:
_data = register_at_broker(_address=_address, _type="agent", _server="https://" + _broker_url,
_username=_username, _password=_password, _verify_SSL=_verify_SSL)
write_srvc_dbg("Agent tried registering, data returned: " + str(_data))
except Exception as e:
if _retry_count > 1:
write_to_log("Failed to register at the broker, will retry " + str(
_retry_count - 1) + " more times, error:" + str(e),
_category=EC_SERVICE, _severity=SEV_INFO)
else:
write_to_log( "Failed to register at the broker, will not retry any more times, error:" +
str(e),
_category=EC_SERVICE, _severity=SEV_FATAL)
if _data:
break
else:
if _retry_count > 1:
time.sleep(3)
_retry_count -= 1
if not _data:
return False
_session_id = _data["session_id"]
write_srvc_dbg("Register session at broker done")
_peers[_session_id] = {
"address": "broker",
"session_id": _session_id,
"queue": Queue()
}
if _connect:
return connect_to_websocket()
else:
return True
def get_peers(self, **kwargs):
"""
Returns a list of all logged in peers
:param kwargs: Unused here so far, but injected by get session
:return: A list of all logged in peers
"""
# TODO: This should probably not be in admin. However listing peers does seems slightly administrative.
_result = []
# Filter out the unserializable web socket
for _session in self.root.peers.values():
_new_session = copy.copy(_session)
_new_session["web_socket"] = "removed for serialization"
_new_session["queue"] = "removed for serialization"
_result.append(_new_session)
write_to_log(_process_id=self.process_id, _category=EC_NOTIFICATION, _severity=SEV_DEBUG,
_data="Returning a list of peers:" + str(_result))
return _result
def unregister(self, **kwargs):
_session_id = kwargs["_session_id"]
if "session_id" in cherrypy.request.cookie:
cherrypy.response.cookie = cherrypy_logout(_session_id)
_peer = self.peers[_session_id]
if "websocket" in _peer:
try:
_peer["websocker"].close()
write_to_log(_data="Unregister: Closed websocket for address " + _peer["address"] + ".",
_category=EC_COMMUNICATION, _severity=SEV_DEBUG)
except:
write_to_log(_data="Unregister: Failed closing websocket for address " + _peer["address"] + ".",
_category=EC_COMMUNICATION, _severity=SEV_ERROR)
del self.peers[_session_id]
return {}
def connect_to_websocket():
global _broker_url, _session_id
write_srvc_dbg("Connecting web socket to broker")
try:
# Initiate the web socket connection to the broker
_web_socket = AgentWebSocket(url="wss://" + _broker_url + "/socket",
_session_id=_session_id,
_stop_agent=stop_agent,
_register_agent = register_agent)
_web_socket.connect()
_web_socket.run_forever()
except Exception as e:
write_to_log("Fatal: An error occurred establishing the web socket:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL)
return False
write_srvc_dbg("Connecting web socket to broker done")
return True
def step_impl(context):
"""
:type context: behave.runner.Context
"""
global _global_params, _global_err_param, logging_callback
_global_params = None
of.common.logging.callback = local_test_log_writer
try:
write_to_log(*_global_err_param)
ok_(_global_params == _global_err_param, "Global params didn't match!\nResult:" + str(_global_params) + "\nComparison: " + str(_global_err_param))
_global_params = None
of.common.logging.callback = None
except Exception as e:
# Be sure to reset globals.<
_global_params = None
of.common.logging.callback = None
raise Exception(e)
# TODO: Add test for sparse message
def __init__(self, _json_schema_folders=[], _uri_handlers=None):
"""
Initiate the SchemaTools class
:param _json_schema_folders: A list of folders where schema files are stored
:param : _uri_handlers: A dict of uri_handlers, resolves a URI prefix to a actual schema.
"""
if not _json_schema_folders:
_json_schema_folders = []
if not _uri_handlers:
self.uri_handlers = {}
else:
self.uri_handlers = _uri_handlers
# All methods that have no handlers should use the cache handler.
for _curr_key, _curr_value in _uri_handlers.items():
if _curr_value is None:
_uri_handlers[_curr_key] = self.cache_handler
self.resolver = RefResolver(base_uri="",
handlers=self.uri_handlers, referrer=None, cache_remote=True)
self.mongodb_validator = MongodbValidator(resolver= self.resolver)
self.json_schema_objects = {}
# Load application specific schemas
for _curr_folder in _json_schema_folders:
_loaded_uris = self.load_schemas_from_directory(os.path.abspath(_curr_folder))
# Resolve all the schemas
for _curr_uri in _loaded_uris:
self.json_schema_objects[_curr_uri] = self.resolveSchema(self.json_schema_objects[_curr_uri])
write_to_log("Schemas loaded and resolved: " +
str.join(", ", ["\"" +_curr_schema["title"] + "\"" for _curr_schema in self.json_schema_objects.values()])
, _category=EC_NOTIFICATION, _severity=SEV_DEBUG)
def monitor_message_queue(self):
"""
Monitors the messages queue. Stops when the queue-threads' terminated attribute is set to True
"""
self.monitor_message_queue_thread.terminated = False
while not self.monitor_message_queue_thread.terminated:
try:
_message = self.message_queue.get(True, .1)
try:
self.send_message(_message)
except Exception as e:
write_to_log(self.log_prefix + "Error sending message:" + str(e) + "\nTraceback:" + traceback.format_exc(),
_category=EC_COMMUNICATION, _severity=SEV_ERROR)
except Empty:
pass
except Exception as e:
write_to_log(self.log_prefix + "Error accessing send queue:" + str(e) + "\nTraceback:" + traceback.format_exc(),
_category=EC_INTERNAL, _severity=SEV_ERROR)
self.write_dbg_info(" stopped message queue monitoring. Exiting thread \"" +
str(self.monitor_message_queue_thread.name) + "\"")
def agent_control(self, _address, _command, _reason, _user):
"""
Controls an agent's running state
:param _address: The address of the agent to control
:param _command: Can be "stop" or "restart".
:param _user: A user instance
"""
write_to_log("Control.agent_control: Got the command " + str(_command), _category=EC_SERVICE, _severity=SEV_DEBUG)
self.send_queue.put([None, agent_control(_destination=_address,
_destination_process_id=zero_object_id,
_command=_command,
_reason=_reason,
_user_id=_user["_id"],
_source=self.address,
_source_process_id=self.process_id,
_message_id=1
)])
return {}
def start(self):
"""
Start monitoring the queue
"""
if self.monitor_thread and not self.monitor_thread.terminated:
raise Exception(write_to_log(self.log_prefix + "The queue monitor is already running.",
_category=EC_INTERNAL, _severity=SEV_ERROR))
self.monitor_thread = MonitorThread(_monitor=self)
self.monitor_thread.terminated = False
self.monitor_thread.start()
self.write_dbg_info("Running, monitoring thread: " + str(self.monitor_thread.name))
def call_hook(self, _hook_name, **kwargs):
self.write_debug_info("Running hook " + _hook_name)
for _curr_plugin_name, _curr_plugin in self.plugins.items():
if "failed" in _curr_plugin and _curr_plugin["failed"]:
self.write_debug_info("Plugin " + _curr_plugin_name + " marked failed, will not call its hook.")
continue
if "hooks_instance" in _curr_plugin:
_hooks_instance = _curr_plugin["hooks_instance"]
if hasattr(_hooks_instance, _hook_name):
try:
self.write_debug_info("Calling " + _hook_name + " in " + _curr_plugin["description"])
getattr(_hooks_instance, _hook_name)(**kwargs)
except Exception as e:
write_to_log(
"An error occurred " + "Calling " + _hook_name + " in " + _curr_plugin_name + ":" + str(e),
_category=EC_SERVICE,
_severity=SEV_ERROR,
)
if "failOnError" in _curr_plugin and _curr_plugin["failOnError"]:
write_to_log(
"Setting "
+ _curr_plugin_name
+ " as Failed. No more hooks will be called for this plugin.",
_category=EC_SERVICE,
_severity=SEV_INFO,
)
_curr_plugin["failed"] = True
else:
write_to_log(
"Ignores error, this plugin will continue to attempt initialization.",
_category=EC_SERVICE,
_severity=SEV_INFO,
)
def register_at_broker(_address, _type, _server, _username, _password, _log_prefix="", _verify_SSL=True):
_log_prefix = make_log_prefix(_log_prefix)
_data = {
"credentials": {
"usernamePassword": {
"username": _username,
"password": _password
}
},
"environment": get_environment_data(),
"peerType": _type,
"address": _address
}
write_dbg_info(_log_prefix + "[" + str(datetime.datetime.utcnow()) + "] Registering at broker API.")
_headers = {'content-type': 'application/json'}
_response = requests.post(_server + "/register", data=json.dumps(_data), auth=('user', 'pass'), headers=_headers,
verify=_verify_SSL)
if _response.status_code == 500:
write_dbg_info(_log_prefix + "Broker login failed with internal server error! Exiting.")
return False
if _response.status_code != 200:
write_dbg_info(_log_prefix + "Broker login failed with error + "+ str(_response.status_code) + "! Exiting.")
return False
_response_dict = _response.json()
if _response_dict is not None:
_data = _response_dict
if "session_id" in _data:
write_dbg_info(_log_prefix + "Got a session id:" + _data["session_id"])
return _data
else:
write_to_log(_log_prefix + "Broker login failed! Exiting.", _category=EC_SERVICE, _severity=SEV_ERROR)
return False
else:
write_to_log(_log_prefix + "Broker login failed! Exiting.", _category=EC_SERVICE, _severity=SEV_ERROR)
return False
def save_process(self, **kwargs):
"""
Save a process structure into a source file
:param kwargs: A parameter object
"""
# TODO: Document the structure of the process parameters, perhaps create a schema?(ORG-110)
has_right(id_right_admin_everything, kwargs["_user"])
_tokens = ProcessTokens(_keywords=self.keywords, _namespaces=self.namespaces)
_tokens.documentation = cherrypy.request.json["documentation"]
_verbs = _tokens.json_to_verbs(_json=cherrypy.request.json["verbs"])
if "documentation" in cherrypy.request.json:
_tokens.documentation = cherrypy.request.json["documentation"]
_process_id = cherrypy.request.json["processId"]
_repo_path = os.path.join(os.path.expanduser(self.repository_parent_folder), _process_id)
_filename = os.path.join(_repo_path, "main.py")
# Store in repository
if not os.path.exists(_repo_path):
# No process definition, it is a new process, create a folder.
write_to_log("Initating new repo at " + _repo_path, EC_NOTIFICATION, SEV_INFO)
os.makedirs(_repo_path)
# TODO: Init git repo (PROD-11)
_namespaces, _map = _tokens.encode_process(_verbs=_verbs, _filename=_filename)
_filename_namespaces = os.path.join(_repo_path, "namespaces.json")
with open(_filename_namespaces, "w") as f:
json.dump(_namespaces, f)
_filename_map = os.path.join(_repo_path, "map.json")
with open(_filename_map, "w") as f:
json.dump(_map, f)
_filename_data = os.path.join(_repo_path, "data.json")
with open(_filename_data, "w") as f:
json.dump(cherrypy.request.json["paramData"], f)
write_to_log("save_process to " + _repo_path + ", done", _category=EC_NOTIFICATION, _severity=SEV_DEBUG)
def __init__(self, _process_id, _address, _stop_broker, _root_object,
_plugins, _web_config):
write_to_log(_category=EC_SERVICE, _severity=SEV_DEBUG, _process_id=_process_id,
_data="Initializing administrative REST API.")
self.stop_broker = _stop_broker
self.process_id = _process_id
self.address = _address
self.root = _root_object
self.plugins = _plugins
# Mount the static content at a mount point /admin
_web_config.update({
"/admin" : {
"tools.staticdir.on": True,
"tools.staticdir.dir": os.path.join(os.path.dirname(__file__), "../ui"),
"tools.trailing_slash.on": True
}
})
self.refresh_static(_web_config=_web_config)
def broker_ctrl(self, _command, _reason, _user):
"""
Controls the broker's running state
:param _command: Can be "stop" or "restart".
:param _user: A user instance
"""
write_to_log("broker.broker_control: Got the command " + str(_command), _category=EC_SERVICE,
_process_id=self.process_id)
# TODO: There should be a log item written with reason and userid.(PROD-32)
# TODO: UserId should also be appended to reason below.(PROD-32)
def _command_local(_local_command):
if _local_command == "restart":
self.stop_broker(_reason=_reason, _restart=True)
if _local_command == "stop":
self.stop_broker(_reason=_reason, _restart=False)
_restart_thread = threading.Thread(target=_command_local, args=[_command])
_restart_thread.start()
return {}
def monitor(self):
"""
Monitors the queue. Stops when the queue-threads' terminated attribute is set to True
"""
self.write_dbg_info("In monitor thread monitoring queue.")
while not self.monitor_thread.terminated:
try:
_item = self.queue.get(True, .1)
try:
self.handler.handle(_item)
except Exception as e:
write_to_log(self.log_prefix + "Error handling item:" + str(e) + "\nData:\n" + str(_item) +
"\nTraceback:" + traceback.format_exc(), _category=EC_INTERNAL,
_severity=SEV_ERROR)
except Empty:
pass
except Exception as e:
self.monitor_thread.terminated = True
raise Exception(write_to_log(self.log_prefix + "Terminating, error accessing own queue:" + str(e),
_category=EC_INTERNAL, _severity=SEV_ERROR))
self.after_get_queue()
self.write_dbg_info("Monitor thread stopped.")
def init(self, _session_id):
"""
Initialize the socket, set the session id an register the web socket with the handler, start monitoring queue.
:param _session_id: The sessionid
"""
self.session_id = _session_id
self.log_prefix = str(os.getpid()) + "-" + self.__class__.__name__ + "(No address yet): "
self.connected = False
try:
monitor.handler.register_web_socket(self)
except Exception as e:
raise Exception(write_to_log("OptimalWebSocket: Exception registering web socket:" + str(e),
_category=EC_COMMUNICATION, _severity=SEV_ERROR))
# Generate log prefix string for performance
self.log_prefix = str(os.getpid()) + "-" + self.__class__.__name__ + "(" + str(self.address) + "): "
self.start_monitoring_message_queue()
def write_dbg_info(self, _data):
write_to_log(self.log_prefix + str(_data),
_category=EC_COMMUNICATION, _severity=SEV_DEBUG, _process_id=self.process_id)
def start_bpm_process(self, _message):
"""
Initializes and starts a BPM process based on the content of the message
:param _message: A message_bpm_process_start-message
"""
try:
# Extract caller information
self.bpm_process_id = _message["processId"]
self.bpm_source = str(message_is_none(_message, "source", ""))
# Execution context
_entry_point = message_is_none(_message, "entryPoint", None)
_globals = message_is_none(_message, "globals", {})
# Create the source path to the module
_source_path = os.path.join(os.path.expanduser(self.repo_base_folder), _message["processDefinitionId"])
self.source_path = _source_path
self.write_dbg_info("Source path: " + _source_path)
if _entry_point:
# If there is an entry point, it is a function call
_module_name = _entry_point["moduleName"]
if "functionName" in _entry_point:
# Add repository location to sys.path to be able to import
sys.path.append(self.source_path)
_module = importlib.import_module(_module_name)
_function_name = _entry_point["functionName"]
self.write_dbg_info("In start_bpm_process, initializing: " + str(_message))
# Find function
_function = getattr(_module, _function_name)
else:
_function = None
else:
# If there is no entry point, it is a module
_module_name = "main"
_function = None
self.script_path = os.path.join(_source_path, _module_name + ".py")
# Send an instance message to the broker. This tells the broker that a worker has instantiated a process.
self.send_queue.put(
[None, store_bpm_process_instance_message(_start_message=_message, _worker_process_id=self.process_id)])
self.write_dbg_info("In start_bpm_process, initializing of " + self.script_path + " done.")
except Exception as e:
_error_message = self.log_prefix + "error initiating BPM process. processDefinitionId: " + \
message_is_none(_message, "processDefinitionId", "not set") + \
", process_id: " + message_is_none(_message, "processId", "not set") + \
", Error: " + str(e)
write_to_log(_error_message, _category=EC_NOTIFICATION, _severity=SEV_ERROR)
self.send_queue.put([None, log_process_state_message(
_changed_by=_message["userId"],
_state="failed",
_reason=_error_message,
_process_id=self.bpm_process_id)])
else:
try:
_thread_name = "BPM Process " + self.bpm_process_id + " thread"
# Load data
self.load_data(_source_path=_source_path)
# Add callbacks to scope
_globals.update(
{
"report_error": self.report_error,
"report_result": self.report_result,
"log_progress": self.log_progress,
"log_message": self.log_message,
"pause": self.pause,
"get_data": self.get_data
}
)
# Add the name space modules to the process globals
self.load_namespaces(_source_path=_source_path, _dest_globals=_globals)
if _function:
_function.__globals__.update(_globals)
self.bpm_process_thread = BPMTread(target=self.run_function,
name=_thread_name, args=[_function],
_user_id=_message["userId"],
_start_message=_message)
else:
self.bpm_process_thread = BPMTread(target=self.run_module,
name=_thread_name,
args=[_module_name, _globals],
_user_id=_message["userId"],
_start_message=_message)
# Send running state.
if "reason" in _message:
_reason = _message["reason"]
else:
_reason = "No reason"
self.send_queue.put([None, log_process_state_message(
_changed_by=self.bpm_process_thread.user_id,
_state="running",
_process_id=self.bpm_process_id,
_reason=_reason)])
threading.settrace(self.trace_calls)
self.bpm_process_thread.start()
except Exception as e:
_error_message = self.log_prefix + "error initiating module " + self.script_path + '.run() - ' + str(e) +\
" processDefinitionId: " + message_is_none(_message, "processDefinitionId", "not set") + \
", process_id: " + message_is_none(_message, "processId", "not set") + \
", Error: " + str(e)
write_to_log(_error_message, _category=EC_NOTIFICATION, _severity=SEV_ERROR)
self.send_queue.put([None, log_process_state_message(
_changed_by= _message["userId"],
_state="failed",
_reason=_error_message,
_process_id=self.bpm_process_id)])
def write_srvc_dbg(_data):
global process_id
write_to_log(_data, _category=EC_SERVICE, _severity=SEV_DEBUG, _process_id=process_id)
def start_agent(_cfg_filename = None):
"""
Starts the agent; Loads settings, connects to database, registers process and starts the web server.
"""
global process_id, _control_monitor, _terminated, _address, _process_queue_manager, _broker_url, \
_username, _password, _peers, _log_to_database_severity, _verify_SSL
_process_id = str(ObjectId())
of.common.logging.callback = log_locally
_terminated = False
# Handle multiprocessing on windows
freeze_support()
write_srvc_dbg("=====start_agent===============================")
try:
if _cfg_filename is None:
_cfg_filename = resolve_config_path()
_settings = JSONXPath(_cfg_filename)
except Exception as e:
write_to_log("Error loading settings: " + str(e), _category=EC_SERVICE, _severity=SEV_FATAL,
_process_id=_process_id)
return
of.common.logging.severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/severityLevel", _default="warning"))
_log_to_database_severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/brokerLevel", _default="warning"))
write_srvc_dbg("===register signal handlers===")
register_signals(stop_agent)
# An _address is completely necessary.
_address = _settings.get("agent/address", _default=None)
if not _address or _address == "":
raise Exception(write_to_log(
"Fatal error: Agent cannot start, missing [agent] _address setting in configuration file.",
_category=EC_SERVICE, _severity=SEV_FATAL))
# An _address is completely necessary.
_verify_SSL = _settings.get("agent/verifySSL", _default=True)
# Gather credentials
_broker_url = _settings.get("agent/brokerUrl", _default="127.0.0.1:8080")
_username = _settings.get("agent/username")
if not _username:
raise Exception(write_to_log("Username must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_password = _settings.get("agent/password")
if not _password:
raise Exception(write_to_log("Password must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_retries = int(_settings.get("agent/connectionRetries", 5))
# Register at the broker
if not register_agent(_retries):
raise Exception(write_to_log("Fatal: The agent failed to register with the broker, tried " + str(
_retries + 1) + " time(s), quitting.", _category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
of.common.logging.callback = log_to_database
_repository_base_folder = _settings.get_path("agent/repositoryFolder",
_default=os.path.join(os.path.dirname(__file__), "repositories"))
write_srvc_dbg("Load schema tool")
try:
# Initiate a schema tools instance for validation other purposes.
_schema_tools = SchemaTools(_json_schema_folders=[of_schema_folder(),
os.path.abspath(os.path.join(script_dir, "..", "schemas", "namespaces"))
],
_uri_handlers={"ref": None})
except Exception as e:
raise Exception(write_to_log("An error occurred while loading schema tools:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
return
write_srvc_dbg("Load schema tool done")
try:
write_srvc_dbg("Initializing monitors")
# Init the monitor for incoming messages
_message_monitor = Monitor(
_handler=AgentWebSocketHandler(_process_id=_process_id,
_peers=_peers,
_schema_tools=_schema_tools,
_address=_address,
_broker_address="broker"))
# The manager for the process queue
_process_queue_manager = multiprocessing.Manager()
# Init the monitor for the worker queue
_worker_monitor = Monitor(
_handler=WorkerSupervisor(_process_id=_process_id,
_message_monitor=_message_monitor,
_repo_base_folder=_repository_base_folder,
_severity=of.common.logging.severity),
_queue=_process_queue_manager.Queue())
# Init the monitor for the agent queue
_control_monitor = Monitor(
_handler=ControlHandler(_process_id=_process_id,
_message_monitor=_message_monitor,
_worker_monitor=_worker_monitor,
_stop_agent=stop_agent
))
# The global variable for handling websockets. TODO: Could this be done without globals? (PROD-33)
of.common.messaging.websocket.monitor = _message_monitor
write_srvc_dbg("Initializing monitors done")
except Exception as e:
raise Exception(write_to_log("Fatal: An error occurred while initiating the monitors and handlers:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
# Try to connect to websocket, quit on failure
if not connect_to_websocket():
os._exit(1)
write_srvc_dbg("Register agent system process")
_control_monitor.handler.message_monitor.queue.put(
[None, store_process_system_document(_process_id=_process_id,
_name="Agent instance(" + _address + ")")])
write_srvc_dbg("Log agent system state")
_control_monitor.handler.message_monitor.queue.put([None,
log_process_state_message(_changed_by=zero_object_id,
_state="running",
_process_id=_process_id,
_reason="Agent starting up at " +
_address)])
# Security check to remind broker if it is unsecured
if not _verify_SSL:
try:
call_api("https://"+ _broker_url + "/status", _data={}, _session_id= _session_id, _verify_SSL=True)
except SSLError as e:
write_to_log("There is a problem with the security certificate:\n" + str(e) + "\n"
"This is a security risk, and and important thing to _address.",
_category=EC_NOTIFICATION, _severity=SEV_WARNING)
except Exception as e:
write_to_log("An error occured while checking status of broker and SSL certificate:" + str(e),
_category=EC_NOTIFICATION, _severity=SEV_ERROR)
write_srvc_dbg("Agent up and running.")
while not _terminated:
time.sleep(0.1)
write_srvc_dbg("Exiting main thread")
def stop_agent(_reason, _restart=False):
"""
Shuts down the agent
:param _reason: The reason for shutting down
:param _restart: If set, the agent will restart
"""
global _process_queue_manager, _start_pid
# Make sure this is not a child process also calling signal handler
if _start_pid != os.getpid():
write_srvc_dbg("Ignoring child processes' signal call to stop_agent().")
return
if _restart is True:
write_srvc_dbg( "--------------AGENT WAS TOLD TO RESTART------------")
else:
write_srvc_dbg( "--------------AGENT WAS TERMINATED, shutting down orderly------------")
write_srvc_dbg( "Reason:" + str(_reason))
write_srvc_dbg("Process Id: " + str(process_id))
try:
write_srvc_dbg("try and tell the broker about shutting down")
_control_monitor.handler.message_monitor.queue.put([None,
log_process_state_message(_changed_by=zero_object_id,
_state="stopped",
_process_id=process_id,
_reason="Agent stopped at " +
_address)])
# Give some time for it to get there
time.sleep(0.1)
write_srvc_dbg("try and tell the broker about shutting down, done")
except Exception as e:
write_to_log("Tried and tell the broker about shutting down, failed, error:" + str(e),
_category=EC_COMMUNICATION, _severity=SEV_ERROR)
write_srvc_dbg( "Stop the control monitor.")
_control_monitor.stop(_reverse_order=True)
time.sleep(0.4)
write_srvc_dbg("Control monitor stopped.")
_exit_status = 0
_process_queue_manager.shutdown()
write_srvc_dbg("Process queue manager shut down.")
if _restart is True:
write_to_log("Agent was told to restart, so now it starts a new agent instance.", _category=EC_SERVICE, _severity=SEV_INFO )
#set_start_method("spawn", force=True)
_agent_process = Process(target=run_agent, name="optimalbpm_agent", daemon=False)
_agent_process.start()
# On the current process (source) must still exist while the new process runs if its to be run using without
# pOpen. TODO: Investigate if it really is impossible to create standalone(non-child) processes using Process.
write_srvc_dbg("Pid of new instance: " + str(_agent_process.pid))
_agent_process.join()
global _terminated
_terminated = True
write_srvc_dbg("Agent exiting with exit status " + str(_exit_status))
if os.name == "nt":
return _exit_status
else:
os._exit(_exit_status)
