def write_dbg_info(self, _data): """ Shortcut to writing debug information :param _message: The message :param _severity: The severity of the log information, a constant defined in the built-in logging module """ write_to_log(self.log_prefix + _data, _category=EC_NOTIFICATION, _severity=SEV_DEBUG, _process_id=self.process_id)
def run_worker_process(_parent_process_id, _process_id, _queue, _send_queue, _repo_base_folder, _severity): """ This function is the first thing that is called when the worker process is initialized :param _parent_process_id: The process_id of the parent process. :param _process_id: The process id of this worker :param _queue: The instance of multiprocess queue on which the worker expects process messages to appear. :param _send_queue: The queue the process uses for external communication :param _repo_base_folder: The repository base folder. """ global send_queue, process_id of.common.logging.callback = log_to_queue of.common.logging.severity = _severity process_id = _process_id send_queue = _send_queue write_to_log("Run.Run_worker_process: Started a new worker process.") # Create a worker handler using the supplied parameters _worker_handler = WorkerHandler(_process_id=_process_id, _parent_process_id=_parent_process_id, _send_queue=_send_queue, _repo_base_folder=_repo_base_folder) # Start an monitor for that the inbound queue, specify handler _worker_monitor = Monitor(_handler=_worker_handler, _queue=_queue) # Run until terminated while not _worker_handler.terminated: time.sleep(0.1) write_to_log("Run.Run_worker_process: Exiting worker process.")
def handle(self, _item): """ Called when the monitor has gotten a message from a worker process """ if _item[0]: # If the source websocket is set, it is not from a worker process, raise error. raise Exception("The process handler only handles messages from worker processes.") else: # TODO: Should any filtering be done here? (PROD-27) _message_data = _item[1] if not isinstance(_message_data, dict): write_to_log("A worker process sent message data that is not a dict, this might be a an attack.", _category=EC_BREAKIN, _severity=SEV_ERROR) raise Exception(self.log_prefix + "A worker process sent message data that is not a dict, this " "might be a an attack: " + str(_message_data)) if "schemaRef" not in _message_data: raise Exception(self.log_prefix + "Missing schemaRef: " + str(_message_data)) if _message_data["schemaRef"] == "ref://bpm.message.bpm.process.result": # A process result message implies that the worker is done and available for new jobs self.release_worker(_message_data["sourceProcessId"]) elif _message_data["schemaRef"] == "ref://of.log.process_state" and \ _message_data["processId"] in self.workers and \ _message_data["state"] in ["killed"]: # If a worker is logging that it is being killed, it should be remove from the workers self.write_dbg_info(self.log_prefix + "Worker " + _message_data["processId"] + " shut down, removing from workers.") del self.workers[_message_data["processId"]] self.write_dbg_info("Forwarding " + str(_message_data)) # Pass the message on to the message queue, heading for the last destination self.message_monitor.queue.put([_item[0], _item[1]])
def handle_message(self, _source_web_socket, _message_data): """ Handle inbound and outbound messages. :param _source_web_socket: The web socket of the source, if none, it is outbound :param _message_data: The message data """ if _source_web_socket is None: # This is an outbound message self.outbound_message_count += 1 # TODO: Handle multiple broker peers(other peers?), be informed trough some kind of messaging.(PROD-25) # For now, however, always have one broker. _message_data["source"] = self.address self.send_to_address(self.broker_address, _message_data) else: self.inbound_message_count += 1 # This is an inbound message try: self.schema_tools.validate(_message_data) except Exception as e: _error = "An error occurred validating an inbound message:" + str( e) write_to_log(_error, _category=EC_COMMUNICATION, _severity=SEV_ERROR) # Respond to sender with an error message self.send_to_address( self.broker_address, reply_with_error_message(self, _message_data, _error)) else: self.process_handler.forward_message(_message_data)
def write_to_log(self, **kwargs): """ This function provides a way for registered peers to write to the log even if they have no web socket. :param kwargs: :return: """ _session_id = kwargs["_session_id"] # Only registered peers get to write to the log, other trying is a probe if _session_id not in self.peers: write_to_log("Client that wasn't registered as peer tried to write to log ", _category=EC_PROBE, _severity=SEV_WARNING) return None _message = cherrypy.request.json write_to_log(_data=_message.get("data"), _category=category_identifiers.index(_message.get("category")), _severity=severity_identifiers.index(_message.get("severity")), _occurred_when=_message.get("occurred_when"), _address=_message.get("address"), _process_id=_message.get("process_id"), _user_id=_message.get("user_id"), _pid=_message.get("pid"), _uid=_message.get("uid"), _node_id=_message.get("node_id") ) return "{}"
def handle_message(self, _source_web_socket, _message_data): """ Handle inbound and outbound messages. :param _source_web_socket: The web socket of the source, if none, it is outbound :param _message_data: The message data """ if _source_web_socket is None: # This is an outbound message self.outbound_message_count += 1 # TODO: Handle multiple broker peers(other peers?), be informed trough some kind of messaging.(PROD-25) # For now, however, always have one broker. _message_data["source"] = self.address self.send_to_address(self.broker_address, _message_data) else: self.inbound_message_count += 1 # This is an inbound message try: self.schema_tools.validate(_message_data) except Exception as e: _error = "An error occurred validating an inbound message:" + str(e) write_to_log(_error, _category=EC_COMMUNICATION, _severity=SEV_ERROR) # Respond to sender with an error message self.send_to_address(self.broker_address, reply_with_error_message(self, _message_data, _error)) else: self.process_handler.forward_message(_message_data)
def agent_control(self, _address, _command, _reason, _user): """ Controls an agent's running state :param _address: The address of the agent to control :param _command: Can be "stop" or "restart". :param _user: A user instance """ write_to_log("Control.agent_control: Got the command " + str(_command), _category=EC_SERVICE, _severity=SEV_DEBUG) self.send_queue.put([ None, agent_control(_destination=_address, _destination_process_id=zero_object_id, _command=_command, _reason=_reason, _user_id=_user["_id"], _source=self.address, _source_process_id=self.process_id, _message_id=1) ]) return {}
def run_module(self, _module_name, _globals): """ Run a BPM process module :param _module_name: Full path to the model. :param _globals: The globals. """ self.write_dbg_info("->>>>>>>>>>>> Running module " + _module_name + ", globals: " + str(_globals)) try: # Call the run_module, it returns the globals after execution _new_globals = runpy.run_path(os.path.join(self.source_path, _module_name + ".py"), init_globals=_globals) except TerminationException as e: write_to_log("TerminationException running the process:" + str(e), _category=EC_NOTIFICATION, _severity=SEV_ERROR) # The process was terminated _result = self.report_termination(_globals) _new_globals = _globals except Exception as e: # An unhandled error occured running the process write_to_log("Error running the process:" + str(e), _category=EC_NOTIFICATION, _severity=SEV_ERROR) _result = self.report_error(self.log_prefix + "Error in module " + _module_name + ".py:" + str(e), e) _new_globals = None else: # The process ended normally self.report_finished() # A module has no return value _result = None self.write_dbg_info(self.log_prefix + "<<<<<<<<<<<<- Done calling " + self.script_path + ".") self.report_result(_globals=_new_globals, _result=_result) self.job_running = False
def restart_agent(self, _reason): try: # Wait 3 seconds and then restart the agent. time.sleep(3) self.stop_agent(_reason=_reason, _restart=True) except Exception as e: write_to_log("Error terminating self:" + str(e), _category=EC_SERVICE, _severity=SEV_ERROR)
def agent_control(self, **kwargs): write_to_log("Got an agent control call, command "+ cherrypy.request.json["command"] + ", reason: " + cherrypy.request.json["reason"] + ", address: " + cherrypy.request.json["address"] + str(cherrypy.request.json)) return self._control.agent_control(cherrypy.request.json["address"], cherrypy.request.json["command"], cherrypy.request.json["reason"], kwargs["_user"])
def error_handler(self, exception): """ If an unhandled error occurs, unregister at the broker, and close the socket """ write_to_log(self.log_prefix + "An exception handle in the socket, closing. Error: " + str(exception), _category=EC_COMMUNICATION, _severity=SEV_ERROR) monitor.handler.unregister_web_socket(self) # Tell the socket to close return False
def refresh_static(self, _web_config): """ This function regenerates all the static content that is used to initialize the user interface plugins. :param _web_config: An instance of the CherryPy web configuration """ self.admin_ui_hooks = self.refresh_hooks() _imports = "" _systemjs = "" _admin_menus = [] # has_right(id_right_admin_everything, kwargs["user"]) for _curr_plugin_key, _curr_plugin_info in self.plugins.items(): # Add any plugin configuration for the Admin user interface if "admin-ui" in _curr_plugin_info: _curr_ui_def = _curr_plugin_info["admin-ui"] if "mountpoint" not in _curr_ui_def: write_to_log("Error loading admin-ui for " + _curr_plugin_key + " no mount point.", _category=EC_SERVICE, _severity=SEV_ERROR) continue _mount_point = _curr_ui_def["mountpoint"] if _mount_point[0] == "/": write_to_log( "Not mounting " + _mount_point + ", cannot mount admin-specific ui under root(root can " "never depend on admin), use root-ui instead.", _category=EC_SERVICE, _severity=SEV_ERROR) continue # Mount the static content at a mount point somewhere under /admin _web_config.update({ "/admin/" + _mount_point: { "tools.staticdir.on": True, "tools.staticdir.dir": os.path.join(_curr_plugin_info["baseDirectoryName"], _mount_point), "tools.trailing_slash.on": True } }) _systemjs += "System.config({\"packages\": {\"" + _mount_point + "\": {\"defaultExtension\": \"ts\"}}});\n" # Add menus if "menus" in _curr_ui_def: _admin_menus += _curr_ui_def["menus"] self.admin_systemjs_init = _systemjs self.admin_menus = _admin_menus
def handle_error(self, _error_message, _message): """ A generic function for handling errors :param _error: The error message """ write_to_log( "An error occured, replying with an error message. Error: " + str(_error_message), _category=EC_SERVICE, _severity=SEV_ERROR) self.message_monitor.queue.put( [None, reply_with_error_message(self, _message, _error_message)])
def closed(self, code, reason=None): """ Called when the socket is closed. :param code: A web socket error code as defined in: http://tools.ietf.org/html/rfc6455#section-7.4.1 :param reason: A string describing the reason for closing the connection """ # TODO: Handle the rest of the possible web socket error codes self.connected = False if code == ABNORMAL_CLOSE: write_to_log(self.log_prefix + "The connection to " + self.address + " has been abnormally closed.", _category=EC_COMMUNICATION, _severity=SEV_ERROR) self.close(code=code, reason=reason) else: self.monitor_message_queue_thread.terminated = True self.write_dbg_info(self.log_prefix + "Closed, code: " + str(code) + ", reason: " + str(reason))
def __init__(self, _process_id, _address, _database_access): """ Initializes the broker web service and includes and initiates the other parts of the API as well :param _database_access: A DatabaseAccess instance for database connectivity :param _process_id: The system process id of the broker :param _address: The peer address of the broker :param _stop_broker: A callback to a function that shuts down the broker """ write_to_log(_process_id=_process_id, _category=EC_SERVICE, _severity=SEV_DEBUG, _data="Initializing broker REST API.") self.peers = {} self.process_id = _process_id self.address = _address self.node = CherryPyNode(_database_access=_database_access)
def handle_error( self, _error, _category=EC_COMMUNICATION, _severity=SEV_ERROR, _web_socket=None, _close_socket=None, _message_to_reply_to=None, ): """ :param _error: The error message :param _category: The category of the error :param _severity: Error severity :param _web_socket: If the sender was external, the web socket :param _close_socket: Close the web socket :param _message_to_reply_to: The source message, if set, the error message is sent to the sender :return: """ """ Handles and logs errors, decides whether to keep the connection open if an error occurrs """ _error = write_to_log( _data=self.log_prefix + "In handler.handle_error error with error :" + str(_error), _category=_category, _severity=_severity, ) if _web_socket: if _message_to_reply_to: _web_socket.send_message(reply_with_error_message(self, _message_to_reply_to, _error)) if _close_socket: _web_socket.close(code=PROTOCOL_ERROR, reason=_error)
def register_agent(_retries, _connect=False): global _broker_url, _username, _password, _peers, _session_id, _verify_SSL _retry_count = _retries + 1 write_srvc_dbg("Register agent session (adress : " + _address + ") at broker(URL: https://" + _broker_url + ")") # Register session at the broker _data = None while _retry_count > 0: try: _data = register_at_broker(_address=_address, _type="agent", _server="https://" + _broker_url, _username=_username, _password=_password, _verify_SSL=_verify_SSL) write_srvc_dbg("Agent tried registering, data returned: " + str(_data)) except Exception as e: if _retry_count > 1: write_to_log("Failed to register at the broker, will retry " + str( _retry_count - 1) + " more times, error:" + str(e), _category=EC_SERVICE, _severity=SEV_INFO) else: write_to_log( "Failed to register at the broker, will not retry any more times, error:" + str(e), _category=EC_SERVICE, _severity=SEV_FATAL) if _data: break else: if _retry_count > 1: time.sleep(3) _retry_count -= 1 if not _data: return False _session_id = _data["session_id"] write_srvc_dbg("Register session at broker done") _peers[_session_id] = { "address": "broker", "session_id": _session_id, "queue": Queue() } if _connect: return connect_to_websocket() else: return True
def get_peers(self, **kwargs): """ Returns a list of all logged in peers :param kwargs: Unused here so far, but injected by get session :return: A list of all logged in peers """ # TODO: This should probably not be in admin. However listing peers does seems slightly administrative. _result = [] # Filter out the unserializable web socket for _session in self.root.peers.values(): _new_session = copy.copy(_session) _new_session["web_socket"] = "removed for serialization" _new_session["queue"] = "removed for serialization" _result.append(_new_session) write_to_log(_process_id=self.process_id, _category=EC_NOTIFICATION, _severity=SEV_DEBUG, _data="Returning a list of peers:" + str(_result)) return _result
def unregister(self, **kwargs): _session_id = kwargs["_session_id"] if "session_id" in cherrypy.request.cookie: cherrypy.response.cookie = cherrypy_logout(_session_id) _peer = self.peers[_session_id] if "websocket" in _peer: try: _peer["websocker"].close() write_to_log(_data="Unregister: Closed websocket for address " + _peer["address"] + ".", _category=EC_COMMUNICATION, _severity=SEV_DEBUG) except: write_to_log(_data="Unregister: Failed closing websocket for address " + _peer["address"] + ".", _category=EC_COMMUNICATION, _severity=SEV_ERROR) del self.peers[_session_id] return {}
def connect_to_websocket(): global _broker_url, _session_id write_srvc_dbg("Connecting web socket to broker") try: # Initiate the web socket connection to the broker _web_socket = AgentWebSocket(url="wss://" + _broker_url + "/socket", _session_id=_session_id, _stop_agent=stop_agent, _register_agent = register_agent) _web_socket.connect() _web_socket.run_forever() except Exception as e: write_to_log("Fatal: An error occurred establishing the web socket:" + str(e), _category=EC_SERVICE, _severity=SEV_FATAL) return False write_srvc_dbg("Connecting web socket to broker done") return True
def step_impl(context): """ :type context: behave.runner.Context """ global _global_params, _global_err_param, logging_callback _global_params = None of.common.logging.callback = local_test_log_writer try: write_to_log(*_global_err_param) ok_(_global_params == _global_err_param, "Global params didn't match!\nResult:" + str(_global_params) + "\nComparison: " + str(_global_err_param)) _global_params = None of.common.logging.callback = None except Exception as e: # Be sure to reset globals.< _global_params = None of.common.logging.callback = None raise Exception(e) # TODO: Add test for sparse message
def __init__(self, _json_schema_folders=[], _uri_handlers=None): """ Initiate the SchemaTools class :param _json_schema_folders: A list of folders where schema files are stored :param : _uri_handlers: A dict of uri_handlers, resolves a URI prefix to a actual schema. """ if not _json_schema_folders: _json_schema_folders = [] if not _uri_handlers: self.uri_handlers = {} else: self.uri_handlers = _uri_handlers # All methods that have no handlers should use the cache handler. for _curr_key, _curr_value in _uri_handlers.items(): if _curr_value is None: _uri_handlers[_curr_key] = self.cache_handler self.resolver = RefResolver(base_uri="", handlers=self.uri_handlers, referrer=None, cache_remote=True) self.mongodb_validator = MongodbValidator(resolver= self.resolver) self.json_schema_objects = {} # Load application specific schemas for _curr_folder in _json_schema_folders: _loaded_uris = self.load_schemas_from_directory(os.path.abspath(_curr_folder)) # Resolve all the schemas for _curr_uri in _loaded_uris: self.json_schema_objects[_curr_uri] = self.resolveSchema(self.json_schema_objects[_curr_uri]) write_to_log("Schemas loaded and resolved: " + str.join(", ", ["\"" +_curr_schema["title"] + "\"" for _curr_schema in self.json_schema_objects.values()]) , _category=EC_NOTIFICATION, _severity=SEV_DEBUG)
def monitor_message_queue(self): """ Monitors the messages queue. Stops when the queue-threads' terminated attribute is set to True """ self.monitor_message_queue_thread.terminated = False while not self.monitor_message_queue_thread.terminated: try: _message = self.message_queue.get(True, .1) try: self.send_message(_message) except Exception as e: write_to_log(self.log_prefix + "Error sending message:" + str(e) + "\nTraceback:" + traceback.format_exc(), _category=EC_COMMUNICATION, _severity=SEV_ERROR) except Empty: pass except Exception as e: write_to_log(self.log_prefix + "Error accessing send queue:" + str(e) + "\nTraceback:" + traceback.format_exc(), _category=EC_INTERNAL, _severity=SEV_ERROR) self.write_dbg_info(" stopped message queue monitoring. Exiting thread \"" + str(self.monitor_message_queue_thread.name) + "\"")
def agent_control(self, _address, _command, _reason, _user): """ Controls an agent's running state :param _address: The address of the agent to control :param _command: Can be "stop" or "restart". :param _user: A user instance """ write_to_log("Control.agent_control: Got the command " + str(_command), _category=EC_SERVICE, _severity=SEV_DEBUG) self.send_queue.put([None, agent_control(_destination=_address, _destination_process_id=zero_object_id, _command=_command, _reason=_reason, _user_id=_user["_id"], _source=self.address, _source_process_id=self.process_id, _message_id=1 )]) return {}
def start(self): """ Start monitoring the queue """ if self.monitor_thread and not self.monitor_thread.terminated: raise Exception(write_to_log(self.log_prefix + "The queue monitor is already running.", _category=EC_INTERNAL, _severity=SEV_ERROR)) self.monitor_thread = MonitorThread(_monitor=self) self.monitor_thread.terminated = False self.monitor_thread.start() self.write_dbg_info("Running, monitoring thread: " + str(self.monitor_thread.name))
def call_hook(self, _hook_name, **kwargs): self.write_debug_info("Running hook " + _hook_name) for _curr_plugin_name, _curr_plugin in self.plugins.items(): if "failed" in _curr_plugin and _curr_plugin["failed"]: self.write_debug_info("Plugin " + _curr_plugin_name + " marked failed, will not call its hook.") continue if "hooks_instance" in _curr_plugin: _hooks_instance = _curr_plugin["hooks_instance"] if hasattr(_hooks_instance, _hook_name): try: self.write_debug_info("Calling " + _hook_name + " in " + _curr_plugin["description"]) getattr(_hooks_instance, _hook_name)(**kwargs) except Exception as e: write_to_log( "An error occurred " + "Calling " + _hook_name + " in " + _curr_plugin_name + ":" + str(e), _category=EC_SERVICE, _severity=SEV_ERROR, ) if "failOnError" in _curr_plugin and _curr_plugin["failOnError"]: write_to_log( "Setting " + _curr_plugin_name + " as Failed. No more hooks will be called for this plugin.", _category=EC_SERVICE, _severity=SEV_INFO, ) _curr_plugin["failed"] = True else: write_to_log( "Ignores error, this plugin will continue to attempt initialization.", _category=EC_SERVICE, _severity=SEV_INFO, )
def register_at_broker(_address, _type, _server, _username, _password, _log_prefix="", _verify_SSL=True): _log_prefix = make_log_prefix(_log_prefix) _data = { "credentials": { "usernamePassword": { "username": _username, "password": _password } }, "environment": get_environment_data(), "peerType": _type, "address": _address } write_dbg_info(_log_prefix + "[" + str(datetime.datetime.utcnow()) + "] Registering at broker API.") _headers = {'content-type': 'application/json'} _response = requests.post(_server + "/register", data=json.dumps(_data), auth=('user', 'pass'), headers=_headers, verify=_verify_SSL) if _response.status_code == 500: write_dbg_info(_log_prefix + "Broker login failed with internal server error! Exiting.") return False if _response.status_code != 200: write_dbg_info(_log_prefix + "Broker login failed with error + "+ str(_response.status_code) + "! Exiting.") return False _response_dict = _response.json() if _response_dict is not None: _data = _response_dict if "session_id" in _data: write_dbg_info(_log_prefix + "Got a session id:" + _data["session_id"]) return _data else: write_to_log(_log_prefix + "Broker login failed! Exiting.", _category=EC_SERVICE, _severity=SEV_ERROR) return False else: write_to_log(_log_prefix + "Broker login failed! Exiting.", _category=EC_SERVICE, _severity=SEV_ERROR) return False
def save_process(self, **kwargs): """ Save a process structure into a source file :param kwargs: A parameter object """ # TODO: Document the structure of the process parameters, perhaps create a schema?(ORG-110) has_right(id_right_admin_everything, kwargs["_user"]) _tokens = ProcessTokens(_keywords=self.keywords, _namespaces=self.namespaces) _tokens.documentation = cherrypy.request.json["documentation"] _verbs = _tokens.json_to_verbs(_json=cherrypy.request.json["verbs"]) if "documentation" in cherrypy.request.json: _tokens.documentation = cherrypy.request.json["documentation"] _process_id = cherrypy.request.json["processId"] _repo_path = os.path.join(os.path.expanduser(self.repository_parent_folder), _process_id) _filename = os.path.join(_repo_path, "main.py") # Store in repository if not os.path.exists(_repo_path): # No process definition, it is a new process, create a folder. write_to_log("Initating new repo at " + _repo_path, EC_NOTIFICATION, SEV_INFO) os.makedirs(_repo_path) # TODO: Init git repo (PROD-11) _namespaces, _map = _tokens.encode_process(_verbs=_verbs, _filename=_filename) _filename_namespaces = os.path.join(_repo_path, "namespaces.json") with open(_filename_namespaces, "w") as f: json.dump(_namespaces, f) _filename_map = os.path.join(_repo_path, "map.json") with open(_filename_map, "w") as f: json.dump(_map, f) _filename_data = os.path.join(_repo_path, "data.json") with open(_filename_data, "w") as f: json.dump(cherrypy.request.json["paramData"], f) write_to_log("save_process to " + _repo_path + ", done", _category=EC_NOTIFICATION, _severity=SEV_DEBUG)
def __init__(self, _process_id, _address, _stop_broker, _root_object, _plugins, _web_config): write_to_log(_category=EC_SERVICE, _severity=SEV_DEBUG, _process_id=_process_id, _data="Initializing administrative REST API.") self.stop_broker = _stop_broker self.process_id = _process_id self.address = _address self.root = _root_object self.plugins = _plugins # Mount the static content at a mount point /admin _web_config.update({ "/admin" : { "tools.staticdir.on": True, "tools.staticdir.dir": os.path.join(os.path.dirname(__file__), "../ui"), "tools.trailing_slash.on": True } }) self.refresh_static(_web_config=_web_config)
def broker_ctrl(self, _command, _reason, _user): """ Controls the broker's running state :param _command: Can be "stop" or "restart". :param _user: A user instance """ write_to_log("broker.broker_control: Got the command " + str(_command), _category=EC_SERVICE, _process_id=self.process_id) # TODO: There should be a log item written with reason and userid.(PROD-32) # TODO: UserId should also be appended to reason below.(PROD-32) def _command_local(_local_command): if _local_command == "restart": self.stop_broker(_reason=_reason, _restart=True) if _local_command == "stop": self.stop_broker(_reason=_reason, _restart=False) _restart_thread = threading.Thread(target=_command_local, args=[_command]) _restart_thread.start() return {}
def monitor(self): """ Monitors the queue. Stops when the queue-threads' terminated attribute is set to True """ self.write_dbg_info("In monitor thread monitoring queue.") while not self.monitor_thread.terminated: try: _item = self.queue.get(True, .1) try: self.handler.handle(_item) except Exception as e: write_to_log(self.log_prefix + "Error handling item:" + str(e) + "\nData:\n" + str(_item) + "\nTraceback:" + traceback.format_exc(), _category=EC_INTERNAL, _severity=SEV_ERROR) except Empty: pass except Exception as e: self.monitor_thread.terminated = True raise Exception(write_to_log(self.log_prefix + "Terminating, error accessing own queue:" + str(e), _category=EC_INTERNAL, _severity=SEV_ERROR)) self.after_get_queue() self.write_dbg_info("Monitor thread stopped.")
def init(self, _session_id): """ Initialize the socket, set the session id an register the web socket with the handler, start monitoring queue. :param _session_id: The sessionid """ self.session_id = _session_id self.log_prefix = str(os.getpid()) + "-" + self.__class__.__name__ + "(No address yet): " self.connected = False try: monitor.handler.register_web_socket(self) except Exception as e: raise Exception(write_to_log("OptimalWebSocket: Exception registering web socket:" + str(e), _category=EC_COMMUNICATION, _severity=SEV_ERROR)) # Generate log prefix string for performance self.log_prefix = str(os.getpid()) + "-" + self.__class__.__name__ + "(" + str(self.address) + "): " self.start_monitoring_message_queue()
def write_dbg_info(self, _data): write_to_log(self.log_prefix + str(_data), _category=EC_COMMUNICATION, _severity=SEV_DEBUG, _process_id=self.process_id)
def start_bpm_process(self, _message): """ Initializes and starts a BPM process based on the content of the message :param _message: A message_bpm_process_start-message """ try: # Extract caller information self.bpm_process_id = _message["processId"] self.bpm_source = str(message_is_none(_message, "source", "")) # Execution context _entry_point = message_is_none(_message, "entryPoint", None) _globals = message_is_none(_message, "globals", {}) # Create the source path to the module _source_path = os.path.join(os.path.expanduser(self.repo_base_folder), _message["processDefinitionId"]) self.source_path = _source_path self.write_dbg_info("Source path: " + _source_path) if _entry_point: # If there is an entry point, it is a function call _module_name = _entry_point["moduleName"] if "functionName" in _entry_point: # Add repository location to sys.path to be able to import sys.path.append(self.source_path) _module = importlib.import_module(_module_name) _function_name = _entry_point["functionName"] self.write_dbg_info("In start_bpm_process, initializing: " + str(_message)) # Find function _function = getattr(_module, _function_name) else: _function = None else: # If there is no entry point, it is a module _module_name = "main" _function = None self.script_path = os.path.join(_source_path, _module_name + ".py") # Send an instance message to the broker. This tells the broker that a worker has instantiated a process. self.send_queue.put( [None, store_bpm_process_instance_message(_start_message=_message, _worker_process_id=self.process_id)]) self.write_dbg_info("In start_bpm_process, initializing of " + self.script_path + " done.") except Exception as e: _error_message = self.log_prefix + "error initiating BPM process. processDefinitionId: " + \ message_is_none(_message, "processDefinitionId", "not set") + \ ", process_id: " + message_is_none(_message, "processId", "not set") + \ ", Error: " + str(e) write_to_log(_error_message, _category=EC_NOTIFICATION, _severity=SEV_ERROR) self.send_queue.put([None, log_process_state_message( _changed_by=_message["userId"], _state="failed", _reason=_error_message, _process_id=self.bpm_process_id)]) else: try: _thread_name = "BPM Process " + self.bpm_process_id + " thread" # Load data self.load_data(_source_path=_source_path) # Add callbacks to scope _globals.update( { "report_error": self.report_error, "report_result": self.report_result, "log_progress": self.log_progress, "log_message": self.log_message, "pause": self.pause, "get_data": self.get_data } ) # Add the name space modules to the process globals self.load_namespaces(_source_path=_source_path, _dest_globals=_globals) if _function: _function.__globals__.update(_globals) self.bpm_process_thread = BPMTread(target=self.run_function, name=_thread_name, args=[_function], _user_id=_message["userId"], _start_message=_message) else: self.bpm_process_thread = BPMTread(target=self.run_module, name=_thread_name, args=[_module_name, _globals], _user_id=_message["userId"], _start_message=_message) # Send running state. if "reason" in _message: _reason = _message["reason"] else: _reason = "No reason" self.send_queue.put([None, log_process_state_message( _changed_by=self.bpm_process_thread.user_id, _state="running", _process_id=self.bpm_process_id, _reason=_reason)]) threading.settrace(self.trace_calls) self.bpm_process_thread.start() except Exception as e: _error_message = self.log_prefix + "error initiating module " + self.script_path + '.run() - ' + str(e) +\ " processDefinitionId: " + message_is_none(_message, "processDefinitionId", "not set") + \ ", process_id: " + message_is_none(_message, "processId", "not set") + \ ", Error: " + str(e) write_to_log(_error_message, _category=EC_NOTIFICATION, _severity=SEV_ERROR) self.send_queue.put([None, log_process_state_message( _changed_by= _message["userId"], _state="failed", _reason=_error_message, _process_id=self.bpm_process_id)])
def write_srvc_dbg(_data): global process_id write_to_log(_data, _category=EC_SERVICE, _severity=SEV_DEBUG, _process_id=process_id)
def start_agent(_cfg_filename = None): """ Starts the agent; Loads settings, connects to database, registers process and starts the web server. """ global process_id, _control_monitor, _terminated, _address, _process_queue_manager, _broker_url, \ _username, _password, _peers, _log_to_database_severity, _verify_SSL _process_id = str(ObjectId()) of.common.logging.callback = log_locally _terminated = False # Handle multiprocessing on windows freeze_support() write_srvc_dbg("=====start_agent===============================") try: if _cfg_filename is None: _cfg_filename = resolve_config_path() _settings = JSONXPath(_cfg_filename) except Exception as e: write_to_log("Error loading settings: " + str(e), _category=EC_SERVICE, _severity=SEV_FATAL, _process_id=_process_id) return of.common.logging.severity = of.common.logging.severity_identifiers.index( _settings.get("agent/logging/severityLevel", _default="warning")) _log_to_database_severity = of.common.logging.severity_identifiers.index( _settings.get("agent/logging/brokerLevel", _default="warning")) write_srvc_dbg("===register signal handlers===") register_signals(stop_agent) # An _address is completely necessary. _address = _settings.get("agent/address", _default=None) if not _address or _address == "": raise Exception(write_to_log( "Fatal error: Agent cannot start, missing [agent] _address setting in configuration file.", _category=EC_SERVICE, _severity=SEV_FATAL)) # An _address is completely necessary. _verify_SSL = _settings.get("agent/verifySSL", _default=True) # Gather credentials _broker_url = _settings.get("agent/brokerUrl", _default="127.0.0.1:8080") _username = _settings.get("agent/username") if not _username: raise Exception(write_to_log("Username must be configured", _category=EC_SERVICE, _severity=SEV_FATAL)) _password = _settings.get("agent/password") if not _password: raise Exception(write_to_log("Password must be configured", _category=EC_SERVICE, _severity=SEV_FATAL)) _retries = int(_settings.get("agent/connectionRetries", 5)) # Register at the broker if not register_agent(_retries): raise Exception(write_to_log("Fatal: The agent failed to register with the broker, tried " + str( _retries + 1) + " time(s), quitting.", _category=EC_SERVICE, _severity=SEV_FATAL)) os._exit(1) of.common.logging.callback = log_to_database _repository_base_folder = _settings.get_path("agent/repositoryFolder", _default=os.path.join(os.path.dirname(__file__), "repositories")) write_srvc_dbg("Load schema tool") try: # Initiate a schema tools instance for validation other purposes. _schema_tools = SchemaTools(_json_schema_folders=[of_schema_folder(), os.path.abspath(os.path.join(script_dir, "..", "schemas", "namespaces")) ], _uri_handlers={"ref": None}) except Exception as e: raise Exception(write_to_log("An error occurred while loading schema tools:" + str(e), _category=EC_SERVICE, _severity=SEV_FATAL)) os._exit(1) return write_srvc_dbg("Load schema tool done") try: write_srvc_dbg("Initializing monitors") # Init the monitor for incoming messages _message_monitor = Monitor( _handler=AgentWebSocketHandler(_process_id=_process_id, _peers=_peers, _schema_tools=_schema_tools, _address=_address, _broker_address="broker")) # The manager for the process queue _process_queue_manager = multiprocessing.Manager() # Init the monitor for the worker queue _worker_monitor = Monitor( _handler=WorkerSupervisor(_process_id=_process_id, _message_monitor=_message_monitor, _repo_base_folder=_repository_base_folder, _severity=of.common.logging.severity), _queue=_process_queue_manager.Queue()) # Init the monitor for the agent queue _control_monitor = Monitor( _handler=ControlHandler(_process_id=_process_id, _message_monitor=_message_monitor, _worker_monitor=_worker_monitor, _stop_agent=stop_agent )) # The global variable for handling websockets. TODO: Could this be done without globals? (PROD-33) of.common.messaging.websocket.monitor = _message_monitor write_srvc_dbg("Initializing monitors done") except Exception as e: raise Exception(write_to_log("Fatal: An error occurred while initiating the monitors and handlers:" + str(e), _category=EC_SERVICE, _severity=SEV_FATAL)) os._exit(1) # Try to connect to websocket, quit on failure if not connect_to_websocket(): os._exit(1) write_srvc_dbg("Register agent system process") _control_monitor.handler.message_monitor.queue.put( [None, store_process_system_document(_process_id=_process_id, _name="Agent instance(" + _address + ")")]) write_srvc_dbg("Log agent system state") _control_monitor.handler.message_monitor.queue.put([None, log_process_state_message(_changed_by=zero_object_id, _state="running", _process_id=_process_id, _reason="Agent starting up at " + _address)]) # Security check to remind broker if it is unsecured if not _verify_SSL: try: call_api("https://"+ _broker_url + "/status", _data={}, _session_id= _session_id, _verify_SSL=True) except SSLError as e: write_to_log("There is a problem with the security certificate:\n" + str(e) + "\n" "This is a security risk, and and important thing to _address.", _category=EC_NOTIFICATION, _severity=SEV_WARNING) except Exception as e: write_to_log("An error occured while checking status of broker and SSL certificate:" + str(e), _category=EC_NOTIFICATION, _severity=SEV_ERROR) write_srvc_dbg("Agent up and running.") while not _terminated: time.sleep(0.1) write_srvc_dbg("Exiting main thread")
def stop_agent(_reason, _restart=False): """ Shuts down the agent :param _reason: The reason for shutting down :param _restart: If set, the agent will restart """ global _process_queue_manager, _start_pid # Make sure this is not a child process also calling signal handler if _start_pid != os.getpid(): write_srvc_dbg("Ignoring child processes' signal call to stop_agent().") return if _restart is True: write_srvc_dbg( "--------------AGENT WAS TOLD TO RESTART------------") else: write_srvc_dbg( "--------------AGENT WAS TERMINATED, shutting down orderly------------") write_srvc_dbg( "Reason:" + str(_reason)) write_srvc_dbg("Process Id: " + str(process_id)) try: write_srvc_dbg("try and tell the broker about shutting down") _control_monitor.handler.message_monitor.queue.put([None, log_process_state_message(_changed_by=zero_object_id, _state="stopped", _process_id=process_id, _reason="Agent stopped at " + _address)]) # Give some time for it to get there time.sleep(0.1) write_srvc_dbg("try and tell the broker about shutting down, done") except Exception as e: write_to_log("Tried and tell the broker about shutting down, failed, error:" + str(e), _category=EC_COMMUNICATION, _severity=SEV_ERROR) write_srvc_dbg( "Stop the control monitor.") _control_monitor.stop(_reverse_order=True) time.sleep(0.4) write_srvc_dbg("Control monitor stopped.") _exit_status = 0 _process_queue_manager.shutdown() write_srvc_dbg("Process queue manager shut down.") if _restart is True: write_to_log("Agent was told to restart, so now it starts a new agent instance.", _category=EC_SERVICE, _severity=SEV_INFO ) #set_start_method("spawn", force=True) _agent_process = Process(target=run_agent, name="optimalbpm_agent", daemon=False) _agent_process.start() # On the current process (source) must still exist while the new process runs if its to be run using without # pOpen. TODO: Investigate if it really is impossible to create standalone(non-child) processes using Process. write_srvc_dbg("Pid of new instance: " + str(_agent_process.pid)) _agent_process.join() global _terminated _terminated = True write_srvc_dbg("Agent exiting with exit status " + str(_exit_status)) if os.name == "nt": return _exit_status else: os._exit(_exit_status)