def read_from_file(self, filepath): config = config_instance.create(filepath, raw=True) # todo: use file lock self._start_time = config.get("DEFAULT", "start_time") try: self._start_time = datetime.datetime.strptime( self._start_time, "%Y-%m-%dT%H:%M:%S") except BaseException: self._start_time = datetime.datetime.strptime( "2000-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S") self._processbot_cmd_line = config.get("DEFAULT", "cmd_line") if self._processbot_cmd_line is not None and self._processbot_cmd_line != '': self._processbot_cmd_line = json.loads(self._processbot_cmd_line) self._unique_id = config.get("DEFAULT", "unique_id") self._sandbox_path = config.get_path("DEFAULT", "sendbox_path") try: self._processbot_pid = config.getint("DEFAULT", "processbot_pid") except ValueError: self._processbot_pid = config.get("DEFAULT", "processbot_pid") self._spawned_process_pids = config.get("DEFAULT", "spawned_process_pids") if self._spawned_process_pids is not None and self._spawned_process_pids != '': self._spawned_process_pids = json.loads(self._spawned_process_pids) self._spawned_process_cmd = config.get("DEFAULT", "spawned_process_cmd") if self._spawned_process_cmd is not None and self._spawned_process_cmd != '': self._spawned_process_cmd = json.loads(self._spawned_process_cmd)
def _setup_resource_monitor(self): # read remote config file self._remote_config = config_instance.create( self._remote_config_filepath) try: self._process_blacklist = json.loads( self._remote_config.get("DEFAULT", "process_blacklist")) except BaseException: self._process_blacklist = [] try: capacity = self._remote_config.getint("DEFAULT", "capacity") except BaseException: capacity = 100 try: load_threshold = self._remote_config.getint( "DEFAULT", "load_threshold") except BaseException: load_threshold = 100 try: load_average_scan_minutes = self._remote_config.getint( "DEFAULT", "load_average_scan_minutes") except BaseException: load_average_scan_minutes = 15 resource_monitor.ResourceMonitor.capacity = capacity resource_monitor.ResourceMonitor.load_threshold = load_threshold resource_monitor.ResourceMonitor.load_average_scan_minutes = load_average_scan_minutes
def __init__(self, args): super(WPSAgentProcess, self).__init__(args) try: config_dir_service = path.path(args.serviceconfig).dirname() # deserilize pickled object with process startup info to get the unique_id # to create the sand box work dir for the process execution self.exe_msg = busIndependentMessages.ExecuteMessage.deserialize( args.params) # read the service config file with interpolation=true (raw=False) to get # the proper sand box work dir using the unique id as input parameter # args.remoteconfig, args.serviceconfig service_config = config_instance.create( args.serviceconfig, case_sensitive=True, variables={'unique_exe_id': self.exe_msg.UniqueId()}, raw=False) work_dir = service_config.get_path("DEFAULT", "workdir") # ensure outdir exists if not work_dir.exists(): work_dir.mkdir() # now i can create a logger with output log file in the sandbox dir WPSAgent.create_logger( self.find_logger_property_file(config_dir_service), work_dir, self.verbose) except Exception as e: # here log is not available use log_bootstrap_error func to log somewhere # else (e.g. operating system error log, tmp file, stdout) msg = "Failure during bot bootstrap due to : " + str(e) WPSAgent.log_bootstrap_error(msg, traceback.format_exc()) sys.exit(100) # start execution self.run()
def __init__(self, remote_config_filepath, service_config_filepath, execute_message): self._uniqueExeId = execute_message.UniqueId() self._remote_wps_endpoint = execute_message.originator() self._remote_wps_baseurl = execute_message.BaseURL() self._input_values = execute_message.variables() # read remote config remote_config = config_instance.create(remote_config_filepath) bus_class_name = remote_config.get("DEFAULT", "bus_class_name") uploader_class_name = None try: uploader_class_name = remote_config.get("UPLOADER", "uploader_class_name") except BaseException: pass self._resource_file_dir = remote_config.get_path( "DEFAULT", "resource_file_dir") if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): self._wps_execution_shared_dir = remote_config.get_path( "DEFAULT", "wps_execution_shared_dir") # ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None # the config file is read with raw=False because the unique_exe_id value # will be used (interpolated) in the config service_config = config_instance.create( service_config_filepath, case_sensitive=True, variables={ 'unique_exe_id': self._uniqueExeId, 'wps_execution_shared_dir': self._wps_execution_shared_dir }, raw=False) self.service = service_config.get("DEFAULT", "service") # todo: what is? self.namespace = service_config.get("DEFAULT", "namespace") self.description = service_config.get("DEFAULT", "description") self._active = service_config.get("DEFAULT", "active").lower() == "true" # True self._executable_path = service_config.get("DEFAULT", "executable_path") self._executable_cmd = service_config.get("DEFAULT", "executable_cmd") if not os.path.isabs(self._executable_path): full_executable_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), self._executable_path) self._executable_cmd = self._executable_cmd.replace( self._executable_path, full_executable_path) self._executable_path = full_executable_path self._stdout_parser = service_config.get_list("Logging", "stdout_parser") self._stdout_action = service_config.get_list("Logging", "stdout_action") self._output_dir = service_config.get_path("DEFAULT", "output_dir") if not os.path.isabs(self._output_dir): self._output_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), self._output_dir) self._max_running_time = datetime.timedelta( seconds=service_config.getint("DEFAULT", "max_running_time_seconds")) # create the concrete uploader object if uploader_class_name: uploader_host = remote_config.get("UPLOADER", "uploader_host") uploader_username = remote_config.get("UPLOADER", "uploader_username") uploader_password = remote_config.get("UPLOADER", "uploader_password") if remote_config.has_option("UPLOADER", "uploader_private_rsa_key") and \ remote_config.has_option("UPLOADER", "uploader_passphrase"): uploader_private_rsa_key = remote_config.get( "UPLOADER", "uploader_private_rsa_key") uploader_passphrase = remote_config.get( "UPLOADER", "uploader_passphrase") privatekey = open(uploader_private_rsa_key, "r") rsa_key = RSA.importKey(privatekey, passphrase=uploader_passphrase) uploader_password = rsa_key.decrypt( base64.b64decode(uploader_password)) self._uploader = introspection.get_class_four_arg( uploader_class_name, uploader_host, uploader_username, uploader_password, self._uniqueExeId) else: self._uploader = None input_sections = OrderedDict() for input_section in [ s for s in service_config.sections() if 'input' in s.lower() or 'const' in s.lower() ]: input_sections[ input_section] = service_config.items_without_defaults( input_section, raw=False) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections) output_sections = OrderedDict() for output_section in [ s for s in service_config.sections() if 'output' in s.lower() ]: output_sections[ output_section] = service_config.items_without_defaults( output_section, raw=False) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir, self._uploader) action_sections = OrderedDict() for action_section in [ s for s in service_config.sections() if 'action' in s.lower() ]: action_sections[ action_section] = service_config.items_without_defaults( action_section, raw=False) self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config( action_sections) self.bus = introspection.get_class_four_arg(bus_class_name, remote_config, self.service, self.namespace, self._uniqueExeId) self._finished = False # event handlers self.bus.RegisterMessageCallback(busIndependentMessages.FinishMessage, self.handle_finish) self.bus.RegisterMessageCallback(busIndependentMessages.AbortMessage, self.handle_abort)
def __init__(self, remote_config_filepath, service_config_filepath): # read remote config file self._remote_config_filepath = remote_config_filepath self._remote_config = config_instance.create( self._remote_config_filepath) # identify the class implementation of the cominication bus bus_class_name = self._remote_config.get("DEFAULT", "bus_class_name") # directory used to store file for resource cleaner self._resource_file_dir = self._remote_config.get_path( "DEFAULT", "resource_file_dir") if self._remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): # directory used to store the process encoded outputs (usually on a shared fs) self._wps_execution_shared_dir = self._remote_config.get_path( "DEFAULT", "wps_execution_shared_dir") # ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None # read service config, with raw=true that is without config file's value # interpolation. Interpolation values are produced only for the process bot # (request hanlder); for example the unique execution id value to craete # the sand box directory self._service_config_file = service_config_filepath self._service_config = config_instance.create( service_config_filepath, case_sensitive=True, variables={ 'wps_execution_shared_dir': self._wps_execution_shared_dir }, raw=True) self.service = self._service_config.get("DEFAULT", "service") # WPS service name? self.namespace = self._service_config.get("DEFAULT", "namespace") self.description = self._service_config.get( "DEFAULT", "description") # WPS service description self._active = self._service_config.get( "DEFAULT", "active").lower() == "true" # True self._output_dir = self._service_config.get_path( "DEFAULT", "output_dir") self._max_running_time = datetime.timedelta( seconds=self._service_config.getint("DEFAULT", "max_running_time_seconds")) _sections = self._service_config.sections() input_sections = OrderedDict() for input_section in [ s for s in _sections if 'input' in s.lower() or 'const' in s.lower() ]: # service bot doesn't have yet the execution unique id, thus the # service_config is read with raw=True to avoid config file variables # interpolation input_sections[ input_section] = self._service_config.items_without_defaults( input_section, raw=True) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections) output_sections = OrderedDict() for output_section in [ s for s in self._service_config.sections() if 'output' in s.lower() ]: output_sections[ output_section] = self._service_config.items_without_defaults( output_section, raw=True) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir) # create the concrete bus object self.bus = introspection.get_class_three_arg(bus_class_name, self._remote_config, self.service, self.namespace) self.bus.RegisterMessageCallback(busIndependentMessages.InviteMessage, self.handle_invite) self.bus.RegisterMessageCallback(busIndependentMessages.ExecuteMessage, self.handle_execute) # -- Register here the callback to the "getloadavg" message self.bus.RegisterMessageCallback( busIndependentMessages.GetLoadAverageMessage, self.handle_getloadavg) # self._lock_running_process = thread.allocate_lock() #critical section # to access running_process from separate threads self.running_process = {} # send the process bot (aka request handler) stdout to service bot (remote wps agent) log file self._redirect_process_stdout_to_logger = True self._remote_wps_endpoint = None # Allocate and start a Resource Monitoring Thread self._initialize_resource_monitor()
def handle_execute(self, execute_message): """Handler for WPS execute message.""" logger = logging.getLogger("servicebot.handle_execute") # read service config, with raw=true that is without config file's value # interpolation. Interpolation values are prodice only for the process bot # (request hanlder); for example the unique execution id value to craete # the sand box directory self._service_config = config_instance.create( self._service_config_file, case_sensitive=True, variables={ 'wps_execution_shared_dir': self._wps_execution_shared_dir }, raw=True) # save execute messsage to tmp file to enable the process bot to read the inputs tmp_file = tempfile.NamedTemporaryFile(prefix='wps_params_', suffix=".tmp", delete=False) execute_message.serialize(tmp_file) param_filepath = tmp_file.name tmp_file.close() logger.debug("save parameters file for executing process " + self.service + " in " + param_filepath) # check if there are enough resources available to keep the exec-request in charge _can_execute = True # if the load average is above threshold then we cannot run another process # and we should answer with a proper message if self._resource_monitor.resource_load > self._resource_monitor.load_threshold: _can_execute = False # if there is at least a single blacklisted process running there is no avail # capacity to run another process and we should answer with a proper message if self._resource_monitor.proc_load > self._resource_monitor.load_threshold: _can_execute = False # compute residual capacity _residual_capacity = self._resource_monitor.capacity - self._resource_monitor.load if _residual_capacity <= 0: _can_execute = False # no residual capacity left, no execution # Do we have enough capacity? PCk = None try: process_weight_class_name = self._service_config.get( "DEFAULT", "process_weight") PCk = introspection.get_class_no_arg(process_weight_class_name) except BaseException: PCk = None if not PCk: try: process_weight = json.loads( self._service_config.get("DEFAULT", "process_weight")) except BaseException: process_weight = {"weight": "0", "coefficient": "1.0"} PCk = resource_monitor.ProcessWeight(process_weight) logger.info( " ---------------Async Request Management------------------- ") logger.info(PCk.__class__) _request_load = PCk.request_weight(execute_message) logger.info("Request Load: %s " % _request_load) logger.info( " ---------------------------------------------------------- ") if _residual_capacity < _request_load: _can_execute = False # no residual capacity left, no execution if _can_execute: # create the Resource Cleaner file containing the process info. The # "invoked_process.pid" will be set by the spawned process itself try: r = resource_cleaner.Resource() # create a resource... r.set_from_servicebot( execute_message.UniqueId(), self._output_dir / execute_message.UniqueId()) # ... and save to file logger.info("Start the resource cleaner!") r.write() except BaseException as ex: logger.exception("Resource Cleaner initialization error: " + str(ex)) # invoke the process bot (aka request handler) asynchronously cmd = 'python wpsagent.py -r ' + self._remote_config_filepath + ' -s ' + \ self._service_config_file + ' -p ' + param_filepath + ' process' invoked_process = subprocess.Popen(args=cmd.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logger.info("created process %s with PId %s and cmd: %s" % (self.service, str(invoked_process.pid), cmd)) # use a parallel thread to wait the end of the request handler process and # get the exit code of the just created asynchronous process computation unique_id = execute_message.UniqueId() thread.start_new_thread(self.output_parser_verbose, ( unique_id, invoked_process, param_filepath, )) # Update the resource_monitor load self._resource_monitor.running_procs_load[ unique_id] = _request_load self._resource_monitor.load += _request_load else: # Send the message back to the WPS outputs = dict() outputs['UniqueId'] = execute_message.UniqueId() try: if self.bus.state() != 'connected': self.bus.xmpp.reconnect() self.bus.xmpp.send_presence() self.bus.SendMessage( busIndependentMessages.CannotExecuteMessage( execute_message.originator(), outputs)) except BaseException: logger.info( "[XMPP Disconnected]: Service " + str(self.service) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) logger.info( "end of execute message handler, going back in listening mode")