def __init__(self, remote_config_filepath, service_config_filepath, execute_message): self._uniqueExeId = execute_message.UniqueId() self._remote_wps_endpoint = execute_message.originator() self._remote_wps_baseurl = execute_message.BaseURL() self._input_values = execute_message.variables() #read remote config remote_config = configInstance.create(remote_config_filepath) bus_class_name = remote_config.get("DEFAULT", "bus_class_name") self._resource_file_dir = remote_config.get_path("DEFAULT", "resource_file_dir") if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): self._wps_execution_shared_dir = remote_config.get_path("DEFAULT", "wps_execution_shared_dir") #ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None #the config file is read with raw=False because the unique_exe_id value will be used (interpolated) in the config serviceConfig = configInstance.create(service_config_filepath, case_sensitive=True, variables = {'unique_exe_id' : self._uniqueExeId, 'wps_execution_shared_dir' : self._wps_execution_shared_dir}, raw=False) self.service = serviceConfig.get("DEFAULT", "service") #todo: what is? self.namespace = serviceConfig.get("DEFAULT", "namespace") self.description = serviceConfig.get("DEFAULT", "description") self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" #True self._executable_path = serviceConfig.get("DEFAULT", "executable_path") self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd") self._stdout_parser = serviceConfig.get_list("Logging", "stdout_parser") self._stdout_action = serviceConfig.get_list("Logging", "stdout_action") self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir") self._max_running_time = datetime.timedelta( seconds = serviceConfig.getint("DEFAULT", "max_running_time_seconds") ) input_sections = OrderedDict() for input_section in [s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower()]: input_sections[input_section] = serviceConfig.items_without_defaults(input_section, raw=False) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections ) output_sections=OrderedDict() for output_section in [s for s in serviceConfig.sections() if 'output' in s.lower()]: output_sections[output_section] = serviceConfig.items_without_defaults(output_section, raw=False) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir ) action_sections=OrderedDict() for action_section in [s for s in serviceConfig.sections() if 'action' in s.lower()]: action_sections[action_section] = serviceConfig.items_without_defaults(action_section, raw=False) self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config( action_sections ) #create the concrete bus object self._lock_bus = thread.allocate_lock() self.bus = introspection.get_class_four_arg(bus_class_name, remote_config, self.service, self.namespace, self._uniqueExeId) #event handlers self.bus.RegisterMessageCallback(busIndipendentMessages.FinishMessage, self.handle_finish) self.bus.RegisterMessageCallback(busIndipendentMessages.AbortMessage, self.handle_abort)
def __init__(self, remote_config_filepath, service_config_filepath): #read remote config file self._remote_config_filepath = remote_config_filepath remote_config = configInstance.create(self._remote_config_filepath) bus_class_name = remote_config.get("DEFAULT", "bus_class_name") #identify the class implementation of the cominication bus self._resource_file_dir = remote_config.get_path("DEFAULT", "resource_file_dir") # directory used to store file for resource cleaner if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): self._wps_execution_shared_dir = remote_config.get_path("DEFAULT", "wps_execution_shared_dir") # directory used to store the process encoded outputs (usually on a shared fs) #ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None #read service config, with raw=true that is without config file's value interpolation. Interpolation values are prodice only for the process bot (request hanlder); for example the unique execution id value to craete the sand box directory self._service_config_file = service_config_filepath serviceConfig = configInstance.create(service_config_filepath, case_sensitive=True, variables = {'wps_execution_shared_dir' : self._wps_execution_shared_dir}, raw=True) self.service = serviceConfig.get("DEFAULT", "service") #WPS service name? self.namespace = serviceConfig.get("DEFAULT", "namespace") self.description = serviceConfig.get("DEFAULT", "description") #WPS service description self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" #True self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir") self._max_running_time = datetime.timedelta( seconds = serviceConfig.getint("DEFAULT", "max_running_time_seconds") ) input_sections = OrderedDict() for input_section in [s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower()]: #service bot doesn't have yet the execution unique id, thus the serviceConfig is read with raw=True to avoid config file variables interpolation input_sections[input_section] = serviceConfig.items_without_defaults(input_section, raw=True) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections ) output_sections=OrderedDict() for output_section in [s for s in serviceConfig.sections() if 'output' in s.lower()]: output_sections[output_section] = serviceConfig.items_without_defaults(output_section, raw=True) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir ) #create the concrete bus object self.bus = introspection.get_class_three_arg(bus_class_name, remote_config, self.service, self.namespace) self.bus.RegisterMessageCallback(busIndipendentMessages.InviteMessage, self.handle_invite) self.bus.RegisterMessageCallback(busIndipendentMessages.ExecuteMessage, self.handle_execute) #self._lock_running_process = thread.allocate_lock() #critical section to access running_process from separate threads self.running_process={} self._redirect_process_stdout_to_logger = False #send the process bot (aka request handler) stdout to service bot (remote wps agent) log file self._remote_wps_endpoint = None
def __init__(self, args): super(WPSAgentProcess, self).__init__(args) try: config_dir_service = path.path(args.serviceconfig).dirname() # deserilize pickled object with process startup info to get the unique_id # to create the sand box work dir for the process execution self.exe_msg = busIndipendentMessages.ExecuteMessage.deserialize(args.params) # read the service config file with interpolation=true (raw=False) to get # the proper sand box work dir using the unique id as input parameter # args.remoteconfig, args.serviceconfig serviceConfig = configInstance.create(args.serviceconfig, case_sensitive=True, variables={ 'unique_exe_id': self.exe_msg.UniqueId() }, raw=False) work_dir = serviceConfig.get_path("DEFAULT", "workdir") # ensure outdir exists if not work_dir.exists(): work_dir.mkdir() # now i can create a logger with output log file in the sandbox dir WPSAgent.create_logger(self.find_logger_property_file(config_dir_service), work_dir, self.verbose) except Exception as e: # here log is not available use log_bootstrap_error func to log somewhere # else (e.g. operating system error log, tmp file, stdout) msg = "Failure during bot bootstrap due to : " + str(e) WPSAgent.log_bootstrap_error(msg, traceback.format_exc()) sys.exit(100) # start execution self.run()
def read_from_file(self, filepath): config = configInstance.create(filepath, raw=True) #todo: use file lock self._start_time = config.get("DEFAULT", "start_time") try: self._start_time = datetime.datetime.strptime( self._start_time, "%Y-%m-%dT%H:%M:%S") except: self._start_time = datetime.datetime.strptime( "2000-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S") self._processbot_cmd_line = config.get("DEFAULT", "cmd_line") if self._processbot_cmd_line != None and self._processbot_cmd_line != '': self._processbot_cmd_line = json.loads(self._processbot_cmd_line) self._unique_id = config.get("DEFAULT", "unique_id") self._sandbox_path = config.get_path("DEFAULT", "sendbox_path") try: self._processbot_pid = config.getint("DEFAULT", "processbot_pid") except ValueError: self._processbot_pid = config.get("DEFAULT", "processbot_pid") self._spawned_process_pids = config.get("DEFAULT", "spawned_process_pids") if self._spawned_process_pids != None and self._spawned_process_pids != '': self._spawned_process_pids = json.loads(self._spawned_process_pids) self._spawned_process_cmd = config.get("DEFAULT", "spawned_process_cmd") if self._spawned_process_cmd != None and self._spawned_process_cmd != '': self._spawned_process_cmd = json.loads(self._spawned_process_cmd)
def __init__(self, args): super(WPSAgentProcess, self).__init__(args) try: config_dir_service = path.path(args.serviceconfig).dirname() #deserilize pickled object with process startup info to get the unique_id to create the sand box work dir for the process execution self.exe_msg = busIndipendentMessages.ExecuteMessage.deserialize( args.params ) #read the service config file with interpolation=true (raw=False) to get the proper sand box work dir using the unique id as input parameter #args.remoteconfig, args.serviceconfig serviceConfig = configInstance.create(args.serviceconfig , case_sensitive=True, variables = {'unique_exe_id' : self.exe_msg.UniqueId()}, raw=False) work_dir = serviceConfig.get_path("DEFAULT", "workdir") #ensure outdir exists if not work_dir.exists(): work_dir.mkdir() #now i can create a logger with output log file in the sandbox dir WPSAgent.create_logger( self.find_logger_property_file(config_dir_service), work_dir, self.verbose) except Exception as e: #here log is not available use log_bootstrap_error func to log somewhere else (e.g. operating system error log, tmp file, stdout) msg = "Failure during bot bootstrap due to : " + str(e) self.log_bootstrap_error(msg, traceback.format_exc()) sys.exit(100) #start execution self.run()
def read_from_file(self, filepath): config = configInstance.create( filepath, raw= True) #todo: use file lock self._start_time = config.get("DEFAULT", "start_time") try: self._start_time = datetime.datetime.strptime( self._start_time, "%Y-%m-%dT%H:%M:%S" ) except: self._start_time = datetime.datetime.strptime( "2000-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S" ) self._processbot_cmd_line = config.get("DEFAULT", "cmd_line") if self._processbot_cmd_line != None and self._processbot_cmd_line !='': self._processbot_cmd_line = json.loads(self._processbot_cmd_line) self._unique_id = config.get("DEFAULT", "unique_id") self._sandbox_path = config.get_path("DEFAULT", "sendbox_path") try: self._processbot_pid = config.getint("DEFAULT", "processbot_pid") except ValueError: self._processbot_pid = config.get("DEFAULT", "processbot_pid") self._spawned_process_pids = config.get("DEFAULT", "spawned_process_pids") if self._spawned_process_pids != None and self._spawned_process_pids !='': self._spawned_process_pids = json.loads( self._spawned_process_pids ) self._spawned_process_cmd = config.get("DEFAULT", "spawned_process_cmd") if self._spawned_process_cmd != None and self._spawned_process_cmd !='': self._spawned_process_cmd = json.loads(self._spawned_process_cmd)
def __init__(self, remote_config_filepath, service_config_filepath, execute_message): self._uniqueExeId = execute_message.UniqueId() self._remote_wps_endpoint = execute_message.originator() self._remote_wps_baseurl = execute_message.BaseURL() self._input_values = execute_message.variables() #read remote config remote_config = configInstance.create(remote_config_filepath) bus_class_name = remote_config.get("DEFAULT", "bus_class_name") uploader_class_name = None try: uploader_class_name = remote_config.get("UPLOADER", "uploader_class_name") except: pass self._resource_file_dir = remote_config.get_path( "DEFAULT", "resource_file_dir") if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): self._wps_execution_shared_dir = remote_config.get_path( "DEFAULT", "wps_execution_shared_dir") #ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None #the config file is read with raw=False because the unique_exe_id value will be used (interpolated) in the config serviceConfig = configInstance.create( service_config_filepath, case_sensitive=True, variables={ 'unique_exe_id': self._uniqueExeId, 'wps_execution_shared_dir': self._wps_execution_shared_dir }, raw=False) self.service = serviceConfig.get("DEFAULT", "service") #todo: what is? self.namespace = serviceConfig.get("DEFAULT", "namespace") self.description = serviceConfig.get("DEFAULT", "description") self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" #True self._executable_path = serviceConfig.get("DEFAULT", "executable_path") self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd") self._stdout_parser = serviceConfig.get_list("Logging", "stdout_parser") self._stdout_action = serviceConfig.get_list("Logging", "stdout_action") self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir") self._max_running_time = datetime.timedelta( seconds=serviceConfig.getint("DEFAULT", "max_running_time_seconds")) #create the concrete uploader object if uploader_class_name: uploader_host = remote_config.get("UPLOADER", "uploader_host") uploader_username = remote_config.get("UPLOADER", "uploader_username") uploader_password = remote_config.get("UPLOADER", "uploader_password") if remote_config.has_option( "UPLOADER", "uploader_private_rsa_key") and remote_config.has_option( "UPLOADER", "uploader_passphrase"): uploader_private_rsa_key = remote_config.get( "UPLOADER", "uploader_private_rsa_key") uploader_passphrase = remote_config.get( "UPLOADER", "uploader_passphrase") privatekey = open(uploader_private_rsa_key, "r") rsa_key = RSA.importKey(privatekey, passphrase=uploader_passphrase) uploader_password = rsa_key.decrypt( base64.b64decode(uploader_password)) self._uploader = introspection.get_class_four_arg( uploader_class_name, uploader_host, uploader_username, uploader_password, self._uniqueExeId) else: self._uploader = None input_sections = OrderedDict() for input_section in [ s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower() ]: input_sections[ input_section] = serviceConfig.items_without_defaults( input_section, raw=False) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections) output_sections = OrderedDict() for output_section in [ s for s in serviceConfig.sections() if 'output' in s.lower() ]: output_sections[ output_section] = serviceConfig.items_without_defaults( output_section, raw=False) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir, self._uploader) action_sections = OrderedDict() for action_section in [ s for s in serviceConfig.sections() if 'action' in s.lower() ]: action_sections[ action_section] = serviceConfig.items_without_defaults( action_section, raw=False) self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config( action_sections) #create the concrete bus object self._lock_bus = thread.allocate_lock() self.bus = introspection.get_class_four_arg(bus_class_name, remote_config, self.service, self.namespace, self._uniqueExeId) self._finished = False #event handlers self.bus.RegisterMessageCallback(busIndipendentMessages.FinishMessage, self.handle_finish) self.bus.RegisterMessageCallback(busIndipendentMessages.AbortMessage, self.handle_abort)
def __init__(self, remote_config_filepath, service_config_filepath): #read remote config file self._remote_config_filepath = remote_config_filepath remote_config = configInstance.create(self._remote_config_filepath) bus_class_name = remote_config.get( "DEFAULT", "bus_class_name" ) #identify the class implementation of the cominication bus self._resource_file_dir = remote_config.get_path( "DEFAULT", "resource_file_dir" ) # directory used to store file for resource cleaner if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): self._wps_execution_shared_dir = remote_config.get_path( "DEFAULT", "wps_execution_shared_dir" ) # directory used to store the process encoded outputs (usually on a shared fs) #ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None #read service config, with raw=true that is without config file's value interpolation. Interpolation values are prodice only for the process bot (request hanlder); for example the unique execution id value to craete the sand box directory self._service_config_file = service_config_filepath serviceConfig = configInstance.create( service_config_filepath, case_sensitive=True, variables={ 'wps_execution_shared_dir': self._wps_execution_shared_dir }, raw=True) self.service = serviceConfig.get("DEFAULT", "service") #WPS service name? self.namespace = serviceConfig.get("DEFAULT", "namespace") self.description = serviceConfig.get( "DEFAULT", "description") #WPS service description self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" #True self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir") self._max_running_time = datetime.timedelta( seconds=serviceConfig.getint("DEFAULT", "max_running_time_seconds")) try: import json self._process_blacklist = json.loads( serviceConfig.get("DEFAULT", "process_blacklist")) except: self._process_blacklist = [] input_sections = OrderedDict() for input_section in [ s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower() ]: #service bot doesn't have yet the execution unique id, thus the serviceConfig is read with raw=True to avoid config file variables interpolation input_sections[ input_section] = serviceConfig.items_without_defaults( input_section, raw=True) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections) output_sections = OrderedDict() for output_section in [ s for s in serviceConfig.sections() if 'output' in s.lower() ]: output_sections[ output_section] = serviceConfig.items_without_defaults( output_section, raw=True) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir) #create the concrete bus object self.bus = introspection.get_class_three_arg(bus_class_name, remote_config, self.service, self.namespace) self.bus.RegisterMessageCallback(busIndipendentMessages.InviteMessage, self.handle_invite) self.bus.RegisterMessageCallback(busIndipendentMessages.ExecuteMessage, self.handle_execute) # -- Register here the callback to the "getloadavg" message self.bus.RegisterMessageCallback( busIndipendentMessages.GetLoadAverageMessage, self.handle_getloadavg) #self._lock_running_process = thread.allocate_lock() #critical section to access running_process from separate threads self.running_process = {} self._redirect_process_stdout_to_logger = True #send the process bot (aka request handler) stdout to service bot (remote wps agent) log file self._remote_wps_endpoint = None # Allocate and start a Resource Monitoring Thread try: load_average_scan_minutes = serviceConfig.getint( "DEFAULT", "load_average_scan_minutes") except: load_average_scan_minutes = 15 self._resource_monitor = resource_monitor.ResourceMonitor( load_average_scan_minutes) self._resource_monitor.start()
def __init__(self, remote_config_filepath, service_config_filepath, execute_message): self._uniqueExeId = execute_message.UniqueId() self._remote_wps_endpoint = execute_message.originator() self._remote_wps_baseurl = execute_message.BaseURL() self._input_values = execute_message.variables() # read remote config remote_config = configInstance.create(remote_config_filepath) bus_class_name = remote_config.get("DEFAULT", "bus_class_name") uploader_class_name = None try: uploader_class_name = remote_config.get("UPLOADER", "uploader_class_name") except BaseException: pass self._resource_file_dir = remote_config.get_path("DEFAULT", "resource_file_dir") if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"): self._wps_execution_shared_dir = remote_config.get_path("DEFAULT", "wps_execution_shared_dir") # ensure outdir exists if not self._wps_execution_shared_dir.exists(): self._wps_execution_shared_dir.mkdir() else: self._wps_execution_shared_dir = None # the config file is read with raw=False because the unique_exe_id value # will be used (interpolated) in the config serviceConfig = configInstance.create(service_config_filepath, case_sensitive=True, variables={ 'unique_exe_id': self._uniqueExeId, 'wps_execution_shared_dir': self._wps_execution_shared_dir }, raw=False) self.service = serviceConfig.get("DEFAULT", "service") # todo: what is? self.namespace = serviceConfig.get("DEFAULT", "namespace") self.description = serviceConfig.get("DEFAULT", "description") self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" # True self._executable_path = serviceConfig.get("DEFAULT", "executable_path") self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd") if not os.path.isabs(self._executable_path): full_executable_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self._executable_path) self._executable_cmd = self._executable_cmd.replace(self._executable_path, full_executable_path) self._executable_path = full_executable_path self._stdout_parser = serviceConfig.get_list("Logging", "stdout_parser") self._stdout_action = serviceConfig.get_list("Logging", "stdout_action") self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir") if not os.path.isabs(self._output_dir): self._output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), self._output_dir) self._max_running_time = datetime.timedelta(seconds=serviceConfig.getint("DEFAULT", "max_running_time_seconds")) # create the concrete uploader object if uploader_class_name: uploader_host = remote_config.get("UPLOADER", "uploader_host") uploader_username = remote_config.get("UPLOADER", "uploader_username") uploader_password = remote_config.get("UPLOADER", "uploader_password") if remote_config.has_option("UPLOADER", "uploader_private_rsa_key") and \ remote_config.has_option("UPLOADER", "uploader_passphrase"): uploader_private_rsa_key = remote_config.get("UPLOADER", "uploader_private_rsa_key") uploader_passphrase = remote_config.get("UPLOADER", "uploader_passphrase") privatekey = open(uploader_private_rsa_key, "r") rsa_key = RSA.importKey(privatekey, passphrase=uploader_passphrase) uploader_password = rsa_key.decrypt(base64.b64decode(uploader_password)) self._uploader = introspection.get_class_four_arg(uploader_class_name, uploader_host, uploader_username, uploader_password, self._uniqueExeId) else: self._uploader = None input_sections = OrderedDict() for input_section in [s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower()]: input_sections[input_section] = serviceConfig.items_without_defaults(input_section, raw=False) self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config(input_sections) output_sections = OrderedDict() for output_section in [s for s in serviceConfig.sections() if 'output' in s.lower()]: output_sections[output_section] = serviceConfig.items_without_defaults(output_section, raw=False) self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir, self._uploader) action_sections = OrderedDict() for action_section in [s for s in serviceConfig.sections() if 'action' in s.lower()]: action_sections[action_section] = serviceConfig.items_without_defaults(action_section, raw=False) self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config( action_sections) self.bus = introspection.get_class_four_arg(bus_class_name, remote_config, self.service, self.namespace, self._uniqueExeId) self._finished = False # event handlers self.bus.RegisterMessageCallback(busIndipendentMessages.FinishMessage, self.handle_finish) self.bus.RegisterMessageCallback(busIndipendentMessages.AbortMessage, self.handle_abort)