def __init__(self, remote_config_filepath, service_config_filepath, execute_message):
self._uniqueExeId = execute_message.UniqueId()
self._remote_wps_endpoint = execute_message.originator()
self._remote_wps_baseurl = execute_message.BaseURL()
self._input_values = execute_message.variables()
#read remote config
remote_config = configInstance.create(remote_config_filepath)
bus_class_name = remote_config.get("DEFAULT", "bus_class_name")
self._resource_file_dir = remote_config.get_path("DEFAULT", "resource_file_dir")
if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"):
self._wps_execution_shared_dir = remote_config.get_path("DEFAULT", "wps_execution_shared_dir")
#ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
#the config file is read with raw=False because the unique_exe_id value will be used (interpolated) in the config
serviceConfig = configInstance.create(service_config_filepath, case_sensitive=True, variables = {'unique_exe_id' : self._uniqueExeId, 'wps_execution_shared_dir' : self._wps_execution_shared_dir}, raw=False)
self.service = serviceConfig.get("DEFAULT", "service") #todo: what is?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
self.description = serviceConfig.get("DEFAULT", "description")
self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" #True
self._executable_path = serviceConfig.get("DEFAULT", "executable_path")
self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd")
self._stdout_parser = serviceConfig.get_list("Logging", "stdout_parser")
self._stdout_action = serviceConfig.get_list("Logging", "stdout_action")
self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir")
self._max_running_time = datetime.timedelta( seconds = serviceConfig.getint("DEFAULT", "max_running_time_seconds") )
input_sections = OrderedDict()
for input_section in [s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower()]:
input_sections[input_section] = serviceConfig.items_without_defaults(input_section, raw=False)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections )
output_sections=OrderedDict()
for output_section in [s for s in serviceConfig.sections() if 'output' in s.lower()]:
output_sections[output_section] = serviceConfig.items_without_defaults(output_section, raw=False)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir )
action_sections=OrderedDict()
for action_section in [s for s in serviceConfig.sections() if 'action' in s.lower()]:
action_sections[action_section] = serviceConfig.items_without_defaults(action_section, raw=False)
self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config( action_sections )
#create the concrete bus object
self._lock_bus = thread.allocate_lock()
self.bus = introspection.get_class_four_arg(bus_class_name, remote_config, self.service, self.namespace, self._uniqueExeId)
#event handlers
self.bus.RegisterMessageCallback(busIndipendentMessages.FinishMessage, self.handle_finish)
self.bus.RegisterMessageCallback(busIndipendentMessages.AbortMessage, self.handle_abort)
def __init__(self, remote_config_filepath, service_config_filepath):
#read remote config file
self._remote_config_filepath = remote_config_filepath
remote_config = configInstance.create(self._remote_config_filepath)
bus_class_name = remote_config.get("DEFAULT", "bus_class_name") #identify the class implementation of the cominication bus
self._resource_file_dir = remote_config.get_path("DEFAULT", "resource_file_dir") # directory used to store file for resource cleaner
if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"):
self._wps_execution_shared_dir = remote_config.get_path("DEFAULT", "wps_execution_shared_dir") # directory used to store the process encoded outputs (usually on a shared fs)
#ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
#read service config, with raw=true that is without config file's value interpolation. Interpolation values are prodice only for the process bot (request hanlder); for example the unique execution id value to craete the sand box directory
self._service_config_file = service_config_filepath
serviceConfig = configInstance.create(service_config_filepath, case_sensitive=True, variables = {'wps_execution_shared_dir' : self._wps_execution_shared_dir}, raw=True)
self.service = serviceConfig.get("DEFAULT", "service") #WPS service name?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
self.description = serviceConfig.get("DEFAULT", "description") #WPS service description
self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" #True
self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir")
self._max_running_time = datetime.timedelta( seconds = serviceConfig.getint("DEFAULT", "max_running_time_seconds") )
input_sections = OrderedDict()
for input_section in [s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower()]:
#service bot doesn't have yet the execution unique id, thus the serviceConfig is read with raw=True to avoid config file variables interpolation
input_sections[input_section] = serviceConfig.items_without_defaults(input_section, raw=True)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config( input_sections )
output_sections=OrderedDict()
for output_section in [s for s in serviceConfig.sections() if 'output' in s.lower()]:
output_sections[output_section] = serviceConfig.items_without_defaults(output_section, raw=True)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config( output_sections, self._wps_execution_shared_dir )
#create the concrete bus object
self.bus = introspection.get_class_three_arg(bus_class_name, remote_config, self.service, self.namespace)
self.bus.RegisterMessageCallback(busIndipendentMessages.InviteMessage, self.handle_invite)
self.bus.RegisterMessageCallback(busIndipendentMessages.ExecuteMessage, self.handle_execute)
#self._lock_running_process = thread.allocate_lock() #critical section to access running_process from separate threads
self.running_process={}
self._redirect_process_stdout_to_logger = False #send the process bot (aka request handler) stdout to service bot (remote wps agent) log file
self._remote_wps_endpoint = None
def __init__(self, args):
super(WPSAgentProcess, self).__init__(args)
try:
config_dir_service = path.path(args.serviceconfig).dirname()
# deserilize pickled object with process startup info to get the unique_id
# to create the sand box work dir for the process execution
self.exe_msg = busIndipendentMessages.ExecuteMessage.deserialize(args.params)
# read the service config file with interpolation=true (raw=False) to get
# the proper sand box work dir using the unique id as input parameter
# args.remoteconfig, args.serviceconfig
serviceConfig = configInstance.create(args.serviceconfig,
case_sensitive=True,
variables={
'unique_exe_id': self.exe_msg.UniqueId()
},
raw=False)
work_dir = serviceConfig.get_path("DEFAULT", "workdir")
# ensure outdir exists
if not work_dir.exists():
work_dir.mkdir()
# now i can create a logger with output log file in the sandbox dir
WPSAgent.create_logger(self.find_logger_property_file(config_dir_service), work_dir, self.verbose)
except Exception as e:
# here log is not available use log_bootstrap_error func to log somewhere
# else (e.g. operating system error log, tmp file, stdout)
msg = "Failure during bot bootstrap due to : " + str(e)
WPSAgent.log_bootstrap_error(msg, traceback.format_exc())
sys.exit(100)
# start execution
self.run()
def read_from_file(self, filepath):
config = configInstance.create(filepath,
raw=True) #todo: use file lock
self._start_time = config.get("DEFAULT", "start_time")
try:
self._start_time = datetime.datetime.strptime(
self._start_time, "%Y-%m-%dT%H:%M:%S")
except:
self._start_time = datetime.datetime.strptime(
"2000-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S")
self._processbot_cmd_line = config.get("DEFAULT", "cmd_line")
if self._processbot_cmd_line != None and self._processbot_cmd_line != '':
self._processbot_cmd_line = json.loads(self._processbot_cmd_line)
self._unique_id = config.get("DEFAULT", "unique_id")
self._sandbox_path = config.get_path("DEFAULT", "sendbox_path")
try:
self._processbot_pid = config.getint("DEFAULT", "processbot_pid")
except ValueError:
self._processbot_pid = config.get("DEFAULT", "processbot_pid")
self._spawned_process_pids = config.get("DEFAULT",
"spawned_process_pids")
if self._spawned_process_pids != None and self._spawned_process_pids != '':
self._spawned_process_pids = json.loads(self._spawned_process_pids)
self._spawned_process_cmd = config.get("DEFAULT",
"spawned_process_cmd")
if self._spawned_process_cmd != None and self._spawned_process_cmd != '':
self._spawned_process_cmd = json.loads(self._spawned_process_cmd)
def __init__(self, args):
super(WPSAgentProcess, self).__init__(args)
try:
config_dir_service = path.path(args.serviceconfig).dirname()
#deserilize pickled object with process startup info to get the unique_id to create the sand box work dir for the process execution
self.exe_msg = busIndipendentMessages.ExecuteMessage.deserialize( args.params )
#read the service config file with interpolation=true (raw=False) to get the proper sand box work dir using the unique id as input parameter
#args.remoteconfig, args.serviceconfig
serviceConfig = configInstance.create(args.serviceconfig , case_sensitive=True, variables = {'unique_exe_id' : self.exe_msg.UniqueId()}, raw=False)
work_dir = serviceConfig.get_path("DEFAULT", "workdir")
#ensure outdir exists
if not work_dir.exists():
work_dir.mkdir()
#now i can create a logger with output log file in the sandbox dir
WPSAgent.create_logger( self.find_logger_property_file(config_dir_service), work_dir, self.verbose)
except Exception as e:
#here log is not available use log_bootstrap_error func to log somewhere else (e.g. operating system error log, tmp file, stdout)
msg = "Failure during bot bootstrap due to : " + str(e)
self.log_bootstrap_error(msg, traceback.format_exc())
sys.exit(100)
#start execution
self.run()
def read_from_file(self, filepath):
config = configInstance.create( filepath, raw= True) #todo: use file lock
self._start_time = config.get("DEFAULT", "start_time")
try:
self._start_time = datetime.datetime.strptime( self._start_time, "%Y-%m-%dT%H:%M:%S" )
except:
self._start_time = datetime.datetime.strptime( "2000-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S" )
self._processbot_cmd_line = config.get("DEFAULT", "cmd_line")
if self._processbot_cmd_line != None and self._processbot_cmd_line !='':
self._processbot_cmd_line = json.loads(self._processbot_cmd_line)
self._unique_id = config.get("DEFAULT", "unique_id")
self._sandbox_path = config.get_path("DEFAULT", "sendbox_path")
try:
self._processbot_pid = config.getint("DEFAULT", "processbot_pid")
except ValueError:
self._processbot_pid = config.get("DEFAULT", "processbot_pid")
self._spawned_process_pids = config.get("DEFAULT", "spawned_process_pids")
if self._spawned_process_pids != None and self._spawned_process_pids !='':
self._spawned_process_pids = json.loads( self._spawned_process_pids )
self._spawned_process_cmd = config.get("DEFAULT", "spawned_process_cmd")
if self._spawned_process_cmd != None and self._spawned_process_cmd !='':
self._spawned_process_cmd = json.loads(self._spawned_process_cmd)
def __init__(self, remote_config_filepath, service_config_filepath,
execute_message):
self._uniqueExeId = execute_message.UniqueId()
self._remote_wps_endpoint = execute_message.originator()
self._remote_wps_baseurl = execute_message.BaseURL()
self._input_values = execute_message.variables()
#read remote config
remote_config = configInstance.create(remote_config_filepath)
bus_class_name = remote_config.get("DEFAULT", "bus_class_name")
uploader_class_name = None
try:
uploader_class_name = remote_config.get("UPLOADER",
"uploader_class_name")
except:
pass
self._resource_file_dir = remote_config.get_path(
"DEFAULT", "resource_file_dir")
if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"):
self._wps_execution_shared_dir = remote_config.get_path(
"DEFAULT", "wps_execution_shared_dir")
#ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
#the config file is read with raw=False because the unique_exe_id value will be used (interpolated) in the config
serviceConfig = configInstance.create(
service_config_filepath,
case_sensitive=True,
variables={
'unique_exe_id': self._uniqueExeId,
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=False)
self.service = serviceConfig.get("DEFAULT", "service") #todo: what is?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
self.description = serviceConfig.get("DEFAULT", "description")
self._active = serviceConfig.get("DEFAULT",
"active").lower() == "true" #True
self._executable_path = serviceConfig.get("DEFAULT", "executable_path")
self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd")
self._stdout_parser = serviceConfig.get_list("Logging",
"stdout_parser")
self._stdout_action = serviceConfig.get_list("Logging",
"stdout_action")
self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir")
self._max_running_time = datetime.timedelta(
seconds=serviceConfig.getint("DEFAULT",
"max_running_time_seconds"))
#create the concrete uploader object
if uploader_class_name:
uploader_host = remote_config.get("UPLOADER", "uploader_host")
uploader_username = remote_config.get("UPLOADER",
"uploader_username")
uploader_password = remote_config.get("UPLOADER",
"uploader_password")
if remote_config.has_option(
"UPLOADER",
"uploader_private_rsa_key") and remote_config.has_option(
"UPLOADER", "uploader_passphrase"):
uploader_private_rsa_key = remote_config.get(
"UPLOADER", "uploader_private_rsa_key")
uploader_passphrase = remote_config.get(
"UPLOADER", "uploader_passphrase")
privatekey = open(uploader_private_rsa_key, "r")
rsa_key = RSA.importKey(privatekey,
passphrase=uploader_passphrase)
uploader_password = rsa_key.decrypt(
base64.b64decode(uploader_password))
self._uploader = introspection.get_class_four_arg(
uploader_class_name, uploader_host, uploader_username,
uploader_password, self._uniqueExeId)
else:
self._uploader = None
input_sections = OrderedDict()
for input_section in [
s for s in serviceConfig.sections()
if 'input' in s.lower() or 'const' in s.lower()
]:
input_sections[
input_section] = serviceConfig.items_without_defaults(
input_section, raw=False)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config(
input_sections)
output_sections = OrderedDict()
for output_section in [
s for s in serviceConfig.sections() if 'output' in s.lower()
]:
output_sections[
output_section] = serviceConfig.items_without_defaults(
output_section, raw=False)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config(
output_sections, self._wps_execution_shared_dir, self._uploader)
action_sections = OrderedDict()
for action_section in [
s for s in serviceConfig.sections() if 'action' in s.lower()
]:
action_sections[
action_section] = serviceConfig.items_without_defaults(
action_section, raw=False)
self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config(
action_sections)
#create the concrete bus object
self._lock_bus = thread.allocate_lock()
self.bus = introspection.get_class_four_arg(bus_class_name,
remote_config,
self.service,
self.namespace,
self._uniqueExeId)
self._finished = False
#event handlers
self.bus.RegisterMessageCallback(busIndipendentMessages.FinishMessage,
self.handle_finish)
self.bus.RegisterMessageCallback(busIndipendentMessages.AbortMessage,
self.handle_abort)
def __init__(self, remote_config_filepath, service_config_filepath):
#read remote config file
self._remote_config_filepath = remote_config_filepath
remote_config = configInstance.create(self._remote_config_filepath)
bus_class_name = remote_config.get(
"DEFAULT", "bus_class_name"
) #identify the class implementation of the cominication bus
self._resource_file_dir = remote_config.get_path(
"DEFAULT", "resource_file_dir"
) # directory used to store file for resource cleaner
if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"):
self._wps_execution_shared_dir = remote_config.get_path(
"DEFAULT", "wps_execution_shared_dir"
) # directory used to store the process encoded outputs (usually on a shared fs)
#ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
#read service config, with raw=true that is without config file's value interpolation. Interpolation values are prodice only for the process bot (request hanlder); for example the unique execution id value to craete the sand box directory
self._service_config_file = service_config_filepath
serviceConfig = configInstance.create(
service_config_filepath,
case_sensitive=True,
variables={
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=True)
self.service = serviceConfig.get("DEFAULT",
"service") #WPS service name?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
self.description = serviceConfig.get(
"DEFAULT", "description") #WPS service description
self._active = serviceConfig.get("DEFAULT",
"active").lower() == "true" #True
self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir")
self._max_running_time = datetime.timedelta(
seconds=serviceConfig.getint("DEFAULT",
"max_running_time_seconds"))
try:
import json
self._process_blacklist = json.loads(
serviceConfig.get("DEFAULT", "process_blacklist"))
except:
self._process_blacklist = []
input_sections = OrderedDict()
for input_section in [
s for s in serviceConfig.sections()
if 'input' in s.lower() or 'const' in s.lower()
]:
#service bot doesn't have yet the execution unique id, thus the serviceConfig is read with raw=True to avoid config file variables interpolation
input_sections[
input_section] = serviceConfig.items_without_defaults(
input_section, raw=True)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config(
input_sections)
output_sections = OrderedDict()
for output_section in [
s for s in serviceConfig.sections() if 'output' in s.lower()
]:
output_sections[
output_section] = serviceConfig.items_without_defaults(
output_section, raw=True)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config(
output_sections, self._wps_execution_shared_dir)
#create the concrete bus object
self.bus = introspection.get_class_three_arg(bus_class_name,
remote_config,
self.service,
self.namespace)
self.bus.RegisterMessageCallback(busIndipendentMessages.InviteMessage,
self.handle_invite)
self.bus.RegisterMessageCallback(busIndipendentMessages.ExecuteMessage,
self.handle_execute)
# -- Register here the callback to the "getloadavg" message
self.bus.RegisterMessageCallback(
busIndipendentMessages.GetLoadAverageMessage,
self.handle_getloadavg)
#self._lock_running_process = thread.allocate_lock() #critical section to access running_process from separate threads
self.running_process = {}
self._redirect_process_stdout_to_logger = True #send the process bot (aka request handler) stdout to service bot (remote wps agent) log file
self._remote_wps_endpoint = None
# Allocate and start a Resource Monitoring Thread
try:
load_average_scan_minutes = serviceConfig.getint(
"DEFAULT", "load_average_scan_minutes")
except:
load_average_scan_minutes = 15
self._resource_monitor = resource_monitor.ResourceMonitor(
load_average_scan_minutes)
self._resource_monitor.start()
def __init__(self, remote_config_filepath, service_config_filepath, execute_message):
self._uniqueExeId = execute_message.UniqueId()
self._remote_wps_endpoint = execute_message.originator()
self._remote_wps_baseurl = execute_message.BaseURL()
self._input_values = execute_message.variables()
# read remote config
remote_config = configInstance.create(remote_config_filepath)
bus_class_name = remote_config.get("DEFAULT", "bus_class_name")
uploader_class_name = None
try:
uploader_class_name = remote_config.get("UPLOADER", "uploader_class_name")
except BaseException:
pass
self._resource_file_dir = remote_config.get_path("DEFAULT", "resource_file_dir")
if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"):
self._wps_execution_shared_dir = remote_config.get_path("DEFAULT", "wps_execution_shared_dir")
# ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
# the config file is read with raw=False because the unique_exe_id value
# will be used (interpolated) in the config
serviceConfig = configInstance.create(service_config_filepath,
case_sensitive=True,
variables={
'unique_exe_id': self._uniqueExeId,
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=False)
self.service = serviceConfig.get("DEFAULT", "service") # todo: what is?
self.namespace = serviceConfig.get("DEFAULT", "namespace")
self.description = serviceConfig.get("DEFAULT", "description")
self._active = serviceConfig.get("DEFAULT", "active").lower() == "true" # True
self._executable_path = serviceConfig.get("DEFAULT", "executable_path")
self._executable_cmd = serviceConfig.get("DEFAULT", "executable_cmd")
if not os.path.isabs(self._executable_path):
full_executable_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self._executable_path)
self._executable_cmd = self._executable_cmd.replace(self._executable_path, full_executable_path)
self._executable_path = full_executable_path
self._stdout_parser = serviceConfig.get_list("Logging", "stdout_parser")
self._stdout_action = serviceConfig.get_list("Logging", "stdout_action")
self._output_dir = serviceConfig.get_path("DEFAULT", "output_dir")
if not os.path.isabs(self._output_dir):
self._output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), self._output_dir)
self._max_running_time = datetime.timedelta(seconds=serviceConfig.getint("DEFAULT", "max_running_time_seconds"))
# create the concrete uploader object
if uploader_class_name:
uploader_host = remote_config.get("UPLOADER", "uploader_host")
uploader_username = remote_config.get("UPLOADER", "uploader_username")
uploader_password = remote_config.get("UPLOADER", "uploader_password")
if remote_config.has_option("UPLOADER", "uploader_private_rsa_key") and \
remote_config.has_option("UPLOADER", "uploader_passphrase"):
uploader_private_rsa_key = remote_config.get("UPLOADER", "uploader_private_rsa_key")
uploader_passphrase = remote_config.get("UPLOADER", "uploader_passphrase")
privatekey = open(uploader_private_rsa_key, "r")
rsa_key = RSA.importKey(privatekey, passphrase=uploader_passphrase)
uploader_password = rsa_key.decrypt(base64.b64decode(uploader_password))
self._uploader = introspection.get_class_four_arg(uploader_class_name,
uploader_host,
uploader_username,
uploader_password,
self._uniqueExeId)
else:
self._uploader = None
input_sections = OrderedDict()
for input_section in [s for s in serviceConfig.sections() if 'input' in s.lower() or 'const' in s.lower()]:
input_sections[input_section] = serviceConfig.items_without_defaults(input_section, raw=False)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config(input_sections)
output_sections = OrderedDict()
for output_section in [s for s in serviceConfig.sections() if 'output' in s.lower()]:
output_sections[output_section] = serviceConfig.items_without_defaults(output_section, raw=False)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config(
output_sections, self._wps_execution_shared_dir, self._uploader)
action_sections = OrderedDict()
for action_section in [s for s in serviceConfig.sections() if 'action' in s.lower()]:
action_sections[action_section] = serviceConfig.items_without_defaults(action_section, raw=False)
self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config(
action_sections)
self.bus = introspection.get_class_four_arg(bus_class_name,
remote_config,
self.service,
self.namespace,
self._uniqueExeId)
self._finished = False
# event handlers
self.bus.RegisterMessageCallback(busIndipendentMessages.FinishMessage, self.handle_finish)
self.bus.RegisterMessageCallback(busIndipendentMessages.AbortMessage, self.handle_abort)