def read_from_file(self, filepath):
config = config_instance.create(filepath,
raw=True) # todo: use file lock
self._start_time = config.get("DEFAULT", "start_time")
try:
self._start_time = datetime.datetime.strptime(
self._start_time, "%Y-%m-%dT%H:%M:%S")
except BaseException:
self._start_time = datetime.datetime.strptime(
"2000-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S")
self._processbot_cmd_line = config.get("DEFAULT", "cmd_line")
if self._processbot_cmd_line is not None and self._processbot_cmd_line != '':
self._processbot_cmd_line = json.loads(self._processbot_cmd_line)
self._unique_id = config.get("DEFAULT", "unique_id")
self._sandbox_path = config.get_path("DEFAULT", "sendbox_path")
try:
self._processbot_pid = config.getint("DEFAULT", "processbot_pid")
except ValueError:
self._processbot_pid = config.get("DEFAULT", "processbot_pid")
self._spawned_process_pids = config.get("DEFAULT",
"spawned_process_pids")
if self._spawned_process_pids is not None and self._spawned_process_pids != '':
self._spawned_process_pids = json.loads(self._spawned_process_pids)
self._spawned_process_cmd = config.get("DEFAULT",
"spawned_process_cmd")
if self._spawned_process_cmd is not None and self._spawned_process_cmd != '':
self._spawned_process_cmd = json.loads(self._spawned_process_cmd)
def _setup_resource_monitor(self):
# read remote config file
self._remote_config = config_instance.create(
self._remote_config_filepath)
try:
self._process_blacklist = json.loads(
self._remote_config.get("DEFAULT", "process_blacklist"))
except BaseException:
self._process_blacklist = []
try:
capacity = self._remote_config.getint("DEFAULT", "capacity")
except BaseException:
capacity = 100
try:
load_threshold = self._remote_config.getint(
"DEFAULT", "load_threshold")
except BaseException:
load_threshold = 100
try:
load_average_scan_minutes = self._remote_config.getint(
"DEFAULT", "load_average_scan_minutes")
except BaseException:
load_average_scan_minutes = 15
resource_monitor.ResourceMonitor.capacity = capacity
resource_monitor.ResourceMonitor.load_threshold = load_threshold
resource_monitor.ResourceMonitor.load_average_scan_minutes = load_average_scan_minutes
def __init__(self, args):
super(WPSAgentProcess, self).__init__(args)
try:
config_dir_service = path.path(args.serviceconfig).dirname()
# deserilize pickled object with process startup info to get the unique_id
# to create the sand box work dir for the process execution
self.exe_msg = busIndependentMessages.ExecuteMessage.deserialize(
args.params)
# read the service config file with interpolation=true (raw=False) to get
# the proper sand box work dir using the unique id as input parameter
# args.remoteconfig, args.serviceconfig
service_config = config_instance.create(
args.serviceconfig,
case_sensitive=True,
variables={'unique_exe_id': self.exe_msg.UniqueId()},
raw=False)
work_dir = service_config.get_path("DEFAULT", "workdir")
# ensure outdir exists
if not work_dir.exists():
work_dir.mkdir()
# now i can create a logger with output log file in the sandbox dir
WPSAgent.create_logger(
self.find_logger_property_file(config_dir_service), work_dir,
self.verbose)
except Exception as e:
# here log is not available use log_bootstrap_error func to log somewhere
# else (e.g. operating system error log, tmp file, stdout)
msg = "Failure during bot bootstrap due to : " + str(e)
WPSAgent.log_bootstrap_error(msg, traceback.format_exc())
sys.exit(100)
# start execution
self.run()
def __init__(self, remote_config_filepath, service_config_filepath,
execute_message):
self._uniqueExeId = execute_message.UniqueId()
self._remote_wps_endpoint = execute_message.originator()
self._remote_wps_baseurl = execute_message.BaseURL()
self._input_values = execute_message.variables()
# read remote config
remote_config = config_instance.create(remote_config_filepath)
bus_class_name = remote_config.get("DEFAULT", "bus_class_name")
uploader_class_name = None
try:
uploader_class_name = remote_config.get("UPLOADER",
"uploader_class_name")
except BaseException:
pass
self._resource_file_dir = remote_config.get_path(
"DEFAULT", "resource_file_dir")
if remote_config.has_option("DEFAULT", "wps_execution_shared_dir"):
self._wps_execution_shared_dir = remote_config.get_path(
"DEFAULT", "wps_execution_shared_dir")
# ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
# the config file is read with raw=False because the unique_exe_id value
# will be used (interpolated) in the config
service_config = config_instance.create(
service_config_filepath,
case_sensitive=True,
variables={
'unique_exe_id': self._uniqueExeId,
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=False)
self.service = service_config.get("DEFAULT",
"service") # todo: what is?
self.namespace = service_config.get("DEFAULT", "namespace")
self.description = service_config.get("DEFAULT", "description")
self._active = service_config.get("DEFAULT",
"active").lower() == "true" # True
self._executable_path = service_config.get("DEFAULT",
"executable_path")
self._executable_cmd = service_config.get("DEFAULT", "executable_cmd")
if not os.path.isabs(self._executable_path):
full_executable_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
self._executable_path)
self._executable_cmd = self._executable_cmd.replace(
self._executable_path, full_executable_path)
self._executable_path = full_executable_path
self._stdout_parser = service_config.get_list("Logging",
"stdout_parser")
self._stdout_action = service_config.get_list("Logging",
"stdout_action")
self._output_dir = service_config.get_path("DEFAULT", "output_dir")
if not os.path.isabs(self._output_dir):
self._output_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), self._output_dir)
self._max_running_time = datetime.timedelta(
seconds=service_config.getint("DEFAULT",
"max_running_time_seconds"))
# create the concrete uploader object
if uploader_class_name:
uploader_host = remote_config.get("UPLOADER", "uploader_host")
uploader_username = remote_config.get("UPLOADER",
"uploader_username")
uploader_password = remote_config.get("UPLOADER",
"uploader_password")
if remote_config.has_option("UPLOADER", "uploader_private_rsa_key") and \
remote_config.has_option("UPLOADER", "uploader_passphrase"):
uploader_private_rsa_key = remote_config.get(
"UPLOADER", "uploader_private_rsa_key")
uploader_passphrase = remote_config.get(
"UPLOADER", "uploader_passphrase")
privatekey = open(uploader_private_rsa_key, "r")
rsa_key = RSA.importKey(privatekey,
passphrase=uploader_passphrase)
uploader_password = rsa_key.decrypt(
base64.b64decode(uploader_password))
self._uploader = introspection.get_class_four_arg(
uploader_class_name, uploader_host, uploader_username,
uploader_password, self._uniqueExeId)
else:
self._uploader = None
input_sections = OrderedDict()
for input_section in [
s for s in service_config.sections()
if 'input' in s.lower() or 'const' in s.lower()
]:
input_sections[
input_section] = service_config.items_without_defaults(
input_section, raw=False)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config(
input_sections)
output_sections = OrderedDict()
for output_section in [
s for s in service_config.sections() if 'output' in s.lower()
]:
output_sections[
output_section] = service_config.items_without_defaults(
output_section, raw=False)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config(
output_sections, self._wps_execution_shared_dir, self._uploader)
action_sections = OrderedDict()
for action_section in [
s for s in service_config.sections() if 'action' in s.lower()
]:
action_sections[
action_section] = service_config.items_without_defaults(
action_section, raw=False)
self._input_params_actions = computational_job_input_actions.ComputationalJobInputActions.create_from_config(
action_sections)
self.bus = introspection.get_class_four_arg(bus_class_name,
remote_config,
self.service,
self.namespace,
self._uniqueExeId)
self._finished = False
# event handlers
self.bus.RegisterMessageCallback(busIndependentMessages.FinishMessage,
self.handle_finish)
self.bus.RegisterMessageCallback(busIndependentMessages.AbortMessage,
self.handle_abort)
def __init__(self, remote_config_filepath, service_config_filepath):
# read remote config file
self._remote_config_filepath = remote_config_filepath
self._remote_config = config_instance.create(
self._remote_config_filepath)
# identify the class implementation of the cominication bus
bus_class_name = self._remote_config.get("DEFAULT", "bus_class_name")
# directory used to store file for resource cleaner
self._resource_file_dir = self._remote_config.get_path(
"DEFAULT", "resource_file_dir")
if self._remote_config.has_option("DEFAULT",
"wps_execution_shared_dir"):
# directory used to store the process encoded outputs (usually on a shared fs)
self._wps_execution_shared_dir = self._remote_config.get_path(
"DEFAULT", "wps_execution_shared_dir")
# ensure outdir exists
if not self._wps_execution_shared_dir.exists():
self._wps_execution_shared_dir.mkdir()
else:
self._wps_execution_shared_dir = None
# read service config, with raw=true that is without config file's value
# interpolation. Interpolation values are produced only for the process bot
# (request hanlder); for example the unique execution id value to craete
# the sand box directory
self._service_config_file = service_config_filepath
self._service_config = config_instance.create(
service_config_filepath,
case_sensitive=True,
variables={
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=True)
self.service = self._service_config.get("DEFAULT",
"service") # WPS service name?
self.namespace = self._service_config.get("DEFAULT", "namespace")
self.description = self._service_config.get(
"DEFAULT", "description") # WPS service description
self._active = self._service_config.get(
"DEFAULT", "active").lower() == "true" # True
self._output_dir = self._service_config.get_path(
"DEFAULT", "output_dir")
self._max_running_time = datetime.timedelta(
seconds=self._service_config.getint("DEFAULT",
"max_running_time_seconds"))
_sections = self._service_config.sections()
input_sections = OrderedDict()
for input_section in [
s for s in _sections
if 'input' in s.lower() or 'const' in s.lower()
]:
# service bot doesn't have yet the execution unique id, thus the
# service_config is read with raw=True to avoid config file variables
# interpolation
input_sections[
input_section] = self._service_config.items_without_defaults(
input_section, raw=True)
self._input_parameters_defs = computation_job_inputs.ComputationJobInputs.create_from_config(
input_sections)
output_sections = OrderedDict()
for output_section in [
s for s in self._service_config.sections()
if 'output' in s.lower()
]:
output_sections[
output_section] = self._service_config.items_without_defaults(
output_section, raw=True)
self._output_parameters_defs = output_parameters.OutputParameters.create_from_config(
output_sections, self._wps_execution_shared_dir)
# create the concrete bus object
self.bus = introspection.get_class_three_arg(bus_class_name,
self._remote_config,
self.service,
self.namespace)
self.bus.RegisterMessageCallback(busIndependentMessages.InviteMessage,
self.handle_invite)
self.bus.RegisterMessageCallback(busIndependentMessages.ExecuteMessage,
self.handle_execute)
# -- Register here the callback to the "getloadavg" message
self.bus.RegisterMessageCallback(
busIndependentMessages.GetLoadAverageMessage,
self.handle_getloadavg)
# self._lock_running_process = thread.allocate_lock() #critical section
# to access running_process from separate threads
self.running_process = {}
# send the process bot (aka request handler) stdout to service bot (remote wps agent) log file
self._redirect_process_stdout_to_logger = True
self._remote_wps_endpoint = None
# Allocate and start a Resource Monitoring Thread
self._initialize_resource_monitor()
def handle_execute(self, execute_message):
"""Handler for WPS execute message."""
logger = logging.getLogger("servicebot.handle_execute")
# read service config, with raw=true that is without config file's value
# interpolation. Interpolation values are prodice only for the process bot
# (request hanlder); for example the unique execution id value to craete
# the sand box directory
self._service_config = config_instance.create(
self._service_config_file,
case_sensitive=True,
variables={
'wps_execution_shared_dir': self._wps_execution_shared_dir
},
raw=True)
# save execute messsage to tmp file to enable the process bot to read the inputs
tmp_file = tempfile.NamedTemporaryFile(prefix='wps_params_',
suffix=".tmp",
delete=False)
execute_message.serialize(tmp_file)
param_filepath = tmp_file.name
tmp_file.close()
logger.debug("save parameters file for executing process " +
self.service + " in " + param_filepath)
# check if there are enough resources available to keep the exec-request in charge
_can_execute = True
# if the load average is above threshold then we cannot run another process
# and we should answer with a proper message
if self._resource_monitor.resource_load > self._resource_monitor.load_threshold:
_can_execute = False
# if there is at least a single blacklisted process running there is no avail
# capacity to run another process and we should answer with a proper message
if self._resource_monitor.proc_load > self._resource_monitor.load_threshold:
_can_execute = False
# compute residual capacity
_residual_capacity = self._resource_monitor.capacity - self._resource_monitor.load
if _residual_capacity <= 0:
_can_execute = False # no residual capacity left, no execution
# Do we have enough capacity?
PCk = None
try:
process_weight_class_name = self._service_config.get(
"DEFAULT", "process_weight")
PCk = introspection.get_class_no_arg(process_weight_class_name)
except BaseException:
PCk = None
if not PCk:
try:
process_weight = json.loads(
self._service_config.get("DEFAULT", "process_weight"))
except BaseException:
process_weight = {"weight": "0", "coefficient": "1.0"}
PCk = resource_monitor.ProcessWeight(process_weight)
logger.info(
" ---------------Async Request Management------------------- ")
logger.info(PCk.__class__)
_request_load = PCk.request_weight(execute_message)
logger.info("Request Load: %s " % _request_load)
logger.info(
" ---------------------------------------------------------- ")
if _residual_capacity < _request_load:
_can_execute = False # no residual capacity left, no execution
if _can_execute:
# create the Resource Cleaner file containing the process info. The
# "invoked_process.pid" will be set by the spawned process itself
try:
r = resource_cleaner.Resource()
# create a resource...
r.set_from_servicebot(
execute_message.UniqueId(),
self._output_dir / execute_message.UniqueId())
# ... and save to file
logger.info("Start the resource cleaner!")
r.write()
except BaseException as ex:
logger.exception("Resource Cleaner initialization error: " +
str(ex))
# invoke the process bot (aka request handler) asynchronously
cmd = 'python wpsagent.py -r ' + self._remote_config_filepath + ' -s ' + \
self._service_config_file + ' -p ' + param_filepath + ' process'
invoked_process = subprocess.Popen(args=cmd.split(),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
logger.info("created process %s with PId %s and cmd: %s" %
(self.service, str(invoked_process.pid), cmd))
# use a parallel thread to wait the end of the request handler process and
# get the exit code of the just created asynchronous process computation
unique_id = execute_message.UniqueId()
thread.start_new_thread(self.output_parser_verbose, (
unique_id,
invoked_process,
param_filepath,
))
# Update the resource_monitor load
self._resource_monitor.running_procs_load[
unique_id] = _request_load
self._resource_monitor.load += _request_load
else:
# Send the message back to the WPS
outputs = dict()
outputs['UniqueId'] = execute_message.UniqueId()
try:
if self.bus.state() != 'connected':
self.bus.xmpp.reconnect()
self.bus.xmpp.send_presence()
self.bus.SendMessage(
busIndependentMessages.CannotExecuteMessage(
execute_message.originator(), outputs))
except BaseException:
logger.info(
"[XMPP Disconnected]: Service " + str(self.service) +
" Could not send info message to GeoServer Endpoint " +
str(self._remote_wps_endpoint))
logger.info(
"end of execute message handler, going back in listening mode")