def get_executable_arguments_for_sandbox(self, transform_metadata, input_json, unevaluated_points):
sandbox = None
if 'sandbox' in transform_metadata:
sandbox = transform_metadata['sandbox']
executable = transform_metadata['executable'].strip()
arguments = transform_metadata['arguments'].strip()
if executable == 'docker' and sandbox:
if 'workdir' in transform_metadata:
docker_workdir = transform_metadata['workdir']
else:
docker_workdir = None
arg_pre = 'run --rm -v $(pwd):%s %s ' % (docker_workdir, sandbox)
arguments = arg_pre + arguments
sandbox = None
output_json = None
if 'output_json' in transform_metadata:
output_json = transform_metadata['output_json']
max_points = self.get_max_points(transform_metadata)
if 'num_points_per_generation' in transform_metadata:
num_points = transform_metadata['num_points_per_generation']
else:
num_points = self.max_unevaluated_points - unevaluated_points
param_values = {'MAX_POINTS': max_points,
'NUM_POINTS': num_points,
'IN': input_json,
'OUT': output_json}
executable = replace_parameters_with_values(executable, param_values)
arguments = replace_parameters_with_values(arguments, param_values)
return 0, None, sandbox, executable, arguments, input_json, output_json, False
def get_executable_arguments_for_sandbox(self, transform_metadata,
input_json, unevaluated_points):
sandbox = None
if 'sandbox' in transform_metadata:
sandbox = transform_metadata['sandbox']
executable = transform_metadata['executable']
arguments = transform_metadata['arguments']
output_json = None
if 'output_json' in transform_metadata:
output_json = transform_metadata['output_json']
if 'max_points' in transform_metadata:
max_points = transform_metadata['max_points']
else:
max_points = self.max_points
if 'num_points_per_generation' in transform_metadata:
num_points = transform_metadata['num_points_per_generation']
else:
num_points = self.max_unevaluated_points - unevaluated_points
param_values = {
'MAX_POINTS': max_points,
'NUM_POINTS': num_points,
'IN': input_json,
'OUT': output_json
}
executable = replace_parameters_with_values(executable, param_values)
arguments = replace_parameters_with_values(arguments, param_values)
return 0, None, sandbox, executable, arguments, input_json, output_json, False
def generate_processing_script_container(self, processing):
param_values = {
'MAX_POINTS': self.max_points,
'NUM_POINTS': self.points_to_generate,
'IN': self.input_json,
'OUT': self.output_json
}
proxy_filename = 'x509up'
if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']:
proxy_filename = os.path.basename(os.environ['X509_USER_PROXY'])
param_values['X509_USER_PROXY_FULLNAME'] = os.environ[
'X509_USER_PROXY']
param_values['X509_USER_PROXY_BASENAME'] = proxy_filename
executable = replace_parameters_with_values(self.executable,
param_values)
arguments = replace_parameters_with_values(self.arguments,
param_values)
script = "#!/bin/bash\n\n"
script += "executable=%s\n" % str(executable)
script += "arguments=%s\n" % str(arguments)
script += "input_json=%s\n" % str(self.input_json)
script += "output_json=%s\n" % str(self.output_json)
script += "\n"
script += "env\n"
script += "echo $X509_USER_PROXY\n"
script += "\n"
script += "echo 'user id:'\n"
script += "id\n"
script += "\n"
if self.sandbox and 'docker' in executable:
arguments = 'run --rm -v $(pwd):%s -v /cvmfs:/cvmfs -e X509_USER_PROXY=%s/%s %s ' % (
self.container_workdir, self.container_workdir, proxy_filename,
self.sandbox) + arguments
script += "echo '%s' '%s'\n" % (str(executable), str(arguments))
script += '%s %s\n' % (str(executable), str(arguments))
if self.sandbox and 'docker' in executable:
script += 'docker image rm -f %s\n' % self.sandbox
script += '\n'
long_id = self.get_long_id(processing)
script_name = 'processing_%s.sh' % long_id
script_name = os.path.join(self.get_working_dir(processing),
script_name)
with open(script_name, 'w') as f:
f.write(script)
run_command("chmod +x %s" % script_name)
return script_name
def generate_processing_script_sandbox(self, processing):
param_values = {
'MAX_POINTS': self.max_points,
'NUM_POINTS': self.points_to_generate,
'IN': self.input_json,
'OUT': self.output_json
}
if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']:
proxy_filename = os.path.basename(os.environ['X509_USER_PROXY'])
param_values['X509_USER_PROXY_FULLNAME'] = os.environ[
'X509_USER_PROXY']
param_values['X509_USER_PROXY_BASENAME'] = proxy_filename
executable = replace_parameters_with_values(self.executable,
param_values)
arguments = replace_parameters_with_values(self.arguments,
param_values)
script = "#!/bin/bash\n\n"
script += "sandbox=%s\n" % str(self.sandbox)
script += "executable=%s\n" % str(executable)
script += "arguments=%s\n" % str(arguments)
script += "input_json=%s\n" % str(self.input_json)
script += "output_json=%s\n" % str(self.output_json)
script += "\n"
script += "env\n"
script += "echo $X509_USER_PROXY\n"
script += "\n"
script += "echo 'user id:'\n"
script += "id\n"
script += "\n"
script += "wget $sandbox\n"
script += 'base_sandbox="$(basename -- $sandbox)"\n'
script += 'tar xzf $base_sandbox\n'
script += 'chmod +x %s\n' % str(executable)
script += "echo '%s' '%s'\n" % (str(executable), str(arguments))
script += '%s %s\n' % (str(executable), str(arguments))
script += '\n'
long_id = self.get_long_id(processing)
script_name = 'processing_%s.sh' % long_id
script_name = os.path.join(self.get_working_dir(processing),
script_name)
with open(script_name, 'w') as f:
f.write(script)
run_command("chmod +x %s" % script_name)
return script_name
def get_executable_arguments_for_method(self, transform_metadata,
input_json, unevaluated_points):
method = transform_metadata['method']
method_executable = '%s.executable' % method
method_arguments = '%s.arguments' % method
method_output_json = '%s.output_json' % method
method_should_transfer_executable = '%s.should_transfer_executable' % method
if not hasattr(self, method_executable) or not hasattr(
self, method_arguments) or not hasattr(self,
method_output_json):
return -1, "method %s is not support" % method, None, None, None, None, None
else:
executable = getattr(self, method_executable)
arguments = getattr(self, method_arguments)
output_json = getattr(self, method_output_json)
should_transfer_executable = False
if hasattr(self, method_should_transfer_executable):
should_transfer_executable = getattr(
self, method_should_transfer_executable)
if type(should_transfer_executable) in [str]:
should_transfer_executable = should_transfer_executable.upper(
)
if should_transfer_executable in [True, 'TRUE']:
should_transfer_executable = True
else:
should_transfer_executable = False
if 'max_points' in transform_metadata:
max_points = transform_metadata['max_points']
else:
max_points = self.max_points
if 'num_points_per_generation' in transform_metadata:
num_points = transform_metadata['num_points_per_generation']
else:
num_points = self.max_unevaluated_points - unevaluated_points
param_values = {
'MAX_POINTS': max_points,
'NUM_POINTS': num_points,
'IN': input_json,
'OUT': output_json
}
executable = replace_parameters_with_values(
executable, param_values)
arguments = replace_parameters_with_values(arguments, param_values)
return 0, None, None, executable, arguments, input_json, output_json, should_transfer_executable
def generate_processing_script_nevergrad(self, processing):
executable = self.agent_attributes['nevergrad']['executable']
arguments = self.agent_attributes['nevergrad']['arguments']
param_values = {
'MAX_POINTS': self.max_points,
'NUM_POINTS': self.points_to_generate,
'IN': self.input_json,
'OUT': self.output_json
}
if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']:
proxy_filename = os.path.basename(os.environ['X509_USER_PROXY'])
param_values['X509_USER_PROXY_FULLNAME'] = os.environ[
'X509_USER_PROXY']
param_values['X509_USER_PROXY_BASENAME'] = proxy_filename
arguments = replace_parameters_with_values(arguments, param_values)
script = "#!/bin/bash\n\n"
script += "executable=%s\n" % os.path.basename(executable)
script += "arguments='%s'\n" % str(arguments)
script += "input_json=%s\n" % str(self.input_json)
script += "output_json=%s\n" % str(self.output_json)
script += "\n"
script += "env\n"
script += "echo $X509_USER_PROXY\n"
script += "\n"
script += "echo 'user id:'\n"
script += "id\n"
script += "\n"
script += "echo '%s' '%s'\n" % (os.path.basename(executable),
str(arguments))
script += '%s %s\n' % (os.path.basename(executable), str(arguments))
script += '\n'
long_id = self.get_long_id(processing)
script_name = 'processing_%s.sh' % long_id
script_name = os.path.join(self.get_working_dir(processing),
script_name)
with open(script_name, 'w') as f:
f.write(script)
run_command("chmod +x %s" % script_name)
return script_name
def __call__(self, processing, transform, input_collection,
output_collection):
try:
contents = core_catalog.get_contents_by_coll_id_status(
coll_id=output_collection['coll_id'])
points = []
unevaluated_points = 0
for content in contents:
point = content['content_metadata']['point']
points.append(point)
if not content['status'] == ContentStatus.Available:
unevaluated_points += 1
if unevaluated_points >= self.min_unevaluated_points:
# not submit the job
processing_metadata = processing['processing_metadata']
processing_metadata['unevaluated_points'] = unevaluated_points
ret = {
'processing_id': processing['processing_id'],
'status': ProcessingStatus.New,
'processing_metadata': processing_metadata
}
return ret
job_dir = self.get_job_dir(processing['processing_id'])
input_json = 'idds_input.json'
with open(os.path.join(job_dir, input_json), 'w') as f:
json.dump(points, f)
sandbox = None
if 'sandbox' in transform['transform_metadata']:
sandbox = transform['transform_metadata']['sandbox']
executable = transform['transform_metadata']['executable']
arguments = transform['transform_metadata']['arguments']
output_json = None
if 'output_json' in transform['transform_metadata']:
output_json = transform['transform_metadata']['output_json']
param_values = {
'NUM_POINTS': self.max_unevaluated_points - unevaluated_points,
'IN': 'input_json',
'OUT': output_json
}
executable = replace_parameters_with_values(
executable, param_values)
arguments = replace_parameters_with_values(arguments, param_values)
input_list = None
job_id, outputs = self.submit_job(processing['processing_id'],
sandbox, executable, arguments,
input_list, input_json,
output_json)
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = job_id
processing_metadata['submitter'] = self.name
if not job_id:
processing_metadata['submit_errors'] = outputs
else:
processing_metadata['submit_errors'] = None
ret = {
'processing_id': processing['processing_id'],
'status': ProcessingStatus.Submitted,
'processing_metadata': processing_metadata
}
return ret
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.AgentPluginError(
'%s: %s' % (str(ex), traceback.format_exc()))