def get_executable_arguments_for_sandbox(self, transform_metadata, input_json, unevaluated_points): sandbox = None if 'sandbox' in transform_metadata: sandbox = transform_metadata['sandbox'] executable = transform_metadata['executable'].strip() arguments = transform_metadata['arguments'].strip() if executable == 'docker' and sandbox: if 'workdir' in transform_metadata: docker_workdir = transform_metadata['workdir'] else: docker_workdir = None arg_pre = 'run --rm -v $(pwd):%s %s ' % (docker_workdir, sandbox) arguments = arg_pre + arguments sandbox = None output_json = None if 'output_json' in transform_metadata: output_json = transform_metadata['output_json'] max_points = self.get_max_points(transform_metadata) if 'num_points_per_generation' in transform_metadata: num_points = transform_metadata['num_points_per_generation'] else: num_points = self.max_unevaluated_points - unevaluated_points param_values = {'MAX_POINTS': max_points, 'NUM_POINTS': num_points, 'IN': input_json, 'OUT': output_json} executable = replace_parameters_with_values(executable, param_values) arguments = replace_parameters_with_values(arguments, param_values) return 0, None, sandbox, executable, arguments, input_json, output_json, False
def get_executable_arguments_for_sandbox(self, transform_metadata, input_json, unevaluated_points): sandbox = None if 'sandbox' in transform_metadata: sandbox = transform_metadata['sandbox'] executable = transform_metadata['executable'] arguments = transform_metadata['arguments'] output_json = None if 'output_json' in transform_metadata: output_json = transform_metadata['output_json'] if 'max_points' in transform_metadata: max_points = transform_metadata['max_points'] else: max_points = self.max_points if 'num_points_per_generation' in transform_metadata: num_points = transform_metadata['num_points_per_generation'] else: num_points = self.max_unevaluated_points - unevaluated_points param_values = { 'MAX_POINTS': max_points, 'NUM_POINTS': num_points, 'IN': input_json, 'OUT': output_json } executable = replace_parameters_with_values(executable, param_values) arguments = replace_parameters_with_values(arguments, param_values) return 0, None, sandbox, executable, arguments, input_json, output_json, False
def generate_processing_script_container(self, processing): param_values = { 'MAX_POINTS': self.max_points, 'NUM_POINTS': self.points_to_generate, 'IN': self.input_json, 'OUT': self.output_json } proxy_filename = 'x509up' if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) param_values['X509_USER_PROXY_FULLNAME'] = os.environ[ 'X509_USER_PROXY'] param_values['X509_USER_PROXY_BASENAME'] = proxy_filename executable = replace_parameters_with_values(self.executable, param_values) arguments = replace_parameters_with_values(self.arguments, param_values) script = "#!/bin/bash\n\n" script += "executable=%s\n" % str(executable) script += "arguments=%s\n" % str(arguments) script += "input_json=%s\n" % str(self.input_json) script += "output_json=%s\n" % str(self.output_json) script += "\n" script += "env\n" script += "echo $X509_USER_PROXY\n" script += "\n" script += "echo 'user id:'\n" script += "id\n" script += "\n" if self.sandbox and 'docker' in executable: arguments = 'run --rm -v $(pwd):%s -v /cvmfs:/cvmfs -e X509_USER_PROXY=%s/%s %s ' % ( self.container_workdir, self.container_workdir, proxy_filename, self.sandbox) + arguments script += "echo '%s' '%s'\n" % (str(executable), str(arguments)) script += '%s %s\n' % (str(executable), str(arguments)) if self.sandbox and 'docker' in executable: script += 'docker image rm -f %s\n' % self.sandbox script += '\n' long_id = self.get_long_id(processing) script_name = 'processing_%s.sh' % long_id script_name = os.path.join(self.get_working_dir(processing), script_name) with open(script_name, 'w') as f: f.write(script) run_command("chmod +x %s" % script_name) return script_name
def generate_processing_script_sandbox(self, processing): param_values = { 'MAX_POINTS': self.max_points, 'NUM_POINTS': self.points_to_generate, 'IN': self.input_json, 'OUT': self.output_json } if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) param_values['X509_USER_PROXY_FULLNAME'] = os.environ[ 'X509_USER_PROXY'] param_values['X509_USER_PROXY_BASENAME'] = proxy_filename executable = replace_parameters_with_values(self.executable, param_values) arguments = replace_parameters_with_values(self.arguments, param_values) script = "#!/bin/bash\n\n" script += "sandbox=%s\n" % str(self.sandbox) script += "executable=%s\n" % str(executable) script += "arguments=%s\n" % str(arguments) script += "input_json=%s\n" % str(self.input_json) script += "output_json=%s\n" % str(self.output_json) script += "\n" script += "env\n" script += "echo $X509_USER_PROXY\n" script += "\n" script += "echo 'user id:'\n" script += "id\n" script += "\n" script += "wget $sandbox\n" script += 'base_sandbox="$(basename -- $sandbox)"\n' script += 'tar xzf $base_sandbox\n' script += 'chmod +x %s\n' % str(executable) script += "echo '%s' '%s'\n" % (str(executable), str(arguments)) script += '%s %s\n' % (str(executable), str(arguments)) script += '\n' long_id = self.get_long_id(processing) script_name = 'processing_%s.sh' % long_id script_name = os.path.join(self.get_working_dir(processing), script_name) with open(script_name, 'w') as f: f.write(script) run_command("chmod +x %s" % script_name) return script_name
def get_executable_arguments_for_method(self, transform_metadata, input_json, unevaluated_points): method = transform_metadata['method'] method_executable = '%s.executable' % method method_arguments = '%s.arguments' % method method_output_json = '%s.output_json' % method method_should_transfer_executable = '%s.should_transfer_executable' % method if not hasattr(self, method_executable) or not hasattr( self, method_arguments) or not hasattr(self, method_output_json): return -1, "method %s is not support" % method, None, None, None, None, None else: executable = getattr(self, method_executable) arguments = getattr(self, method_arguments) output_json = getattr(self, method_output_json) should_transfer_executable = False if hasattr(self, method_should_transfer_executable): should_transfer_executable = getattr( self, method_should_transfer_executable) if type(should_transfer_executable) in [str]: should_transfer_executable = should_transfer_executable.upper( ) if should_transfer_executable in [True, 'TRUE']: should_transfer_executable = True else: should_transfer_executable = False if 'max_points' in transform_metadata: max_points = transform_metadata['max_points'] else: max_points = self.max_points if 'num_points_per_generation' in transform_metadata: num_points = transform_metadata['num_points_per_generation'] else: num_points = self.max_unevaluated_points - unevaluated_points param_values = { 'MAX_POINTS': max_points, 'NUM_POINTS': num_points, 'IN': input_json, 'OUT': output_json } executable = replace_parameters_with_values( executable, param_values) arguments = replace_parameters_with_values(arguments, param_values) return 0, None, None, executable, arguments, input_json, output_json, should_transfer_executable
def generate_processing_script_nevergrad(self, processing): executable = self.agent_attributes['nevergrad']['executable'] arguments = self.agent_attributes['nevergrad']['arguments'] param_values = { 'MAX_POINTS': self.max_points, 'NUM_POINTS': self.points_to_generate, 'IN': self.input_json, 'OUT': self.output_json } if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) param_values['X509_USER_PROXY_FULLNAME'] = os.environ[ 'X509_USER_PROXY'] param_values['X509_USER_PROXY_BASENAME'] = proxy_filename arguments = replace_parameters_with_values(arguments, param_values) script = "#!/bin/bash\n\n" script += "executable=%s\n" % os.path.basename(executable) script += "arguments='%s'\n" % str(arguments) script += "input_json=%s\n" % str(self.input_json) script += "output_json=%s\n" % str(self.output_json) script += "\n" script += "env\n" script += "echo $X509_USER_PROXY\n" script += "\n" script += "echo 'user id:'\n" script += "id\n" script += "\n" script += "echo '%s' '%s'\n" % (os.path.basename(executable), str(arguments)) script += '%s %s\n' % (os.path.basename(executable), str(arguments)) script += '\n' long_id = self.get_long_id(processing) script_name = 'processing_%s.sh' % long_id script_name = os.path.join(self.get_working_dir(processing), script_name) with open(script_name, 'w') as f: f.write(script) run_command("chmod +x %s" % script_name) return script_name
def __call__(self, processing, transform, input_collection, output_collection): try: contents = core_catalog.get_contents_by_coll_id_status( coll_id=output_collection['coll_id']) points = [] unevaluated_points = 0 for content in contents: point = content['content_metadata']['point'] points.append(point) if not content['status'] == ContentStatus.Available: unevaluated_points += 1 if unevaluated_points >= self.min_unevaluated_points: # not submit the job processing_metadata = processing['processing_metadata'] processing_metadata['unevaluated_points'] = unevaluated_points ret = { 'processing_id': processing['processing_id'], 'status': ProcessingStatus.New, 'processing_metadata': processing_metadata } return ret job_dir = self.get_job_dir(processing['processing_id']) input_json = 'idds_input.json' with open(os.path.join(job_dir, input_json), 'w') as f: json.dump(points, f) sandbox = None if 'sandbox' in transform['transform_metadata']: sandbox = transform['transform_metadata']['sandbox'] executable = transform['transform_metadata']['executable'] arguments = transform['transform_metadata']['arguments'] output_json = None if 'output_json' in transform['transform_metadata']: output_json = transform['transform_metadata']['output_json'] param_values = { 'NUM_POINTS': self.max_unevaluated_points - unevaluated_points, 'IN': 'input_json', 'OUT': output_json } executable = replace_parameters_with_values( executable, param_values) arguments = replace_parameters_with_values(arguments, param_values) input_list = None job_id, outputs = self.submit_job(processing['processing_id'], sandbox, executable, arguments, input_list, input_json, output_json) processing_metadata = processing['processing_metadata'] processing_metadata['job_id'] = job_id processing_metadata['submitter'] = self.name if not job_id: processing_metadata['submit_errors'] = outputs else: processing_metadata['submit_errors'] = None ret = { 'processing_id': processing['processing_id'], 'status': ProcessingStatus.Submitted, 'processing_metadata': processing_metadata } return ret except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.AgentPluginError( '%s: %s' % (str(ex), traceback.format_exc()))