def master_submit(self, rjobs, subjobspecs, buildjobspec): '''Submit jobs''' from pandatools import Client from pandatools import MiscUtils from Ganga.Core.exceptions import IncompleteJobSubmissionError from Ganga.Utility.logging import log_user_exception job = self.getJobObject() # job name jobName = 'ganga.%s' % MiscUtils.wrappedUuidGen() jobspecs = {} if buildjobspec: jobspecs = buildjobspec else: jobspecs = subjobspecs logger.debug(jobspecs) # submit task for subjob in rjobs: subjob.updateStatus('submitting') logger.info("Submitting to Jedi ...") verbose = logger.isEnabledFor(10) status, tmpOut = Client.insertTaskParams(jobspecs, verbose) logger.debug(tmpOut) if status != 0: logger.error("Task submission to Jedi failed with %s " % status) return False if tmpOut[0] == False: logger.error("Task submission to Jedi failed %s" % tmpOut[1]) return False logger.info("Task submission to Jedi suceeded with new jediTaskID=%s" % tmpOut[1]) #if buildjobspec: # job.backend.buildjob = PandaBuildJob() # job.backend.buildjob.id = jobids[0][0] # job.backend.buildjob.url = 'http://panda.cern.ch/?job=%d'%jobids[0][0] # del jobids[0] for subjob in rjobs: subjob.backend.id = tmpOut[1] subjob.backend.url = 'http://pandamon.cern.ch/jedi/taskinfo?days=20&task=%d' % tmpOut[ 1] subjob.updateStatus('submitted') logger.info("Panda monitor url: %s" % subjob.backend.url) return True
def master_submit(self,rjobs,subjobspecs,buildjobspec): '''Submit jobs''' from pandatools import Client from pandatools import MiscUtils from Ganga.Core import IncompleteJobSubmissionError from Ganga.Utility.logging import log_user_exception job = self.getJobObject() # job name jobName = 'ganga.%s' % MiscUtils.wrappedUuidGen() jobspecs = {} if buildjobspec: jobspecs = buildjobspec else: jobspecs = subjobspecs logger.debug(jobspecs) # submit task for subjob in rjobs: subjob.updateStatus('submitting') logger.info("Submitting to Jedi ...") verbose = logger.isEnabledFor(10) with inject_proxy(self.credential_requirements): status, tmpOut = Client.insertTaskParams(jobspecs, verbose) logger.debug(tmpOut) if status != 0: logger.error("Task submission to Jedi failed with %s " %status) return False if tmpOut[0] == False: logger.error("Task submission to Jedi failed %s" %tmpOut[1]) return False logger.info("Task submission to Jedi suceeded with new jediTaskID=%s" %tmpOut[1]) #if buildjobspec: # job.backend.buildjob = PandaBuildJob() # job.backend.buildjob.id = jobids[0][0] # job.backend.buildjob.url = 'http://panda.cern.ch/?job=%d'%jobids[0][0] # del jobids[0] for subjob in rjobs: subjob.backend.id = tmpOut[1] subjob.backend.url = 'http://pandamon.cern.ch/jedi/taskinfo?days=20&task=%d'%tmpOut[1] subjob.updateStatus('submitted') logger.info("Panda monitor url: %s" %subjob.backend.url) return True
def submit_panda_task(self, processing): try: from pandatools import Client proc = processing['processing_metadata']['processing'] task_param = proc.processing_metadata['task_param'] return_code = Client.insertTaskParams(task_param, verbose=True) if return_code[0] == 0: return return_code[1][1] else: self.logger.warn("submit_panda_task, return_code: %s" % str(return_code)) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) # raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc())) return None
def submit_panda_task(self, processing): try: from pandatools import Client status, tmpOut = Client.insertTaskParams(self.panda_task_paramsmap, False, True) if status == 0: tmp_status, tmp_output = tmpOut m = re.search("jediTaskID=(\d+)", tmp_output) # noqa W605 task_id = int(m.group(1)) processing.workload_id = task_id else: self.add_errors(tmpOut) raise Exception(tmpOut) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
options.evaluationMetrics), }, ]) if options.noSubmit: if options.noSubmit: if options.verbose: tmpLog.debug("==== taskParams ====") tmpKeys = list(taskParamMap) tmpKeys.sort() for tmpKey in tmpKeys: print('%s : %s' % (tmpKey, taskParamMap[tmpKey])) sys.exit(0) tmpLog.info("submit {0}".format(options.outDS)) tmpStat, tmpOut = Client.insertTaskParams(taskParamMap, options.verbose, True) # result taskID = None exitCode = None if tmpStat != 0: tmpStr = "task submission failed with {0}".format(tmpStat) tmpLog.error(tmpStr) exitCode = 1 if tmpOut[0] in [0, 3]: tmpStr = tmpOut[1] tmpLog.info(tmpStr) try: m = re.search('jediTaskID=(\d+)', tmpStr) taskID = int(m.group(1)) except Exception: pass
taskParamMap['workingGroup'] = 'EIC' taskParamMap['nFilesPerJob'] = 1 inFileList = [str(i) for i in range(0, LIMIT)] taskParamMap['nFiles'] = len(inFileList) taskParamMap['noInput'] = True taskParamMap['pfnList'] = inFileList taskParamMap['taskName'] = 'test EIC submission' taskParamMap['userName'] = '******' taskParamMap['processingType'] = 'step1' taskParamMap['prodSourceLabel'] = 'test' taskParamMap['taskType'] = 'test' taskParamMap['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c' taskParamMap['taskPriority'] = 900 taskParamMap['architecture'] = '' taskParamMap['transUses'] = '' taskParamMap['transHome'] = None #taskParamMap['coreCount'] = 1 taskParamMap['ramCount'] = 2000 taskParamMap['skipScout'] = True taskParamMap['cloud'] = 'US' taskParamMap['jobParameters'] = [ {'type':'constant', 'value':"""export EIC_LEVEL=dev; source /cvmfs/eic.opensciencegrid.org/x8664_sl7/MCEG/releases/etc/eic_bash.sh; echo """+task_command_line+"""; """+task_command_line+""";""" }, ] print(Client.insertTaskParams(taskParamMap,verbose=True))
'destination': 'local', 'value': '{0}.${{SN}}.log.tgz'.format(logDatasetName) } outDatasetName = 'panda.jeditest.{0}'.format(uuid.uuid4()) taskParamMap['jobParameters'] = [ { 'type': 'constant', 'value': '-i "${IN/T}"', }, { 'type': 'constant', 'value': 'ecmEnergy=8000 runNumber=12345' }, { 'type': 'template', 'param_type': 'output', 'token': 'local', 'destination': 'local', 'value': 'outputEVNTFile={0}.${{SN}}.root'.format(outDatasetName), 'dataset': outDatasetName, 'offset': 1000, }, { 'type': 'constant', 'value': 'aaaa', }, ] print(Client.insertTaskParams(taskParamMap))