def master_submit(self, rjobs, subjobspecs, buildjobspec):
'''Submit jobs'''
from pandatools import Client
from pandatools import MiscUtils
from Ganga.Core.exceptions import IncompleteJobSubmissionError
from Ganga.Utility.logging import log_user_exception
job = self.getJobObject()
# job name
jobName = 'ganga.%s' % MiscUtils.wrappedUuidGen()
jobspecs = {}
if buildjobspec:
jobspecs = buildjobspec
else:
jobspecs = subjobspecs
logger.debug(jobspecs)
# submit task
for subjob in rjobs:
subjob.updateStatus('submitting')
logger.info("Submitting to Jedi ...")
verbose = logger.isEnabledFor(10)
status, tmpOut = Client.insertTaskParams(jobspecs, verbose)
logger.debug(tmpOut)
if status != 0:
logger.error("Task submission to Jedi failed with %s " % status)
return False
if tmpOut[0] == False:
logger.error("Task submission to Jedi failed %s" % tmpOut[1])
return False
logger.info("Task submission to Jedi suceeded with new jediTaskID=%s" %
tmpOut[1])
#if buildjobspec:
# job.backend.buildjob = PandaBuildJob()
# job.backend.buildjob.id = jobids[0][0]
# job.backend.buildjob.url = 'http://panda.cern.ch/?job=%d'%jobids[0][0]
# del jobids[0]
for subjob in rjobs:
subjob.backend.id = tmpOut[1]
subjob.backend.url = 'http://pandamon.cern.ch/jedi/taskinfo?days=20&task=%d' % tmpOut[
1]
subjob.updateStatus('submitted')
logger.info("Panda monitor url: %s" % subjob.backend.url)
return True
def master_submit(self,rjobs,subjobspecs,buildjobspec):
'''Submit jobs'''
from pandatools import Client
from pandatools import MiscUtils
from Ganga.Core import IncompleteJobSubmissionError
from Ganga.Utility.logging import log_user_exception
job = self.getJobObject()
# job name
jobName = 'ganga.%s' % MiscUtils.wrappedUuidGen()
jobspecs = {}
if buildjobspec:
jobspecs = buildjobspec
else:
jobspecs = subjobspecs
logger.debug(jobspecs)
# submit task
for subjob in rjobs:
subjob.updateStatus('submitting')
logger.info("Submitting to Jedi ...")
verbose = logger.isEnabledFor(10)
with inject_proxy(self.credential_requirements):
status, tmpOut = Client.insertTaskParams(jobspecs, verbose)
logger.debug(tmpOut)
if status != 0:
logger.error("Task submission to Jedi failed with %s " %status)
return False
if tmpOut[0] == False:
logger.error("Task submission to Jedi failed %s" %tmpOut[1])
return False
logger.info("Task submission to Jedi suceeded with new jediTaskID=%s" %tmpOut[1])
#if buildjobspec:
# job.backend.buildjob = PandaBuildJob()
# job.backend.buildjob.id = jobids[0][0]
# job.backend.buildjob.url = 'http://panda.cern.ch/?job=%d'%jobids[0][0]
# del jobids[0]
for subjob in rjobs:
subjob.backend.id = tmpOut[1]
subjob.backend.url = 'http://pandamon.cern.ch/jedi/taskinfo?days=20&task=%d'%tmpOut[1]
subjob.updateStatus('submitted')
logger.info("Panda monitor url: %s" %subjob.backend.url)
return True
def submit_panda_task(self, processing):
try:
from pandatools import Client
proc = processing['processing_metadata']['processing']
task_param = proc.processing_metadata['task_param']
return_code = Client.insertTaskParams(task_param, verbose=True)
if return_code[0] == 0:
return return_code[1][1]
else:
self.logger.warn("submit_panda_task, return_code: %s" %
str(return_code))
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
# raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc()))
return None
def submit_panda_task(self, processing):
try:
from pandatools import Client
status, tmpOut = Client.insertTaskParams(self.panda_task_paramsmap,
False, True)
if status == 0:
tmp_status, tmp_output = tmpOut
m = re.search("jediTaskID=(\d+)", tmp_output) # noqa W605
task_id = int(m.group(1))
processing.workload_id = task_id
else:
self.add_errors(tmpOut)
raise Exception(tmpOut)
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
options.evaluationMetrics),
},
])
if options.noSubmit:
if options.noSubmit:
if options.verbose:
tmpLog.debug("==== taskParams ====")
tmpKeys = list(taskParamMap)
tmpKeys.sort()
for tmpKey in tmpKeys:
print('%s : %s' % (tmpKey, taskParamMap[tmpKey]))
sys.exit(0)
tmpLog.info("submit {0}".format(options.outDS))
tmpStat, tmpOut = Client.insertTaskParams(taskParamMap, options.verbose, True)
# result
taskID = None
exitCode = None
if tmpStat != 0:
tmpStr = "task submission failed with {0}".format(tmpStat)
tmpLog.error(tmpStr)
exitCode = 1
if tmpOut[0] in [0, 3]:
tmpStr = tmpOut[1]
tmpLog.info(tmpStr)
try:
m = re.search('jediTaskID=(\d+)', tmpStr)
taskID = int(m.group(1))
except Exception:
pass
taskParamMap['workingGroup'] = 'EIC'
taskParamMap['nFilesPerJob'] = 1
inFileList = [str(i) for i in range(0, LIMIT)]
taskParamMap['nFiles'] = len(inFileList)
taskParamMap['noInput'] = True
taskParamMap['pfnList'] = inFileList
taskParamMap['taskName'] = 'test EIC submission'
taskParamMap['userName'] = '******'
taskParamMap['processingType'] = 'step1'
taskParamMap['prodSourceLabel'] = 'test'
taskParamMap['taskType'] = 'test'
taskParamMap['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c'
taskParamMap['taskPriority'] = 900
taskParamMap['architecture'] = ''
taskParamMap['transUses'] = ''
taskParamMap['transHome'] = None
#taskParamMap['coreCount'] = 1
taskParamMap['ramCount'] = 2000
taskParamMap['skipScout'] = True
taskParamMap['cloud'] = 'US'
taskParamMap['jobParameters'] = [
{'type':'constant',
'value':"""export EIC_LEVEL=dev;
source /cvmfs/eic.opensciencegrid.org/x8664_sl7/MCEG/releases/etc/eic_bash.sh;
echo """+task_command_line+""";
"""+task_command_line+""";"""
},
]
print(Client.insertTaskParams(taskParamMap,verbose=True))
'destination': 'local',
'value': '{0}.${{SN}}.log.tgz'.format(logDatasetName)
}
outDatasetName = 'panda.jeditest.{0}'.format(uuid.uuid4())
taskParamMap['jobParameters'] = [
{
'type': 'constant',
'value': '-i "${IN/T}"',
},
{
'type': 'constant',
'value': 'ecmEnergy=8000 runNumber=12345'
},
{
'type': 'template',
'param_type': 'output',
'token': 'local',
'destination': 'local',
'value': 'outputEVNTFile={0}.${{SN}}.root'.format(outDatasetName),
'dataset': outDatasetName,
'offset': 1000,
},
{
'type': 'constant',
'value': 'aaaa',
},
]
print(Client.insertTaskParams(taskParamMap))
