def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_kill(self):
'''kill the master job to the grid'''
job = self.getJobObject()
if not job.master and len(job.subjobs) == 0:
return IBackend.master_kill(self)
elif job.master:
return IBackend.master_kill(self)
else:
return self.master_bulk_kill()
def master_kill(self):
'''kill the master job to the grid'''
job = self.getJobObject()
if not job.master and len(job.subjobs) == 0:
return IBackend.master_kill(self)
elif job.master:
return IBackend.master_kill(self)
else:
return self.master_bulk_kill()
def __mt_job_prepare__(self, rjobs, subjobconfigs, masterjobconfig):
'''preparing jobs in multiple threads'''
logger.warning(
'preparing %d subjobs ... it may take a while' % len(rjobs))
# prepare the master job (i.e. create shared inputsandbox, etc.)
master_input_sandbox = IBackend.master_prepare(self, masterjobconfig)
# uploading the master job if it's over the WMS sandbox limitation
for f in master_input_sandbox:
master_input_idx = self.__check_and_prestage_inputfile__(f)
if not master_input_idx:
logger.error('master input sandbox perparation failed: %s' % f)
return None
# the algorithm for preparing a single bulk job
class MyAlgorithm(Algorithm):
def __init__(self):
Algorithm.__init__(self)
def process(self, sj_info):
my_sc = sj_info[0]
my_sj = sj_info[1]
try:
logger.debug("preparing job %s" % my_sj.getFQID('.'))
jdlpath = my_sj.backend.preparejob(
my_sc, master_input_sandbox)
if (not jdlpath) or (not os.path.exists(jdlpath)):
raise GangaException(
'job %s not properly prepared' % my_sj.getFQID('.'))
self.__appendResult__(my_sj.id, jdlpath)
return True
except Exception as x:
log_user_exception()
return False
mt_data = []
for sc, sj in zip(subjobconfigs, rjobs):
mt_data.append([sc, sj])
myAlg = MyAlgorithm()
myData = Data(collection=mt_data)
runner = MTRunner(
name='lcg_jprepare', algorithm=myAlg, data=myData, numThread=10)
runner.start()
runner.join(-1)
if len(runner.getDoneList()) < len(mt_data):
return None
else:
# return a JDL file dictionary with subjob ids as keys, JDL file
# paths as values
return runner.getResults()
def __mt_job_prepare__(self, rjobs, subjobconfigs, masterjobconfig):
'''preparing jobs in multiple threads'''
logger.warning(
'preparing %d subjobs ... it may take a while' % len(rjobs))
# prepare the master job (i.e. create shared inputsandbox, etc.)
master_input_sandbox = IBackend.master_prepare(self, masterjobconfig)
# uploading the master job if it's over the WMS sandbox limitation
for f in master_input_sandbox:
master_input_idx = self.__check_and_prestage_inputfile__(f)
if not master_input_idx:
logger.error('master input sandbox perparation failed: %s' % f)
return None
# the algorithm for preparing a single bulk job
class MyAlgorithm(Algorithm):
def __init__(self):
Algorithm.__init__(self)
def process(self, sj_info):
my_sc = sj_info[0]
my_sj = sj_info[1]
try:
logger.debug("preparing job %s" % my_sj.getFQID('.'))
jdlpath = my_sj.backend.preparejob(
my_sc, master_input_sandbox)
if (not jdlpath) or (not os.path.exists(jdlpath)):
raise GangaException(
'job %s not properly prepared' % my_sj.getFQID('.'))
self.__appendResult__(my_sj.id, jdlpath)
return True
except Exception as x:
log_user_exception()
return False
mt_data = []
for sc, sj in zip(subjobconfigs, rjobs):
mt_data.append([sc, sj])
myAlg = MyAlgorithm()
myData = Data(collection=mt_data)
runner = MTRunner(
name='lcg_jprepare', algorithm=myAlg, data=myData, numThread=10)
runner.start()
runner.join(-1)
if len(runner.getDoneList()) < len(mt_data):
return None
else:
# return a JDL file dictionary with subjob ids as keys, JDL file
# paths as values
return runner.getResults()
def master_submit(self,
rjobs,
subjobconfigs,
masterjobconfig,
keep_going=False):
""" Overload master_submit to avoid parallel submission with Interactive backend"""
return IBackend.master_submit(self, rjobs, subjobconfigs,
masterjobconfig, keep_going)
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(
self.CE, self.delegation_id, self.credential_requirements)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('CREAM bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('CREAM bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
'''Submit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
# finding ARC CE endpoint for job submission
#allowed_celist = []
# try:
# allowed_celist = self.requirements.getce()
# if not self.CE and allowed_celist:
# self.CE = allowed_celist[0]
# except:
# logger.warning('ARC CE assigment from ARCRequirements failed.')
# if self.CE and allowed_celist:
# if self.CE not in allowed_celist:
# logger.warning('submission to CE not allowed: %s, use %s instead' % ( self.CE, allowed_celist[0] ) )
# self.CE = allowed_celist[0]
# use arc info to check for any endpoints recorded in the config file
rc, output = grids['GLITE'].arc_info()
if not self.CE and rc != 0:
raise GangaException(
"ARC CE endpoint not set and no default settings in '%s'. " % config['ArcConfigFile'])
elif self.CE:
logger.info('ARC CE endpoint set to: ' + str(self.CE))
else:
logger.info("Using ARC CE endpoints defined in '%s'" %
config['ArcConfigFile'])
# delegate proxy to ARC CE
# if not grids['GLITE'].arc_proxy_delegation(self.CE):
# logger.warning('proxy delegation to %s failed' % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(
self, rjobs, subjobconfigs, masterjobconfig)
else:
ick = self.master_bulk_submit(
rjobs, subjobconfigs, masterjobconfig)
profiler.check('==> master_submit() elapsed time')
return ick
def master_resubmit(self, rjobs):
"""Resubmit the master job to the grid"""
profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
profiler.start()
job = self.getJobObject()
ick = False
# delegate proxy to CREAM CE
if not grids["GLITE"].cream_proxy_delegation(self.CE):
logger.warning("proxy delegation to %s failed" % self.CE)
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug("rjobs: %s" % str(rjobs))
logger.debug("mode: master job normal resubmission")
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug("mode: individual subjob resubmission")
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug("mode: master job resubmission")
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException("CREAM bulk submission failure")
profiler.check("job re-submission elapsed time")
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
'''Submit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
# finding CREAM CE endpoint for job submission
allowed_celist = []
try:
allowed_celist = self.requirements.getce()
if not self.CE and allowed_celist:
self.CE = allowed_celist[0]
except:
logger.warning(
'CREAM CE assigment from AtlasCREAMRequirements failed.')
if self.CE and allowed_celist:
if self.CE not in allowed_celist:
logger.warning(
'submission to CE not allowed: %s, use %s instead' %
(self.CE, allowed_celist[0]))
self.CE = allowed_celist[0]
if not self.CE:
raise GangaException('CREAM CE endpoint not set')
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(
self.CE, self.delegation_id, self.credential_requirements)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(self, rjobs, subjobconfigs,
masterjobconfig)
else:
ick = self.master_bulk_submit(rjobs, subjobconfigs,
masterjobconfig)
profiler.check('==> master_submit() elapsed time')
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
'''Submit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
# finding CREAM CE endpoint for job submission
allowed_celist = []
try:
allowed_celist = self.requirements.getce()
if not self.CE and allowed_celist:
self.CE = allowed_celist[0]
except:
logger.warning(
'CREAM CE assigment from AtlasCREAMRequirements failed.')
if self.CE and allowed_celist:
if self.CE not in allowed_celist:
logger.warning('submission to CE not allowed: %s, use %s instead' % (
self.CE, allowed_celist[0]))
self.CE = allowed_celist[0]
if not self.CE:
raise GangaException('CREAM CE endpoint not set')
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(
self, rjobs, subjobconfigs, masterjobconfig)
else:
ick = self.master_bulk_submit(
rjobs, subjobconfigs, masterjobconfig)
profiler.check('==> master_submit() elapsed time')
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
"""Submit the master job to the grid"""
profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
profiler.start()
job = self.getJobObject()
# finding CREAM CE endpoint for job submission
allowed_celist = []
try:
allowed_celist = self.requirements.getce()
if not self.CE and allowed_celist:
self.CE = allowed_celist[0]
except:
logger.warning("CREAM CE assigment from AtlasCREAMRequirements failed.")
if self.CE and allowed_celist:
if self.CE not in allowed_celist:
logger.warning("submission to CE not allowed: %s, use %s instead" % (self.CE, allowed_celist[0]))
self.CE = allowed_celist[0]
if not self.CE:
raise GangaException("CREAM CE endpoint not set")
# delegate proxy to CREAM CE
if not grids["GLITE"].cream_proxy_delegation(self.CE):
logger.warning("proxy delegation to %s failed" % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig)
else:
ick = self.master_bulk_submit(rjobs, subjobconfigs, masterjobconfig)
profiler.check("==> master_submit() elapsed time")
return ick
def __mt_job_prepare__(self, rjobs, subjobconfigs, masterjobconfig):
'''preparing jobs in multiple threads'''
logger.warning('preparing %d subjobs ... it may take a while' % len(rjobs))
# prepare the master job (i.e. create shared inputsandbox, etc.)
master_input_sandbox=IBackend.master_prepare(self,masterjobconfig)
## uploading the master job if it's over the WMS sandbox limitation
for f in master_input_sandbox:
master_input_idx = self.__check_and_prestage_inputfile__(f)
if not master_input_idx:
logger.error('master input sandbox perparation failed: %s' % f)
return None
# the algorithm for preparing a single bulk job
class MyAlgorithm(Algorithm):
def __init__(self):
Algorithm.__init__(self)
def process(self, sj_info):
my_sc = sj_info[0]
my_sj = sj_info[1]
try:
logger.debug("preparing job %s" % my_sj.getFQID('.'))
jdlpath = my_sj.backend.preparejob(my_sc, master_input_sandbox)
if (not jdlpath) or (not os.path.exists(jdlpath)):
raise GangaException('job %s not properly prepared' % my_sj.getFQID('.'))
self.__appendResult__( my_sj.id, jdlpath )
return True
except Exception,x:
log_user_exception()
return False
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
#if self.error.method == 'master_submit':
self.error.trigger()
return IBackend.master_submit(self, rjobs, subjobconfigs,
masterjobconfig)
def master_submit(self,rjobs,subjobconfigs,masterjobconfig):
#if self.error.method == 'master_submit':
self.error.trigger()
return IBackend.master_submit(self,rjobs,subjobconfigs,masterjobconfig)
def master_submit(self, rjobs, subjobconfigs, masterjobconfig, keep_going=False):
""" Overload master_submit to avoid parallel submission with Interactive backend"""
return IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig, keep_going)