def jobIsCancelled(self, ID): L = Client.getPandIDsWithJobID(ID)[1] if L is None: result = False else: result = False for item in L: if L[item][0] == 'cancelled': result = True pass return result
def jobIsFinished(self, ID): L = Client.getPandIDsWithJobID(ID)[1] if L is None: result = False else: result = True for item in L: if L[item][0] != 'finished': result = False pass return result
def status(self, JobID, forceUpdate=False):
# get logger
tmpLog = PLogger.getPandaLogger()
# check proxy
self.gridPassPhrase, self.vomsFQAN = PsubUtils.checkGridProxy(
self.gridPassPhrase, False, self.verbose)
# get job info from local repository
job = self.getJobInfo(JobID)
if job == None:
# not found
return None
# update if needed
if job.dbStatus != 'frozen' or forceUpdate:
tmpLog.info("Getting status for JobID=%s ..." % JobID)
# get status from Panda server
status, pandaIDstatus = Client.getPandIDsWithJobID(
JobID, verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get status for JobID=%s" % JobID)
return None
# get one job to set computingSite which may have changed due to rebrokerage
pandaJob = None
if pandaIDstatus != {}:
tmpPandaIDs = pandaIDstatus.keys()
tmpPandaIDs.sort()
status, tmpPandaJobs = Client.getFullJobStatus(
tmpPandaIDs[:1], verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get PandaJobs for %s" % JobID)
return None
pandaJob = tmpPandaJobs[0]
# convert to local job spec
job = PdbUtils.convertPtoD([],
pandaIDstatus,
job,
pandaJobForSiteID=pandaJob)
# update DB
try:
PdbUtils.updateJobDB(job, self.verbose)
except:
tmpLog.error("Failed to update local repository for JobID=%s" %
JobID)
return None
tmpLog.info("Updated JobID=%s" % JobID)
# return
return job
TotalRealTime = 0
TotalJobs = 0
tmpLog = PLogger.getPandaLogger()
gridPassPhrase, vomsFQAN = PsubUtils.checkGridProxy("", False,
prog_options.verbose)
bookConf = BookConfig.getConfig()
if prog_options.verbose > 0:
verbose2 = prog_options.verbose - 1
else:
verbose2 = 0
for strid in prog_args:
print "Retrieving information for Jobset:", strid
status, pandaIDstatus = Client.getPandIDsWithJobID(int(strid))
if prog_options.verbose:
print status, pandaIDstatus
if status != 0:
print "Skip..."
continue
pandaJob = None
if pandaIDstatus != {}:
tmpPandaIDs = pandaIDstatus.keys()
tmpPandaIDs.sort()
nIDs = len(tmpPandaIDs)
nIDs_step = (nIDs + 9) / 10
for i, tmpID in enumerate(tmpPandaIDs):
if prog_options.verbose:
def status(self,JobID,forceUpdate=False):
# get logger
tmpLog = PLogger.getPandaLogger()
# check proxy
self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy(
self.gridPassPhrase,
False,
self.verbose,
useCache=True)
# get job info from local repository
job = self.getJobInfo(JobID)
if job == None:
# not found
return None
# update if needed
if job.dbStatus != 'frozen' or forceUpdate:
if not job.isJEDI():
tmpLog.info("Getting status for JobID=%s ..." % JobID)
# get status from Panda server
status,pandaIDstatus = Client.getPandIDsWithJobID(JobID,verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get status for ID=%s" % JobID)
return None
# get one job to set computingSite which may have changed due to rebrokerage
pandaJob = None
if pandaIDstatus != {}:
tmpPandaIDs = pandaIDstatus.keys()
tmpPandaIDs.sort()
status,tmpPandaJobs = Client.getFullJobStatus(
tmpPandaIDs[:1],
verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get PandaJobs for %s" % JobID)
return None
pandaJob = tmpPandaJobs[0]
# convert to local job spec
job = PdbUtils.convertPtoD([],pandaIDstatus,job,pandaJobForSiteID=pandaJob)
# check merge job generation
status = self.setMergeJobStatus(job,forceUpdate)
if not status:
return None
else:
tmpLog.info("Getting status for TaskID=%s ..." % job.jediTaskID)
# get JEDI task
status,jediTaskDict = Client.getJediTaskDetails(
{'jediTaskID':job.jediTaskID},
False,
True,
verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get task details for %s" % JobID)
return
# convert JEDI task
job = PdbUtils.convertJTtoD(jediTaskDict,job)
# update DB
try:
PdbUtils.updateJobDB(job,self.verbose)
except:
tmpLog.error("Failed to update local repository for JobID=%s" % JobID)
return None
if not job.isJEDI():
tmpLog.info("Updated JobID=%s" % JobID)
else:
tmpLog.info("Updated TaskID=%s ..." % job.jediTaskID)
# return
return job
def retrieveMergeJobs(job, pandaJobDefId):
'''
methods for retrieving panda job ids of merging jobs given a jobDefId
'''
from pandatools import Client
ick = False
status = ''
num_mjobs = 0
(ec, info) = Client.checkMergeGenerationStatus(pandaJobDefId)
if ec == 0:
try:
status = info['status']
mergeJobDefIds = info['mergeIDs']
if status == 'NA':
logger.warning('No merging jobs expected')
job.backend.mergejobs = []
elif status == 'generating':
logger.debug('merging jobs are generating')
job.backend.mergejobs = []
elif status == 'standby':
logger.debug('merging jobs to be created')
job.backend.mergejobs = []
elif status == 'generated':
logger.debug('merging jobs are generated')
for id in mergeJobDefIds:
logger.debug("merging jobDefId: %d" % id)
## retrieve merging job id,status given the jobDefId
(ec2, mjs) = Client.getPandIDsWithJobID(id)
if ec2 == 0:
for jid,jinfo in mjs.items():
mjobj = PandaMergeJob()
mjobj.id = jid
#mjobj.status = jinfo[0]
mjobj.url = 'http://panda.cern.ch/?job=%d' % jid
if mjobj not in job.backend.mergejobs:
job.backend.mergejobs.append(mjobj)
else:
logger.debug("merging job %s already exists locally" % mjobj.id)
num_mjobs += 1
else:
logger.warning("getPandIDsWithJobID returns non-zero exit code: %d" % ec2)
ick = True
except KeyError:
logger.error('unexpected job information: %s' % repr(info))
except Exception as e:
logger.error('general merge job information retrieval error')
raise e
else:
logger.error('checkMergeGenerationStatus returns non-zero exit code: %d' % ec)
return (ick, status, num_mjobs)
def retrieveMergeJobs(job, pandaJobDefId):
'''
methods for retrieving panda job ids of merging jobs given a jobDefId
'''
from pandatools import Client
ick = False
status = ''
num_mjobs = 0
(ec, info) = Client.checkMergeGenerationStatus(pandaJobDefId)
if ec == 0:
try:
status = info['status']
mergeJobDefIds = info['mergeIDs']
if status == 'NA':
logger.warning('No merging jobs expected')
job.backend.mergejobs = []
elif status == 'generating':
logger.debug('merging jobs are generating')
job.backend.mergejobs = []
elif status == 'standby':
logger.debug('merging jobs to be created')
job.backend.mergejobs = []
elif status == 'generated':
logger.debug('merging jobs are generated')
for id in mergeJobDefIds:
logger.debug("merging jobDefId: %d" % id)
## retrieve merging job id,status given the jobDefId
(ec2, mjs) = Client.getPandIDsWithJobID(id)
if ec2 == 0:
for jid, jinfo in mjs.items():
mjobj = PandaMergeJob()
mjobj.id = jid
#mjobj.status = jinfo[0]
mjobj.url = 'http://panda.cern.ch/?job=%d' % jid
if mjobj not in job.backend.mergejobs:
job.backend.mergejobs.append(mjobj)
else:
logger.debug(
"merging job %s already exists locally" %
mjobj.id)
num_mjobs += 1
else:
logger.warning(
"getPandIDsWithJobID returns non-zero exit code: %d"
% ec2)
ick = True
except KeyError:
logger.error('unexpected job information: %s' % repr(info))
except Exception as e:
logger.error('general merge job information retrieval error')
raise e
else:
logger.error(
'checkMergeGenerationStatus returns non-zero exit code: %d' % ec)
return (ick, status, num_mjobs)
def sync(self):
# get logger
tmpLog = PLogger.getPandaLogger()
tmpLog.info("Synchronizing local repository ...")
# check proxy
self.gridPassPhrase, self.vomsFQAN = PsubUtils.checkGridProxy(
self.gridPassPhrase, False, self.verbose)
# get JobIDs in local repository
localJobIDs = PdbUtils.getListOfJobIDs()
# get recent JobIDs from panda server
syncTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
# set sync time for the first attempt
bookConf = BookConfig.getConfig()
if self.restoreDB:
# reset last_synctime to restore database
bookConf.last_synctime = ''
# disable
self.restoreDB = False
tmpLog.info(
"It may take several minutes to restore local repository ...")
if bookConf.last_synctime == '':
bookConf.last_synctime = datetime.datetime.utcnow(
) - datetime.timedelta(days=180)
bookConf.last_synctime = bookConf.last_synctime.strftime(
'%Y-%m-%d %H:%M:%S')
status, remoteJobIDs = Client.getJobIDsInTimeRange(
bookConf.last_synctime, verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get JobIDs from panda server")
return
tmpLog.info("Got %s jobs to be updated" % len(remoteJobIDs))
# insert if missing
for remoteJobID in remoteJobIDs:
# check local status
job = None
if remoteJobID in localJobIDs:
# get job info from local repository
job = PdbUtils.readJobDB(remoteJobID, self.verbose)
# skip if frozen
if job.dbStatus == 'frozen':
continue
tmpLog.info("Updating JobID=%s ..." % remoteJobID)
# get PandaIDs
status, pandaIDstatus = Client.getPandIDsWithJobID(
remoteJobID, verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get PandaIDs for %s" % remoteJobID)
return
pandaIDs = pandaIDstatus.keys()
pandaIDs.sort()
# get full JobSpec
pandaJobs = []
pandaFileInfo = {}
pandaJobForSiteID = None
if job == None:
tmpIDs = [pandaIDs[0], pandaIDs[-1]]
status, pandaJobs = Client.getFullJobStatus(
tmpIDs, verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get PandaJobs for %s" %
remoteJobID)
return
# get slimmed file info
status, pandaFileInfo = Client.getSlimmedFileInfoPandaIDs(
pandaIDs, verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get file info for %s" %
remoteJobID)
return
else:
# get one job to set computingSite which may have changed due to rebrokerage
status, tmpPandaJobs = Client.getFullJobStatus(
[pandaIDs[0]], verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get PandaJobs for %s" %
remoteJobID)
return
pandaJobForSiteID = tmpPandaJobs[0]
# convert to local job spec
localJob = PdbUtils.convertPtoD(pandaJobs, pandaIDstatus, job,
pandaFileInfo, pandaJobForSiteID)
# update database
if not remoteJobID in localJobIDs:
# insert to DB
try:
PdbUtils.insertJobDB(localJob, self.verbose)
except:
tmpLog.error(
"Failed to insert JobID=%s to local repository" %
remoteJobID)
return
# set retryID
if not localJob.provenanceID in [0, '0']:
try:
PdbUtils.setRetryID(localJob, self.verbose)
except:
tmpLog.error(
"Failed to set retryID for JobID=%s in local repository"
% remoteJobID)
return
else:
# update
try:
PdbUtils.updateJobDB(localJob, self.verbose)
except:
tmpLog.error(
"Failed to update local repository for JobID=%s" %
remoteJobID)
return
# update sync time
bookConf = BookConfig.getConfig()
bookConf.last_synctime = syncTime
BookConfig.updateConfig(bookConf)
tmpLog.info("Synchronization Completed")
