def sync(self):
# get logger
tmpLog = PLogger.getPandaLogger()
tmpLog.info("Synchronizing local repository ...")
# check proxy
self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy(
self.gridPassPhrase,
False,
self.verbose,
useCache=True)
# get nickname
nickName = PsubUtils.getNickname()
# set Rucio accounting
PsubUtils.setRucioAccount(nickName,'pbook',True)
# get JobIDs in local repository
localJobIDs = PdbUtils.getListOfJobIDs()
# get recent JobIDs from panda server
syncTimeRaw = datetime.datetime.utcnow()
syncTime = syncTimeRaw.strftime('%Y-%m-%d %H:%M:%S')
# set sync time for the first attempt
bookConf = BookConfig.getConfig()
if self.restoreDB:
# reset last_synctime to restore database
bookConf.last_synctime = ''
# disable
self.restoreDB = False
tmpLog.info("It may take several minutes to restore local repository ...")
if bookConf.last_synctime == '':
bookConf.last_synctime = datetime.datetime.utcnow()-datetime.timedelta(days=180)
bookConf.last_synctime = bookConf.last_synctime.strftime('%Y-%m-%d %H:%M:%S')
maxTaskID = None
while True:
status, jediTaskDicts = Client.getJobIDsJediTasksInTimeRange(bookConf.last_synctime,
minTaskID=maxTaskID,
verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get tasks from panda server")
return
if len(jediTaskDicts) == 0:
break
tmpLog.info("Got %s tasks to be updated" % len(jediTaskDicts))
# insert if missing
for remoteJobID in jediTaskDicts.keys():
taskID = jediTaskDicts[remoteJobID]['jediTaskID']
# get max
if maxTaskID is None or taskID > maxTaskID:
maxTaskID = taskID
# check local status
job = None
if remoteJobID in localJobIDs:
# get job info from local repository
job = PdbUtils.readJobDB(remoteJobID, self.verbose)
# skip if frozen
if job.dbStatus == 'frozen':
continue
tmpLog.info("Updating taskID=%s ..." % taskID)
# convert JEDI task
localJob = PdbUtils.convertJTtoD(jediTaskDicts[remoteJobID],job)
# update database
if not remoteJobID in localJobIDs:
# insert to DB
try:
PdbUtils.insertJobDB(localJob,self.verbose)
except:
tmpLog.error("Failed to insert taskID=%s to local repository" % taskID)
return
else:
# update
try:
PdbUtils.updateJobDB(localJob,self.verbose,syncTimeRaw)
except:
tmpLog.error("Failed to update local repository for taskID=%s" % taskID)
return
# update sync time
bookConf = BookConfig.getConfig()
bookConf.last_synctime = syncTime
BookConfig.updateConfig(bookConf)
self.updateTaskJobsetMap()
tmpLog.info("Synchronization Completed")
def status(self,JobID,forceUpdate=False):
# get logger
tmpLog = PLogger.getPandaLogger()
# check proxy
self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy(
self.gridPassPhrase,
False,
self.verbose,
useCache=True)
# get job info from local repository
job = self.getJobInfo(JobID)
if job == None:
# not found
return None
# update if needed
if job.dbStatus != 'frozen' or forceUpdate:
if not job.isJEDI():
tmpLog.info("Getting status for JobID=%s ..." % JobID)
# get status from Panda server
status,pandaIDstatus = Client.getPandIDsWithJobID(JobID,verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get status for ID=%s" % JobID)
return None
# get one job to set computingSite which may have changed due to rebrokerage
pandaJob = None
if pandaIDstatus != {}:
tmpPandaIDs = pandaIDstatus.keys()
tmpPandaIDs.sort()
status,tmpPandaJobs = Client.getFullJobStatus(
tmpPandaIDs[:1],
verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get PandaJobs for %s" % JobID)
return None
pandaJob = tmpPandaJobs[0]
# convert to local job spec
job = PdbUtils.convertPtoD([],pandaIDstatus,job,pandaJobForSiteID=pandaJob)
# check merge job generation
status = self.setMergeJobStatus(job,forceUpdate)
if not status:
return None
else:
tmpLog.info("Getting status for TaskID=%s ..." % job.jediTaskID)
# get JEDI task
status,jediTaskDict = Client.getJediTaskDetails(
{'jediTaskID':job.jediTaskID},
False,
True,
verbose=self.verbose)
if status != 0:
tmpLog.error("Failed to get task details for %s" % JobID)
return
# convert JEDI task
job = PdbUtils.convertJTtoD(jediTaskDict,job)
# update DB
try:
PdbUtils.updateJobDB(job,self.verbose)
except:
tmpLog.error("Failed to update local repository for JobID=%s" % JobID)
return None
if not job.isJEDI():
tmpLog.info("Updated JobID=%s" % JobID)
else:
tmpLog.info("Updated TaskID=%s ..." % job.jediTaskID)
# return
return job