def getJobIDs(self, timestamp): date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(timestamp)) status, output = Client.getJobIDsInTimeRange(date) if status != 0: print 'Error: panda !' return [] output.sort() return output
def sync(self): # get logger tmpLog = PLogger.getPandaLogger() tmpLog.info("Synchronizing local repository ...") # check proxy self.gridPassPhrase, self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose) # get JobIDs in local repository localJobIDs = PdbUtils.getListOfJobIDs() # get recent JobIDs from panda server syncTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') # set sync time for the first attempt bookConf = BookConfig.getConfig() if self.restoreDB: # reset last_synctime to restore database bookConf.last_synctime = '' # disable self.restoreDB = False tmpLog.info( "It may take several minutes to restore local repository ...") if bookConf.last_synctime == '': bookConf.last_synctime = datetime.datetime.utcnow( ) - datetime.timedelta(days=180) bookConf.last_synctime = bookConf.last_synctime.strftime( '%Y-%m-%d %H:%M:%S') status, remoteJobIDs = Client.getJobIDsInTimeRange( bookConf.last_synctime, verbose=self.verbose) if status != 0: tmpLog.error("Failed to get JobIDs from panda server") return tmpLog.info("Got %s jobs to be updated" % len(remoteJobIDs)) # insert if missing for remoteJobID in remoteJobIDs: # check local status job = None if remoteJobID in localJobIDs: # get job info from local repository job = PdbUtils.readJobDB(remoteJobID, self.verbose) # skip if frozen if job.dbStatus == 'frozen': continue tmpLog.info("Updating JobID=%s ..." % remoteJobID) # get PandaIDs status, pandaIDstatus = Client.getPandIDsWithJobID( remoteJobID, verbose=self.verbose) if status != 0: tmpLog.error("Failed to get PandaIDs for %s" % remoteJobID) return pandaIDs = pandaIDstatus.keys() pandaIDs.sort() # get full JobSpec pandaJobs = [] pandaFileInfo = {} pandaJobForSiteID = None if job == None: tmpIDs = [pandaIDs[0], pandaIDs[-1]] status, pandaJobs = Client.getFullJobStatus( tmpIDs, verbose=self.verbose) if status != 0: tmpLog.error("Failed to get PandaJobs for %s" % remoteJobID) return # get slimmed file info status, pandaFileInfo = Client.getSlimmedFileInfoPandaIDs( pandaIDs, verbose=self.verbose) if status != 0: tmpLog.error("Failed to get file info for %s" % remoteJobID) return else: # get one job to set computingSite which may have changed due to rebrokerage status, tmpPandaJobs = Client.getFullJobStatus( [pandaIDs[0]], verbose=self.verbose) if status != 0: tmpLog.error("Failed to get PandaJobs for %s" % remoteJobID) return pandaJobForSiteID = tmpPandaJobs[0] # convert to local job spec localJob = PdbUtils.convertPtoD(pandaJobs, pandaIDstatus, job, pandaFileInfo, pandaJobForSiteID) # update database if not remoteJobID in localJobIDs: # insert to DB try: PdbUtils.insertJobDB(localJob, self.verbose) except: tmpLog.error( "Failed to insert JobID=%s to local repository" % remoteJobID) return # set retryID if not localJob.provenanceID in [0, '0']: try: PdbUtils.setRetryID(localJob, self.verbose) except: tmpLog.error( "Failed to set retryID for JobID=%s in local repository" % remoteJobID) return else: # update try: PdbUtils.updateJobDB(localJob, self.verbose) except: tmpLog.error( "Failed to update local repository for JobID=%s" % remoteJobID) return # update sync time bookConf = BookConfig.getConfig() bookConf.last_synctime = syncTime BookConfig.updateConfig(bookConf) tmpLog.info("Synchronization Completed")