def sync(self): # get logger tmpLog = PLogger.getPandaLogger() tmpLog.info("Synchronizing local repository ...") # check proxy self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose, useCache=True) # get nickname nickName = PsubUtils.getNickname() # set Rucio accounting PsubUtils.setRucioAccount(nickName,'pbook',True) # get JobIDs in local repository localJobIDs = PdbUtils.getListOfJobIDs() # get recent JobIDs from panda server syncTimeRaw = datetime.datetime.utcnow() syncTime = syncTimeRaw.strftime('%Y-%m-%d %H:%M:%S') # set sync time for the first attempt bookConf = BookConfig.getConfig() if self.restoreDB: # reset last_synctime to restore database bookConf.last_synctime = '' # disable self.restoreDB = False tmpLog.info("It may take several minutes to restore local repository ...") if bookConf.last_synctime == '': bookConf.last_synctime = datetime.datetime.utcnow()-datetime.timedelta(days=180) bookConf.last_synctime = bookConf.last_synctime.strftime('%Y-%m-%d %H:%M:%S') maxTaskID = None while True: status, jediTaskDicts = Client.getJobIDsJediTasksInTimeRange(bookConf.last_synctime, minTaskID=maxTaskID, verbose=self.verbose) if status != 0: tmpLog.error("Failed to get tasks from panda server") return if len(jediTaskDicts) == 0: break tmpLog.info("Got %s tasks to be updated" % len(jediTaskDicts)) # insert if missing for remoteJobID in jediTaskDicts.keys(): taskID = jediTaskDicts[remoteJobID]['jediTaskID'] # get max if maxTaskID is None or taskID > maxTaskID: maxTaskID = taskID # check local status job = None if remoteJobID in localJobIDs: # get job info from local repository job = PdbUtils.readJobDB(remoteJobID, self.verbose) # skip if frozen if job.dbStatus == 'frozen': continue tmpLog.info("Updating taskID=%s ..." % taskID) # convert JEDI task localJob = PdbUtils.convertJTtoD(jediTaskDicts[remoteJobID],job) # update database if not remoteJobID in localJobIDs: # insert to DB try: PdbUtils.insertJobDB(localJob,self.verbose) except: tmpLog.error("Failed to insert taskID=%s to local repository" % taskID) return else: # update try: PdbUtils.updateJobDB(localJob,self.verbose,syncTimeRaw) except: tmpLog.error("Failed to update local repository for taskID=%s" % taskID) return # update sync time bookConf = BookConfig.getConfig() bookConf.last_synctime = syncTime BookConfig.updateConfig(bookConf) self.updateTaskJobsetMap() tmpLog.info("Synchronization Completed")