def loadLFNs(self, **dbsContacts): """ Get the list of LFNs from the DBS """ for i in self: self.remove(i) dbsUrl = dbsContacts.get('DBSURL', None) if dbsUrl == None: dbsUrl = getLocalDBSURL() reader = DBSReader(dbsUrl) fileList = reader.getFiles(self.dataset) for block in fileList.values(): result = [ x['LogicalFileName'] for x in block['Files']] self.extend(result) return
def processingComplete(self): """ _processingComplete_ look at the processing jobs for the workflow, and return True if all processing jobs are complete """ intermediateDBS = self.workflowSpec.parameters['DBSURL'] outputDataset = self.workflowSpec.outputDatasets()[0].name() allJobs = WEUtils.jobsForWorkflow(self.workflow, "Merge") finishedJobs = WEUtils.jobsForWorkflow(self.workflow, "Merge", "finished") totalProcessing = len(allJobs) totalComplete = len(finishedJobs) logging.info("%s: %s/%s jobs complete" % (self.workflow,totalComplete,totalProcessing)) if totalProcessing == 0: # Protection for non-sensical situation return False if totalComplete < totalProcessing: return False # Check to make sure local DBS knows about all output files try: reader = DBSReader(intermediateDBS) blockList = reader.getFiles(dataset = outputDataset) except: logging.info("Dataset not in DBS yet") return False totalRegistered = 0 for block in blockList: totalRegistered += len(blockList[block]['Files']) logging.info("%s: %s/%s jobs registered" % (self.workflow,totalRegistered,totalProcessing)) if totalRegistered < totalProcessing: return False return True