def generateFTS3Job(sourceSE, targetSE, lfns, multiHopSE=None): """Utility to create a new FTS3Job object with some FTS3Files The FileIDs are filled in order, starting with 1 :param src sourceSE: source SE Name :param src targetSE: target SE Name :param list lfns: list of lfns (str) :param src multiHopSE: hop SE Name """ newJob = FTS3Job() newJob.type = "Transfer" newJob.sourceSE = sourceSE newJob.targetSE = targetSE newJob.multiHopSE = multiHopSE filesToSubmit = [] for i, lfn in enumerate(lfns, start=1): ftsFile = FTS3File() ftsFile.fileID = i ftsFile.checksum = lfn ftsFile.lfn = lfn filesToSubmit.append(ftsFile) newJob.filesToSubmit = filesToSubmit newJob.operationID = 123 newJob.rmsReqID = 456 return newJob
def setUp(self): self.f1 = FTS3File() self.f1.fakeAttr_possibleSources = ['Src1', 'Src2'] self.f1.lfn = 'f1' self.f1.targetSE = 'target1' self.f2 = FTS3File() self.f2.fakeAttr_possibleSources = ['Src2', 'Src3'] self.f2.lfn = 'f2' self.f2.targetSE = 'target2' self.f3 = FTS3File() self.f3.fakeAttr_possibleSources = ['Src4'] self.f3.lfn = 'f3' self.f3.targetSE = 'target1' # File does not exist :-) self.f4 = FTS3File() self.f4.lfn = 'f4' self.f4.targetSE = 'target3' self.allFiles = [self.f1, self.f2, self.f3, self.f4]
def setUp(self): self.f1 = FTS3File() self.f1.fakeAttr_possibleSources = ["Src1", "Src2"] self.f1.lfn = "f1" self.f1.targetSE = "target1" self.f2 = FTS3File() self.f2.fakeAttr_possibleSources = ["Src2", "Src3"] self.f2.lfn = "f2" self.f2.targetSE = "target2" self.f3 = FTS3File() self.f3.fakeAttr_possibleSources = ["Src4"] self.f3.lfn = "f3" self.f3.targetSE = "target1" # File does not exist :-) self.f4 = FTS3File() self.f4.lfn = "f4" self.f4.targetSE = "target3" self.allFiles = [self.f1, self.f2, self.f3, self.f4]
def generateOperation(self, opType, nbFiles, dests, sources=None): """ Generate one FTS3Operation object with FTS3Files in it""" op = None if opType == 'Transfer': op = FTS3TransferOperation() elif opType == 'Staging': op = FTS3StagingOperation() op.username = "******" op.userGroup = "Floyd" op.sourceSEs = sources for _i in xrange(nbFiles * len(dests)): self.fileCounter += 1 for dest in dests: ftsFile = FTS3File() ftsFile.lfn = 'lfn%s' % self.fileCounter ftsFile.targetSE = dest op.ftsFiles.append(ftsFile) return op
def generateOperation(self, opType, nbFiles, dests, sources=None): """Generate one FTS3Operation object with FTS3Files in it""" op = None if opType == "Transfer": op = FTS3TransferOperation() elif opType == "Staging": op = FTS3StagingOperation() proxyInfo = getProxyInfo()["Value"] op.username = proxyInfo["username"] op.userGroup = proxyInfo["group"] op.sourceSEs = sources for _i in range(nbFiles * len(dests)): self.fileCounter += 1 for dest in dests: ftsFile = FTS3File() ftsFile.lfn = "lfn%s" % self.fileCounter ftsFile.targetSE = dest op.ftsFiles.append(ftsFile) return op
def _makeFile(): f = FTS3File() f.targetSE = "targetSE" return f
def fts3Transfer(self): """ replicate and register using FTS3 """ self.log.info("scheduling files in FTS3...") # Check first if we do not have ongoing transfers res = self._checkExistingFTS3Operations() if not res['OK']: return res # if res['Value'] is False # it means that there are ongoing transfers # and we should stop here if res['Value'] is False: # return S_OK such that the request is put back return S_OK() fts3Files = [] toSchedule = {} # Dict which maps the FileID to the object rmsFilesIds = {} for opFile in self.getWaitingFilesList(): rmsFilesIds[opFile.FileID] = opFile opFile.Error = '' gMonitor.addMark("FTSScheduleAtt") # # check replicas replicas = self._filterReplicas(opFile) if not replicas["OK"]: continue replicas = replicas["Value"] validReplicas = replicas["Valid"] noMetaReplicas = replicas["NoMetadata"] noReplicas = replicas['NoReplicas'] badReplicas = replicas['Bad'] noPFN = replicas['NoPFN'] if validReplicas: validTargets = list( set(self.operation.targetSEList) - set(validReplicas)) if not validTargets: self.log.info("file %s is already present at all targets" % opFile.LFN) opFile.Status = "Done" else: toSchedule[opFile.LFN] = [opFile, validTargets] else: gMonitor.addMark("FTSScheduleFail") if noMetaReplicas: self.log.warn( "unable to schedule '%s', couldn't get metadata at %s" % (opFile.LFN, ','.join(noMetaReplicas))) opFile.Error = "Couldn't get metadata" elif noReplicas: self.log.error( "Unable to schedule transfer", "File %s doesn't exist at %s" % (opFile.LFN, ','.join(noReplicas))) opFile.Error = 'No replicas found' opFile.Status = 'Failed' elif badReplicas: self.log.error( "Unable to schedule transfer", "File %s, all replicas have a bad checksum at %s" % (opFile.LFN, ','.join(badReplicas))) opFile.Error = 'All replicas have a bad checksum' opFile.Status = 'Failed' elif noPFN: self.log.warn( "unable to schedule %s, could not get a PFN at %s" % (opFile.LFN, ','.join(noPFN))) res = self._addMetadataToFiles(toSchedule) if not res['OK']: return res else: filesToSchedule = res['Value'] for lfn in filesToSchedule: opFile = filesToSchedule[lfn] validTargets = toSchedule[lfn][1] for targetSE in validTargets: ftsFile = FTS3File.fromRMSFile(opFile, targetSE) fts3Files.append(ftsFile) if fts3Files: res = Registry.getUsernameForDN(self.request.OwnerDN) if not res['OK']: self.log.error( "Cannot get username for DN", "%s %s" % (self.request.OwnerDN, res['Message'])) return res username = res['Value'] fts3Operation = FTS3TransferOperation.fromRMSObjects( self.request, self.operation, username) fts3Operation.ftsFiles = fts3Files ftsSchedule = FTS3Client().persistOperation(fts3Operation) if not ftsSchedule["OK"]: self.log.error("Completely failed to schedule to FTS3:", ftsSchedule["Message"]) return ftsSchedule # might have nothing to schedule ftsSchedule = ftsSchedule["Value"] self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule) self.log.info("%d files have been scheduled to FTS3" % len(fts3Files)) for ftsFile in fts3Files: opFile = rmsFilesIds[ftsFile.rmsFileID] gMonitor.addMark("FTSScheduleOK", 1) opFile.Status = "Scheduled" self.log.debug("%s has been scheduled for FTS" % opFile.LFN) else: self.log.info("No files to schedule after metadata checks") # Just in case some transfers could not be scheduled, try them with RM return self.dmTransfer(fromFTS=True)
def fts3Transfer(self): """ replicate and register using FTS3 """ self.log.info("scheduling files in FTS3...") # Check first if we do not have ongoing transfers res = self._checkExistingFTS3Operations() if not res['OK']: return res # if res['Value'] is False # it means that there are ongoing transfers # and we should stop here if res['Value'] is False: # return S_OK such that the request is put back return S_OK() fts3Files = [] toSchedule = {} # Dict which maps the FileID to the object rmsFilesIds = {} for opFile in self.getWaitingFilesList(): rmsFilesIds[opFile.FileID] = opFile opFile.Error = '' gMonitor.addMark("FTSScheduleAtt") # # check replicas replicas = self._filterReplicas(opFile) if not replicas["OK"]: continue replicas = replicas["Value"] validReplicas = replicas["Valid"] noMetaReplicas = replicas["NoMetadata"] noReplicas = replicas['NoReplicas'] badReplicas = replicas['Bad'] noPFN = replicas['NoPFN'] if validReplicas: validTargets = list(set(self.operation.targetSEList) - set(validReplicas)) if not validTargets: self.log.info("file %s is already present at all targets" % opFile.LFN) opFile.Status = "Done" else: toSchedule[opFile.LFN] = [opFile, validTargets] else: gMonitor.addMark("FTSScheduleFail") if noMetaReplicas: self.log.warn("unable to schedule '%s', couldn't get metadata at %s" % (opFile.LFN, ','.join(noMetaReplicas))) opFile.Error = "Couldn't get metadata" elif noReplicas: self.log.error( "Unable to schedule transfer", "File %s doesn't exist at %s" % (opFile.LFN, ','.join(noReplicas))) opFile.Error = 'No replicas found' opFile.Status = 'Failed' elif badReplicas: self.log.error( "Unable to schedule transfer", "File %s, all replicas have a bad checksum at %s" % (opFile.LFN, ','.join(badReplicas))) opFile.Error = 'All replicas have a bad checksum' opFile.Status = 'Failed' elif noPFN: self.log.warn( "unable to schedule %s, could not get a PFN at %s" % (opFile.LFN, ','.join(noPFN))) res = self._addMetadataToFiles(toSchedule) if not res['OK']: return res else: filesToSchedule = res['Value'] for lfn in filesToSchedule: opFile = filesToSchedule[lfn] validTargets = toSchedule[lfn][1] for targetSE in validTargets: ftsFile = FTS3File.fromRMSFile(opFile, targetSE) fts3Files.append(ftsFile) if fts3Files: res = Registry.getUsernameForDN(self.request.OwnerDN) if not res['OK']: self.log.error( "Cannot get username for DN", "%s %s" % (self.request.OwnerDN, res['Message'])) return res username = res['Value'] fts3Operation = FTS3TransferOperation.fromRMSObjects(self.request, self.operation, username) fts3Operation.ftsFiles = fts3Files ftsSchedule = FTS3Client().persistOperation(fts3Operation) if not ftsSchedule["OK"]: self.log.error("Completely failed to schedule to FTS3:", ftsSchedule["Message"]) return ftsSchedule # might have nothing to schedule ftsSchedule = ftsSchedule["Value"] self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule) self.log.info("%d files have been scheduled to FTS3" % len(fts3Files)) for ftsFile in fts3Files: opFile = rmsFilesIds[ftsFile.rmsFileID] gMonitor.addMark("FTSScheduleOK", 1) opFile.Status = "Scheduled" self.log.debug("%s has been scheduled for FTS" % opFile.LFN) else: self.log.info("No files to schedule after metadata checks") # Just in case some transfers could not be scheduled, try them with RM return self.dmTransfer(fromFTS=True)
def fts3Transfer(self): """replicate and register using FTS3""" self.log.info("scheduling files in FTS3...") # Check first if we do not have ongoing transfers res = self._checkExistingFTS3Operations() if not res["OK"]: return res # if res['Value'] is False # it means that there are ongoing transfers # and we should stop here if res["Value"] is False: # return S_OK such that the request is put back return S_OK() fts3Files = [] toSchedule = {} # Dict which maps the FileID to the object rmsFilesIds = {} if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Attempted", len(self.getWaitingFilesList()))) for opFile in self.getWaitingFilesList(): rmsFilesIds[opFile.FileID] = opFile opFile.Error = "" # # check replicas replicas = self._filterReplicas(opFile) if not replicas["OK"]: continue replicas = replicas["Value"] validReplicas = replicas["Valid"] noMetaReplicas = replicas["NoMetadata"] noReplicas = replicas["NoReplicas"] badReplicas = replicas["Bad"] noPFN = replicas["NoPFN"] if validReplicas: validTargets = list( set(self.operation.targetSEList) - set(validReplicas)) if not validTargets: self.log.info("file %s is already present at all targets" % opFile.LFN) opFile.Status = "Done" else: toSchedule[opFile.LFN] = [opFile, validTargets] else: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Failed", 1)) if noMetaReplicas: self.log.warn( "unable to schedule file", "'%s': couldn't get metadata at %s" % (opFile.LFN, ",".join(noMetaReplicas)), ) opFile.Error = "Couldn't get metadata" elif noReplicas: self.log.error( "Unable to schedule transfer", "File %s doesn't exist at %s" % (opFile.LFN, ",".join(noReplicas)), ) opFile.Error = "No replicas found" opFile.Status = "Failed" elif badReplicas: self.log.error( "Unable to schedule transfer", "File %s, all replicas have a bad checksum at %s" % (opFile.LFN, ",".join(badReplicas)), ) opFile.Error = "All replicas have a bad checksum" opFile.Status = "Failed" elif noPFN: self.log.warn( "unable to schedule %s, could not get a PFN at %s" % (opFile.LFN, ",".join(noPFN))) if self.rmsMonitoring: self.rmsMonitoringReporter.commit() res = self._addMetadataToFiles(toSchedule) if not res["OK"]: return res else: filesToSchedule = res["Value"] for lfn in filesToSchedule: opFile = filesToSchedule[lfn] validTargets = toSchedule[lfn][1] for targetSE in validTargets: ftsFile = FTS3File.fromRMSFile(opFile, targetSE) fts3Files.append(ftsFile) if fts3Files: res = Registry.getUsernameForDN(self.request.OwnerDN) if not res["OK"]: self.log.error( "Cannot get username for DN", "%s %s" % (self.request.OwnerDN, res["Message"])) return res username = res["Value"] fts3Operation = FTS3TransferOperation.fromRMSObjects( self.request, self.operation, username) fts3Operation.ftsFiles = fts3Files try: if not fts3Operation.activity: vo = getVOfromProxyGroup().get("Value") fts3Plugin = getFTS3Plugin(vo=vo) fts3Operation.activity = fts3Plugin.inferFTSActivity( fts3Operation, self.request, self.operation) except Exception: pass ftsSchedule = FTS3Client().persistOperation(fts3Operation) if not ftsSchedule["OK"]: self.log.error("Completely failed to schedule to FTS3:", ftsSchedule["Message"]) return ftsSchedule # might have nothing to schedule ftsSchedule = ftsSchedule["Value"] self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule) self.log.info("%d files have been scheduled to FTS3" % len(fts3Files)) if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Successful", len(fts3Files))) for ftsFile in fts3Files: opFile = rmsFilesIds[ftsFile.rmsFileID] opFile.Status = "Scheduled" self.log.debug("%s has been scheduled for FTS" % opFile.LFN) else: self.log.info("No files to schedule after metadata checks") if self.rmsMonitoring: self.rmsMonitoringReporter.commit() # Just in case some transfers could not be scheduled, try them with RM return self.dmTransfer(fromFTS=True)