def __needsMultiHopStaging(self, sourceSEName, destSEName): """ Checks whether transfers between the two SE given as parameters need a multi hop transfer to stage with a different protocol than the transfer one. :param str sourceSEName: source storage element name :param str destSEName: destination storage element name :returns: boolean """ srcSE = StorageElement(sourceSEName, vo=self.vo) dstSE = StorageElement(destSEName, vo=self.vo) srcIsTape = srcSE.getStatus()['Value'].get('TapeSE', True) if not srcIsTape: return False # To know if we will need a multihop staging transfer, # we check whether we can generate transfer URLs # for a fake LFN, and see if the protocol we get # is compatible with staging tpcProtocols = DMSHelpers(vo=self.vo).getThirdPartyProtocols() res = dstSE.generateTransferURLsBetweenSEs('/%s/fakeLFN' % self.vo, srcSE, protocols=tpcProtocols) # There is an error, but let's ignore it, # it will be dealt with in the FTS3Job logic if not res['OK']: return False srcProto, _destProto = res['Value']['Protocols'] if srcProto not in srcSE.localStageProtocolList: return True return False
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """ Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructTransferJob/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) res = self.__fetchSpaceToken(self.sourceSE) if not res['OK']: return res source_spacetoken = res['Value'] failedLFNs = set() dstSE = StorageElement(self.targetSE, vo=self.vo) srcSE = StorageElement(self.sourceSE, vo=self.vo) # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res['Value']['Successful'] transfers = [] fileIDsInTheJob = [] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = 'Defunct' continue trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=getattr(ftsFile, 'fileID'), activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(getattr(ftsFile, 'fileID')) # If the source is not an tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = 86400 if sourceIsTape else None if not transfers: log.error("No transfer possible!") return S_ERROR("No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE } job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """ Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructTransferJob/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) res = self.__fetchSpaceToken(self.sourceSE, self.vo) if not res['OK']: return res source_spacetoken = res['Value'] failedLFNs = set() dstSE = StorageElement(self.targetSE, vo=self.vo) srcSE = StorageElement(self.sourceSE, vo=self.vo) # If the source is not a tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE, self.vo) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = BRING_ONLINE_TIMEOUT if sourceIsTape else None # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].items(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res['Value']['Successful'] # This contains the staging URLs if they are different from the transfer URLs # (CTA...) allStageURLs = dict() # In case we are transfering from a tape system, and the stage protocol # is not the same as the transfer protocol, we generate the staging URLs # to do a multihop transfer. See below. if sourceIsTape: srcProto, _destProto = res['Value']['Protocols'] if srcProto not in srcSE.localStageProtocolList: # As of version 3.10, FTS can only handle one file per multi hop # job. If we are here, that means that we need one, so make sure that # we only have a single file to transfer (this should have been checked # at the job construction step in FTS3Operation). # This test is important, because multiple files would result in the source # being deleted ! if len(allLFNs) != 1: log.debug( "Multihop job has %s files while only 1 allowed" % len(allLFNs)) return S_ERROR( errno.E2BIG, "Trying multihop job with more than one file !") res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].items(): failedLFNs.add(lfn) log.error("Could not get stage SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs.pop(lfn) allStageURLs = res['Value']['Successful'] transfers = [] fileIDsInTheJob = [] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] stageURL = allStageURLs.get(ftsFile.lfn) if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = 'Defunct' continue ftsFileID = getattr(ftsFile, 'fileID') # Under normal circumstances, we simply submit an fts transfer as such: # * srcProto://myFile -> destProto://myFile # # Even in case of the source storage being a tape system, it works fine. # However, if the staging and transfer protocols are different (which might be the case for CTA), # we use the multihop machinery to submit two sequential fts transfers: # one to stage, one to transfer. # It looks like such # * stageProto://myFile -> stageProto://myFile # * srcProto://myFile -> destProto://myFile if stageURL: # We do not set a fileID in the metadata # such that we do not update the DB when monitoring stageTrans_metadata = {'desc': 'PreStage %s' % ftsFileID} stageTrans = fts3.new_transfer(stageURL, stageURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=stageTrans_metadata, activity=self.activity) transfers.append(stageTrans) trans_metadata = { 'desc': 'Transfer %s' % ftsFileID, 'fileID': ftsFileID } trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(ftsFileID) if not transfers: log.error("No transfer possible!") return S_ERROR("No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'rmsReqID': self.rmsReqID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE } job = fts3.new_job( transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, verify_checksum= 'target', # Only check target vs specified, since we verify the source earlier multihop=bool( allStageURLs), # if we have stage urls, then we need multihop metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """ Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructTransferJob/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) res = self.__fetchSpaceToken(self.sourceSE) if not res['OK']: return res source_spacetoken = res['Value'] failedLFNs = set() dstSE = StorageElement(self.targetSE, vo=self.vo) srcSE = StorageElement(self.sourceSE, vo=self.vo) # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res['Value']['Successful'] transfers = [] fileIDsInTheJob = [] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = 'Defunct' continue trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=getattr(ftsFile, 'fileID'), activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(getattr(ftsFile, 'fileID')) # If the source is not an tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = 86400 if sourceIsTape else None if not transfers: log.error("No transfer possible!") return S_ERROR("No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE} job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * multiHopSE (optional) * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) Note that, because of FTS limitations (and also because it anyway would be "not very smart"), multiHop can only use non-SRM disk storage as hops. :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}") isMultiHop = False # Check if it is a multiHop transfer if self.multiHopSE: if len(allLFNs) != 1: log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs)) return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !") allHops = [(self.sourceSE, self.multiHopSE), (self.multiHopSE, self.targetSE)] isMultiHop = True else: allHops = [(self.sourceSE, self.targetSE)] nbOfHops = len(allHops) res = self.__fetchSpaceToken(self.sourceSE, self.vo) if not res["OK"]: return res source_spacetoken = res["Value"] failedLFNs = set() copy_pin_lifetime = None bring_online = None archive_timeout = None transfers = [] fileIDsInTheJob = set() for hopId, (hopSrcSEName, hopDstSEName) in enumerate(allHops, start=1): # Again, this is relevant only for the very initial source # but code factorization is more important hopSrcIsTape = self.__isTapeSE(hopSrcSEName, self.vo) dstSE = StorageElement(hopDstSEName, vo=self.vo) srcSE = StorageElement(hopSrcSEName, vo=self.vo) # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res["OK"]: return res for lfn, reason in res["Value"]["Failed"].items(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res["Value"]["Successful"] srcProto, destProto = res["Value"]["Protocols"] # If the source is a tape SE, we should set the # copy_pin_lifetime and bring_online params # In case of multihop, this is relevant only for the # original source, but again, code factorization is more important if hopSrcIsTape: copy_pin_lifetime = pinTime bring_online = srcSE.options.get("BringOnlineTimeout", BRING_ONLINE_TIMEOUT) # If the destination is a tape, and the protocol supports it, # check if we want to have an archive timeout # In case of multihop, this is relevant only for the # final target, but again, code factorization is more important dstIsTape = self.__isTapeSE(hopDstSEName, self.vo) if dstIsTape and destProto in dstSE.localStageProtocolList: archive_timeout = dstSE.options.get("ArchiveTimeout") # This contains the staging URLs if they are different from the transfer URLs # (CTA...) allStageURLs = dict() # In case we are transfering from a tape system, and the stage protocol # is not the same as the transfer protocol, we generate the staging URLs # to do a multihop transfer. See below. if hopSrcIsTape and srcProto not in srcSE.localStageProtocolList: isMultiHop = True # As of version 3.10, FTS can only handle one file per multi hop # job. If we are here, that means that we need one, so make sure that # we only have a single file to transfer (this should have been checked # at the job construction step in FTS3Operation). # This test is important, because multiple files would result in the source # being deleted ! if len(allLFNs) != 1: log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs)) return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !") res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList) if not res["OK"]: return res for lfn, reason in res["Value"]["Failed"].items(): failedLFNs.add(lfn) log.error("Could not get stage SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs.pop(lfn) allStageURLs = res["Value"]["Successful"] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] stageURL = allStageURLs.get(ftsFile.lfn) if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = "Defunct" continue ftsFileID = getattr(ftsFile, "fileID") # Under normal circumstances, we simply submit an fts transfer as such: # * srcProto://myFile -> destProto://myFile # # Even in case of the source storage being a tape system, it works fine. # However, if the staging and transfer protocols are different (which might be the case for CTA), # we use the multihop machinery to submit two sequential fts transfers: # one to stage, one to transfer. # It looks like such # * stageProto://myFile -> stageProto://myFile # * srcProto://myFile -> destProto://myFile if stageURL: # We do not set a fileID in the metadata # such that we do not update the DB when monitoring stageTrans_metadata = {"desc": "PreStage %s" % ftsFileID} # If we use an activity, also set it as file metadata # for WLCG monitoring purposes # https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14? if self.activity: stageTrans_metadata["activity"] = self.activity stageTrans = fts3.new_transfer( stageURL, stageURL, checksum="ADLER32:%s" % ftsFile.checksum, filesize=ftsFile.size, metadata=stageTrans_metadata, activity=self.activity, ) transfers.append(stageTrans) # If it is the last hop only, we set the fileID metadata # for monitoring if hopId == nbOfHops: trans_metadata = {"desc": "Transfer %s" % ftsFileID, "fileID": ftsFileID} else: trans_metadata = {"desc": "MultiHop %s" % ftsFileID} # If we use an activity, also set it as file metadata # for WLCG monitoring purposes # https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14? if self.activity: trans_metadata["activity"] = self.activity # because of an xroot bug (https://github.com/xrootd/xrootd/issues/1433) # the checksum needs to be lowercase. It does not impact the other # protocol, so it's fine to put it here. # I only add it in this transfer and not the "staging" one above because it # impacts only root -> root transfers trans = fts3.new_transfer( sourceSURL, targetSURL, checksum="ADLER32:%s" % ftsFile.checksum.lower(), filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity, ) transfers.append(trans) fileIDsInTheJob.add(ftsFileID) if not transfers: log.error("No transfer possible!") return S_ERROR(errno.ENODATA, "No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { "operationID": self.operationID, "rmsReqID": self.rmsReqID, "sourceSE": self.sourceSE, "targetSE": self.targetSE, } if self.activity: job_metadata["activity"] = self.activity job = fts3.new_job( transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, verify_checksum="target", # Only check target vs specified, since we verify the source earlier multihop=isMultiHop, metadata=job_metadata, priority=self.priority, archive_timeout=archive_timeout, ) return S_OK((job, fileIDsInTheJob))