def submitFTS3( self, pinTime = False ): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR( "FTSJob already has been submitted" ) transfers = [] for ftsFile in self: trans = fts3.new_transfer( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum = ftsFile.Checksum, filesize = ftsFile.Size ) transfers.append( trans ) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job( transfers = transfers, overwrite = True, source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken, bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 ) try: if not self._fts3context: self._fts3context = fts3.Context( endpoint = self.FTSServer ) context = self._fts3context self.FTSGUID = fts3.submit( context, job ) except Exception, e: return S_ERROR( "Error at submission: %s" % e )
def get_transfer_status(self, batch_id): #override if self.server_id == 0: self._set_server_id() results = self._get_status(batch_id, 'transfer') staged_tasks = [] for task_id, status, exitcode, msg, start_time, finish_time in self._get_status(batch_id, 'staging'): if status == FileQuery.STAT_DONE: staged_tasks.append(task_id) results.append((task_id, FileQuery.STAT_QUEUED, -1, None, None, None)) else: # these tasks won't appear in results from _get_status('transfer') # because no transfer jobs have been submitted yet results.append((task_id, status, exitcode, None, start_time, finish_time)) if len(staged_tasks) != 0: transfers = [] pfn_to_tid = {} for task_id, source_pfn, dest_pfn, checksum, filesize in self.db.select_many('fts_staging_queue', ('id', 'source', 'destination', 'checksum', 'size'), 'id', staged_tasks): transfers.append(fts3.new_transfer(source_pfn, dest_pfn, checksum = checksum, filesize = filesize)) pfn_to_tid[dest_pfn] = task_id if self.checksum_algorithm: verify_checksum = 'target' else: verify_checksum = None job = fts3.new_job(transfers, retry = self.fts_retry, overwrite = False, verify_checksum = verify_checksum, metadata = self.metadata_string) success = self._submit_job(job, 'transfer', batch_id, pfn_to_tid) if success and not self._read_only: self.db.delete_many('fts_staging_queue', 'id', pfn_to_tid.values()) return results
def _fts_submit_job(source_url, dest_url, src_filenames, dst_filenames, checksum, overwrite, testing_folder, context, metadata): """ https://gitlab.cern.ch/fts/fts-rest/-/blob/develop/src/fts3/rest/client/easy/submission.py#L106 """ transfers = [] for i in xrange(len(src_filenames)): source_file = os.path.join(source_url, testing_folder, "src", src_filenames[i]) dest_file = os.path.join(dest_url, testing_folder, "dest", dst_filenames[i]) transfer = fts3.new_transfer(source=source_file, destination=dest_file) transfers.append(transfer) # create job job = fts3.new_job(transfers, verify_checksum=checksum, overwrite=overwrite, timeout=3600, metadata=metadata) # submit job while True: try: job_id = fts3.submit(context, job) break except fts3_client_exceptions.ClientError as e: _flush_logging_msg(e) return -1 return job_id
def submit(proxy, toTrans, source, destination): # prepare rest job with 200 files per job transfers = [] for files in chunks(toTrans, 200): c = pycurl.Curl() # create destination and source pfns for job for lfn in files: print(lfn) transfers.append( fts3.new_transfer(apply_tfc_to_lfn(source, lfn, c), apply_tfc_to_lfn(destination, lfn, c))) c.close() # Submit fts job context = fts3.Context('https://fts3.cern.ch:8446', proxy, proxy, verify=True) print(fts3.delegate(context, lifetime=timedelta(hours=48), force=False)) job = fts3.new_job(transfers) #print("Monitor link: https://fts3.cern.ch:8449/fts3/ftsmon/#/job/"+fts3.submit(context, job)) jobid = fts3.submit(context, job) #for file in (fts3.get_job_status(context, jobid, list_files=True))["files"]: for key, value in (fts3.get_job_status(context, jobid, list_files=True)).iteritems(): print key
def submitTheFTSJob(ftsFile): ### First way : Random choice of two servers # ftsServ = random.choice([ftsServ1, ftsServ2]) ### Second way : Weighted choice of two servers # rndValue = random.uniform(0.0,1.0) # ftsServ = ftsServ1 # if rndValue < 0.7 : ftsServ = ftsServ2 ### Third way : Random choice of three servers fList = [ftsServ1, ftsServ2, ftsServ3] ftsServ = random.choice(fList) # context = fts3.Context(ftsServ) # Open the file and stop the processing. listOfPairs[:] = open(ceBase + "DOING/" + ftsFile).read().split("\n") # listOfPairs[:] = open(ceBase + "TODO/" + ftsFile).read().split("\n") # All the threading bit is here to check in parallel whether the files we are looking are okay in castor # Once the function is done, the list "okayFiles" should be filled transfers = [] if checkStatus: checkStatusOnCastor() else: okayFiles[:] = [] for onePair in listOfPairs: if len(onePair)<10: continue (sourceSURL, targetSURL) = onePair.split(" ") okayFiles.append((sourceSURL, targetSURL)) if len(okayFiles)>0: for oneSet in okayFiles: transf = fts3.new_transfer(oneSet[0], oneSet[1]) transfers.append(transf) job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi ftsJobID = fts3.submit(context, job) return ftsJobID, ftsServ else: # None of the files in this lot were good! return "-1", "-1"
def submitFTS3(self, pinTime=False): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR("FTSJob already has been submitted") transfers = [] for ftsFile in self: trans = fts3.new_transfer(ftsFile.SourceSURL, ftsFile.TargetSURL, checksum=ftsFile.Checksum, filesize=ftsFile.Size) transfers.append(trans) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=dest_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3) try: context = fts3.Context(self.FTSServer) self.FTSGUID = fts3.submit(context, job) except Exception, e: return S_ERROR("Error at submission: %s" % e)
def run(self): """ """ self.threadLock.acquire() self.log.info("Processing transfers from: %s" % self.source) # create destination and source pfns for job transfers = [] for lfn in self.files: transfers.append(fts3.new_transfer(lfn[0], lfn[1], metadata={'oracleId': lfn[2]} ) ) self.log.info("Submitting %s transfers to FTS server" % len(self.files)) # Submit fts job job = fts3.new_job(transfers, overwrite=True, verify_checksum=True, metadata={"issuer": "ASO", "userDN": self.files[0][4], "taskname": self.files[0][5]}, copy_pin_lifetime=-1, bring_online=None, source_spacetoken=None, spacetoken=None, # max time for job in the fts queue in seconds. # Usually, it should take O(s) for healthy situations max_time_in_queue=600, retry=3, # seconds after which the transfer is retried # reduced under FTS suggestion w.r.t. the 3hrs of asov1 retry_delay=600 # timeout on the single transfer process # TODO: not clear if we may need it # timeout = 1300 ) jobid = fts3.submit(self.ftsContext, job) self.jobids.append(jobid) # TODO: manage exception here, what we should do? fileDoc = dict() fileDoc['asoworker'] = 'asoless' fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = [x[2] for x in self.files] fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files] fileDoc['list_of_fts_instance'] = ['https://fts3.cern.ch:8446/' for _ in self.files] fileDoc['list_of_fts_id'] = [jobid for _ in self.files] self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids']))) self.toUpdate.append(fileDoc) self.threadLock.release()
def get_transfer_status(self, batch_id): #override if self.server_id == 0: self._set_server_id() results = self._get_status(batch_id, 'transfer') staged_tasks = [] for task_id, status, exitcode, msg, start_time, finish_time in self._get_status(batch_id, 'staging'): if status == FileQuery.STAT_DONE: staged_tasks.append(task_id) results.append((task_id, FileQuery.STAT_QUEUED, -1, None, None, None)) else: # these tasks won't appear in results from _get_status('transfer') # because no transfer jobs have been submitted yet results.append((task_id, status, exitcode, None, start_time, finish_time)) if len(staged_tasks) != 0: if self.checksum_algorithm: verify_checksum = 'target' else: verify_checksum = None which_dest = "" transfers = [] pfn_to_tid = {} for task_id, source_pfn, dest_pfn, checksum, filesize in self.db.select_many('fts_staging_queue', ('id', 'source', 'destination', 'checksum', 'size'), 'id', staged_tasks): for s in inventory.sites.itervalues(): if s.backend in dest_pfn and s.backend != "": which_dest = s #transfers = [] #pfn_to_tid = {} transfers.append(fts3.new_transfer(source_pfn, dest_pfn, checksum = checksum, filesize = filesize)) pfn_to_tid[dest_pfn] = task_id job = fts3.new_job(transfers, retry = self.fts_retry, overwrite = True, verify_checksum = verify_checksum, metadata = self.metadata_string) #which_dest = "" #for s in inventory.sites.itervalues(): # if s.backend in dest_pfn and s.backend != "": # which_dest = s LOG.info("Submitting job from buffer to site %s with proxy %s." % (which_dest.name, which_dest.x509proxy)) #LOG.info("dest_pfn: %s" % dest_pfn) success = self._submit_job(job, 'transfer', batch_id, pfn_to_tid, x509=which_dest.x509proxy) if success and not self._read_only: self.db.delete_many('fts_staging_queue', 'id', pfn_to_tid.values()) return results
def submitTheFTSJob(ftsFile): ### First way : Random choice of two servers # ftsServ = random.choice([ftsServ1, ftsServ2]) ### Second way : Weighted choice of two servers # rndValue = random.uniform(0.0,1.0) # ftsServ = ftsServ1 # if rndValue < 0.7 : ftsServ = ftsServ2 ### Third way : Random choice of three servers fList = [ftsServ1, ftsServ2, ftsServ3] ftsServ = random.choice(fList) # context = fts3.Context(ftsServ) filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n") transfers = [] for ftra in filecontent: if len(ftra) < 10: continue (sourceSURL, targetSURL) = ftra.split(" ") comm = "gfal-stat " + sourceSURL runComm = subprocess.Popen(comm, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) theInfo = runComm.communicate()[1].strip() if theInfo.startswith( "gfal-stat error: 2 (No such file or directory)"): bFTS = open(ceBase + "DONE/badFileList.txt", "a") bFTS.write(ftra + "\n") bFTS.close() else: transf = fts3.new_transfer(sourceSURL, targetSURL) transfers.append(transf) # transf = fts3.new_transfer(sourceSURL, targetSURL) # transfers.append(transf) if len(transfers) > 0: # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5) # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi job = fts3.new_job( transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=0) # To avoid deleted files snarling up the system for hours ftsJobID = fts3.submit(context, job, delegation_lifetime=fts3.timedelta(hours=72)) return ftsJobID, ftsServ else: return "-1", "-1"
def get_transfer_status(self, batch_id): #override if self.server_id == 0: self._set_server_id() results = self._get_status(batch_id, 'transfer') staged_tasks = [] for task_id, status, exitcode, msg, start_time, finish_time in self._get_status( batch_id, 'staging'): if status == FileQuery.STAT_DONE: staged_tasks.append(task_id) results.append( (task_id, FileQuery.STAT_QUEUED, -1, None, None, None)) else: # these tasks won't appear in results from _get_status('transfer') # because no transfer jobs have been submitted yet results.append( (task_id, status, exitcode, None, start_time, finish_time)) if len(staged_tasks) != 0: transfers = [] pfn_to_tid = {} for task_id, source_pfn, dest_pfn, checksum, filesize in self.db.select_many( 'fts_staging_queue', ('id', 'source', 'destination', 'checksum', 'size'), 'id', staged_tasks): transfers.append( fts3.new_transfer(source_pfn, dest_pfn, checksum=checksum, filesize=filesize)) pfn_to_tid[dest_pfn] = task_id if self.checksum_algorithm: verify_checksum = 'target' else: verify_checksum = None job = fts3.new_job(transfers, retry=self.fts_retry, overwrite=True, verify_checksum=verify_checksum, metadata=self.metadata_string) success = self._submit_job(job, 'transfer', batch_id, pfn_to_tid) if success and not self._read_only: self.db.delete_many('fts_staging_queue', 'id', pfn_to_tid.values()) return results
def submitFTS3(self, pinTime=False): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR("FTSJob already has been submitted") transfers = [] for ftsFile in self: trans = fts3.new_transfer(ftsFile.SourceSURL, ftsFile.TargetSURL, checksum='ADLER32:%s' % ftsFile.Checksum, filesize=ftsFile.Size) transfers.append(trans) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=dest_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3) try: if not self._fts3context: self._fts3context = fts3.Context(endpoint=self.FTSServer, request_class=ftsSSLRequest, verify=False) context = self._fts3context self.FTSGUID = fts3.submit(context, job) except Exception as e: return S_ERROR("Error at submission: %s" % e) self.Status = "Submitted" self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True) for ftsFile in self: ftsFile.FTSGUID = self.FTSGUID ftsFile.Status = "Submitted" return S_OK()
def submitFTS3( self, pinTime = False ): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR( "FTSJob already has been submitted" ) transfers = [] for ftsFile in self: trans = fts3.new_transfer( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum = 'ADLER32:%s'%ftsFile.Checksum, filesize = ftsFile.Size ) transfers.append( trans ) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job( transfers = transfers, overwrite = True, source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken, bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 ) try: if not self._fts3context: self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False ) context = self._fts3context self.FTSGUID = fts3.submit( context, job ) except Exception as e: return S_ERROR( "Error at submission: %s" % e ) self.Status = "Submitted" self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True ) for ftsFile in self: ftsFile.FTSGUID = self.FTSGUID ftsFile.Status = "Submitted" return S_OK()
def submitTheFTSJob(context, ftsFile): ftsServ = random.choice([ftsServ1, ftsServ2]) with open(ceBase + "DOING/" + ftsFile) as fF: filecontent = fF.read().split("\n") # filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n") transfers = [] for ftra in filecontent: if len(ftra) < 10: continue (sourceSURL, targetSURL) = ftra.split(" ") transf = fts3.new_transfer(sourceSURL, targetSURL) transfers.append(transf) if len(transfers) > 0: # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5) job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi ftsJobID = fts3.submit(context, job) return ftsJobID, ftsServ else: return "-1", "-1"
transferedFiles = [f['dest_surl'] for f in job_status['files'] if f['file_state'] in ['FINISHED'] or f['reason'] == 'DESTINATION file already exists and overwrite is not enabled'] transferList = [] for filename in transferedFiles: fs = filename.split('/') try: lfcURI = '/'.join(fs[fs.index('grid'):]) except ValueError: # substring 'grid' not found # get the last occurrence of auger in the path lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):]) # note that this path construction is not particularly liable and should be upgraded in the future transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI) transferList.append(transfer) if len(transferList) > 0: job = fts3.new_job(transferList, metadata="Registration of the files transfered by job " + job_id, overwrite=True) if options.register == 1 or options.register == 0 and query_yes_no("The registration job is ready now, do you wish to submit it?"): jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit(context, job) print "The registration job ID is " + jobID with open(options.regJobIdFile, 'a') as f: f.write('\t' + jobID + '\n') notFinishedTransfers = [fts3.new_transfer(f['source_surl'], f['dest_surl']) for f in job_status['files'] if f['file_state'] in ['FAILED', 'CANCELED'] and f['reason'] != 'DESTINATION file already exists and overwrite is not enabled'] if len(notFinishedTransfers) < 1:
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """ Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructTransferJob/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) res = self.__fetchSpaceToken(self.sourceSE, self.vo) if not res['OK']: return res source_spacetoken = res['Value'] failedLFNs = set() dstSE = StorageElement(self.targetSE, vo=self.vo) srcSE = StorageElement(self.sourceSE, vo=self.vo) # If the source is not a tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE, self.vo) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = BRING_ONLINE_TIMEOUT if sourceIsTape else None # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].items(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res['Value']['Successful'] # This contains the staging URLs if they are different from the transfer URLs # (CTA...) allStageURLs = dict() # In case we are transfering from a tape system, and the stage protocol # is not the same as the transfer protocol, we generate the staging URLs # to do a multihop transfer. See below. if sourceIsTape: srcProto, _destProto = res['Value']['Protocols'] if srcProto not in srcSE.localStageProtocolList: # As of version 3.10, FTS can only handle one file per multi hop # job. If we are here, that means that we need one, so make sure that # we only have a single file to transfer (this should have been checked # at the job construction step in FTS3Operation). # This test is important, because multiple files would result in the source # being deleted ! if len(allLFNs) != 1: log.debug( "Multihop job has %s files while only 1 allowed" % len(allLFNs)) return S_ERROR( errno.E2BIG, "Trying multihop job with more than one file !") res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].items(): failedLFNs.add(lfn) log.error("Could not get stage SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs.pop(lfn) allStageURLs = res['Value']['Successful'] transfers = [] fileIDsInTheJob = [] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] stageURL = allStageURLs.get(ftsFile.lfn) if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = 'Defunct' continue ftsFileID = getattr(ftsFile, 'fileID') # Under normal circumstances, we simply submit an fts transfer as such: # * srcProto://myFile -> destProto://myFile # # Even in case of the source storage being a tape system, it works fine. # However, if the staging and transfer protocols are different (which might be the case for CTA), # we use the multihop machinery to submit two sequential fts transfers: # one to stage, one to transfer. # It looks like such # * stageProto://myFile -> stageProto://myFile # * srcProto://myFile -> destProto://myFile if stageURL: # We do not set a fileID in the metadata # such that we do not update the DB when monitoring stageTrans_metadata = {'desc': 'PreStage %s' % ftsFileID} stageTrans = fts3.new_transfer(stageURL, stageURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=stageTrans_metadata, activity=self.activity) transfers.append(stageTrans) trans_metadata = { 'desc': 'Transfer %s' % ftsFileID, 'fileID': ftsFileID } trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(ftsFileID) if not transfers: log.error("No transfer possible!") return S_ERROR("No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'rmsReqID': self.rmsReqID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE } job = fts3.new_job( transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, verify_checksum= 'target', # Only check target vs specified, since we verify the source earlier multihop=bool( allStageURLs), # if we have stage urls, then we need multihop metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
destinationPrefix = args[1] fileArgInd = 2 else: fileArgInd = 0 for filename in args[fileArgInd:]: with open(filename) as inputF: currentFileLines = inputF.readlines() numOfTransfers = 0 numOfJobs = 1 transferList = [] for curLine in currentFileLines: if options.replication: sourceURI, destinationURI = getReplicationPaths(curLine, sourcePrefix, destinationPrefix) transfer = fts3.new_transfer(sourceURI, destinationURI) else: lineArr = curLine.split() sourceURI = lineArr[0] destinationURI = lineArr[-1] transfer = fts3.new_transfer(sourceURI, destinationURI) if len(lineArr) > 2: for lineIndex in range(1, len(lineArr)-1): fts3.add_alternative_source(transfer, lineArr[lineIndex]) transferList.append(transfer) numOfTransfers += 1 if numOfTransfers > MAX_NUM_OF_TRANSFERS: numOfTransfers = 0 job = createTransferJob(transferList, filename, str(numOfJobs), options.overwriteFlag)
def submitToFTS(logger, ftsContext, files, jobids, toUpdate): """ actual FTS job submission INPUT PARAMS: :param logger: logging object :param ftsContext: FTS context :param files: [ [source_pfn, dest_pfn, file oracle id, source site, username, taskname, file size, checksum], ...] OUTPUT PARAMS: :param jobids: collect the list of job ids when submitted :param toUpdate: list of oracle ids to update RETURNS: the submitted jobid (a string) """ transfers = [] for lfn in files: transfers.append( fts3.new_transfer(lfn[0], lfn[1], filesize=lfn[6], metadata={'oracleId': lfn[2]})) logger.info("Submitting %s transfers to FTS server" % len(files)) #SB# for lfn in files: logger.info("%s %s %s %s", lfn[0], lfn[1], lfn[6], {'oracleId': lfn[2]}) #SB# # Submit fts job job = fts3.new_job( transfers, overwrite=True, verify_checksum=True, metadata={ "issuer": "ASO", "userDN": files[0][4], "taskname": files[0][5] }, copy_pin_lifetime=-1, bring_online=None, source_spacetoken=None, spacetoken=None, # max time for job in the FTS queue in hours. From FTS experts in # https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329 # The max_time_in_queue applies per job, not per retry. # The max_time_in_queue is a timeout for how much the job can stay in # a SUBMITTED, ACTIVE or STAGING state. # When a job's max_time_in_queue is reached, the job and all of its # transfers not yet in a terminal state are marked as CANCELED # StefanoB: I see that we hit this at times with 6h, causing job resubmissions, # so will try to make it longer to give FTS maximum chances within our # 24h overall limit (which takes care also of non-FTS related issues !) # ASO transfers never require STAGING so jobs can spend max_time_in_queue only # as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated) max_time_in_queue=10, # from same cern.service-now.com ticket as above: # The number of retries applies to each transfer within that job. # A transfer is granted the first execution + number_of_retries. # E.g.: retry=3 --> first execution + 3 retries # so retry=3 means each transfer has 4 chances at most during the 6h # max_time_in_queue retry=3, reuse=ftsReuse, # seconds after which the transfer is retried # this is a transfer that fails, gets put to SUBMITTED right away, # but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay # reduced under FTS suggestion w.r.t. the 3hrs of asov1 # StefanoB: indeed 10 minutes makes much more sense for storage server glitches retry_delay=600 # timeout on the single transfer process # TODO: not clear if we may need it # timeout = 1300 ) jobid = fts3.submit(ftsContext, job) jobids.append(jobid) fileDoc = dict() fileDoc['asoworker'] = asoworker fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = [x[2] for x in files] fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in files] fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in files] fileDoc['list_of_fts_id'] = [jobid for _ in files] logger.info("Will mark as submitted %s files" % (len(fileDoc['list_of_ids']))) toUpdate.append(fileDoc) return jobid
def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken): """ Build a job for staging Some attributes of the job are expected to be set * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: List of LFNs to stage :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructStagingJob/%s/%s" % (self.operationID, self.targetSE), True) transfers = [] fileIDsInTheJob = [] # Set of LFNs for which we did not get an SRM URL failedLFNs = set() # getting all the target surls res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs, protocol='srm') if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): failedLFNs.add(lfn) log.error("Could not get target SURL", "%s %s" % (lfn, reason)) allTargetSURLs = res['Value']['Successful'] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn] trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=getattr(ftsFile, 'fileID'), activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(getattr(ftsFile, 'fileID')) # If the source is not an tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = 86400 if sourceIsTape else None # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE} job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=target_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
def _constructStagingJob(self, context, pinTime, allTargetSURLs, failedLFNs, target_spacetoken): """ Build a job for staging Some attributes of the job are expected to be set * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param context: fts3 context :param pinTime: pining time in case staging is needed :param allTargetSURLs: dict {lfn:surl} for the target :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructStagingJob/%s/%s" % (self.operationID, self.targetSE), True) transfers = [] fileIDsInTheJob = [] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn] trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=getattr(ftsFile, 'fileID'), activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(getattr(ftsFile, 'fileID')) # If the source is not an tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = 86400 if sourceIsTape else None # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE} job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=target_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
def start_transfers(self, batch_id, batch_tasks): #override result = {} stage_files = [] transfers = [] s_pfn_to_task = {} t_pfn_to_task = {} for task in batch_tasks: sub = task.subscription lfn = sub.file.lfn dest_pfn = sub.destination.to_pfn(lfn, 'gfal2') source_pfn = task.source.to_pfn(lfn, 'gfal2') if dest_pfn is None or source_pfn is None: # either gfal2 is not supported or lfn could not be mapped LOG.warning('Could not obtain PFN for %s at %s or %s', lfn, sub.destination.name, task.source.name) result[task] = False continue if self.checksum_algorithm: checksum = '%s:%s' % ( self.checksum_algorithm, str(sub.file.checksum[self.checksum_index])) verify_checksum = 'target' else: checksum = None verify_checksum = False if task.source.storage_type == Site.TYPE_MSS: LOG.debug('Staging %s at %s', lfn, task.source.name) # need to stage first stage_files.append( (source_pfn, dest_pfn, checksum, sub.file.size)) # task identified by the source PFN s_pfn_to_task[source_pfn] = task else: LOG.debug('Submitting transfer of %s from %s to %s to FTS', lfn, task.source.name, sub.destination.name) transfers.append( fts3.new_transfer(source_pfn, dest_pfn, checksum=checksum, filesize=sub.file.size)) # there should be only one task per destination pfn t_pfn_to_task[dest_pfn] = task if len(stage_files) != 0: LOG.debug('Submit new staging job for %d files', len(stage_files)) job = fts3.new_staging_job([ff[0] for ff in stage_files], bring_online=36000, metadata=self.metadata_string) success = self._submit_job( job, 'staging', batch_id, dict( (pfn, task.id) for pfn, task in s_pfn_to_task.iteritems())) for source_pfn, _, _, _ in stage_files: result[s_pfn_to_task[source_pfn]] = success if success and not self._read_only: LOG.debug('Recording staging queue') fields = ('id', 'source', 'destination', 'checksum', 'size') mapping = lambda ff: (s_pfn_to_task[ff[0]].id, ) + ff if not self._read_only: self.db.insert_many('fts_staging_queue', fields, mapping, stage_files) if len(transfers) != 0: LOG.debug('Submit new transfer job for %d files', len(transfers)) job = fts3.new_job(transfers, retry=self.fts_retry, overwrite=True, verify_checksum=verify_checksum, metadata=self.metadata_string) success = self._submit_job( job, 'transfer', batch_id, dict( (pfn, task.id) for pfn, task in t_pfn_to_task.iteritems())) for transfer in transfers: dest_pfn = transfer['destinations'][0] result[t_pfn_to_task[dest_pfn]] = success return result
def start_transfers(self, batch_id, batch_tasks): #override result = {} stage_files = [] transfers = [] s_pfn_to_task = {} t_pfn_to_task = {} for task in batch_tasks: sub = task.subscription lfn = sub.file.lfn dest_pfn = sub.destination.to_pfn(lfn, 'gfal2') source_pfn = task.source.to_pfn(lfn, 'gfal2') self.x509proxy = sub.destination.x509proxy if task.source.storage_type == Site.TYPE_MSS: self.x509proxy = task.source.x509proxy if dest_pfn is None or source_pfn is None: # either gfal2 is not supported or lfn could not be mapped LOG.warning('Could not obtain PFN for %s at %s or %s', lfn, sub.destination.name, task.source.name) result[task] = False continue if self.checksum_algorithm: checksum = '%s:%s' % (self.checksum_algorithm, str(sub.file.checksum[self.checksum_index])) verify_checksum = 'target' else: checksum = None verify_checksum = False if task.source.storage_type == Site.TYPE_MSS: LOG.debug('Staging %s at %s', lfn, task.source.name) # need to stage first stage_files.append((source_pfn, dest_pfn, checksum, sub.file.size)) # task identified by the source PFN s_pfn_to_task[source_pfn] = task else: LOG.info("Here we are") LOG.info('Submitting transfer of %s from %s to %s to FTS', lfn, task.source.name, sub.destination.name) transfers.append(fts3.new_transfer(source_pfn, dest_pfn, checksum = checksum, filesize = sub.file.size)) # there should be only one task per destination pfn t_pfn_to_task[dest_pfn] = task if len(stage_files) != 0: LOG.debug('Submit new staging job for %d files', len(stage_files)) job = fts3.new_staging_job([ff[0] for ff in stage_files], bring_online = 36000, metadata = self.metadata_string) success = self._submit_job(job, 'staging', batch_id, dict((pfn, task.id) for pfn, task in s_pfn_to_task.iteritems()), x509=task.source.x509proxy) for source_pfn, _, _, _ in stage_files: result[s_pfn_to_task[source_pfn]] = success if success and not self._read_only: LOG.debug('Recording staging queue') fields = ('id', 'source', 'destination', 'checksum', 'size') mapping = lambda ff: (s_pfn_to_task[ff[0]].id,) + ff if not self._read_only: self.db.insert_many('fts_staging_queue', fields, mapping, stage_files) if len(transfers) != 0: LOG.debug('Submit new transfer job for %d files', len(transfers)) LOG.info("Submitting transfer job from disk to site %s with proxy %s." % (sub.destination.name, sub.destination.x509proxy)) job = fts3.new_job(transfers, retry = self.fts_retry, overwrite = True, verify_checksum = verify_checksum, metadata = self.metadata_string) success = self._submit_job(job, 'transfer', batch_id, dict((pfn, task.id) for pfn, task in t_pfn_to_task.iteritems()), x509=sub.destination.x509proxy) for transfer in transfers: dest_pfn = transfer['destinations'][0] result[t_pfn_to_task[dest_pfn]] = success return result
fileArgInd = 2 else: fileArgInd = 0 for filename in args[fileArgInd:]: with open(filename) as inputF: currentFileLines = inputF.readlines() numOfTransfers = 0 numOfJobs = 1 transferList = [] for curLine in currentFileLines: if options.replication: sourceURI, destinationURI = getReplicationPaths( curLine, sourcePrefix, destinationPrefix) transfer = fts3.new_transfer(sourceURI, destinationURI) else: lineArr = curLine.split() sourceURI = lineArr[0] destinationURI = lineArr[-1] transfer = fts3.new_transfer(sourceURI, destinationURI) if len(lineArr) > 2: for lineIndex in range(1, len(lineArr) - 1): fts3.add_alternative_source(transfer, lineArr[lineIndex]) transferList.append(transfer) numOfTransfers += 1 if numOfTransfers > MAX_NUM_OF_TRANSFERS: numOfTransfers = 0
def run(self): """ """ self.threadLock.acquire() self.log.info("Processing transfers from: %s" % self.source) # create destination and source pfns for job transfers = [] for lfn in self.files: transfers.append(fts3.new_transfer(lfn[0], lfn[1], filesize=lfn[6], metadata={'oracleId': lfn[2]} ) ) self.log.info("Submitting %s transfers to FTS server" % len(self.files)) # Submit fts job job = fts3.new_job(transfers, overwrite=True, verify_checksum=True, metadata={"issuer": "ASO", "userDN": self.files[0][4], "taskname": self.files[0][5]}, copy_pin_lifetime=-1, bring_online=None, source_spacetoken=None, spacetoken=None, # max time for job in the FTS queue in hours. From FTS experts in # https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329 # The max_time_in_queue applies per job, not per retry. # The max_time_in_queue is a timeout for how much the job can stay in # a SUBMITTED, ACTIVE or STAGING state. # When a job's max_time_in_queue is reached, the job and all of its # transfers not yet in a terminal state are marked as CANCELED # StefanoB: I see that we hit this at times with 6h, causing job resubmissions, # so will try to make it longer to give FTS maximum chances within our # 24h overall limit (which takes care also of non-FTS related issues !) # ASO transfers never require STAGING so jobs can spend max_time_in_queue only # as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated) max_time_in_queue=10, # from same cern.service-now.com ticket as above: # The number of retries applies to each transfer within that job. # A transfer is granted the first execution + number_of_retries. # E.g.: retry=3 --> first execution + 3 retries # so retry=3 means each transfer has 4 chances at most during the 6h # max_time_in_queue retry=3, reuse=ftsReuse, # seconds after which the transfer is retried # this is a transfer that fails, gets put to SUBMITTED right away, # but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay # reduced under FTS suggestion w.r.t. the 3hrs of asov1 # StefanoB: indeed 10 minutes makes much more sense for storage server glitches retry_delay=600 # timeout on the single transfer process # TODO: not clear if we may need it # timeout = 1300 ) jobid = fts3.submit(self.ftsContext, job) self.jobids.append(jobid) # TODO: manage exception here, what we should do? fileDoc = dict() fileDoc['asoworker'] = asoworker fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = [x[2] for x in self.files] fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files] fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in self.files] fileDoc['list_of_fts_id'] = [jobid for _ in self.files] self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids']))) self.toUpdate.append(fileDoc) self.threadLock.release()
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """ Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructTransferJob/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) res = self.__fetchSpaceToken(self.sourceSE) if not res['OK']: return res source_spacetoken = res['Value'] failedLFNs = set() dstSE = StorageElement(self.targetSE, vo=self.vo) srcSE = StorageElement(self.sourceSE, vo=self.vo) # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res['Value']['Successful'] transfers = [] fileIDsInTheJob = [] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = 'Defunct' continue trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=getattr(ftsFile, 'fileID'), activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(getattr(ftsFile, 'fileID')) # If the source is not an tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = BRING_ONLINE_TIMEOUT if sourceIsTape else None if not transfers: log.error("No transfer possible!") return S_ERROR("No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE} job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
opts = OptionParser() opts.add_option('-s', '--endpoint', dest='endpoint', default='https://fts3-pilot.cern.ch:8446') opts.add_option('--dry-run', dest='dry_run', default=False, action='store_true') (options, args) = opts.parse_args() if len(args) < 2: raise Exception('Need a source and a destination') source = args[0] destination = args[1] checksum = None if len(args) > 2: checksum = args[2] logging.getLogger('fts3.rest.client').setLevel(logging.DEBUG) # Build the job transfer = fts3.new_transfer(source, destination, checksum=checksum, filesize=None, metadata='Test submission') job = fts3.new_job([transfer], verify_checksum=True, metadata='Test job', retry=1, priority =3) # Submit or just print if options.dry_run: print json.dumps(job, indent=2) else: context = fts3.Context(options.endpoint) print fts3.submit(context, job)
def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken): """ Build a job for staging Some attributes of the job are expected to be set * targetSE * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) :param pinTime: pining time in case staging is needed :param allLFNs: List of LFNs to stage :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger( "constructStagingJob/%s/%s" % (self.operationID, self.targetSE), True) transfers = [] fileIDsInTheJob = [] # Set of LFNs for which we did not get an SRM URL failedLFNs = set() # getting all the target surls res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs, protocol='srm') if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): failedLFNs.add(lfn) log.error("Could not get target SURL", "%s %s" % (lfn, reason)) allTargetSURLs = res['Value']['Successful'] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn] trans = fts3.new_transfer(sourceSURL, targetSURL, checksum='ADLER32:%s' % ftsFile.checksum, filesize=ftsFile.size, metadata=getattr(ftsFile, 'fileID'), activity=self.activity) transfers.append(trans) fileIDsInTheJob.append(getattr(ftsFile, 'fileID')) # If the source is not an tape SE, we should set the # copy_pin_lifetime and bring_online params to None, # otherwise they will do an extra useless queue in FTS sourceIsTape = self.__isTapeSE(self.sourceSE) copy_pin_lifetime = pinTime if sourceIsTape else None bring_online = 86400 if sourceIsTape else None # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { 'operationID': self.operationID, 'sourceSE': self.sourceSE, 'targetSE': self.targetSE } job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=target_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority) return S_OK((job, fileIDsInTheJob))
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): """Build a job for transfer Some attributes of the job are expected to be set * sourceSE * targetSE * multiHopSE (optional) * activity (optional) * priority (optional) * filesToSubmit * operationID (optional, used as metadata for the job) Note that, because of FTS limitations (and also because it anyway would be "not very smart"), multiHop can only use non-SRM disk storage as hops. :param pinTime: pining time in case staging is needed :param allLFNs: list of LFNs to transfer :param failedLFNs: set of LFNs in filesToSubmit for which there was a problem :param target_spacetoken: the space token of the target :param protocols: list of protocols to restrict the protocol choice for the transfer :return: S_OK( (job object, list of ftsFileIDs in the job)) """ log = gLogger.getSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}") isMultiHop = False # Check if it is a multiHop transfer if self.multiHopSE: if len(allLFNs) != 1: log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs)) return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !") allHops = [(self.sourceSE, self.multiHopSE), (self.multiHopSE, self.targetSE)] isMultiHop = True else: allHops = [(self.sourceSE, self.targetSE)] nbOfHops = len(allHops) res = self.__fetchSpaceToken(self.sourceSE, self.vo) if not res["OK"]: return res source_spacetoken = res["Value"] failedLFNs = set() copy_pin_lifetime = None bring_online = None archive_timeout = None transfers = [] fileIDsInTheJob = set() for hopId, (hopSrcSEName, hopDstSEName) in enumerate(allHops, start=1): # Again, this is relevant only for the very initial source # but code factorization is more important hopSrcIsTape = self.__isTapeSE(hopSrcSEName, self.vo) dstSE = StorageElement(hopDstSEName, vo=self.vo) srcSE = StorageElement(hopSrcSEName, vo=self.vo) # getting all the (source, dest) surls res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols) if not res["OK"]: return res for lfn, reason in res["Value"]["Failed"].items(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs = res["Value"]["Successful"] srcProto, destProto = res["Value"]["Protocols"] # If the source is a tape SE, we should set the # copy_pin_lifetime and bring_online params # In case of multihop, this is relevant only for the # original source, but again, code factorization is more important if hopSrcIsTape: copy_pin_lifetime = pinTime bring_online = srcSE.options.get("BringOnlineTimeout", BRING_ONLINE_TIMEOUT) # If the destination is a tape, and the protocol supports it, # check if we want to have an archive timeout # In case of multihop, this is relevant only for the # final target, but again, code factorization is more important dstIsTape = self.__isTapeSE(hopDstSEName, self.vo) if dstIsTape and destProto in dstSE.localStageProtocolList: archive_timeout = dstSE.options.get("ArchiveTimeout") # This contains the staging URLs if they are different from the transfer URLs # (CTA...) allStageURLs = dict() # In case we are transfering from a tape system, and the stage protocol # is not the same as the transfer protocol, we generate the staging URLs # to do a multihop transfer. See below. if hopSrcIsTape and srcProto not in srcSE.localStageProtocolList: isMultiHop = True # As of version 3.10, FTS can only handle one file per multi hop # job. If we are here, that means that we need one, so make sure that # we only have a single file to transfer (this should have been checked # at the job construction step in FTS3Operation). # This test is important, because multiple files would result in the source # being deleted ! if len(allLFNs) != 1: log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs)) return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !") res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList) if not res["OK"]: return res for lfn, reason in res["Value"]["Failed"].items(): failedLFNs.add(lfn) log.error("Could not get stage SURL", "%s %s" % (lfn, reason)) allSrcDstSURLs.pop(lfn) allStageURLs = res["Value"]["Successful"] for ftsFile in self.filesToSubmit: if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] stageURL = allStageURLs.get(ftsFile.lfn) if sourceSURL == targetSURL: log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = "Defunct" continue ftsFileID = getattr(ftsFile, "fileID") # Under normal circumstances, we simply submit an fts transfer as such: # * srcProto://myFile -> destProto://myFile # # Even in case of the source storage being a tape system, it works fine. # However, if the staging and transfer protocols are different (which might be the case for CTA), # we use the multihop machinery to submit two sequential fts transfers: # one to stage, one to transfer. # It looks like such # * stageProto://myFile -> stageProto://myFile # * srcProto://myFile -> destProto://myFile if stageURL: # We do not set a fileID in the metadata # such that we do not update the DB when monitoring stageTrans_metadata = {"desc": "PreStage %s" % ftsFileID} # If we use an activity, also set it as file metadata # for WLCG monitoring purposes # https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14? if self.activity: stageTrans_metadata["activity"] = self.activity stageTrans = fts3.new_transfer( stageURL, stageURL, checksum="ADLER32:%s" % ftsFile.checksum, filesize=ftsFile.size, metadata=stageTrans_metadata, activity=self.activity, ) transfers.append(stageTrans) # If it is the last hop only, we set the fileID metadata # for monitoring if hopId == nbOfHops: trans_metadata = {"desc": "Transfer %s" % ftsFileID, "fileID": ftsFileID} else: trans_metadata = {"desc": "MultiHop %s" % ftsFileID} # If we use an activity, also set it as file metadata # for WLCG monitoring purposes # https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14? if self.activity: trans_metadata["activity"] = self.activity # because of an xroot bug (https://github.com/xrootd/xrootd/issues/1433) # the checksum needs to be lowercase. It does not impact the other # protocol, so it's fine to put it here. # I only add it in this transfer and not the "staging" one above because it # impacts only root -> root transfers trans = fts3.new_transfer( sourceSURL, targetSURL, checksum="ADLER32:%s" % ftsFile.checksum.lower(), filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity, ) transfers.append(trans) fileIDsInTheJob.add(ftsFileID) if not transfers: log.error("No transfer possible!") return S_ERROR(errno.ENODATA, "No transfer possible") # We add a few metadata to the fts job so that we can reuse them later on without # querying our DB. # source and target SE are just used for accounting purpose job_metadata = { "operationID": self.operationID, "rmsReqID": self.rmsReqID, "sourceSE": self.sourceSE, "targetSE": self.targetSE, } if self.activity: job_metadata["activity"] = self.activity job = fts3.new_job( transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, verify_checksum="target", # Only check target vs specified, since we verify the source earlier multihop=isMultiHop, metadata=job_metadata, priority=self.priority, archive_timeout=archive_timeout, ) return S_OK((job, fileIDsInTheJob))
def submit(self, files, job_params, timeout=None): """ Submit a transfer to FTS3 via JSON. :param external_host: FTS server as a string. :param files: List of dictionary which for a transfer. :param job_params: Dictionary containing key/value pairs, for all transfers. :param user_transfer: boolean for user tranfer submission :returns: FTS transfer identifier. """ usercert = self.prepare_credentials(files) if not usercert: logging.error('Unable to prepare credentials.') return None logging.info('Proxy %s is ready.', usercert) # FTS3 expects 'davs' as the scheme identifier instead of https for file in files: if not file['sources'] or file['sources'] == []: os.unlink(usercert) raise Exception('No sources defined') new_src_urls = [] new_dst_urls = [] for url in file['sources']: if url.startswith('https'): new_src_urls.append(':'.join(['davs'] + url.split(':')[1:])) else: new_src_urls.append(url) for url in file['destinations']: if url.startswith('https'): new_dst_urls.append(':'.join(['davs'] + url.split(':')[1:])) else: new_dst_urls.append(url) file['sources'] = new_src_urls file['destinations'] = new_dst_urls transfer_id = None expected_transfer_id = None if self.deterministic: job_params = job_params.copy() job_params["id_generator"] = "deterministic" job_params["sid"] = files[0]['metadata']['request_id'] expected_transfer_id = self.__get_deterministic_id( job_params["sid"]) logging.debug( "Submit bulk transfers in deterministic mode, sid %s, expected transfer id: %s", job_params["sid"], expected_transfer_id) # bulk submission jobID = None transfers = [] for _file in files: for source, destination in zip(_file["sources"], _file["destinations"]): transfers.append( fts.new_transfer(source, destination, activity=_file['activity'], metadata=_file['metadata'], filesize=_file['filesize'], checksum=_file['checksum'])) try: context = fts.Context(self.external_host, ucert=usercert, ukey=usercert, verify=True, capath='/etc/grid-security/certificates/') job = fts.new_job( transfers, overwrite=job_params['overwrite'], verify_checksum=job_params['verify_checksum'], metadata=job_params['job_metadata'], copy_pin_lifetime=job_params['copy_pin_lifetime'], bring_online=job_params['bring_online'], source_spacetoken=None, spacetoken=None, priority=job_params['priority']) # submission for bindings checks the delegation, so the usual delegation part is useless here jobID = fts.submit(context, job, delegation_lifetime=timedelta(hours=12), delegate_when_lifetime_lt=timedelta(hours=8)) except ServerError: logging.error("Server side exception during FTS job submission.") return None except ClientError: logging.error("Client side exception during FTS job submission.") return None except BadEndpoint: logging.error("Wrong FTS endpoint: %s", self.external_host) return None if jobID: record_counter( 'transfertool.fts3myproxy.%s.submission.success' % self.__extract_host(self.external_host), len(files)) transfer_id = jobID else: transfer_id = jobID logging.error("Unexpected failure during FTS job submission.") record_counter( 'transfertool.fts3myproxy.%s.submission.failure' % self.__extract_host(self.external_host), len(files)) os.unlink(usercert) return transfer_id
f['dest_surl'] for f in job_status['files'] if f['file_state'] in ['FINISHED'] or f['reason'] == 'DESTINATION file already exists and overwrite is not enabled' ] transferList = [] for filename in transferedFiles: fs = filename.split('/') try: lfcURI = '/'.join(fs[fs.index('grid'):]) except ValueError: # substring 'grid' not found # get the last occurrence of auger in the path lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):]) # note that this path construction is not particularly liable and should be upgraded in the future transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI) transferList.append(transfer) if len(transferList) > 0: job = fts3.new_job( transferList, metadata="Registration of the files transfered by job " + job_id, overwrite=True) if options.register == 1 or options.register == 0 and query_yes_no( "The registration job is ready now, do you wish to submit it?" ): jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit( context, job) print "The registration job ID is " + jobID with open(options.regJobIdFile, 'a') as f: