def submit(proxy, toTrans, source, destination): # prepare rest job with 200 files per job transfers = [] for files in chunks(toTrans, 200): c = pycurl.Curl() # create destination and source pfns for job for lfn in files: print(lfn) transfers.append( fts3.new_transfer(apply_tfc_to_lfn(source, lfn, c), apply_tfc_to_lfn(destination, lfn, c))) c.close() # Submit fts job context = fts3.Context('https://fts3.cern.ch:8446', proxy, proxy, verify=True) print(fts3.delegate(context, lifetime=timedelta(hours=48), force=False)) job = fts3.new_job(transfers) #print("Monitor link: https://fts3.cern.ch:8449/fts3/ftsmon/#/job/"+fts3.submit(context, job)) jobid = fts3.submit(context, job) #for file in (fts3.get_job_status(context, jobid, list_files=True))["files"]: for key, value in (fts3.get_job_status(context, jobid, list_files=True)).iteritems(): print key
def _fts_submit_job(source_url, dest_url, src_filenames, dst_filenames, checksum, overwrite, testing_folder, context, metadata): """ https://gitlab.cern.ch/fts/fts-rest/-/blob/develop/src/fts3/rest/client/easy/submission.py#L106 """ transfers = [] for i in xrange(len(src_filenames)): source_file = os.path.join(source_url, testing_folder, "src", src_filenames[i]) dest_file = os.path.join(dest_url, testing_folder, "dest", dst_filenames[i]) transfer = fts3.new_transfer(source=source_file, destination=dest_file) transfers.append(transfer) # create job job = fts3.new_job(transfers, verify_checksum=checksum, overwrite=overwrite, timeout=3600, metadata=metadata) # submit job while True: try: job_id = fts3.submit(context, job) break except fts3_client_exceptions.ClientError as e: _flush_logging_msg(e) return -1 return job_id
def submitTheFTSJob(ftsFile): ### First way : Random choice of two servers # ftsServ = random.choice([ftsServ1, ftsServ2]) ### Second way : Weighted choice of two servers # rndValue = random.uniform(0.0,1.0) # ftsServ = ftsServ1 # if rndValue < 0.7 : ftsServ = ftsServ2 ### Third way : Random choice of three servers fList = [ftsServ1, ftsServ2, ftsServ3] ftsServ = random.choice(fList) # context = fts3.Context(ftsServ) # Open the file and stop the processing. listOfPairs[:] = open(ceBase + "DOING/" + ftsFile).read().split("\n") # listOfPairs[:] = open(ceBase + "TODO/" + ftsFile).read().split("\n") # All the threading bit is here to check in parallel whether the files we are looking are okay in castor # Once the function is done, the list "okayFiles" should be filled transfers = [] if checkStatus: checkStatusOnCastor() else: okayFiles[:] = [] for onePair in listOfPairs: if len(onePair)<10: continue (sourceSURL, targetSURL) = onePair.split(" ") okayFiles.append((sourceSURL, targetSURL)) if len(okayFiles)>0: for oneSet in okayFiles: transf = fts3.new_transfer(oneSet[0], oneSet[1]) transfers.append(transf) job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi ftsJobID = fts3.submit(context, job) return ftsJobID, ftsServ else: # None of the files in this lot were good! return "-1", "-1"
def submitFTS3( self, pinTime = False ): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR( "FTSJob already has been submitted" ) transfers = [] for ftsFile in self: trans = fts3.new_transfer( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum = ftsFile.Checksum, filesize = ftsFile.Size ) transfers.append( trans ) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job( transfers = transfers, overwrite = True, source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken, bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 ) try: if not self._fts3context: self._fts3context = fts3.Context( endpoint = self.FTSServer ) context = self._fts3context self.FTSGUID = fts3.submit( context, job ) except Exception, e: return S_ERROR( "Error at submission: %s" % e )
def submitFTS3(self, pinTime=False): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR("FTSJob already has been submitted") transfers = [] for ftsFile in self: trans = fts3.new_transfer(ftsFile.SourceSURL, ftsFile.TargetSURL, checksum=ftsFile.Checksum, filesize=ftsFile.Size) transfers.append(trans) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=dest_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3) try: context = fts3.Context(self.FTSServer) self.FTSGUID = fts3.submit(context, job) except Exception, e: return S_ERROR("Error at submission: %s" % e)
def run(self): """ """ self.threadLock.acquire() self.log.info("Processing transfers from: %s" % self.source) # create destination and source pfns for job transfers = [] for lfn in self.files: transfers.append(fts3.new_transfer(lfn[0], lfn[1], metadata={'oracleId': lfn[2]} ) ) self.log.info("Submitting %s transfers to FTS server" % len(self.files)) # Submit fts job job = fts3.new_job(transfers, overwrite=True, verify_checksum=True, metadata={"issuer": "ASO", "userDN": self.files[0][4], "taskname": self.files[0][5]}, copy_pin_lifetime=-1, bring_online=None, source_spacetoken=None, spacetoken=None, # max time for job in the fts queue in seconds. # Usually, it should take O(s) for healthy situations max_time_in_queue=600, retry=3, # seconds after which the transfer is retried # reduced under FTS suggestion w.r.t. the 3hrs of asov1 retry_delay=600 # timeout on the single transfer process # TODO: not clear if we may need it # timeout = 1300 ) jobid = fts3.submit(self.ftsContext, job) self.jobids.append(jobid) # TODO: manage exception here, what we should do? fileDoc = dict() fileDoc['asoworker'] = 'asoless' fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = [x[2] for x in self.files] fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files] fileDoc['list_of_fts_instance'] = ['https://fts3.cern.ch:8446/' for _ in self.files] fileDoc['list_of_fts_id'] = [jobid for _ in self.files] self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids']))) self.toUpdate.append(fileDoc) self.threadLock.release()
def submitTheFTSJob(ftsFile): ### First way : Random choice of two servers # ftsServ = random.choice([ftsServ1, ftsServ2]) ### Second way : Weighted choice of two servers # rndValue = random.uniform(0.0,1.0) # ftsServ = ftsServ1 # if rndValue < 0.7 : ftsServ = ftsServ2 ### Third way : Random choice of three servers fList = [ftsServ1, ftsServ2, ftsServ3] ftsServ = random.choice(fList) # context = fts3.Context(ftsServ) filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n") transfers = [] for ftra in filecontent: if len(ftra) < 10: continue (sourceSURL, targetSURL) = ftra.split(" ") comm = "gfal-stat " + sourceSURL runComm = subprocess.Popen(comm, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) theInfo = runComm.communicate()[1].strip() if theInfo.startswith( "gfal-stat error: 2 (No such file or directory)"): bFTS = open(ceBase + "DONE/badFileList.txt", "a") bFTS.write(ftra + "\n") bFTS.close() else: transf = fts3.new_transfer(sourceSURL, targetSURL) transfers.append(transf) # transf = fts3.new_transfer(sourceSURL, targetSURL) # transfers.append(transf) if len(transfers) > 0: # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5) # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi job = fts3.new_job( transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=0) # To avoid deleted files snarling up the system for hours ftsJobID = fts3.submit(context, job, delegation_lifetime=fts3.timedelta(hours=72)) return ftsJobID, ftsServ else: return "-1", "-1"
def submitFTS3(self, pinTime=False): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR("FTSJob already has been submitted") transfers = [] for ftsFile in self: trans = fts3.new_transfer(ftsFile.SourceSURL, ftsFile.TargetSURL, checksum='ADLER32:%s' % ftsFile.Checksum, filesize=ftsFile.Size) transfers.append(trans) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job(transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, spacetoken=dest_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3) try: if not self._fts3context: self._fts3context = fts3.Context(endpoint=self.FTSServer, request_class=ftsSSLRequest, verify=False) context = self._fts3context self.FTSGUID = fts3.submit(context, job) except Exception as e: return S_ERROR("Error at submission: %s" % e) self.Status = "Submitted" self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True) for ftsFile in self: ftsFile.FTSGUID = self.FTSGUID ftsFile.Status = "Submitted" return S_OK()
def submitFTS3( self, pinTime = False ): """ submit fts job using FTS3 rest API """ if self.FTSGUID: return S_ERROR( "FTSJob already has been submitted" ) transfers = [] for ftsFile in self: trans = fts3.new_transfer( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum = 'ADLER32:%s'%ftsFile.Checksum, filesize = ftsFile.Size ) transfers.append( trans ) source_spacetoken = self.SourceToken if self.SourceToken else None dest_spacetoken = self.TargetToken if self.TargetToken else None copy_pin_lifetime = pinTime if pinTime else None bring_online = 86400 if pinTime else None job = fts3.new_job( transfers = transfers, overwrite = True, source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken, bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 ) try: if not self._fts3context: self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False ) context = self._fts3context self.FTSGUID = fts3.submit( context, job ) except Exception as e: return S_ERROR( "Error at submission: %s" % e ) self.Status = "Submitted" self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True ) for ftsFile in self: ftsFile.FTSGUID = self.FTSGUID ftsFile.Status = "Submitted" return S_OK()
def submitTheFTSJob(context, ftsFile): ftsServ = random.choice([ftsServ1, ftsServ2]) with open(ceBase + "DOING/" + ftsFile) as fF: filecontent = fF.read().split("\n") # filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n") transfers = [] for ftra in filecontent: if len(ftra) < 10: continue (sourceSURL, targetSURL) = ftra.split(" ") transf = fts3.new_transfer(sourceSURL, targetSURL) transfers.append(transf) if len(transfers) > 0: # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5) job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi ftsJobID = fts3.submit(context, job) return ftsJobID, ftsServ else: return "-1", "-1"
# get the last occurrence of auger in the path lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):]) # note that this path construction is not particularly liable and should be upgraded in the future transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI) transferList.append(transfer) if len(transferList) > 0: job = fts3.new_job( transferList, metadata="Registration of the files transfered by job " + job_id, overwrite=True) if options.register == 1 or options.register == 0 and query_yes_no( "The registration job is ready now, do you wish to submit it?" ): jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit( context, job) print "The registration job ID is " + jobID with open(options.regJobIdFile, 'a') as f: f.write('\t' + jobID + '\n') notFinishedTransfers = [ fts3.new_transfer(f['source_surl'], f['dest_surl']) for f in job_status['files'] if f['file_state'] in ['FAILED', 'CANCELED'] and f['reason'] != 'DESTINATION file already exists and overwrite is not enabled' ] if len(notFinishedTransfers) < 1: sys.exit(0) jobChunks = [
def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protocols=None): """ submit the job to the FTS server Some attributes are expected to be defined for the submission to work: * type (set by FTS3Operation) * sourceSE (only for Transfer jobs) * targetSE * activity (optional) * priority (optional) * username * userGroup * filesToSubmit * operationID (optional, used as metadata for the job) We also expect the FTSFiles have an ID defined, as it is given as transfer metadata :param pinTime: Time the file should be pinned on disk (used for transfers and staging) Used only if he source SE is a tape storage :param context: fts3 context. If not given, it is created (see ftsServer & ucert param) :param ftsServer: the address of the fts server to submit to. Used only if context is not given. if not given either, use the ftsServer object attribute :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc) :param protocols: list of protocols from which we should choose the protocol to use :returns S_OK([FTSFiles ids of files submitted]) """ log = gLogger.getSubLogger( "submit/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) if not context: if not ftsServer: ftsServer = self.ftsServer context = fts3.Context(endpoint=ftsServer, ucert=ucert, request_class=ftsSSLRequest, verify=False) # Construct the target SURL res = self.__fetchSpaceToken(self.targetSE) if not res['OK']: return res target_spacetoken = res['Value'] allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit] if self.type == 'Transfer': res = self._constructTransferJob(pinTime, allLFNs, target_spacetoken, protocols=protocols) elif self.type == 'Staging': res = self._constructStagingJob(pinTime, allLFNs, target_spacetoken) # elif self.type == 'Removal': # res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken) if not res['OK']: return res job, fileIDsInTheJob = res['Value'] setFileIdsInTheJob = set(fileIDsInTheJob) try: self.ftsGUID = fts3.submit(context, job) log.info("Got GUID %s" % self.ftsGUID) # Only increase the amount of attempt # if we succeeded in submitting -> no ! Why did I do that ?? for ftsFile in self.filesToSubmit: ftsFile.attempt += 1 if ftsFile.fileID in setFileIdsInTheJob: ftsFile.status = 'Submitted' now = datetime.datetime.utcnow().replace(microsecond=0) self.submitTime = now self.lastUpdate = now self.lastMonitor = now except FTS3ClientException as e: log.exception("Error at submission", repr(e)) return S_ERROR("Error at submission: %s" % e) return S_OK(fileIDsInTheJob)
fs = filename.split('/') try: lfcURI = '/'.join(fs[fs.index('grid'):]) except ValueError: # substring 'grid' not found # get the last occurrence of auger in the path lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):]) # note that this path construction is not particularly liable and should be upgraded in the future transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI) transferList.append(transfer) if len(transferList) > 0: job = fts3.new_job(transferList, metadata="Registration of the files transfered by job " + job_id, overwrite=True) if options.register == 1 or options.register == 0 and query_yes_no("The registration job is ready now, do you wish to submit it?"): jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit(context, job) print "The registration job ID is " + jobID with open(options.regJobIdFile, 'a') as f: f.write('\t' + jobID + '\n') notFinishedTransfers = [fts3.new_transfer(f['source_surl'], f['dest_surl']) for f in job_status['files'] if f['file_state'] in ['FAILED', 'CANCELED'] and f['reason'] != 'DESTINATION file already exists and overwrite is not enabled'] if len(notFinishedTransfers) < 1: sys.exit(0) jobChunks = [notFinishedTransfers[i:i + MAX_NUM_OF_TRANSFERS] for i in xrange(0, len(notFinishedTransfers), MAX_NUM_OF_TRANSFERS)] jobs = [] for chunkOfTransfers in jobChunks:
transfer = fts3.new_transfer(sourceURI, destinationURI) if len(lineArr) > 2: for lineIndex in range(1, len(lineArr)-1): fts3.add_alternative_source(transfer, lineArr[lineIndex]) transferList.append(transfer) numOfTransfers += 1 if numOfTransfers > MAX_NUM_OF_TRANSFERS: numOfTransfers = 0 job = createTransferJob(transferList, filename, str(numOfJobs), options.overwriteFlag) if options.dryRun: print json.dumps(job, indent=2) jobID = 'abc' else: jobID = fts3.submit(context, job) transferJobs.append(jobID) numOfJobs += 1 transferList = [] # submit the last job if transferList: job = createTransferJob(transferList, filename, str(numOfJobs) + " last", options.overwriteFlag) if options.dryRun: print json.dumps(job, indent=2) jobID = 'abc' else: jobID = fts3.submit(context, job) transferJobs.append(jobID) # if some jobs were submitted, note their IDs
def submitToFTS(logger, ftsContext, files, jobids, toUpdate): """ actual FTS job submission INPUT PARAMS: :param logger: logging object :param ftsContext: FTS context :param files: [ [source_pfn, dest_pfn, file oracle id, source site, username, taskname, file size, checksum], ...] OUTPUT PARAMS: :param jobids: collect the list of job ids when submitted :param toUpdate: list of oracle ids to update RETURNS: the submitted jobid (a string) """ transfers = [] for lfn in files: transfers.append( fts3.new_transfer(lfn[0], lfn[1], filesize=lfn[6], metadata={'oracleId': lfn[2]})) logger.info("Submitting %s transfers to FTS server" % len(files)) #SB# for lfn in files: logger.info("%s %s %s %s", lfn[0], lfn[1], lfn[6], {'oracleId': lfn[2]}) #SB# # Submit fts job job = fts3.new_job( transfers, overwrite=True, verify_checksum=True, metadata={ "issuer": "ASO", "userDN": files[0][4], "taskname": files[0][5] }, copy_pin_lifetime=-1, bring_online=None, source_spacetoken=None, spacetoken=None, # max time for job in the FTS queue in hours. From FTS experts in # https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329 # The max_time_in_queue applies per job, not per retry. # The max_time_in_queue is a timeout for how much the job can stay in # a SUBMITTED, ACTIVE or STAGING state. # When a job's max_time_in_queue is reached, the job and all of its # transfers not yet in a terminal state are marked as CANCELED # StefanoB: I see that we hit this at times with 6h, causing job resubmissions, # so will try to make it longer to give FTS maximum chances within our # 24h overall limit (which takes care also of non-FTS related issues !) # ASO transfers never require STAGING so jobs can spend max_time_in_queue only # as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated) max_time_in_queue=10, # from same cern.service-now.com ticket as above: # The number of retries applies to each transfer within that job. # A transfer is granted the first execution + number_of_retries. # E.g.: retry=3 --> first execution + 3 retries # so retry=3 means each transfer has 4 chances at most during the 6h # max_time_in_queue retry=3, reuse=ftsReuse, # seconds after which the transfer is retried # this is a transfer that fails, gets put to SUBMITTED right away, # but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay # reduced under FTS suggestion w.r.t. the 3hrs of asov1 # StefanoB: indeed 10 minutes makes much more sense for storage server glitches retry_delay=600 # timeout on the single transfer process # TODO: not clear if we may need it # timeout = 1300 ) jobid = fts3.submit(ftsContext, job) jobids.append(jobid) fileDoc = dict() fileDoc['asoworker'] = asoworker fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = [x[2] for x in files] fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in files] fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in files] fileDoc['list_of_fts_id'] = [jobid for _ in files] logger.info("Will mark as submitted %s files" % (len(fileDoc['list_of_ids']))) toUpdate.append(fileDoc) return jobid
def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protocols=None): """ submit the job to the FTS server Some attributes are expected to be defined for the submission to work: * type (set by FTS3Operation) * sourceSE (only for Transfer jobs) * targetSE * activity (optional) * priority (optional) * username * userGroup * filesToSubmit * operationID (optional, used as metadata for the job) We also expect the FTSFiles have an ID defined, as it is given as transfer metadata :param pinTime: Time the file should be pinned on disk (used for transfers and staging) Used only if he source SE is a tape storage :param context: fts3 context. If not given, it is created (see ftsServer & ucert param) :param ftsServer: the address of the fts server to submit to. Used only if context is not given. if not given either, use the ftsServer object attribute :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc) :param protocols: list of protocols from which we should choose the protocol to use :returns S_OK([FTSFiles ids of files submitted]) """ log = gLogger.getSubLogger("submit/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) if not context: if not ftsServer: ftsServer = self.ftsServer context = fts3.Context( endpoint=ftsServer, ucert=ucert, request_class=ftsSSLRequest, verify=False) # Construct the target SURL res = self.__fetchSpaceToken(self.targetSE) if not res['OK']: return res target_spacetoken = res['Value'] allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit] if self.type == 'Transfer': res = self._constructTransferJob( pinTime, allLFNs, target_spacetoken, protocols=protocols) elif self.type == 'Staging': res = self._constructStagingJob( pinTime, allLFNs, target_spacetoken) # elif self.type == 'Removal': # res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken) if not res['OK']: return res job, fileIDsInTheJob = res['Value'] setFileIdsInTheJob = set(fileIDsInTheJob) try: self.ftsGUID = fts3.submit(context, job) log.info("Got GUID %s" % self.ftsGUID) # Only increase the amount of attempt # if we succeeded in submitting -> no ! Why did I do that ?? for ftsFile in self.filesToSubmit: ftsFile.attempt += 1 # This should never happen because a file should be "released" # first by the previous job. # But we just print a warning if ftsFile.ftsGUID is not None: log.warn( "FTSFile has a non NULL ftsGUID at job submission time", "FileID: %s existing ftsGUID: %s" % (ftsFile.fileID, ftsFile.ftsGUID)) # `assign` the file to this job ftsFile.ftsGUID = self.ftsGUID if ftsFile.fileID in setFileIdsInTheJob: ftsFile.status = 'Submitted' now = datetime.datetime.utcnow().replace(microsecond=0) self.submitTime = now self.lastUpdate = now self.lastMonitor = now except FTS3ClientException as e: log.exception("Error at submission", repr(e)) return S_ERROR("Error at submission: %s" % e) return S_OK(fileIDsInTheJob)
def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, ): """ submit the job to the FTS server Some attributes are expected to be defined for the submission to work: * type (set by FTS3Operation) * sourceSE (only for Transfer jobs) * targetSE * activity (optional) * priority (optional) * username * userGroup * filesToSubmit * operationID (optional, used as metadata for the job) We also expect the FTSFiles have an ID defined, as it is given as transfer metadata :param pinTime: Time the file should be pinned on disk (used for transfers and staging) Used only if he source SE is a tape storage :param context: fts3 context. If not given, it is created (see ftsServer & ucert param) :param ftsServer: the address of the fts server to submit to. Used only if context is not given. if not given either, use the ftsServer object attribute :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc) :returns S_OK([FTSFiles ids of files submitted]) """ log = gLogger.getSubLogger("submit/%s/%s_%s" % (self.operationID, self.sourceSE, self.targetSE), True) if not context: if not ftsServer: ftsServer = self.ftsServer context = fts3.Context( endpoint=ftsServer, ucert=ucert, request_class=ftsSSLRequest, verify=False) # Construct the target SURL res = self.__fetchSpaceToken(self.targetSE) if not res['OK']: return res target_spacetoken = res['Value'] allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit] failedLFNs = set() # getting all the target surls res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs, protocol='srm') if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): failedLFNs.add(lfn) log.error("Could not get target SURL", "%s %s" % (lfn, reason)) allTargetSURLs = res['Value']['Successful'] if self.type == 'Transfer': res = self._constructTransferJob( context, pinTime, allTargetSURLs, failedLFNs, target_spacetoken) elif self.type == 'Staging': res = self._constructStagingJob( context, pinTime, allTargetSURLs, failedLFNs, target_spacetoken) elif self.type == 'Removal': res = self._constructRemovalJob(context, allTargetSURLs, failedLFNs, target_spacetoken) if not res['OK']: return res job, fileIDsInTheJob = res['Value'] setFileIdsInTheJob = set(fileIDsInTheJob) try: self.ftsGUID = fts3.submit(context, job) log.info("Got GUID %s" % self.ftsGUID) # Only increase the amount of attempt # if we succeeded in submitting -> no ! Why did I do that ?? for ftsFile in self.filesToSubmit: ftsFile.attempt += 1 if ftsFile.fileID in setFileIdsInTheJob: ftsFile.status = 'Submitted' now = datetime.datetime.utcnow().replace(microsecond=0) self.submitTime = now self.lastUpdate = now self.lastMonitor = now except FTS3ClientException as e: log.exception("Error at submission", repr(e)) return S_ERROR("Error at submission: %s" % e) return S_OK(fileIDsInTheJob)
def run(self): """ """ self.threadLock.acquire() self.log.info("Processing transfers from: %s" % self.source) # create destination and source pfns for job transfers = [] for lfn in self.files: transfers.append(fts3.new_transfer(lfn[0], lfn[1], filesize=lfn[6], metadata={'oracleId': lfn[2]} ) ) self.log.info("Submitting %s transfers to FTS server" % len(self.files)) # Submit fts job job = fts3.new_job(transfers, overwrite=True, verify_checksum=True, metadata={"issuer": "ASO", "userDN": self.files[0][4], "taskname": self.files[0][5]}, copy_pin_lifetime=-1, bring_online=None, source_spacetoken=None, spacetoken=None, # max time for job in the FTS queue in hours. From FTS experts in # https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329 # The max_time_in_queue applies per job, not per retry. # The max_time_in_queue is a timeout for how much the job can stay in # a SUBMITTED, ACTIVE or STAGING state. # When a job's max_time_in_queue is reached, the job and all of its # transfers not yet in a terminal state are marked as CANCELED # StefanoB: I see that we hit this at times with 6h, causing job resubmissions, # so will try to make it longer to give FTS maximum chances within our # 24h overall limit (which takes care also of non-FTS related issues !) # ASO transfers never require STAGING so jobs can spend max_time_in_queue only # as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated) max_time_in_queue=10, # from same cern.service-now.com ticket as above: # The number of retries applies to each transfer within that job. # A transfer is granted the first execution + number_of_retries. # E.g.: retry=3 --> first execution + 3 retries # so retry=3 means each transfer has 4 chances at most during the 6h # max_time_in_queue retry=3, reuse=ftsReuse, # seconds after which the transfer is retried # this is a transfer that fails, gets put to SUBMITTED right away, # but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay # reduced under FTS suggestion w.r.t. the 3hrs of asov1 # StefanoB: indeed 10 minutes makes much more sense for storage server glitches retry_delay=600 # timeout on the single transfer process # TODO: not clear if we may need it # timeout = 1300 ) jobid = fts3.submit(self.ftsContext, job) self.jobids.append(jobid) # TODO: manage exception here, what we should do? fileDoc = dict() fileDoc['asoworker'] = asoworker fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = [x[2] for x in self.files] fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files] fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in self.files] fileDoc['list_of_fts_id'] = [jobid for _ in self.files] self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids']))) self.toUpdate.append(fileDoc) self.threadLock.release()
def submit(self, files, job_params, timeout=None): """ Submit a transfer to FTS3 via JSON. :param external_host: FTS server as a string. :param files: List of dictionary which for a transfer. :param job_params: Dictionary containing key/value pairs, for all transfers. :param user_transfer: boolean for user tranfer submission :returns: FTS transfer identifier. """ usercert = self.prepare_credentials(files) if not usercert: logging.error('Unable to prepare credentials.') return None logging.info('Proxy %s is ready.', usercert) # FTS3 expects 'davs' as the scheme identifier instead of https for file in files: if not file['sources'] or file['sources'] == []: os.unlink(usercert) raise Exception('No sources defined') new_src_urls = [] new_dst_urls = [] for url in file['sources']: if url.startswith('https'): new_src_urls.append(':'.join(['davs'] + url.split(':')[1:])) else: new_src_urls.append(url) for url in file['destinations']: if url.startswith('https'): new_dst_urls.append(':'.join(['davs'] + url.split(':')[1:])) else: new_dst_urls.append(url) file['sources'] = new_src_urls file['destinations'] = new_dst_urls transfer_id = None expected_transfer_id = None if self.deterministic: job_params = job_params.copy() job_params["id_generator"] = "deterministic" job_params["sid"] = files[0]['metadata']['request_id'] expected_transfer_id = self.__get_deterministic_id( job_params["sid"]) logging.debug( "Submit bulk transfers in deterministic mode, sid %s, expected transfer id: %s", job_params["sid"], expected_transfer_id) # bulk submission jobID = None transfers = [] for _file in files: for source, destination in zip(_file["sources"], _file["destinations"]): transfers.append( fts.new_transfer(source, destination, activity=_file['activity'], metadata=_file['metadata'], filesize=_file['filesize'], checksum=_file['checksum'])) try: context = fts.Context(self.external_host, ucert=usercert, ukey=usercert, verify=True, capath='/etc/grid-security/certificates/') job = fts.new_job( transfers, overwrite=job_params['overwrite'], verify_checksum=job_params['verify_checksum'], metadata=job_params['job_metadata'], copy_pin_lifetime=job_params['copy_pin_lifetime'], bring_online=job_params['bring_online'], source_spacetoken=None, spacetoken=None, priority=job_params['priority']) # submission for bindings checks the delegation, so the usual delegation part is useless here jobID = fts.submit(context, job, delegation_lifetime=timedelta(hours=12), delegate_when_lifetime_lt=timedelta(hours=8)) except ServerError: logging.error("Server side exception during FTS job submission.") return None except ClientError: logging.error("Client side exception during FTS job submission.") return None except BadEndpoint: logging.error("Wrong FTS endpoint: %s", self.external_host) return None if jobID: record_counter( 'transfertool.fts3myproxy.%s.submission.success' % self.__extract_host(self.external_host), len(files)) transfer_id = jobID else: transfer_id = jobID logging.error("Unexpected failure during FTS job submission.") record_counter( 'transfertool.fts3myproxy.%s.submission.failure' % self.__extract_host(self.external_host), len(files)) os.unlink(usercert) return transfer_id
for lineIndex in range(1, len(lineArr) - 1): fts3.add_alternative_source(transfer, lineArr[lineIndex]) transferList.append(transfer) numOfTransfers += 1 if numOfTransfers > MAX_NUM_OF_TRANSFERS: numOfTransfers = 0 job = createTransferJob(transferList, filename, str(numOfJobs), options.overwriteFlag) if options.dryRun: print json.dumps(job, indent=2) jobID = 'abc' else: jobID = fts3.submit(context, job) transferJobs.append(jobID) numOfJobs += 1 transferList = [] # submit the last job if transferList: job = createTransferJob(transferList, filename, str(numOfJobs) + " last", options.overwriteFlag) if options.dryRun: print json.dumps(job, indent=2) jobID = 'abc' else: jobID = fts3.submit(context, job) transferJobs.append(jobID)
opts = OptionParser() opts.add_option('-s', '--endpoint', dest='endpoint', default='https://fts3-pilot.cern.ch:8446') opts.add_option('--dry-run', dest='dry_run', default=False, action='store_true') (options, args) = opts.parse_args() if len(args) < 2: raise Exception('Need a source and a destination') source = args[0] destination = args[1] checksum = None if len(args) > 2: checksum = args[2] logging.getLogger('fts3.rest.client').setLevel(logging.DEBUG) # Build the job transfer = fts3.new_transfer(source, destination, checksum=checksum, filesize=None, metadata='Test submission') job = fts3.new_job([transfer], verify_checksum=True, metadata='Test job', retry=1, priority =3) # Submit or just print if options.dry_run: print json.dumps(job, indent=2) else: context = fts3.Context(options.endpoint) print fts3.submit(context, job)