def submit(proxy, toTrans, source, destination):
# prepare rest job with 200 files per job
transfers = []
for files in chunks(toTrans, 200):
c = pycurl.Curl()
# create destination and source pfns for job
for lfn in files:
print(lfn)
transfers.append(
fts3.new_transfer(apply_tfc_to_lfn(source, lfn, c),
apply_tfc_to_lfn(destination, lfn, c)))
c.close()
# Submit fts job
context = fts3.Context('https://fts3.cern.ch:8446',
proxy,
proxy,
verify=True)
print(fts3.delegate(context, lifetime=timedelta(hours=48),
force=False))
job = fts3.new_job(transfers)
#print("Monitor link: https://fts3.cern.ch:8449/fts3/ftsmon/#/job/"+fts3.submit(context, job))
jobid = fts3.submit(context, job)
#for file in (fts3.get_job_status(context, jobid, list_files=True))["files"]:
for key, value in (fts3.get_job_status(context, jobid,
list_files=True)).iteritems():
print key
def _fts_submit_job(source_url, dest_url, src_filenames, dst_filenames,
checksum, overwrite, testing_folder, context, metadata):
"""
https://gitlab.cern.ch/fts/fts-rest/-/blob/develop/src/fts3/rest/client/easy/submission.py#L106
"""
transfers = []
for i in xrange(len(src_filenames)):
source_file = os.path.join(source_url, testing_folder, "src",
src_filenames[i])
dest_file = os.path.join(dest_url, testing_folder, "dest",
dst_filenames[i])
transfer = fts3.new_transfer(source=source_file, destination=dest_file)
transfers.append(transfer)
# create job
job = fts3.new_job(transfers,
verify_checksum=checksum,
overwrite=overwrite,
timeout=3600,
metadata=metadata)
# submit job
while True:
try:
job_id = fts3.submit(context, job)
break
except fts3_client_exceptions.ClientError as e:
_flush_logging_msg(e)
return -1
return job_id
def submitTheFTSJob(ftsFile):
### First way : Random choice of two servers
# ftsServ = random.choice([ftsServ1, ftsServ2])
### Second way : Weighted choice of two servers
# rndValue = random.uniform(0.0,1.0)
# ftsServ = ftsServ1
# if rndValue < 0.7 : ftsServ = ftsServ2
### Third way : Random choice of three servers
fList = [ftsServ1, ftsServ2, ftsServ3]
ftsServ = random.choice(fList)
#
context = fts3.Context(ftsServ)
# Open the file and stop the processing.
listOfPairs[:] = open(ceBase + "DOING/" + ftsFile).read().split("\n")
# listOfPairs[:] = open(ceBase + "TODO/" + ftsFile).read().split("\n")
# All the threading bit is here to check in parallel whether the files we are looking are okay in castor
# Once the function is done, the list "okayFiles" should be filled
transfers = []
if checkStatus:
checkStatusOnCastor()
else:
okayFiles[:] = []
for onePair in listOfPairs:
if len(onePair)<10: continue
(sourceSURL, targetSURL) = onePair.split(" ")
okayFiles.append((sourceSURL, targetSURL))
if len(okayFiles)>0:
for oneSet in okayFiles:
transf = fts3.new_transfer(oneSet[0], oneSet[1])
transfers.append(transf)
job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi
ftsJobID = fts3.submit(context, job)
return ftsJobID, ftsServ
else: # None of the files in this lot were good!
return "-1", "-1"
def submitFTS3( self, pinTime = False ):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR( "FTSJob already has been submitted" )
transfers = []
for ftsFile in self:
trans = fts3.new_transfer( ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum = ftsFile.Checksum,
filesize = ftsFile.Size )
transfers.append( trans )
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job( transfers = transfers, overwrite = True,
source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer )
context = self._fts3context
self.FTSGUID = fts3.submit( context, job )
except Exception, e:
return S_ERROR( "Error at submission: %s" % e )
def submitFTS3(self, pinTime=False):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR("FTSJob already has been submitted")
transfers = []
for ftsFile in self:
trans = fts3.new_transfer(ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum=ftsFile.Checksum,
filesize=ftsFile.Size)
transfers.append(trans)
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=dest_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3)
try:
context = fts3.Context(self.FTSServer)
self.FTSGUID = fts3.submit(context, job)
except Exception, e:
return S_ERROR("Error at submission: %s" % e)
def run(self):
"""
"""
self.threadLock.acquire()
self.log.info("Processing transfers from: %s" % self.source)
# create destination and source pfns for job
transfers = []
for lfn in self.files:
transfers.append(fts3.new_transfer(lfn[0],
lfn[1],
metadata={'oracleId': lfn[2]}
)
)
self.log.info("Submitting %s transfers to FTS server" % len(self.files))
# Submit fts job
job = fts3.new_job(transfers,
overwrite=True,
verify_checksum=True,
metadata={"issuer": "ASO",
"userDN": self.files[0][4],
"taskname": self.files[0][5]},
copy_pin_lifetime=-1,
bring_online=None,
source_spacetoken=None,
spacetoken=None,
# max time for job in the fts queue in seconds.
# Usually, it should take O(s) for healthy situations
max_time_in_queue=600,
retry=3,
# seconds after which the transfer is retried
# reduced under FTS suggestion w.r.t. the 3hrs of asov1
retry_delay=600
# timeout on the single transfer process
# TODO: not clear if we may need it
# timeout = 1300
)
jobid = fts3.submit(self.ftsContext, job)
self.jobids.append(jobid)
# TODO: manage exception here, what we should do?
fileDoc = dict()
fileDoc['asoworker'] = 'asoless'
fileDoc['subresource'] = 'updateTransfers'
fileDoc['list_of_ids'] = [x[2] for x in self.files]
fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files]
fileDoc['list_of_fts_instance'] = ['https://fts3.cern.ch:8446/' for _ in self.files]
fileDoc['list_of_fts_id'] = [jobid for _ in self.files]
self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids'])))
self.toUpdate.append(fileDoc)
self.threadLock.release()
def submitTheFTSJob(ftsFile):
### First way : Random choice of two servers
# ftsServ = random.choice([ftsServ1, ftsServ2])
### Second way : Weighted choice of two servers
# rndValue = random.uniform(0.0,1.0)
# ftsServ = ftsServ1
# if rndValue < 0.7 : ftsServ = ftsServ2
### Third way : Random choice of three servers
fList = [ftsServ1, ftsServ2, ftsServ3]
ftsServ = random.choice(fList)
#
context = fts3.Context(ftsServ)
filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n")
transfers = []
for ftra in filecontent:
if len(ftra) < 10: continue
(sourceSURL, targetSURL) = ftra.split(" ")
comm = "gfal-stat " + sourceSURL
runComm = subprocess.Popen(comm,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
theInfo = runComm.communicate()[1].strip()
if theInfo.startswith(
"gfal-stat error: 2 (No such file or directory)"):
bFTS = open(ceBase + "DONE/badFileList.txt", "a")
bFTS.write(ftra + "\n")
bFTS.close()
else:
transf = fts3.new_transfer(sourceSURL, targetSURL)
transfers.append(transf)
# transf = fts3.new_transfer(sourceSURL, targetSURL)
# transfers.append(transf)
if len(transfers) > 0:
# job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5)
# job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi
job = fts3.new_job(
transfers=transfers,
overwrite=True,
verify_checksum=True,
reuse=False,
retry=0) # To avoid deleted files snarling up the system for hours
ftsJobID = fts3.submit(context,
job,
delegation_lifetime=fts3.timedelta(hours=72))
return ftsJobID, ftsServ
else:
return "-1", "-1"
def submitFTS3(self, pinTime=False):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR("FTSJob already has been submitted")
transfers = []
for ftsFile in self:
trans = fts3.new_transfer(ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum='ADLER32:%s' % ftsFile.Checksum,
filesize=ftsFile.Size)
transfers.append(trans)
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=dest_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3)
try:
if not self._fts3context:
self._fts3context = fts3.Context(endpoint=self.FTSServer,
request_class=ftsSSLRequest,
verify=False)
context = self._fts3context
self.FTSGUID = fts3.submit(context, job)
except Exception as e:
return S_ERROR("Error at submission: %s" % e)
self.Status = "Submitted"
self._log = gLogger.getSubLogger(
"req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True)
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def submitFTS3( self, pinTime = False ):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR( "FTSJob already has been submitted" )
transfers = []
for ftsFile in self:
trans = fts3.new_transfer( ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum = 'ADLER32:%s'%ftsFile.Checksum,
filesize = ftsFile.Size )
transfers.append( trans )
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job( transfers = transfers, overwrite = True,
source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
context = self._fts3context
self.FTSGUID = fts3.submit( context, job )
except Exception as e:
return S_ERROR( "Error at submission: %s" % e )
self.Status = "Submitted"
self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def submitTheFTSJob(context, ftsFile):
ftsServ = random.choice([ftsServ1, ftsServ2])
with open(ceBase + "DOING/" + ftsFile) as fF:
filecontent = fF.read().split("\n")
# filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n")
transfers = []
for ftra in filecontent:
if len(ftra) < 10: continue
(sourceSURL, targetSURL) = ftra.split(" ")
transf = fts3.new_transfer(sourceSURL, targetSURL)
transfers.append(transf)
if len(transfers) > 0:
# job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5)
job = fts3.new_job(transfers=transfers,
overwrite=True,
verify_checksum=True,
reuse=False,
retry=5) # requested by Andrea Manzi
ftsJobID = fts3.submit(context, job)
return ftsJobID, ftsServ
else:
return "-1", "-1"
# get the last occurrence of auger in the path
lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):])
# note that this path construction is not particularly liable and should be upgraded in the future
transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI)
transferList.append(transfer)
if len(transferList) > 0:
job = fts3.new_job(
transferList,
metadata="Registration of the files transfered by job " + job_id,
overwrite=True)
if options.register == 1 or options.register == 0 and query_yes_no(
"The registration job is ready now, do you wish to submit it?"
):
jobID = fts3.submit(reg_context,
job) if options.reg_endpoint else fts3.submit(
context, job)
print "The registration job ID is " + jobID
with open(options.regJobIdFile, 'a') as f:
f.write('\t' + jobID + '\n')
notFinishedTransfers = [
fts3.new_transfer(f['source_surl'], f['dest_surl'])
for f in job_status['files']
if f['file_state'] in ['FAILED', 'CANCELED'] and f['reason'] !=
'DESTINATION file already exists and overwrite is not enabled'
]
if len(notFinishedTransfers) < 1:
sys.exit(0)
jobChunks = [
def submit(self,
context=None,
ftsServer=None,
ucert=None,
pinTime=36000,
protocols=None):
""" submit the job to the FTS server
Some attributes are expected to be defined for the submission to work:
* type (set by FTS3Operation)
* sourceSE (only for Transfer jobs)
* targetSE
* activity (optional)
* priority (optional)
* username
* userGroup
* filesToSubmit
* operationID (optional, used as metadata for the job)
We also expect the FTSFiles have an ID defined, as it is given as transfer metadata
:param pinTime: Time the file should be pinned on disk (used for transfers and staging)
Used only if he source SE is a tape storage
:param context: fts3 context. If not given, it is created (see ftsServer & ucert param)
:param ftsServer: the address of the fts server to submit to. Used only if context is
not given. if not given either, use the ftsServer object attribute
:param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc)
:param protocols: list of protocols from which we should choose the protocol to use
:returns S_OK([FTSFiles ids of files submitted])
"""
log = gLogger.getSubLogger(
"submit/%s/%s_%s" %
(self.operationID, self.sourceSE, self.targetSE), True)
if not context:
if not ftsServer:
ftsServer = self.ftsServer
context = fts3.Context(endpoint=ftsServer,
ucert=ucert,
request_class=ftsSSLRequest,
verify=False)
# Construct the target SURL
res = self.__fetchSpaceToken(self.targetSE)
if not res['OK']:
return res
target_spacetoken = res['Value']
allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]
if self.type == 'Transfer':
res = self._constructTransferJob(pinTime,
allLFNs,
target_spacetoken,
protocols=protocols)
elif self.type == 'Staging':
res = self._constructStagingJob(pinTime, allLFNs,
target_spacetoken)
# elif self.type == 'Removal':
# res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken)
if not res['OK']:
return res
job, fileIDsInTheJob = res['Value']
setFileIdsInTheJob = set(fileIDsInTheJob)
try:
self.ftsGUID = fts3.submit(context, job)
log.info("Got GUID %s" % self.ftsGUID)
# Only increase the amount of attempt
# if we succeeded in submitting -> no ! Why did I do that ??
for ftsFile in self.filesToSubmit:
ftsFile.attempt += 1
if ftsFile.fileID in setFileIdsInTheJob:
ftsFile.status = 'Submitted'
now = datetime.datetime.utcnow().replace(microsecond=0)
self.submitTime = now
self.lastUpdate = now
self.lastMonitor = now
except FTS3ClientException as e:
log.exception("Error at submission", repr(e))
return S_ERROR("Error at submission: %s" % e)
return S_OK(fileIDsInTheJob)
fs = filename.split('/')
try:
lfcURI = '/'.join(fs[fs.index('grid'):])
except ValueError: # substring 'grid' not found
# get the last occurrence of auger in the path
lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):])
# note that this path construction is not particularly liable and should be upgraded in the future
transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI)
transferList.append(transfer)
if len(transferList) > 0:
job = fts3.new_job(transferList, metadata="Registration of the files transfered by job " + job_id,
overwrite=True)
if options.register == 1 or options.register == 0 and query_yes_no("The registration job is ready now, do you wish to submit it?"):
jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit(context, job)
print "The registration job ID is " + jobID
with open(options.regJobIdFile, 'a') as f:
f.write('\t' + jobID + '\n')
notFinishedTransfers = [fts3.new_transfer(f['source_surl'], f['dest_surl']) for f in job_status['files']
if f['file_state'] in ['FAILED', 'CANCELED']
and f['reason'] != 'DESTINATION file already exists and overwrite is not enabled']
if len(notFinishedTransfers) < 1:
sys.exit(0)
jobChunks = [notFinishedTransfers[i:i + MAX_NUM_OF_TRANSFERS] for i in
xrange(0, len(notFinishedTransfers), MAX_NUM_OF_TRANSFERS)]
jobs = []
for chunkOfTransfers in jobChunks:
transfer = fts3.new_transfer(sourceURI, destinationURI)
if len(lineArr) > 2:
for lineIndex in range(1, len(lineArr)-1):
fts3.add_alternative_source(transfer, lineArr[lineIndex])
transferList.append(transfer)
numOfTransfers += 1
if numOfTransfers > MAX_NUM_OF_TRANSFERS:
numOfTransfers = 0
job = createTransferJob(transferList, filename, str(numOfJobs), options.overwriteFlag)
if options.dryRun:
print json.dumps(job, indent=2)
jobID = 'abc'
else:
jobID = fts3.submit(context, job)
transferJobs.append(jobID)
numOfJobs += 1
transferList = []
# submit the last job
if transferList:
job = createTransferJob(transferList, filename, str(numOfJobs) + " last", options.overwriteFlag)
if options.dryRun:
print json.dumps(job, indent=2)
jobID = 'abc'
else:
jobID = fts3.submit(context, job)
transferJobs.append(jobID)
# if some jobs were submitted, note their IDs
def submitToFTS(logger, ftsContext, files, jobids, toUpdate):
"""
actual FTS job submission
INPUT PARAMS:
:param logger: logging object
:param ftsContext: FTS context
:param files: [
[source_pfn,
dest_pfn,
file oracle id,
source site,
username,
taskname,
file size,
checksum],
...]
OUTPUT PARAMS:
:param jobids: collect the list of job ids when submitted
:param toUpdate: list of oracle ids to update
RETURNS: the submitted jobid (a string)
"""
transfers = []
for lfn in files:
transfers.append(
fts3.new_transfer(lfn[0],
lfn[1],
filesize=lfn[6],
metadata={'oracleId': lfn[2]}))
logger.info("Submitting %s transfers to FTS server" % len(files))
#SB#
for lfn in files:
logger.info("%s %s %s %s", lfn[0], lfn[1], lfn[6],
{'oracleId': lfn[2]})
#SB#
# Submit fts job
job = fts3.new_job(
transfers,
overwrite=True,
verify_checksum=True,
metadata={
"issuer": "ASO",
"userDN": files[0][4],
"taskname": files[0][5]
},
copy_pin_lifetime=-1,
bring_online=None,
source_spacetoken=None,
spacetoken=None,
# max time for job in the FTS queue in hours. From FTS experts in
# https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329
# The max_time_in_queue applies per job, not per retry.
# The max_time_in_queue is a timeout for how much the job can stay in
# a SUBMITTED, ACTIVE or STAGING state.
# When a job's max_time_in_queue is reached, the job and all of its
# transfers not yet in a terminal state are marked as CANCELED
# StefanoB: I see that we hit this at times with 6h, causing job resubmissions,
# so will try to make it longer to give FTS maximum chances within our
# 24h overall limit (which takes care also of non-FTS related issues !)
# ASO transfers never require STAGING so jobs can spend max_time_in_queue only
# as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated)
max_time_in_queue=10,
# from same cern.service-now.com ticket as above:
# The number of retries applies to each transfer within that job.
# A transfer is granted the first execution + number_of_retries.
# E.g.: retry=3 --> first execution + 3 retries
# so retry=3 means each transfer has 4 chances at most during the 6h
# max_time_in_queue
retry=3,
reuse=ftsReuse,
# seconds after which the transfer is retried
# this is a transfer that fails, gets put to SUBMITTED right away,
# but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay
# reduced under FTS suggestion w.r.t. the 3hrs of asov1
# StefanoB: indeed 10 minutes makes much more sense for storage server glitches
retry_delay=600
# timeout on the single transfer process
# TODO: not clear if we may need it
# timeout = 1300
)
jobid = fts3.submit(ftsContext, job)
jobids.append(jobid)
fileDoc = dict()
fileDoc['asoworker'] = asoworker
fileDoc['subresource'] = 'updateTransfers'
fileDoc['list_of_ids'] = [x[2] for x in files]
fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in files]
fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in files]
fileDoc['list_of_fts_id'] = [jobid for _ in files]
logger.info("Will mark as submitted %s files" %
(len(fileDoc['list_of_ids'])))
toUpdate.append(fileDoc)
return jobid
def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protocols=None):
""" submit the job to the FTS server
Some attributes are expected to be defined for the submission to work:
* type (set by FTS3Operation)
* sourceSE (only for Transfer jobs)
* targetSE
* activity (optional)
* priority (optional)
* username
* userGroup
* filesToSubmit
* operationID (optional, used as metadata for the job)
We also expect the FTSFiles have an ID defined, as it is given as transfer metadata
:param pinTime: Time the file should be pinned on disk (used for transfers and staging)
Used only if he source SE is a tape storage
:param context: fts3 context. If not given, it is created (see ftsServer & ucert param)
:param ftsServer: the address of the fts server to submit to. Used only if context is
not given. if not given either, use the ftsServer object attribute
:param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc)
:param protocols: list of protocols from which we should choose the protocol to use
:returns S_OK([FTSFiles ids of files submitted])
"""
log = gLogger.getSubLogger("submit/%s/%s_%s" %
(self.operationID, self.sourceSE, self.targetSE), True)
if not context:
if not ftsServer:
ftsServer = self.ftsServer
context = fts3.Context(
endpoint=ftsServer,
ucert=ucert,
request_class=ftsSSLRequest,
verify=False)
# Construct the target SURL
res = self.__fetchSpaceToken(self.targetSE)
if not res['OK']:
return res
target_spacetoken = res['Value']
allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]
if self.type == 'Transfer':
res = self._constructTransferJob(
pinTime,
allLFNs,
target_spacetoken,
protocols=protocols)
elif self.type == 'Staging':
res = self._constructStagingJob(
pinTime,
allLFNs,
target_spacetoken)
# elif self.type == 'Removal':
# res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken)
if not res['OK']:
return res
job, fileIDsInTheJob = res['Value']
setFileIdsInTheJob = set(fileIDsInTheJob)
try:
self.ftsGUID = fts3.submit(context, job)
log.info("Got GUID %s" % self.ftsGUID)
# Only increase the amount of attempt
# if we succeeded in submitting -> no ! Why did I do that ??
for ftsFile in self.filesToSubmit:
ftsFile.attempt += 1
# This should never happen because a file should be "released"
# first by the previous job.
# But we just print a warning
if ftsFile.ftsGUID is not None:
log.warn(
"FTSFile has a non NULL ftsGUID at job submission time",
"FileID: %s existing ftsGUID: %s" %
(ftsFile.fileID,
ftsFile.ftsGUID))
# `assign` the file to this job
ftsFile.ftsGUID = self.ftsGUID
if ftsFile.fileID in setFileIdsInTheJob:
ftsFile.status = 'Submitted'
now = datetime.datetime.utcnow().replace(microsecond=0)
self.submitTime = now
self.lastUpdate = now
self.lastMonitor = now
except FTS3ClientException as e:
log.exception("Error at submission", repr(e))
return S_ERROR("Error at submission: %s" % e)
return S_OK(fileIDsInTheJob)
def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, ):
""" submit the job to the FTS server
Some attributes are expected to be defined for the submission to work:
* type (set by FTS3Operation)
* sourceSE (only for Transfer jobs)
* targetSE
* activity (optional)
* priority (optional)
* username
* userGroup
* filesToSubmit
* operationID (optional, used as metadata for the job)
We also expect the FTSFiles have an ID defined, as it is given as transfer metadata
:param pinTime: Time the file should be pinned on disk (used for transfers and staging)
Used only if he source SE is a tape storage
:param context: fts3 context. If not given, it is created (see ftsServer & ucert param)
:param ftsServer: the address of the fts server to submit to. Used only if context is
not given. if not given either, use the ftsServer object attribute
:param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc)
:returns S_OK([FTSFiles ids of files submitted])
"""
log = gLogger.getSubLogger("submit/%s/%s_%s" %
(self.operationID, self.sourceSE, self.targetSE), True)
if not context:
if not ftsServer:
ftsServer = self.ftsServer
context = fts3.Context(
endpoint=ftsServer,
ucert=ucert,
request_class=ftsSSLRequest,
verify=False)
# Construct the target SURL
res = self.__fetchSpaceToken(self.targetSE)
if not res['OK']:
return res
target_spacetoken = res['Value']
allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]
failedLFNs = set()
# getting all the target surls
res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs, protocol='srm')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].iteritems():
failedLFNs.add(lfn)
log.error("Could not get target SURL", "%s %s" % (lfn, reason))
allTargetSURLs = res['Value']['Successful']
if self.type == 'Transfer':
res = self._constructTransferJob(
context,
pinTime,
allTargetSURLs,
failedLFNs,
target_spacetoken)
elif self.type == 'Staging':
res = self._constructStagingJob(
context,
pinTime,
allTargetSURLs,
failedLFNs,
target_spacetoken)
elif self.type == 'Removal':
res = self._constructRemovalJob(context, allTargetSURLs, failedLFNs, target_spacetoken)
if not res['OK']:
return res
job, fileIDsInTheJob = res['Value']
setFileIdsInTheJob = set(fileIDsInTheJob)
try:
self.ftsGUID = fts3.submit(context, job)
log.info("Got GUID %s" % self.ftsGUID)
# Only increase the amount of attempt
# if we succeeded in submitting -> no ! Why did I do that ??
for ftsFile in self.filesToSubmit:
ftsFile.attempt += 1
if ftsFile.fileID in setFileIdsInTheJob:
ftsFile.status = 'Submitted'
now = datetime.datetime.utcnow().replace(microsecond=0)
self.submitTime = now
self.lastUpdate = now
self.lastMonitor = now
except FTS3ClientException as e:
log.exception("Error at submission", repr(e))
return S_ERROR("Error at submission: %s" % e)
return S_OK(fileIDsInTheJob)
def run(self):
"""
"""
self.threadLock.acquire()
self.log.info("Processing transfers from: %s" % self.source)
# create destination and source pfns for job
transfers = []
for lfn in self.files:
transfers.append(fts3.new_transfer(lfn[0],
lfn[1],
filesize=lfn[6],
metadata={'oracleId': lfn[2]}
)
)
self.log.info("Submitting %s transfers to FTS server" % len(self.files))
# Submit fts job
job = fts3.new_job(transfers,
overwrite=True,
verify_checksum=True,
metadata={"issuer": "ASO",
"userDN": self.files[0][4],
"taskname": self.files[0][5]},
copy_pin_lifetime=-1,
bring_online=None,
source_spacetoken=None,
spacetoken=None,
# max time for job in the FTS queue in hours. From FTS experts in
# https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329
# The max_time_in_queue applies per job, not per retry.
# The max_time_in_queue is a timeout for how much the job can stay in
# a SUBMITTED, ACTIVE or STAGING state.
# When a job's max_time_in_queue is reached, the job and all of its
# transfers not yet in a terminal state are marked as CANCELED
# StefanoB: I see that we hit this at times with 6h, causing job resubmissions,
# so will try to make it longer to give FTS maximum chances within our
# 24h overall limit (which takes care also of non-FTS related issues !)
# ASO transfers never require STAGING so jobs can spend max_time_in_queue only
# as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated)
max_time_in_queue=10,
# from same cern.service-now.com ticket as above:
# The number of retries applies to each transfer within that job.
# A transfer is granted the first execution + number_of_retries.
# E.g.: retry=3 --> first execution + 3 retries
# so retry=3 means each transfer has 4 chances at most during the 6h
# max_time_in_queue
retry=3,
reuse=ftsReuse,
# seconds after which the transfer is retried
# this is a transfer that fails, gets put to SUBMITTED right away,
# but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay
# reduced under FTS suggestion w.r.t. the 3hrs of asov1
# StefanoB: indeed 10 minutes makes much more sense for storage server glitches
retry_delay=600
# timeout on the single transfer process
# TODO: not clear if we may need it
# timeout = 1300
)
jobid = fts3.submit(self.ftsContext, job)
self.jobids.append(jobid)
# TODO: manage exception here, what we should do?
fileDoc = dict()
fileDoc['asoworker'] = asoworker
fileDoc['subresource'] = 'updateTransfers'
fileDoc['list_of_ids'] = [x[2] for x in self.files]
fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files]
fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in self.files]
fileDoc['list_of_fts_id'] = [jobid for _ in self.files]
self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids'])))
self.toUpdate.append(fileDoc)
self.threadLock.release()
def submit(self, files, job_params, timeout=None):
"""
Submit a transfer to FTS3 via JSON.
:param external_host: FTS server as a string.
:param files: List of dictionary which for a transfer.
:param job_params: Dictionary containing key/value pairs, for all transfers.
:param user_transfer: boolean for user tranfer submission
:returns: FTS transfer identifier.
"""
usercert = self.prepare_credentials(files)
if not usercert:
logging.error('Unable to prepare credentials.')
return None
logging.info('Proxy %s is ready.', usercert)
# FTS3 expects 'davs' as the scheme identifier instead of https
for file in files:
if not file['sources'] or file['sources'] == []:
os.unlink(usercert)
raise Exception('No sources defined')
new_src_urls = []
new_dst_urls = []
for url in file['sources']:
if url.startswith('https'):
new_src_urls.append(':'.join(['davs'] +
url.split(':')[1:]))
else:
new_src_urls.append(url)
for url in file['destinations']:
if url.startswith('https'):
new_dst_urls.append(':'.join(['davs'] +
url.split(':')[1:]))
else:
new_dst_urls.append(url)
file['sources'] = new_src_urls
file['destinations'] = new_dst_urls
transfer_id = None
expected_transfer_id = None
if self.deterministic:
job_params = job_params.copy()
job_params["id_generator"] = "deterministic"
job_params["sid"] = files[0]['metadata']['request_id']
expected_transfer_id = self.__get_deterministic_id(
job_params["sid"])
logging.debug(
"Submit bulk transfers in deterministic mode, sid %s, expected transfer id: %s",
job_params["sid"], expected_transfer_id)
# bulk submission
jobID = None
transfers = []
for _file in files:
for source, destination in zip(_file["sources"],
_file["destinations"]):
transfers.append(
fts.new_transfer(source,
destination,
activity=_file['activity'],
metadata=_file['metadata'],
filesize=_file['filesize'],
checksum=_file['checksum']))
try:
context = fts.Context(self.external_host,
ucert=usercert,
ukey=usercert,
verify=True,
capath='/etc/grid-security/certificates/')
job = fts.new_job(
transfers,
overwrite=job_params['overwrite'],
verify_checksum=job_params['verify_checksum'],
metadata=job_params['job_metadata'],
copy_pin_lifetime=job_params['copy_pin_lifetime'],
bring_online=job_params['bring_online'],
source_spacetoken=None,
spacetoken=None,
priority=job_params['priority'])
# submission for bindings checks the delegation, so the usual delegation part is useless here
jobID = fts.submit(context,
job,
delegation_lifetime=timedelta(hours=12),
delegate_when_lifetime_lt=timedelta(hours=8))
except ServerError:
logging.error("Server side exception during FTS job submission.")
return None
except ClientError:
logging.error("Client side exception during FTS job submission.")
return None
except BadEndpoint:
logging.error("Wrong FTS endpoint: %s", self.external_host)
return None
if jobID:
record_counter(
'transfertool.fts3myproxy.%s.submission.success' %
self.__extract_host(self.external_host), len(files))
transfer_id = jobID
else:
transfer_id = jobID
logging.error("Unexpected failure during FTS job submission.")
record_counter(
'transfertool.fts3myproxy.%s.submission.failure' %
self.__extract_host(self.external_host), len(files))
os.unlink(usercert)
return transfer_id
for lineIndex in range(1, len(lineArr) - 1):
fts3.add_alternative_source(transfer,
lineArr[lineIndex])
transferList.append(transfer)
numOfTransfers += 1
if numOfTransfers > MAX_NUM_OF_TRANSFERS:
numOfTransfers = 0
job = createTransferJob(transferList, filename, str(numOfJobs),
options.overwriteFlag)
if options.dryRun:
print json.dumps(job, indent=2)
jobID = 'abc'
else:
jobID = fts3.submit(context, job)
transferJobs.append(jobID)
numOfJobs += 1
transferList = []
# submit the last job
if transferList:
job = createTransferJob(transferList, filename,
str(numOfJobs) + " last",
options.overwriteFlag)
if options.dryRun:
print json.dumps(job, indent=2)
jobID = 'abc'
else:
jobID = fts3.submit(context, job)
transferJobs.append(jobID)
opts = OptionParser()
opts.add_option('-s', '--endpoint', dest='endpoint', default='https://fts3-pilot.cern.ch:8446')
opts.add_option('--dry-run', dest='dry_run', default=False, action='store_true')
(options, args) = opts.parse_args()
if len(args) < 2:
raise Exception('Need a source and a destination')
source = args[0]
destination = args[1]
checksum = None
if len(args) > 2:
checksum = args[2]
logging.getLogger('fts3.rest.client').setLevel(logging.DEBUG)
# Build the job
transfer = fts3.new_transfer(source, destination,
checksum=checksum, filesize=None,
metadata='Test submission')
job = fts3.new_job([transfer], verify_checksum=True, metadata='Test job', retry=1, priority =3)
# Submit or just print
if options.dry_run:
print json.dumps(job, indent=2)
else:
context = fts3.Context(options.endpoint)
print fts3.submit(context, job)
