def submitFTS3( self, pinTime = False ):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR( "FTSJob already has been submitted" )
transfers = []
for ftsFile in self:
trans = fts3.new_transfer( ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum = ftsFile.Checksum,
filesize = ftsFile.Size )
transfers.append( trans )
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job( transfers = transfers, overwrite = True,
source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer )
context = self._fts3context
self.FTSGUID = fts3.submit( context, job )
except Exception, e:
return S_ERROR( "Error at submission: %s" % e )
def submit(proxy, toTrans, source, destination):
# prepare rest job with 200 files per job
transfers = []
for files in chunks(toTrans, 200):
c = pycurl.Curl()
# create destination and source pfns for job
for lfn in files:
print(lfn)
transfers.append(
fts3.new_transfer(apply_tfc_to_lfn(source, lfn, c),
apply_tfc_to_lfn(destination, lfn, c)))
c.close()
# Submit fts job
context = fts3.Context('https://fts3.cern.ch:8446',
proxy,
proxy,
verify=True)
print(fts3.delegate(context, lifetime=timedelta(hours=48),
force=False))
job = fts3.new_job(transfers)
#print("Monitor link: https://fts3.cern.ch:8449/fts3/ftsmon/#/job/"+fts3.submit(context, job))
jobid = fts3.submit(context, job)
#for file in (fts3.get_job_status(context, jobid, list_files=True))["files"]:
for key, value in (fts3.get_job_status(context, jobid,
list_files=True)).iteritems():
print key
def _fts_submit_job(source_url, dest_url, src_filenames, dst_filenames,
checksum, overwrite, testing_folder, context, metadata):
"""
https://gitlab.cern.ch/fts/fts-rest/-/blob/develop/src/fts3/rest/client/easy/submission.py#L106
"""
transfers = []
for i in xrange(len(src_filenames)):
source_file = os.path.join(source_url, testing_folder, "src",
src_filenames[i])
dest_file = os.path.join(dest_url, testing_folder, "dest",
dst_filenames[i])
transfer = fts3.new_transfer(source=source_file, destination=dest_file)
transfers.append(transfer)
# create job
job = fts3.new_job(transfers,
verify_checksum=checksum,
overwrite=overwrite,
timeout=3600,
metadata=metadata)
# submit job
while True:
try:
job_id = fts3.submit(context, job)
break
except fts3_client_exceptions.ClientError as e:
_flush_logging_msg(e)
return -1
return job_id
def submitTheFTSJob(ftsFile):
### First way : Random choice of two servers
# ftsServ = random.choice([ftsServ1, ftsServ2])
### Second way : Weighted choice of two servers
# rndValue = random.uniform(0.0,1.0)
# ftsServ = ftsServ1
# if rndValue < 0.7 : ftsServ = ftsServ2
### Third way : Random choice of three servers
fList = [ftsServ1, ftsServ2, ftsServ3]
ftsServ = random.choice(fList)
#
context = fts3.Context(ftsServ)
# Open the file and stop the processing.
listOfPairs[:] = open(ceBase + "DOING/" + ftsFile).read().split("\n")
# listOfPairs[:] = open(ceBase + "TODO/" + ftsFile).read().split("\n")
# All the threading bit is here to check in parallel whether the files we are looking are okay in castor
# Once the function is done, the list "okayFiles" should be filled
transfers = []
if checkStatus:
checkStatusOnCastor()
else:
okayFiles[:] = []
for onePair in listOfPairs:
if len(onePair)<10: continue
(sourceSURL, targetSURL) = onePair.split(" ")
okayFiles.append((sourceSURL, targetSURL))
if len(okayFiles)>0:
for oneSet in okayFiles:
transf = fts3.new_transfer(oneSet[0], oneSet[1])
transfers.append(transf)
job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi
ftsJobID = fts3.submit(context, job)
return ftsJobID, ftsServ
else: # None of the files in this lot were good!
return "-1", "-1"
def get_transfer_status(self, batch_id): #override
if self.server_id == 0:
self._set_server_id()
results = self._get_status(batch_id, 'transfer')
staged_tasks = []
for task_id, status, exitcode, msg, start_time, finish_time in self._get_status(batch_id, 'staging'):
if status == FileQuery.STAT_DONE:
staged_tasks.append(task_id)
results.append((task_id, FileQuery.STAT_QUEUED, -1, None, None, None))
else:
# these tasks won't appear in results from _get_status('transfer')
# because no transfer jobs have been submitted yet
results.append((task_id, status, exitcode, None, start_time, finish_time))
if len(staged_tasks) != 0:
transfers = []
pfn_to_tid = {}
for task_id, source_pfn, dest_pfn, checksum, filesize in self.db.select_many('fts_staging_queue', ('id', 'source', 'destination', 'checksum', 'size'), 'id', staged_tasks):
transfers.append(fts3.new_transfer(source_pfn, dest_pfn, checksum = checksum, filesize = filesize))
pfn_to_tid[dest_pfn] = task_id
if self.checksum_algorithm:
verify_checksum = 'target'
else:
verify_checksum = None
job = fts3.new_job(transfers, retry = self.fts_retry, overwrite = False, verify_checksum = verify_checksum, metadata = self.metadata_string)
success = self._submit_job(job, 'transfer', batch_id, pfn_to_tid)
if success and not self._read_only:
self.db.delete_many('fts_staging_queue', 'id', pfn_to_tid.values())
return results
def submitFTS3(self, pinTime=False):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR("FTSJob already has been submitted")
transfers = []
for ftsFile in self:
trans = fts3.new_transfer(ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum=ftsFile.Checksum,
filesize=ftsFile.Size)
transfers.append(trans)
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=dest_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3)
try:
context = fts3.Context(self.FTSServer)
self.FTSGUID = fts3.submit(context, job)
except Exception, e:
return S_ERROR("Error at submission: %s" % e)
def run(self):
"""
"""
self.threadLock.acquire()
self.log.info("Processing transfers from: %s" % self.source)
# create destination and source pfns for job
transfers = []
for lfn in self.files:
transfers.append(fts3.new_transfer(lfn[0],
lfn[1],
metadata={'oracleId': lfn[2]}
)
)
self.log.info("Submitting %s transfers to FTS server" % len(self.files))
# Submit fts job
job = fts3.new_job(transfers,
overwrite=True,
verify_checksum=True,
metadata={"issuer": "ASO",
"userDN": self.files[0][4],
"taskname": self.files[0][5]},
copy_pin_lifetime=-1,
bring_online=None,
source_spacetoken=None,
spacetoken=None,
# max time for job in the fts queue in seconds.
# Usually, it should take O(s) for healthy situations
max_time_in_queue=600,
retry=3,
# seconds after which the transfer is retried
# reduced under FTS suggestion w.r.t. the 3hrs of asov1
retry_delay=600
# timeout on the single transfer process
# TODO: not clear if we may need it
# timeout = 1300
)
jobid = fts3.submit(self.ftsContext, job)
self.jobids.append(jobid)
# TODO: manage exception here, what we should do?
fileDoc = dict()
fileDoc['asoworker'] = 'asoless'
fileDoc['subresource'] = 'updateTransfers'
fileDoc['list_of_ids'] = [x[2] for x in self.files]
fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files]
fileDoc['list_of_fts_instance'] = ['https://fts3.cern.ch:8446/' for _ in self.files]
fileDoc['list_of_fts_id'] = [jobid for _ in self.files]
self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids'])))
self.toUpdate.append(fileDoc)
self.threadLock.release()
def createTransferJob(transfers, lib, num, overwrite):
label = 'Library %s part %s' % (lib, num)
newjob = fts3.new_job(transfers,
metadata=label,
retry=3,
spacetoken=options.spacetoken,
reuse=False,
verify_checksum=options.computeChecksum,
overwrite=overwrite)
newjob['params']["timeout"] = 7200
return newjob
def get_transfer_status(self, batch_id): #override
if self.server_id == 0:
self._set_server_id()
results = self._get_status(batch_id, 'transfer')
staged_tasks = []
for task_id, status, exitcode, msg, start_time, finish_time in self._get_status(
batch_id, 'staging'):
if status == FileQuery.STAT_DONE:
staged_tasks.append(task_id)
results.append(
(task_id, FileQuery.STAT_QUEUED, -1, None, None, None))
else:
# these tasks won't appear in results from _get_status('transfer')
# because no transfer jobs have been submitted yet
results.append(
(task_id, status, exitcode, None, start_time, finish_time))
if len(staged_tasks) != 0:
transfers = []
pfn_to_tid = {}
for task_id, source_pfn, dest_pfn, checksum, filesize in self.db.select_many(
'fts_staging_queue',
('id', 'source', 'destination', 'checksum', 'size'), 'id',
staged_tasks):
transfers.append(
fts3.new_transfer(source_pfn,
dest_pfn,
checksum=checksum,
filesize=filesize))
pfn_to_tid[dest_pfn] = task_id
if self.checksum_algorithm:
verify_checksum = 'target'
else:
verify_checksum = None
job = fts3.new_job(transfers,
retry=self.fts_retry,
overwrite=True,
verify_checksum=verify_checksum,
metadata=self.metadata_string)
success = self._submit_job(job, 'transfer', batch_id, pfn_to_tid)
if success and not self._read_only:
self.db.delete_many('fts_staging_queue', 'id',
pfn_to_tid.values())
return results
def get_transfer_status(self, batch_id): #override
if self.server_id == 0:
self._set_server_id()
results = self._get_status(batch_id, 'transfer')
staged_tasks = []
for task_id, status, exitcode, msg, start_time, finish_time in self._get_status(batch_id, 'staging'):
if status == FileQuery.STAT_DONE:
staged_tasks.append(task_id)
results.append((task_id, FileQuery.STAT_QUEUED, -1, None, None, None))
else:
# these tasks won't appear in results from _get_status('transfer')
# because no transfer jobs have been submitted yet
results.append((task_id, status, exitcode, None, start_time, finish_time))
if len(staged_tasks) != 0:
if self.checksum_algorithm:
verify_checksum = 'target'
else:
verify_checksum = None
which_dest = ""
transfers = []
pfn_to_tid = {}
for task_id, source_pfn, dest_pfn, checksum, filesize in self.db.select_many('fts_staging_queue', ('id', 'source', 'destination', 'checksum', 'size'), 'id', staged_tasks):
for s in inventory.sites.itervalues():
if s.backend in dest_pfn and s.backend != "":
which_dest = s
#transfers = []
#pfn_to_tid = {}
transfers.append(fts3.new_transfer(source_pfn, dest_pfn, checksum = checksum, filesize = filesize))
pfn_to_tid[dest_pfn] = task_id
job = fts3.new_job(transfers, retry = self.fts_retry, overwrite = True, verify_checksum = verify_checksum, metadata = self.metadata_string)
#which_dest = ""
#for s in inventory.sites.itervalues():
# if s.backend in dest_pfn and s.backend != "":
# which_dest = s
LOG.info("Submitting job from buffer to site %s with proxy %s." % (which_dest.name, which_dest.x509proxy))
#LOG.info("dest_pfn: %s" % dest_pfn)
success = self._submit_job(job, 'transfer', batch_id, pfn_to_tid, x509=which_dest.x509proxy)
if success and not self._read_only:
self.db.delete_many('fts_staging_queue', 'id', pfn_to_tid.values())
return results
def submitTheFTSJob(ftsFile):
### First way : Random choice of two servers
# ftsServ = random.choice([ftsServ1, ftsServ2])
### Second way : Weighted choice of two servers
# rndValue = random.uniform(0.0,1.0)
# ftsServ = ftsServ1
# if rndValue < 0.7 : ftsServ = ftsServ2
### Third way : Random choice of three servers
fList = [ftsServ1, ftsServ2, ftsServ3]
ftsServ = random.choice(fList)
#
context = fts3.Context(ftsServ)
filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n")
transfers = []
for ftra in filecontent:
if len(ftra) < 10: continue
(sourceSURL, targetSURL) = ftra.split(" ")
comm = "gfal-stat " + sourceSURL
runComm = subprocess.Popen(comm,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
theInfo = runComm.communicate()[1].strip()
if theInfo.startswith(
"gfal-stat error: 2 (No such file or directory)"):
bFTS = open(ceBase + "DONE/badFileList.txt", "a")
bFTS.write(ftra + "\n")
bFTS.close()
else:
transf = fts3.new_transfer(sourceSURL, targetSURL)
transfers.append(transf)
# transf = fts3.new_transfer(sourceSURL, targetSURL)
# transfers.append(transf)
if len(transfers) > 0:
# job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5)
# job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi
job = fts3.new_job(
transfers=transfers,
overwrite=True,
verify_checksum=True,
reuse=False,
retry=0) # To avoid deleted files snarling up the system for hours
ftsJobID = fts3.submit(context,
job,
delegation_lifetime=fts3.timedelta(hours=72))
return ftsJobID, ftsServ
else:
return "-1", "-1"
def submitFTS3(self, pinTime=False):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR("FTSJob already has been submitted")
transfers = []
for ftsFile in self:
trans = fts3.new_transfer(ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum='ADLER32:%s' % ftsFile.Checksum,
filesize=ftsFile.Size)
transfers.append(trans)
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=dest_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3)
try:
if not self._fts3context:
self._fts3context = fts3.Context(endpoint=self.FTSServer,
request_class=ftsSSLRequest,
verify=False)
context = self._fts3context
self.FTSGUID = fts3.submit(context, job)
except Exception as e:
return S_ERROR("Error at submission: %s" % e)
self.Status = "Submitted"
self._log = gLogger.getSubLogger(
"req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True)
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def submitFTS3( self, pinTime = False ):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR( "FTSJob already has been submitted" )
transfers = []
for ftsFile in self:
trans = fts3.new_transfer( ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum = 'ADLER32:%s'%ftsFile.Checksum,
filesize = ftsFile.Size )
transfers.append( trans )
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job( transfers = transfers, overwrite = True,
source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
context = self._fts3context
self.FTSGUID = fts3.submit( context, job )
except Exception as e:
return S_ERROR( "Error at submission: %s" % e )
self.Status = "Submitted"
self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def submitTheFTSJob(context, ftsFile):
ftsServ = random.choice([ftsServ1, ftsServ2])
with open(ceBase + "DOING/" + ftsFile) as fF:
filecontent = fF.read().split("\n")
# filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n")
transfers = []
for ftra in filecontent:
if len(ftra) < 10: continue
(sourceSURL, targetSURL) = ftra.split(" ")
transf = fts3.new_transfer(sourceSURL, targetSURL)
transfers.append(transf)
if len(transfers) > 0:
# job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5)
job = fts3.new_job(transfers=transfers,
overwrite=True,
verify_checksum=True,
reuse=False,
retry=5) # requested by Andrea Manzi
ftsJobID = fts3.submit(context, job)
return ftsJobID, ftsServ
else:
return "-1", "-1"
def submit(self, files, job_params, timeout=None):
"""
Submit a transfer to FTS3 via JSON.
:param external_host: FTS server as a string.
:param files: List of dictionary which for a transfer.
:param job_params: Dictionary containing key/value pairs, for all transfers.
:param user_transfer: boolean for user tranfer submission
:returns: FTS transfer identifier.
"""
usercert = self.prepare_credentials(files)
if not usercert:
logging.error('Unable to prepare credentials.')
return None
logging.info('Proxy %s is ready.', usercert)
# FTS3 expects 'davs' as the scheme identifier instead of https
for file in files:
if not file['sources'] or file['sources'] == []:
os.unlink(usercert)
raise Exception('No sources defined')
new_src_urls = []
new_dst_urls = []
for url in file['sources']:
if url.startswith('https'):
new_src_urls.append(':'.join(['davs'] +
url.split(':')[1:]))
else:
new_src_urls.append(url)
for url in file['destinations']:
if url.startswith('https'):
new_dst_urls.append(':'.join(['davs'] +
url.split(':')[1:]))
else:
new_dst_urls.append(url)
file['sources'] = new_src_urls
file['destinations'] = new_dst_urls
transfer_id = None
expected_transfer_id = None
if self.deterministic:
job_params = job_params.copy()
job_params["id_generator"] = "deterministic"
job_params["sid"] = files[0]['metadata']['request_id']
expected_transfer_id = self.__get_deterministic_id(
job_params["sid"])
logging.debug(
"Submit bulk transfers in deterministic mode, sid %s, expected transfer id: %s",
job_params["sid"], expected_transfer_id)
# bulk submission
jobID = None
transfers = []
for _file in files:
for source, destination in zip(_file["sources"],
_file["destinations"]):
transfers.append(
fts.new_transfer(source,
destination,
activity=_file['activity'],
metadata=_file['metadata'],
filesize=_file['filesize'],
checksum=_file['checksum']))
try:
context = fts.Context(self.external_host,
ucert=usercert,
ukey=usercert,
verify=True,
capath='/etc/grid-security/certificates/')
job = fts.new_job(
transfers,
overwrite=job_params['overwrite'],
verify_checksum=job_params['verify_checksum'],
metadata=job_params['job_metadata'],
copy_pin_lifetime=job_params['copy_pin_lifetime'],
bring_online=job_params['bring_online'],
source_spacetoken=None,
spacetoken=None,
priority=job_params['priority'])
# submission for bindings checks the delegation, so the usual delegation part is useless here
jobID = fts.submit(context,
job,
delegation_lifetime=timedelta(hours=12),
delegate_when_lifetime_lt=timedelta(hours=8))
except ServerError:
logging.error("Server side exception during FTS job submission.")
return None
except ClientError:
logging.error("Client side exception during FTS job submission.")
return None
except BadEndpoint:
logging.error("Wrong FTS endpoint: %s", self.external_host)
return None
if jobID:
record_counter(
'transfertool.fts3myproxy.%s.submission.success' %
self.__extract_host(self.external_host), len(files))
transfer_id = jobID
else:
transfer_id = jobID
logging.error("Unexpected failure during FTS job submission.")
record_counter(
'transfertool.fts3myproxy.%s.submission.failure' %
self.__extract_host(self.external_host), len(files))
os.unlink(usercert)
return transfer_id
def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken):
""" Build a job for staging
Some attributes of the job are expected to be set
* targetSE
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
:param pinTime: pining time in case staging is needed
:param allLFNs: List of LFNs to stage
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(
"constructStagingJob/%s/%s" % (self.operationID, self.targetSE),
True)
transfers = []
fileIDsInTheJob = []
# Set of LFNs for which we did not get an SRM URL
failedLFNs = set()
# getting all the target surls
res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs,
protocol='srm')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].iteritems():
failedLFNs.add(lfn)
log.error("Could not get target SURL", "%s %s" % (lfn, reason))
allTargetSURLs = res['Value']['Successful']
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn]
trans = fts3.new_transfer(sourceSURL,
targetSURL,
checksum='ADLER32:%s' % ftsFile.checksum,
filesize=ftsFile.size,
metadata=getattr(ftsFile, 'fileID'),
activity=self.activity)
transfers.append(trans)
fileIDsInTheJob.append(getattr(ftsFile, 'fileID'))
# If the source is not an tape SE, we should set the
# copy_pin_lifetime and bring_online params to None,
# otherwise they will do an extra useless queue in FTS
sourceIsTape = self.__isTapeSE(self.sourceSE)
copy_pin_lifetime = pinTime if sourceIsTape else None
bring_online = 86400 if sourceIsTape else None
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
'operationID': self.operationID,
'sourceSE': self.sourceSE,
'targetSE': self.targetSE
}
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=target_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
metadata=job_metadata,
priority=self.priority)
return S_OK((job, fileIDsInTheJob))
transferList = []
for filename in transferedFiles:
fs = filename.split('/')
try:
lfcURI = '/'.join(fs[fs.index('grid'):])
except ValueError: # substring 'grid' not found
# get the last occurrence of auger in the path
lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):])
# note that this path construction is not particularly liable and should be upgraded in the future
transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI)
transferList.append(transfer)
if len(transferList) > 0:
job = fts3.new_job(transferList, metadata="Registration of the files transfered by job " + job_id,
overwrite=True)
if options.register == 1 or options.register == 0 and query_yes_no("The registration job is ready now, do you wish to submit it?"):
jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit(context, job)
print "The registration job ID is " + jobID
with open(options.regJobIdFile, 'a') as f:
f.write('\t' + jobID + '\n')
notFinishedTransfers = [fts3.new_transfer(f['source_surl'], f['dest_surl']) for f in job_status['files']
if f['file_state'] in ['FAILED', 'CANCELED']
and f['reason'] != 'DESTINATION file already exists and overwrite is not enabled']
if len(notFinishedTransfers) < 1:
sys.exit(0)
jobChunks = [notFinishedTransfers[i:i + MAX_NUM_OF_TRANSFERS] for i in
xrange(0, len(notFinishedTransfers), MAX_NUM_OF_TRANSFERS)]
def _constructTransferJob(self,
pinTime,
allLFNs,
target_spacetoken,
protocols=None):
""" Build a job for transfer
Some attributes of the job are expected to be set
* sourceSE
* targetSE
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
:param pinTime: pining time in case staging is needed
:param allLFNs: list of LFNs to transfer
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:param protocols: list of protocols to restrict the protocol choice for the transfer
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(
"constructTransferJob/%s/%s_%s" %
(self.operationID, self.sourceSE, self.targetSE), True)
res = self.__fetchSpaceToken(self.sourceSE, self.vo)
if not res['OK']:
return res
source_spacetoken = res['Value']
failedLFNs = set()
dstSE = StorageElement(self.targetSE, vo=self.vo)
srcSE = StorageElement(self.sourceSE, vo=self.vo)
# If the source is not a tape SE, we should set the
# copy_pin_lifetime and bring_online params to None,
# otherwise they will do an extra useless queue in FTS
sourceIsTape = self.__isTapeSE(self.sourceSE, self.vo)
copy_pin_lifetime = pinTime if sourceIsTape else None
bring_online = BRING_ONLINE_TIMEOUT if sourceIsTape else None
# getting all the (source, dest) surls
res = dstSE.generateTransferURLsBetweenSEs(allLFNs,
srcSE,
protocols=protocols)
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
failedLFNs.add(lfn)
log.error("Could not get source SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs = res['Value']['Successful']
# This contains the staging URLs if they are different from the transfer URLs
# (CTA...)
allStageURLs = dict()
# In case we are transfering from a tape system, and the stage protocol
# is not the same as the transfer protocol, we generate the staging URLs
# to do a multihop transfer. See below.
if sourceIsTape:
srcProto, _destProto = res['Value']['Protocols']
if srcProto not in srcSE.localStageProtocolList:
# As of version 3.10, FTS can only handle one file per multi hop
# job. If we are here, that means that we need one, so make sure that
# we only have a single file to transfer (this should have been checked
# at the job construction step in FTS3Operation).
# This test is important, because multiple files would result in the source
# being deleted !
if len(allLFNs) != 1:
log.debug(
"Multihop job has %s files while only 1 allowed" %
len(allLFNs))
return S_ERROR(
errno.E2BIG,
"Trying multihop job with more than one file !")
res = srcSE.getURL(allSrcDstSURLs,
protocol=srcSE.localStageProtocolList)
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
failedLFNs.add(lfn)
log.error("Could not get stage SURL",
"%s %s" % (lfn, reason))
allSrcDstSURLs.pop(lfn)
allStageURLs = res['Value']['Successful']
transfers = []
fileIDsInTheJob = []
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn]
stageURL = allStageURLs.get(ftsFile.lfn)
if sourceSURL == targetSURL:
log.error("sourceSURL equals to targetSURL",
"%s" % ftsFile.lfn)
ftsFile.error = "sourceSURL equals to targetSURL"
ftsFile.status = 'Defunct'
continue
ftsFileID = getattr(ftsFile, 'fileID')
# Under normal circumstances, we simply submit an fts transfer as such:
# * srcProto://myFile -> destProto://myFile
#
# Even in case of the source storage being a tape system, it works fine.
# However, if the staging and transfer protocols are different (which might be the case for CTA),
# we use the multihop machinery to submit two sequential fts transfers:
# one to stage, one to transfer.
# It looks like such
# * stageProto://myFile -> stageProto://myFile
# * srcProto://myFile -> destProto://myFile
if stageURL:
# We do not set a fileID in the metadata
# such that we do not update the DB when monitoring
stageTrans_metadata = {'desc': 'PreStage %s' % ftsFileID}
stageTrans = fts3.new_transfer(stageURL,
stageURL,
checksum='ADLER32:%s' %
ftsFile.checksum,
filesize=ftsFile.size,
metadata=stageTrans_metadata,
activity=self.activity)
transfers.append(stageTrans)
trans_metadata = {
'desc': 'Transfer %s' % ftsFileID,
'fileID': ftsFileID
}
trans = fts3.new_transfer(sourceSURL,
targetSURL,
checksum='ADLER32:%s' % ftsFile.checksum,
filesize=ftsFile.size,
metadata=trans_metadata,
activity=self.activity)
transfers.append(trans)
fileIDsInTheJob.append(ftsFileID)
if not transfers:
log.error("No transfer possible!")
return S_ERROR("No transfer possible")
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
'operationID': self.operationID,
'rmsReqID': self.rmsReqID,
'sourceSE': self.sourceSE,
'targetSE': self.targetSE
}
job = fts3.new_job(
transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
verify_checksum=
'target', # Only check target vs specified, since we verify the source earlier
multihop=bool(
allStageURLs), # if we have stage urls, then we need multihop
metadata=job_metadata,
priority=self.priority)
return S_OK((job, fileIDsInTheJob))
def createTransferJob(transfers, lib, num, overwrite):
label = 'Library %s part %s' % (lib, num)
newjob = fts3.new_job(transfers, metadata=label, retry=3, spacetoken=options.spacetoken, reuse=False,
verify_checksum=options.computeChecksum, overwrite=overwrite)
newjob['params']["timeout"] = 7200
return newjob
def submitToFTS(logger, ftsContext, files, jobids, toUpdate):
"""
actual FTS job submission
INPUT PARAMS:
:param logger: logging object
:param ftsContext: FTS context
:param files: [
[source_pfn,
dest_pfn,
file oracle id,
source site,
username,
taskname,
file size,
checksum],
...]
OUTPUT PARAMS:
:param jobids: collect the list of job ids when submitted
:param toUpdate: list of oracle ids to update
RETURNS: the submitted jobid (a string)
"""
transfers = []
for lfn in files:
transfers.append(
fts3.new_transfer(lfn[0],
lfn[1],
filesize=lfn[6],
metadata={'oracleId': lfn[2]}))
logger.info("Submitting %s transfers to FTS server" % len(files))
#SB#
for lfn in files:
logger.info("%s %s %s %s", lfn[0], lfn[1], lfn[6],
{'oracleId': lfn[2]})
#SB#
# Submit fts job
job = fts3.new_job(
transfers,
overwrite=True,
verify_checksum=True,
metadata={
"issuer": "ASO",
"userDN": files[0][4],
"taskname": files[0][5]
},
copy_pin_lifetime=-1,
bring_online=None,
source_spacetoken=None,
spacetoken=None,
# max time for job in the FTS queue in hours. From FTS experts in
# https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329
# The max_time_in_queue applies per job, not per retry.
# The max_time_in_queue is a timeout for how much the job can stay in
# a SUBMITTED, ACTIVE or STAGING state.
# When a job's max_time_in_queue is reached, the job and all of its
# transfers not yet in a terminal state are marked as CANCELED
# StefanoB: I see that we hit this at times with 6h, causing job resubmissions,
# so will try to make it longer to give FTS maximum chances within our
# 24h overall limit (which takes care also of non-FTS related issues !)
# ASO transfers never require STAGING so jobs can spend max_time_in_queue only
# as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated)
max_time_in_queue=10,
# from same cern.service-now.com ticket as above:
# The number of retries applies to each transfer within that job.
# A transfer is granted the first execution + number_of_retries.
# E.g.: retry=3 --> first execution + 3 retries
# so retry=3 means each transfer has 4 chances at most during the 6h
# max_time_in_queue
retry=3,
reuse=ftsReuse,
# seconds after which the transfer is retried
# this is a transfer that fails, gets put to SUBMITTED right away,
# but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay
# reduced under FTS suggestion w.r.t. the 3hrs of asov1
# StefanoB: indeed 10 minutes makes much more sense for storage server glitches
retry_delay=600
# timeout on the single transfer process
# TODO: not clear if we may need it
# timeout = 1300
)
jobid = fts3.submit(ftsContext, job)
jobids.append(jobid)
fileDoc = dict()
fileDoc['asoworker'] = asoworker
fileDoc['subresource'] = 'updateTransfers'
fileDoc['list_of_ids'] = [x[2] for x in files]
fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in files]
fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in files]
fileDoc['list_of_fts_id'] = [jobid for _ in files]
logger.info("Will mark as submitted %s files" %
(len(fileDoc['list_of_ids'])))
toUpdate.append(fileDoc)
return jobid
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None):
"""Build a job for transfer
Some attributes of the job are expected to be set
* sourceSE
* targetSE
* multiHopSE (optional)
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
Note that, because of FTS limitations (and also because it anyway would be "not very smart"),
multiHop can only use non-SRM disk storage as hops.
:param pinTime: pining time in case staging is needed
:param allLFNs: list of LFNs to transfer
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:param protocols: list of protocols to restrict the protocol choice for the transfer
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}")
isMultiHop = False
# Check if it is a multiHop transfer
if self.multiHopSE:
if len(allLFNs) != 1:
log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs))
return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !")
allHops = [(self.sourceSE, self.multiHopSE), (self.multiHopSE, self.targetSE)]
isMultiHop = True
else:
allHops = [(self.sourceSE, self.targetSE)]
nbOfHops = len(allHops)
res = self.__fetchSpaceToken(self.sourceSE, self.vo)
if not res["OK"]:
return res
source_spacetoken = res["Value"]
failedLFNs = set()
copy_pin_lifetime = None
bring_online = None
archive_timeout = None
transfers = []
fileIDsInTheJob = set()
for hopId, (hopSrcSEName, hopDstSEName) in enumerate(allHops, start=1):
# Again, this is relevant only for the very initial source
# but code factorization is more important
hopSrcIsTape = self.__isTapeSE(hopSrcSEName, self.vo)
dstSE = StorageElement(hopDstSEName, vo=self.vo)
srcSE = StorageElement(hopSrcSEName, vo=self.vo)
# getting all the (source, dest) surls
res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols)
if not res["OK"]:
return res
for lfn, reason in res["Value"]["Failed"].items():
failedLFNs.add(lfn)
log.error("Could not get source SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs = res["Value"]["Successful"]
srcProto, destProto = res["Value"]["Protocols"]
# If the source is a tape SE, we should set the
# copy_pin_lifetime and bring_online params
# In case of multihop, this is relevant only for the
# original source, but again, code factorization is more important
if hopSrcIsTape:
copy_pin_lifetime = pinTime
bring_online = srcSE.options.get("BringOnlineTimeout", BRING_ONLINE_TIMEOUT)
# If the destination is a tape, and the protocol supports it,
# check if we want to have an archive timeout
# In case of multihop, this is relevant only for the
# final target, but again, code factorization is more important
dstIsTape = self.__isTapeSE(hopDstSEName, self.vo)
if dstIsTape and destProto in dstSE.localStageProtocolList:
archive_timeout = dstSE.options.get("ArchiveTimeout")
# This contains the staging URLs if they are different from the transfer URLs
# (CTA...)
allStageURLs = dict()
# In case we are transfering from a tape system, and the stage protocol
# is not the same as the transfer protocol, we generate the staging URLs
# to do a multihop transfer. See below.
if hopSrcIsTape and srcProto not in srcSE.localStageProtocolList:
isMultiHop = True
# As of version 3.10, FTS can only handle one file per multi hop
# job. If we are here, that means that we need one, so make sure that
# we only have a single file to transfer (this should have been checked
# at the job construction step in FTS3Operation).
# This test is important, because multiple files would result in the source
# being deleted !
if len(allLFNs) != 1:
log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs))
return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !")
res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList)
if not res["OK"]:
return res
for lfn, reason in res["Value"]["Failed"].items():
failedLFNs.add(lfn)
log.error("Could not get stage SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs.pop(lfn)
allStageURLs = res["Value"]["Successful"]
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn]
stageURL = allStageURLs.get(ftsFile.lfn)
if sourceSURL == targetSURL:
log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn)
ftsFile.error = "sourceSURL equals to targetSURL"
ftsFile.status = "Defunct"
continue
ftsFileID = getattr(ftsFile, "fileID")
# Under normal circumstances, we simply submit an fts transfer as such:
# * srcProto://myFile -> destProto://myFile
#
# Even in case of the source storage being a tape system, it works fine.
# However, if the staging and transfer protocols are different (which might be the case for CTA),
# we use the multihop machinery to submit two sequential fts transfers:
# one to stage, one to transfer.
# It looks like such
# * stageProto://myFile -> stageProto://myFile
# * srcProto://myFile -> destProto://myFile
if stageURL:
# We do not set a fileID in the metadata
# such that we do not update the DB when monitoring
stageTrans_metadata = {"desc": "PreStage %s" % ftsFileID}
# If we use an activity, also set it as file metadata
# for WLCG monitoring purposes
# https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14?
if self.activity:
stageTrans_metadata["activity"] = self.activity
stageTrans = fts3.new_transfer(
stageURL,
stageURL,
checksum="ADLER32:%s" % ftsFile.checksum,
filesize=ftsFile.size,
metadata=stageTrans_metadata,
activity=self.activity,
)
transfers.append(stageTrans)
# If it is the last hop only, we set the fileID metadata
# for monitoring
if hopId == nbOfHops:
trans_metadata = {"desc": "Transfer %s" % ftsFileID, "fileID": ftsFileID}
else:
trans_metadata = {"desc": "MultiHop %s" % ftsFileID}
# If we use an activity, also set it as file metadata
# for WLCG monitoring purposes
# https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14?
if self.activity:
trans_metadata["activity"] = self.activity
# because of an xroot bug (https://github.com/xrootd/xrootd/issues/1433)
# the checksum needs to be lowercase. It does not impact the other
# protocol, so it's fine to put it here.
# I only add it in this transfer and not the "staging" one above because it
# impacts only root -> root transfers
trans = fts3.new_transfer(
sourceSURL,
targetSURL,
checksum="ADLER32:%s" % ftsFile.checksum.lower(),
filesize=ftsFile.size,
metadata=trans_metadata,
activity=self.activity,
)
transfers.append(trans)
fileIDsInTheJob.add(ftsFileID)
if not transfers:
log.error("No transfer possible!")
return S_ERROR(errno.ENODATA, "No transfer possible")
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
"operationID": self.operationID,
"rmsReqID": self.rmsReqID,
"sourceSE": self.sourceSE,
"targetSE": self.targetSE,
}
if self.activity:
job_metadata["activity"] = self.activity
job = fts3.new_job(
transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
verify_checksum="target", # Only check target vs specified, since we verify the source earlier
multihop=isMultiHop,
metadata=job_metadata,
priority=self.priority,
archive_timeout=archive_timeout,
)
return S_OK((job, fileIDsInTheJob))
def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken):
""" Build a job for staging
Some attributes of the job are expected to be set
* targetSE
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
:param pinTime: pining time in case staging is needed
:param allLFNs: List of LFNs to stage
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(
"constructStagingJob/%s/%s" %
(self.operationID, self.targetSE), True)
transfers = []
fileIDsInTheJob = []
# Set of LFNs for which we did not get an SRM URL
failedLFNs = set()
# getting all the target surls
res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs, protocol='srm')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].iteritems():
failedLFNs.add(lfn)
log.error("Could not get target SURL", "%s %s" % (lfn, reason))
allTargetSURLs = res['Value']['Successful']
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn]
trans = fts3.new_transfer(sourceSURL,
targetSURL,
checksum='ADLER32:%s' % ftsFile.checksum,
filesize=ftsFile.size,
metadata=getattr(ftsFile, 'fileID'),
activity=self.activity)
transfers.append(trans)
fileIDsInTheJob.append(getattr(ftsFile, 'fileID'))
# If the source is not an tape SE, we should set the
# copy_pin_lifetime and bring_online params to None,
# otherwise they will do an extra useless queue in FTS
sourceIsTape = self.__isTapeSE(self.sourceSE)
copy_pin_lifetime = pinTime if sourceIsTape else None
bring_online = 86400 if sourceIsTape else None
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
'operationID': self.operationID,
'sourceSE': self.sourceSE,
'targetSE': self.targetSE}
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=target_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
metadata=job_metadata,
priority=self.priority)
return S_OK((job, fileIDsInTheJob))
opts = OptionParser()
opts.add_option('-s', '--endpoint', dest='endpoint', default='https://fts3-pilot.cern.ch:8446')
opts.add_option('--dry-run', dest='dry_run', default=False, action='store_true')
(options, args) = opts.parse_args()
if len(args) < 2:
raise Exception('Need a source and a destination')
source = args[0]
destination = args[1]
checksum = None
if len(args) > 2:
checksum = args[2]
logging.getLogger('fts3.rest.client').setLevel(logging.DEBUG)
# Build the job
transfer = fts3.new_transfer(source, destination,
checksum=checksum, filesize=None,
metadata='Test submission')
job = fts3.new_job([transfer], verify_checksum=True, metadata='Test job', retry=1, priority =3)
# Submit or just print
if options.dry_run:
print json.dumps(job, indent=2)
else:
context = fts3.Context(options.endpoint)
print fts3.submit(context, job)
def _constructStagingJob(self, context, pinTime, allTargetSURLs, failedLFNs, target_spacetoken):
""" Build a job for staging
Some attributes of the job are expected to be set
* targetSE
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
:param context: fts3 context
:param pinTime: pining time in case staging is needed
:param allTargetSURLs: dict {lfn:surl} for the target
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(
"constructStagingJob/%s/%s" %
(self.operationID, self.targetSE), True)
transfers = []
fileIDsInTheJob = []
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn]
trans = fts3.new_transfer(sourceSURL,
targetSURL,
checksum='ADLER32:%s' % ftsFile.checksum,
filesize=ftsFile.size,
metadata=getattr(ftsFile, 'fileID'),
activity=self.activity)
transfers.append(trans)
fileIDsInTheJob.append(getattr(ftsFile, 'fileID'))
# If the source is not an tape SE, we should set the
# copy_pin_lifetime and bring_online params to None,
# otherwise they will do an extra useless queue in FTS
sourceIsTape = self.__isTapeSE(self.sourceSE)
copy_pin_lifetime = pinTime if sourceIsTape else None
bring_online = 86400 if sourceIsTape else None
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
'operationID': self.operationID,
'sourceSE': self.sourceSE,
'targetSE': self.targetSE}
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=target_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
metadata=job_metadata,
priority=self.priority)
return S_OK((job, fileIDsInTheJob))
def start_transfers(self, batch_id, batch_tasks): #override
result = {}
stage_files = []
transfers = []
s_pfn_to_task = {}
t_pfn_to_task = {}
for task in batch_tasks:
sub = task.subscription
lfn = sub.file.lfn
dest_pfn = sub.destination.to_pfn(lfn, 'gfal2')
source_pfn = task.source.to_pfn(lfn, 'gfal2')
self.x509proxy = sub.destination.x509proxy
if task.source.storage_type == Site.TYPE_MSS:
self.x509proxy = task.source.x509proxy
if dest_pfn is None or source_pfn is None:
# either gfal2 is not supported or lfn could not be mapped
LOG.warning('Could not obtain PFN for %s at %s or %s', lfn, sub.destination.name, task.source.name)
result[task] = False
continue
if self.checksum_algorithm:
checksum = '%s:%s' % (self.checksum_algorithm, str(sub.file.checksum[self.checksum_index]))
verify_checksum = 'target'
else:
checksum = None
verify_checksum = False
if task.source.storage_type == Site.TYPE_MSS:
LOG.debug('Staging %s at %s', lfn, task.source.name)
# need to stage first
stage_files.append((source_pfn, dest_pfn, checksum, sub.file.size))
# task identified by the source PFN
s_pfn_to_task[source_pfn] = task
else:
LOG.info("Here we are")
LOG.info('Submitting transfer of %s from %s to %s to FTS', lfn, task.source.name, sub.destination.name)
transfers.append(fts3.new_transfer(source_pfn, dest_pfn, checksum = checksum, filesize = sub.file.size))
# there should be only one task per destination pfn
t_pfn_to_task[dest_pfn] = task
if len(stage_files) != 0:
LOG.debug('Submit new staging job for %d files', len(stage_files))
job = fts3.new_staging_job([ff[0] for ff in stage_files], bring_online = 36000, metadata = self.metadata_string)
success = self._submit_job(job, 'staging', batch_id, dict((pfn, task.id) for pfn, task in s_pfn_to_task.iteritems()), x509=task.source.x509proxy)
for source_pfn, _, _, _ in stage_files:
result[s_pfn_to_task[source_pfn]] = success
if success and not self._read_only:
LOG.debug('Recording staging queue')
fields = ('id', 'source', 'destination', 'checksum', 'size')
mapping = lambda ff: (s_pfn_to_task[ff[0]].id,) + ff
if not self._read_only:
self.db.insert_many('fts_staging_queue', fields, mapping, stage_files)
if len(transfers) != 0:
LOG.debug('Submit new transfer job for %d files', len(transfers))
LOG.info("Submitting transfer job from disk to site %s with proxy %s." % (sub.destination.name, sub.destination.x509proxy))
job = fts3.new_job(transfers, retry = self.fts_retry, overwrite = True,
verify_checksum = verify_checksum, metadata = self.metadata_string)
success = self._submit_job(job, 'transfer', batch_id, dict((pfn, task.id) for pfn, task in t_pfn_to_task.iteritems()), x509=sub.destination.x509proxy)
for transfer in transfers:
dest_pfn = transfer['destinations'][0]
result[t_pfn_to_task[dest_pfn]] = success
return result
def run(self):
"""
"""
self.threadLock.acquire()
self.log.info("Processing transfers from: %s" % self.source)
# create destination and source pfns for job
transfers = []
for lfn in self.files:
transfers.append(fts3.new_transfer(lfn[0],
lfn[1],
filesize=lfn[6],
metadata={'oracleId': lfn[2]}
)
)
self.log.info("Submitting %s transfers to FTS server" % len(self.files))
# Submit fts job
job = fts3.new_job(transfers,
overwrite=True,
verify_checksum=True,
metadata={"issuer": "ASO",
"userDN": self.files[0][4],
"taskname": self.files[0][5]},
copy_pin_lifetime=-1,
bring_online=None,
source_spacetoken=None,
spacetoken=None,
# max time for job in the FTS queue in hours. From FTS experts in
# https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329
# The max_time_in_queue applies per job, not per retry.
# The max_time_in_queue is a timeout for how much the job can stay in
# a SUBMITTED, ACTIVE or STAGING state.
# When a job's max_time_in_queue is reached, the job and all of its
# transfers not yet in a terminal state are marked as CANCELED
# StefanoB: I see that we hit this at times with 6h, causing job resubmissions,
# so will try to make it longer to give FTS maximum chances within our
# 24h overall limit (which takes care also of non-FTS related issues !)
# ASO transfers never require STAGING so jobs can spend max_time_in_queue only
# as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated)
max_time_in_queue=10,
# from same cern.service-now.com ticket as above:
# The number of retries applies to each transfer within that job.
# A transfer is granted the first execution + number_of_retries.
# E.g.: retry=3 --> first execution + 3 retries
# so retry=3 means each transfer has 4 chances at most during the 6h
# max_time_in_queue
retry=3,
reuse=ftsReuse,
# seconds after which the transfer is retried
# this is a transfer that fails, gets put to SUBMITTED right away,
# but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay
# reduced under FTS suggestion w.r.t. the 3hrs of asov1
# StefanoB: indeed 10 minutes makes much more sense for storage server glitches
retry_delay=600
# timeout on the single transfer process
# TODO: not clear if we may need it
# timeout = 1300
)
jobid = fts3.submit(self.ftsContext, job)
self.jobids.append(jobid)
# TODO: manage exception here, what we should do?
fileDoc = dict()
fileDoc['asoworker'] = asoworker
fileDoc['subresource'] = 'updateTransfers'
fileDoc['list_of_ids'] = [x[2] for x in self.files]
fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files]
fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in self.files]
fileDoc['list_of_fts_id'] = [jobid for _ in self.files]
self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids'])))
self.toUpdate.append(fileDoc)
self.threadLock.release()
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None):
""" Build a job for transfer
Some attributes of the job are expected to be set
* sourceSE
* targetSE
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
:param pinTime: pining time in case staging is needed
:param allLFNs: list of LFNs to transfer
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:param protocols: list of protocols to restrict the protocol choice for the transfer
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(
"constructTransferJob/%s/%s_%s" %
(self.operationID, self.sourceSE, self.targetSE), True)
res = self.__fetchSpaceToken(self.sourceSE)
if not res['OK']:
return res
source_spacetoken = res['Value']
failedLFNs = set()
dstSE = StorageElement(self.targetSE, vo=self.vo)
srcSE = StorageElement(self.sourceSE, vo=self.vo)
# getting all the (source, dest) surls
res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols)
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].iteritems():
failedLFNs.add(lfn)
log.error("Could not get source SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs = res['Value']['Successful']
transfers = []
fileIDsInTheJob = []
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn]
if sourceSURL == targetSURL:
log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn)
ftsFile.error = "sourceSURL equals to targetSURL"
ftsFile.status = 'Defunct'
continue
trans = fts3.new_transfer(sourceSURL,
targetSURL,
checksum='ADLER32:%s' % ftsFile.checksum,
filesize=ftsFile.size,
metadata=getattr(ftsFile, 'fileID'),
activity=self.activity)
transfers.append(trans)
fileIDsInTheJob.append(getattr(ftsFile, 'fileID'))
# If the source is not an tape SE, we should set the
# copy_pin_lifetime and bring_online params to None,
# otherwise they will do an extra useless queue in FTS
sourceIsTape = self.__isTapeSE(self.sourceSE)
copy_pin_lifetime = pinTime if sourceIsTape else None
bring_online = BRING_ONLINE_TIMEOUT if sourceIsTape else None
if not transfers:
log.error("No transfer possible!")
return S_ERROR("No transfer possible")
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
'operationID': self.operationID,
'sourceSE': self.sourceSE,
'targetSE': self.targetSE}
job = fts3.new_job(transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
metadata=job_metadata,
priority=self.priority)
return S_OK((job, fileIDsInTheJob))
def start_transfers(self, batch_id, batch_tasks): #override
result = {}
stage_files = []
transfers = []
s_pfn_to_task = {}
t_pfn_to_task = {}
for task in batch_tasks:
sub = task.subscription
lfn = sub.file.lfn
dest_pfn = sub.destination.to_pfn(lfn, 'gfal2')
source_pfn = task.source.to_pfn(lfn, 'gfal2')
if dest_pfn is None or source_pfn is None:
# either gfal2 is not supported or lfn could not be mapped
LOG.warning('Could not obtain PFN for %s at %s or %s', lfn,
sub.destination.name, task.source.name)
result[task] = False
continue
if self.checksum_algorithm:
checksum = '%s:%s' % (
self.checksum_algorithm,
str(sub.file.checksum[self.checksum_index]))
verify_checksum = 'target'
else:
checksum = None
verify_checksum = False
if task.source.storage_type == Site.TYPE_MSS:
LOG.debug('Staging %s at %s', lfn, task.source.name)
# need to stage first
stage_files.append(
(source_pfn, dest_pfn, checksum, sub.file.size))
# task identified by the source PFN
s_pfn_to_task[source_pfn] = task
else:
LOG.debug('Submitting transfer of %s from %s to %s to FTS',
lfn, task.source.name, sub.destination.name)
transfers.append(
fts3.new_transfer(source_pfn,
dest_pfn,
checksum=checksum,
filesize=sub.file.size))
# there should be only one task per destination pfn
t_pfn_to_task[dest_pfn] = task
if len(stage_files) != 0:
LOG.debug('Submit new staging job for %d files', len(stage_files))
job = fts3.new_staging_job([ff[0] for ff in stage_files],
bring_online=36000,
metadata=self.metadata_string)
success = self._submit_job(
job, 'staging', batch_id,
dict(
(pfn, task.id) for pfn, task in s_pfn_to_task.iteritems()))
for source_pfn, _, _, _ in stage_files:
result[s_pfn_to_task[source_pfn]] = success
if success and not self._read_only:
LOG.debug('Recording staging queue')
fields = ('id', 'source', 'destination', 'checksum', 'size')
mapping = lambda ff: (s_pfn_to_task[ff[0]].id, ) + ff
if not self._read_only:
self.db.insert_many('fts_staging_queue', fields, mapping,
stage_files)
if len(transfers) != 0:
LOG.debug('Submit new transfer job for %d files', len(transfers))
job = fts3.new_job(transfers,
retry=self.fts_retry,
overwrite=True,
verify_checksum=verify_checksum,
metadata=self.metadata_string)
success = self._submit_job(
job, 'transfer', batch_id,
dict(
(pfn, task.id) for pfn, task in t_pfn_to_task.iteritems()))
for transfer in transfers:
dest_pfn = transfer['destinations'][0]
result[t_pfn_to_task[dest_pfn]] = success
return result
transferList = []
for filename in transferedFiles:
fs = filename.split('/')
try:
lfcURI = '/'.join(fs[fs.index('grid'):])
except ValueError: # substring 'grid' not found
# get the last occurrence of auger in the path
lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):])
# note that this path construction is not particularly liable and should be upgraded in the future
transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI)
transferList.append(transfer)
if len(transferList) > 0:
job = fts3.new_job(
transferList,
metadata="Registration of the files transfered by job " + job_id,
overwrite=True)
if options.register == 1 or options.register == 0 and query_yes_no(
"The registration job is ready now, do you wish to submit it?"
):
jobID = fts3.submit(reg_context,
job) if options.reg_endpoint else fts3.submit(
context, job)
print "The registration job ID is " + jobID
with open(options.regJobIdFile, 'a') as f:
f.write('\t' + jobID + '\n')
notFinishedTransfers = [
fts3.new_transfer(f['source_surl'], f['dest_surl'])
for f in job_status['files']
if f['file_state'] in ['FAILED', 'CANCELED'] and f['reason'] !=
'DESTINATION file already exists and overwrite is not enabled'
