def submit_transfer(external_host,
job,
submitter='submitter',
process=0,
thread=0,
timeout=None):
"""
Submit a transfer or staging request
:param external_host: FTS server to submit to.
:param job: Job dictionary.
:param submitter: Name of the submitting entity.
:param process: Process which submits.
:param thread: Thread which submits.
:param timeout: Timeout
"""
# prepare submitting
xfers_ret = {}
try:
for file in job['files']:
file_metadata = file['metadata']
request_id = file_metadata['request_id']
log_str = '%s:%s PREPARING REQUEST %s DID %s:%s TO SUBMITTING STATE PREVIOUS %s FROM %s TO %s USING %s ' % (
process, thread, file_metadata['request_id'],
file_metadata['scope'], file_metadata['name'],
file_metadata['previous_attempt_id']
if 'previous_attempt_id' in file_metadata else None,
file['sources'], file['destinations'], external_host)
xfers_ret[request_id] = {
'state': RequestState.SUBMITTING,
'external_host': external_host,
'external_id': None,
'dest_url': file['destinations'][0]
}
logging.info("%s" % (log_str))
xfers_ret[request_id]['file'] = file
logging.debug("%s:%s start to prepare transfer" % (process, thread))
transfer_core.prepare_sources_for_transfers(xfers_ret)
logging.debug("%s:%s finished to prepare transfer" % (process, thread))
except:
logging.error(
"%s:%s Failed to prepare requests %s state to SUBMITTING(Will not submit jobs but return directly) with error: %s"
% (process, thread, xfers_ret.keys(), traceback.format_exc()))
return
# submit the job
eid = None
try:
ts = time.time()
logging.info("%s:%s About to submit job to %s with timeout %s" %
(process, thread, external_host, timeout))
eid = transfer_core.submit_bulk_transfers(external_host,
files=job['files'],
transfertool='fts3',
job_params=job['job_params'],
timeout=timeout)
duration = time.time() - ts
logging.info("%s:%s Submit job %s to %s in %s seconds" %
(process, thread, eid, external_host, duration))
record_timer(
'daemons.conveyor.%s.submit_bulk_transfer.per_file' % submitter,
(time.time() - ts) * 1000 / len(job['files']))
record_counter('daemons.conveyor.%s.submit_bulk_transfer' % submitter,
len(job['files']))
record_timer(
'daemons.conveyor.%s.submit_bulk_transfer.files' % submitter,
len(job['files']))
except Exception, ex:
logging.error("Failed to submit a job with error %s: %s" %
(str(ex), traceback.format_exc()))
def submit_transfer(external_host, job, submitter='submitter', timeout=None, user_transfer_job=False, logger=logging.log, transfertool=TRANSFER_TOOL):
"""
Submit a transfer or staging request
:param external_host: FTS server to submit to.
:param job: Job dictionary.
:param submitter: Name of the submitting entity.
:param timeout: Timeout
:param user_transfer_job: Parameter for transfer with user credentials
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
"""
# Prepare submitting
xfers_ret = {}
try:
for t_file in job['files']:
file_metadata = t_file['metadata']
request_id = file_metadata['request_id']
log_str = 'PREPARING REQUEST %s DID %s:%s TO SUBMITTING STATE PREVIOUS %s FROM %s TO %s USING %s ' % (file_metadata['request_id'],
file_metadata['scope'],
file_metadata['name'],
file_metadata['previous_attempt_id'] if 'previous_attempt_id' in file_metadata else None,
t_file['sources'],
t_file['destinations'],
external_host)
xfers_ret[request_id] = {'state': RequestState.SUBMITTING, 'external_host': external_host, 'external_id': None, 'dest_url': t_file['destinations'][0]}
logger(logging.INFO, "%s", log_str)
xfers_ret[request_id]['file'] = t_file
logger(logging.DEBUG, 'Start to prepare transfer')
transfer_core.prepare_sources_for_transfers(xfers_ret)
logger(logging.DEBUG, 'Finished to prepare transfer')
except RequestNotFound as error:
logger(logging.ERROR, str(error))
return
except Exception:
logger(logging.ERROR, 'Failed to prepare requests %s state to SUBMITTING (Will not submit jobs but return directly) with error: %s' % (list(xfers_ret.keys())), exc_info=True)
return
# Prepare the dictionary for xfers results
xfers_ret = {}
for t_file in job['files']:
file_metadata = t_file['metadata']
request_id = file_metadata['request_id']
xfers_ret[request_id] = {'scope': file_metadata['scope'],
'name': file_metadata['name'],
'external_host': external_host,
'external_id': None,
'request_type': t_file.get('request_type', None),
'dst_rse': file_metadata.get('dst_rse', None),
'src_rse': file_metadata.get('src_rse', None),
'src_rse_id': file_metadata['src_rse_id'],
'metadata': file_metadata}
# Submit the job
try:
start_time = time.time()
logger(logging.INFO, 'About to submit job to %s with timeout %s' % (external_host, timeout))
# A eid is returned if the job is properly submitted otherwise an exception is raised
eid = transfer_core.submit_bulk_transfers(external_host,
files=job['files'],
transfertool=transfertool,
job_params=job['job_params'],
timeout=timeout,
user_transfer_job=user_transfer_job)
duration = time.time() - start_time
logger(logging.INFO, 'Submit job %s to %s in %s seconds' % (eid, external_host, duration))
record_timer('daemons.conveyor.%s.submit_bulk_transfer.per_file' % submitter, (time.time() - start_time) * 1000 / len(job['files']))
record_counter('daemons.conveyor.%s.submit_bulk_transfer' % submitter, len(job['files']))
record_timer('daemons.conveyor.%s.submit_bulk_transfer.files' % submitter, len(job['files']))
# Update all the requests to SUBMITTED and cancel the transfer job in case the update failed
try:
for t_file in job['files']:
file_metadata = t_file['metadata']
request_id = file_metadata['request_id']
xfers_ret[request_id]['state'] = RequestState.SUBMITTED
xfers_ret[request_id]['external_id'] = eid
logger(logging.INFO, 'COPYING REQUEST %s DID %s:%s USING %s with state(%s) with eid(%s)' % (file_metadata['request_id'], file_metadata['scope'], file_metadata['name'], external_host, RequestState.SUBMITTED, eid))
logger(logging.DEBUG, 'Start to bulk register transfer state for eid %s' % eid)
transfer_core.set_transfers_state(xfers_ret, datetime.datetime.utcnow())
logger(logging.DEBUG, 'Finished to register transfer state',)
except Exception:
logger(logging.ERROR, 'Failed to register transfer state with error', exc_info=True)
try:
logger(logging.INFO, 'Cancel transfer %s on %s', eid, external_host)
request.cancel_request_external_id(eid, external_host)
except Exception:
# The job is still submitted in the file transfer service but the request is not updated. Possibility to have a double submission during the next cycle
logger(logging.ERROR, 'Failed to cancel transfers %s on %s with error' % (eid, external_host), exc_info=True)
# This exception is raised if one job is already submitted for one file
except DuplicateFileTransferSubmission as error:
logger(logging.WARNING, 'Failed to submit a job because of duplicate file : %s', str(error))
logger(logging.INFO, 'Submitting files one by one')
try:
# In this loop we submit the jobs and update the requests state one by one
single_xfers_ret = {}
for t_file in job['files']:
single_xfers_ret = {}
single_xfers_ret[request_id] = xfers_ret[request_id]
file_metadata = t_file['metadata']
request_id = file_metadata['request_id']
start_time = time.time()
logger(logging.INFO, 'About to submit job to %s with timeout %s', external_host, timeout)
eid = transfer_core.submit_bulk_transfers(external_host,
files=[t_file],
transfertool=transfertool,
job_params=job['job_params'],
timeout=timeout,
user_transfer_job=user_transfer_job)
duration = time.time() - start_time
logger(logging.INFO, 'Submit job %s to %s in %s seconds' % (eid, external_host, duration))
record_timer('daemons.conveyor.%s.submit_bulk_transfer.per_file' % submitter, (time.time() - start_time) * 1000)
record_counter('daemons.conveyor.%s.submit_bulk_transfer' % submitter, 1)
record_timer('daemons.conveyor.%s.submit_bulk_transfer.files' % submitter, 1)
single_xfers_ret[request_id]['state'] = RequestState.SUBMITTED
single_xfers_ret[request_id]['external_id'] = eid
logger(logging.INFO, 'COPYING REQUEST %s DID %s:%s USING %s with state(%s) with eid(%s)' % (file_metadata['request_id'], file_metadata['scope'], file_metadata['name'], external_host, RequestState.SUBMITTED, eid))
try:
logger(logging.DEBUG, 'Start to register transfer state')
transfer_core.set_transfers_state(single_xfers_ret, datetime.datetime.utcnow())
logger(logging.DEBUG, 'Finished to register transfer state')
except Exception:
logger(logging.ERROR, 'Failed to register transfer state with error', exc_info=True)
try:
logger(logging.INFO, 'Cancel transfer %s on %s', eid, external_host)
request.cancel_request_external_id(eid, external_host)
except Exception:
logger(logging.ERROR, 'Failed to cancel transfers %s on %s with error' % (eid, external_host), exc_info=True)
except (DuplicateFileTransferSubmission, TransferToolTimeout, TransferToolWrongAnswer, Exception) as error:
request_id = single_xfers_ret.keys()[0]
single_xfers_ret[request_id]['state'] = RequestState.SUBMISSION_FAILED
logger(logging.ERROR, 'Cannot submit the job for request %s : %s', request_id, str(error))
try:
logger(logging.DEBUG, 'Update the transfer state to fail')
transfer_core.set_transfers_state(single_xfers_ret, datetime.datetime.utcnow())
logger(logging.DEBUG, 'Finished to register transfer state')
except Exception:
logger(logging.ERROR, 'Failed to register transfer state with error', exc_info=True)
# No need to cancel the job here
# The following exceptions are raised if the job failed to be submitted
except (TransferToolTimeout, TransferToolWrongAnswer) as error:
logger(logging.ERROR, str(error))
try:
for t_file in job['files']:
file_metadata = t_file['metadata']
request_id = file_metadata['request_id']
xfers_ret[request_id]['state'] = RequestState.SUBMISSION_FAILED
logger(logging.DEBUG, 'Start to register transfer state')
transfer_core.set_transfers_state(xfers_ret, datetime.datetime.utcnow())
logger(logging.DEBUG, 'Finished to register transfer state')
except Exception:
logger(logging.ERROR, 'Failed to register transfer state with error', exc_info=True)
# No need to cancel the job here
except Exception as error:
logger(logging.ERROR, 'Failed to submit a job with error %s', str(error), exc_info=True)
def submit_transfer(external_host,
job,
submitter='submitter',
logging_prepend_str='',
timeout=None,
user_transfer_job=False):
"""
Submit a transfer or staging request
:param external_host: FTS server to submit to.
:param job: Job dictionary.
:param submitter: Name of the submitting entity.
:param logging_prepend_str: String to prepend to the logging
:param timeout: Timeout
:param user_transfer_job: Parameter for transfer with user credentials
"""
prepend_str = ''
if logging_prepend_str:
prepend_str = logging_prepend_str
# prepare submitting
xfers_ret = {}
try:
for file in job['files']:
file_metadata = file['metadata']
request_id = file_metadata['request_id']
log_str = prepend_str + 'PREPARING REQUEST %s DID %s:%s TO SUBMITTING STATE PREVIOUS %s FROM %s TO %s USING %s ' % (
file_metadata['request_id'], file_metadata['scope'],
file_metadata['name'], file_metadata['previous_attempt_id']
if 'previous_attempt_id' in file_metadata else None,
file['sources'], file['destinations'], external_host)
xfers_ret[request_id] = {
'state': RequestState.SUBMITTING,
'external_host': external_host,
'external_id': None,
'dest_url': file['destinations'][0]
}
logging.info("%s" % (log_str))
xfers_ret[request_id]['file'] = file
logging.debug(prepend_str + 'Start to prepare transfer')
transfer_core.prepare_sources_for_transfers(xfers_ret)
logging.debug(prepend_str + 'Finished to prepare transfer')
except Exception:
logging.error(
prepend_str +
'Failed to prepare requests %s state to SUBMITTING(Will not submit jobs but return directly) with error: %s'
% (list(xfers_ret.keys()), traceback.format_exc()))
return
# submit the job
eid = None
try:
start_time = time.time()
logging.info(prepend_str +
'About to submit job to %s with timeout %s' %
(external_host, timeout))
eid = transfer_core.submit_bulk_transfers(
external_host,
files=job['files'],
transfertool='fts3',
job_params=job['job_params'],
timeout=timeout,
user_transfer_job=user_transfer_job)
duration = time.time() - start_time
logging.info(prepend_str + 'Submit job %s to %s in %s seconds' %
(eid, external_host, duration))
record_timer(
'daemons.conveyor.%s.submit_bulk_transfer.per_file' % submitter,
(time.time() - start_time) * 1000 / len(job['files']))
record_counter('daemons.conveyor.%s.submit_bulk_transfer' % submitter,
len(job['files']))
record_timer(
'daemons.conveyor.%s.submit_bulk_transfer.files' % submitter,
len(job['files']))
except Exception as error:
logging.error(prepend_str +
'Failed to submit a job with error %s: %s' %
(str(error), traceback.format_exc()))
# register transfer
xfers_ret = {}
try:
for file in job['files']:
file_metadata = file['metadata']
request_id = file_metadata['request_id']
log_str = prepend_str + 'COPYING REQUEST %s DID %s:%s USING %s' % (
file_metadata['request_id'], file_metadata['scope'],
file_metadata['name'], external_host)
if eid:
xfers_ret[request_id] = {
'scope': file_metadata['scope'],
'name': file_metadata['name'],
'state': RequestState.SUBMITTED,
'external_host': external_host,
'external_id': eid,
'request_type': file.get('request_type', None),
'dst_rse': file_metadata.get('dst_rse', None),
'src_rse': file_metadata.get('src_rse', None),
'src_rse_id': file_metadata['src_rse_id'],
'metadata': file_metadata
}
log_str += 'with state(%s) with eid(%s)' % (
RequestState.SUBMITTED, eid)
logging.info("%s" % (log_str))
else:
xfers_ret[request_id] = {
'scope': file_metadata['scope'],
'name': file_metadata['name'],
'state': RequestState.SUBMISSION_FAILED,
'external_host': external_host,
'external_id': None,
'request_type': file.get('request_type', None),
'dst_rse': file_metadata.get('dst_rse', None),
'src_rse': file_metadata.get('src_rse', None),
'src_rse_id': file_metadata['src_rse_id'],
'metadata': file_metadata
}
log_str += 'with state(%s) with eid(%s)' % (
RequestState.SUBMISSION_FAILED, None)
logging.warn("%s" % (log_str))
logging.debug(prepend_str + 'Start to register transfer state')
transfer_core.set_transfers_state(xfers_ret,
datetime.datetime.utcnow())
logging.debug(prepend_str + 'Finished to register transfer state')
except Exception:
logging.error(prepend_str +
'Failed to register transfer state with error: %s' %
(traceback.format_exc()))
try:
if eid:
logging.info(prepend_str + 'Cancel transfer %s on %s' %
(eid, external_host))
request.cancel_request_external_id(eid, external_host)
except Exception:
logging.error(
prepend_str +
'Failed to cancel transfers %s on %s with error: %s' %
(eid, external_host, traceback.format_exc()))
