def __get_stagein_transfers(total_workers=0,
worker_number=0,
failover_schemes=None,
limit=None,
activity=None,
older_than=None,
rses=None,
mock=False,
schemes=None,
bring_online=43200,
retry_other_fts=False,
session=None):
transfers, reqs_no_source = get_stagein_requests_and_source_replicas(
total_workers=total_workers,
worker_number=worker_number,
limit=limit,
activity=activity,
older_than=older_than,
rses=rses,
mock=mock,
schemes=schemes,
bring_online=bring_online,
retry_other_fts=retry_other_fts,
failover_schemes=failover_schemes,
session=session)
set_requests_state(reqs_no_source, RequestState.NO_SOURCES)
return transfers
def __get_transfers(total_workers=0, worker_number=0, failover_schemes=None, limit=None, activity=None, older_than=None,
rses=None, schemes=None, mock=False, max_sources=4, bring_online=43200,
retry_other_fts=False, transfertool=None, logger=logging.log):
"""
Get transfers to process
:param total_workers: Number of total workers.
:param worker_number: Id of the executing worker.
:param failover_schemes: Failover schemes.
:param limit: Integer of requests to retrieve.
:param activity: Activity to be selected.
:param older_than: Only select requests older than this DateTime.
:param rses: List of rse_id to select requests.
:param schemes: Schemes to process.
:param mock: Mock testing.
:param max_sources: Max sources.
:param bring_online: Bring online timeout.
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:param retry_other_fts: Retry other fts servers if needed
:param transfertool: The transfer tool as specified in rucio.cfg
:returns: List of transfers
"""
transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source = transfer_core.get_transfer_requests_and_source_replicas(total_workers=total_workers,
worker_number=worker_number,
limit=limit,
activity=activity,
older_than=older_than,
rses=rses,
schemes=schemes,
bring_online=bring_online,
retry_other_fts=retry_other_fts,
failover_schemes=failover_schemes,
transfertool=transfertool)
request_core.set_requests_state(reqs_no_source, RequestState.NO_SOURCES, logger=logger)
request_core.set_requests_state(reqs_only_tape_source, RequestState.ONLY_TAPE_SOURCES, logger=logger)
request_core.set_requests_state(reqs_scheme_mismatch, RequestState.MISMATCH_SCHEME, logger=logger)
for request_id in transfers:
sources = transfers[request_id]['sources']
sources = __sort_ranking(sources, logger=logger)
if len(sources) > max_sources:
sources = sources[:max_sources]
if not mock:
transfers[request_id]['sources'] = sources
else:
transfers[request_id]['sources'] = __mock_sources(sources)
# remove link_ranking in the final sources
sources = transfers[request_id]['sources']
transfers[request_id]['sources'] = []
for rse, source_url, source_rse_id, ranking, link_ranking in sources:
transfers[request_id]['sources'].append((rse, source_url, source_rse_id, ranking))
transfers[request_id]['file_metadata']['src_rse'] = sources[0][0]
transfers[request_id]['file_metadata']['src_rse_id'] = sources[0][2]
logger(logging.DEBUG, "Transfer for request(%s): %s", request_id, transfers[request_id])
return transfers
def __get_transfers(process=None,
total_processes=None,
thread=None,
total_threads=None,
failover_schemes=None,
limit=None,
activity=None,
older_than=None,
rses=None,
schemes=None,
mock=False,
max_sources=4,
bring_online=43200,
retry_other_fts=False):
"""
Get transfers to process
:param process: Identifier of the caller process as an integer.
:param total_processes: Maximum number of processes as an integer.
:param thread: Identifier of the caller thread as an integer.
:param total_threads: Maximum number of threads as an integer.
:param failover_schemes: Failover schemes.
:param limit: Integer of requests to retrieve.
:param activity: Activity to be selected.
:param older_than: Only select requests older than this DateTime.
:param rses: List of rse_id to select requests.
:param schemes: Schemes to process.
:param mock: Mock testing.
:param max_sources: Max sources.
:bring_online: Bring online timeout.
:retry_other_fts: Retry other fts servers if needed
:returns: List of transfers
"""
transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source = transfer_core.get_transfer_requests_and_source_replicas(
process=process,
total_processes=total_processes,
thread=thread,
total_threads=total_threads,
limit=limit,
activity=activity,
older_than=older_than,
rses=rses,
schemes=schemes,
bring_online=bring_online,
retry_other_fts=retry_other_fts,
failover_schemes=failover_schemes)
request_core.set_requests_state(reqs_no_source, RequestState.NO_SOURCES)
request_core.set_requests_state(reqs_only_tape_source,
RequestState.ONLY_TAPE_SOURCES)
request_core.set_requests_state(reqs_scheme_mismatch,
RequestState.MISMATCH_SCHEME)
for request_id in transfers:
sources = transfers[request_id]['sources']
sources = __sort_ranking(sources)
if len(sources) > max_sources:
sources = sources[:max_sources]
if not mock:
transfers[request_id]['sources'] = sources
else:
transfers[request_id]['sources'] = __mock_sources(sources)
# remove link_ranking in the final sources
sources = transfers[request_id]['sources']
transfers[request_id]['sources'] = []
for source in sources:
rse, source_url, source_rse_id, ranking, link_ranking = source
transfers[request_id]['sources'].append(
(rse, source_url, source_rse_id, ranking))
transfers[request_id]['file_metadata']['src_rse'] = sources[0][0]
transfers[request_id]['file_metadata']['src_rse_id'] = sources[0][2]
logging.debug("Transfer for request(%s): %s" %
(request_id, transfers[request_id]))
return transfers
