def __get_stagein_transfers(total_workers=0, worker_number=0, failover_schemes=None, limit=None, activity=None, older_than=None, rses=None, mock=False, schemes=None, bring_online=43200, retry_other_fts=False, session=None): transfers, reqs_no_source = get_stagein_requests_and_source_replicas( total_workers=total_workers, worker_number=worker_number, limit=limit, activity=activity, older_than=older_than, rses=rses, mock=mock, schemes=schemes, bring_online=bring_online, retry_other_fts=retry_other_fts, failover_schemes=failover_schemes, session=session) set_requests_state(reqs_no_source, RequestState.NO_SOURCES) return transfers
def __get_transfers(total_workers=0, worker_number=0, failover_schemes=None, limit=None, activity=None, older_than=None, rses=None, schemes=None, mock=False, max_sources=4, bring_online=43200, retry_other_fts=False, transfertool=None, logger=logging.log): """ Get transfers to process :param total_workers: Number of total workers. :param worker_number: Id of the executing worker. :param failover_schemes: Failover schemes. :param limit: Integer of requests to retrieve. :param activity: Activity to be selected. :param older_than: Only select requests older than this DateTime. :param rses: List of rse_id to select requests. :param schemes: Schemes to process. :param mock: Mock testing. :param max_sources: Max sources. :param bring_online: Bring online timeout. :param logger: Optional decorated logger that can be passed from the calling daemons or servers. :param retry_other_fts: Retry other fts servers if needed :param transfertool: The transfer tool as specified in rucio.cfg :returns: List of transfers """ transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source = transfer_core.get_transfer_requests_and_source_replicas(total_workers=total_workers, worker_number=worker_number, limit=limit, activity=activity, older_than=older_than, rses=rses, schemes=schemes, bring_online=bring_online, retry_other_fts=retry_other_fts, failover_schemes=failover_schemes, transfertool=transfertool) request_core.set_requests_state(reqs_no_source, RequestState.NO_SOURCES, logger=logger) request_core.set_requests_state(reqs_only_tape_source, RequestState.ONLY_TAPE_SOURCES, logger=logger) request_core.set_requests_state(reqs_scheme_mismatch, RequestState.MISMATCH_SCHEME, logger=logger) for request_id in transfers: sources = transfers[request_id]['sources'] sources = __sort_ranking(sources, logger=logger) if len(sources) > max_sources: sources = sources[:max_sources] if not mock: transfers[request_id]['sources'] = sources else: transfers[request_id]['sources'] = __mock_sources(sources) # remove link_ranking in the final sources sources = transfers[request_id]['sources'] transfers[request_id]['sources'] = [] for rse, source_url, source_rse_id, ranking, link_ranking in sources: transfers[request_id]['sources'].append((rse, source_url, source_rse_id, ranking)) transfers[request_id]['file_metadata']['src_rse'] = sources[0][0] transfers[request_id]['file_metadata']['src_rse_id'] = sources[0][2] logger(logging.DEBUG, "Transfer for request(%s): %s", request_id, transfers[request_id]) return transfers
def __get_transfers(process=None, total_processes=None, thread=None, total_threads=None, failover_schemes=None, limit=None, activity=None, older_than=None, rses=None, schemes=None, mock=False, max_sources=4, bring_online=43200, retry_other_fts=False): """ Get transfers to process :param process: Identifier of the caller process as an integer. :param total_processes: Maximum number of processes as an integer. :param thread: Identifier of the caller thread as an integer. :param total_threads: Maximum number of threads as an integer. :param failover_schemes: Failover schemes. :param limit: Integer of requests to retrieve. :param activity: Activity to be selected. :param older_than: Only select requests older than this DateTime. :param rses: List of rse_id to select requests. :param schemes: Schemes to process. :param mock: Mock testing. :param max_sources: Max sources. :bring_online: Bring online timeout. :retry_other_fts: Retry other fts servers if needed :returns: List of transfers """ transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source = transfer_core.get_transfer_requests_and_source_replicas( process=process, total_processes=total_processes, thread=thread, total_threads=total_threads, limit=limit, activity=activity, older_than=older_than, rses=rses, schemes=schemes, bring_online=bring_online, retry_other_fts=retry_other_fts, failover_schemes=failover_schemes) request_core.set_requests_state(reqs_no_source, RequestState.NO_SOURCES) request_core.set_requests_state(reqs_only_tape_source, RequestState.ONLY_TAPE_SOURCES) request_core.set_requests_state(reqs_scheme_mismatch, RequestState.MISMATCH_SCHEME) for request_id in transfers: sources = transfers[request_id]['sources'] sources = __sort_ranking(sources) if len(sources) > max_sources: sources = sources[:max_sources] if not mock: transfers[request_id]['sources'] = sources else: transfers[request_id]['sources'] = __mock_sources(sources) # remove link_ranking in the final sources sources = transfers[request_id]['sources'] transfers[request_id]['sources'] = [] for source in sources: rse, source_url, source_rse_id, ranking, link_ranking = source transfers[request_id]['sources'].append( (rse, source_url, source_rse_id, ranking)) transfers[request_id]['file_metadata']['src_rse'] = sources[0][0] transfers[request_id]['file_metadata']['src_rse_id'] = sources[0][2] logging.debug("Transfer for request(%s): %s" % (request_id, transfers[request_id])) return transfers