def test_multihop_requests_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that multihop transfers are handled and intermediate request correctly created """ rs0_name, src_rse_id = rse_factory.make_posix_rse() _, intermediate_rse_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() rse_core.add_rse_attribute(intermediate_rse_id, 'available_for_multihop', True) add_distance(src_rse_id, intermediate_rse_id, ranking=10) add_distance(intermediate_rse_id, dst_rse_id, ranking=10) did = did_factory.upload_test_file(rs0_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) [[_, [transfer]] ] = next_transfers_to_submit(rses=rse_factory.created_rses).items() # the intermediate request was correctly created assert request_core.get_request_by_did(rse_id=intermediate_rse_id, **did)
def test_s3s_fts_src(self): """ S3: TPC a file from S3 to storage """ expected_src_url = 's3s://fake-rucio.s3-eu-south-8.amazonaws.com:443/mock/69/3b/file-on-aws' expected_dst_url = 'https://somestorage.ch:1094/my/prefix/mock/69/3b/file-on-aws' rule_id = add_rule(dids=self.files3, account=self.root, copies=1, rse_expression=self.rsenons3, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) [[_, [transfer_path]] ] = next_transfers_to_submit(rses=[self.rsenons3_id]).items() assert transfer_path[0].rws.rule_id == rule_id[0] assert transfer_path[0].legacy_sources[0][1] == expected_src_url assert transfer_path[0].dest_url == expected_dst_url
def run_once(bulk, group_bulk, rse_ids, scheme, failover_scheme, transfertool_kwargs, heartbeat_handler, activity): worker_number, total_workers, logger = heartbeat_handler.live() start_time = time.time() transfers = next_transfers_to_submit( total_workers=total_workers, worker_number=worker_number, failover_schemes=failover_scheme, limit=bulk, activity=activity, rses=rse_ids, schemes=scheme, transfertools_by_name={'fts3': FTS3Transfertool}, older_than=None, request_type=RequestType.STAGEIN, logger=logger, ) total_transfers = len(list(hop for paths in transfers.values() for path in paths for hop in path)) record_timer('daemons.conveyor.stager.get_stagein_transfers.per_transfer', (time.time() - start_time) * 1000 / (total_transfers if transfers else 1)) record_counter('daemons.conveyor.stager.get_stagein_transfers', total_transfers) record_timer('daemons.conveyor.stager.get_stagein_transfers.transfers', total_transfers) logger(logging.INFO, 'Got %s stagein transfers for %s' % (total_transfers, activity)) for builder, transfer_paths in transfers.items(): transfertool_obj = builder.make_transfertool(logger=logger, **transfertool_kwargs.get(builder.transfertool_class, {})) logger(logging.INFO, 'Starting to group transfers for %s (%s)' % (activity, transfertool_obj)) start_time = time.time() grouped_jobs = transfertool_obj.group_into_submit_jobs(transfer_paths) record_timer('daemons.conveyor.stager.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfer_paths) or 1)) logger(logging.INFO, 'Starting to submit transfers for %s (%s)' % (activity, transfertool_obj)) for job in grouped_jobs: worker_number, total_workers, logger = heartbeat_handler.live() submit_transfer(transfertool_obj=transfertool_obj, transfers=job['transfers'], job_params=job['job_params'], submitter='transfer_submitter', logger=logger) queue_empty = False if total_transfers < group_bulk: queue_empty = True logger(logging.INFO, 'Only %s transfers for %s which is less than group bulk %s' % (total_transfers, activity, group_bulk)) return queue_empty
def run_once(bulk, group_bulk, filter_transfertool, transfertools, ignore_availability, rse_ids, scheme, failover_scheme, partition_hash_var, timeout, transfertool_kwargs, heartbeat_handler, activity): worker_number, total_workers, logger = heartbeat_handler.live() start_time = time.time() transfers = next_transfers_to_submit( total_workers=total_workers, worker_number=worker_number, partition_hash_var=partition_hash_var, failover_schemes=failover_scheme, limit=bulk, activity=activity, rses=rse_ids, schemes=scheme, filter_transfertool=filter_transfertool, transfertool_classes=[ TRANSFERTOOL_CLASSES_BY_NAME[transfertool] for transfertool in transfertools ], older_than=None, request_type=RequestType.TRANSFER, ignore_availability=ignore_availability, logger=logger, ) total_transfers = len( list(hop for paths in transfers.values() for path in paths for hop in path)) record_timer( 'daemons.conveyor.transfer_submitter.get_transfers.per_transfer', (time.time() - start_time) * 1000 / (total_transfers or 1)) GET_TRANSFERS_COUNTER.inc(total_transfers) record_timer('daemons.conveyor.transfer_submitter.get_transfers.transfers', total_transfers) logger(logging.INFO, 'Got %s transfers for %s in %s seconds', total_transfers, activity, time.time() - start_time) for builder, transfer_paths in transfers.items(): # Globus Transfertool is not yet production-ready, but we need to partially activate it # in all submitters if we want to enable native multi-hopping between transfertools. # This "if" can be triggered in a FTS submitter if it tries to multi-hop from # a globus-only RSE via a dual-stack RSE towards an FTS-only RSE. # # Just ignore this transfer and keep it in a queued state, so that it's picked up # latter by that special submitter instance dedicated to globus transfers. # # TODO: remove this "if" if transfertools[ 0] != GlobusTransferTool.external_name and builder.transfertool_class == GlobusTransferTool: logger(logging.INFO, 'Skipping submission of following transfers: %s', [transfer_path_str(p) for p in transfer_paths]) continue transfertool_obj = builder.make_transfertool( logger=logger, **transfertool_kwargs.get(builder.transfertool_class, {})) start_time = time.time() logger(logging.DEBUG, 'Starting to group transfers for %s (%s)', activity, transfertool_obj) grouped_jobs = transfertool_obj.group_into_submit_jobs(transfer_paths) record_timer('daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfer_paths) or 1)) logger(logging.DEBUG, 'Starting to submit transfers for %s (%s)', activity, transfertool_obj) for job in grouped_jobs: worker_number, total_workers, logger = heartbeat_handler.live() logger( logging.DEBUG, 'submitjob: transfers=%s, job_params=%s' % ([str(t) for t in job['transfers']], job['job_params'])) submit_transfer(transfertool_obj=transfertool_obj, transfers=job['transfers'], job_params=job['job_params'], submitter='transfer_submitter', timeout=timeout, logger=logger) queue_empty = False if total_transfers < group_bulk: queue_empty = True logger(logging.DEBUG, 'Only %s transfers for %s which is less than group bulk %s', total_transfers, activity, group_bulk) return queue_empty
def test_singlehop_vs_multihop_priority(rse_factory, root_account, mock_scope, core_config_mock, caches_mock): """ On small distance difference, singlehop is prioritized over multihop due to HOP_PENALTY. On big difference, multihop is prioritized """ # +------+ +------+ # | | 10 | | # | RSE0 +--->| RSE1 | # | | | +-+ 10 # +------+ +------+ | +------+ +------+ # +->| | 200 | | # +------+ | RSE3 |<------| RSE4 | # | | 30 +--->| | | | # | RSE2 +-----------+ +------+ +------+ # | | # +------+ _, rse0_id = rse_factory.make_posix_rse() _, rse1_id = rse_factory.make_posix_rse() _, rse2_id = rse_factory.make_posix_rse() rse3_name, rse3_id = rse_factory.make_posix_rse() _, rse4_id = rse_factory.make_posix_rse() add_distance(rse0_id, rse1_id, ranking=10) add_distance(rse1_id, rse3_id, ranking=10) add_distance(rse2_id, rse3_id, ranking=30) add_distance(rse4_id, rse3_id, ranking=200) rse_core.add_rse_attribute(rse1_id, 'available_for_multihop', True) # add same file to two source RSEs file = { 'scope': mock_scope, 'name': 'lfn.' + generate_uuid(), 'type': 'FILE', 'bytes': 1, 'adler32': 'beefdead' } did = {'scope': file['scope'], 'name': file['name']} for rse_id in [rse0_id, rse2_id]: add_replicas(rse_id=rse_id, files=[file], account=root_account) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=rse3_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # The singlehop must be prioritized [[_, [transfer]] ] = next_transfers_to_submit(rses=rse_factory.created_rses).items() assert len(transfer) == 1 assert transfer[0].src.rse.id == rse2_id assert transfer[0].dst.rse.id == rse3_id # add same file to two source RSEs file = { 'scope': mock_scope, 'name': 'lfn.' + generate_uuid(), 'type': 'FILE', 'bytes': 1, 'adler32': 'beefdead' } did = {'scope': file['scope'], 'name': file['name']} for rse_id in [rse0_id, rse4_id]: add_replicas(rse_id=rse_id, files=[file], account=root_account) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=rse3_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # The multihop must be prioritized [[_, transfers] ] = next_transfers_to_submit(rses=rse_factory.created_rses).items() transfer = next(iter(t for t in transfers if t[0].rws.name == file['name'])) assert len(transfer) == 2
def test_disk_vs_tape_priority(rse_factory, root_account, mock_scope): tape1_rse_name, tape1_rse_id = rse_factory.make_posix_rse( rse_type=RSEType.TAPE) tape2_rse_name, tape2_rse_id = rse_factory.make_posix_rse( rse_type=RSEType.TAPE) disk1_rse_name, disk1_rse_id = rse_factory.make_posix_rse( rse_type=RSEType.DISK) disk2_rse_name, disk2_rse_id = rse_factory.make_posix_rse( rse_type=RSEType.DISK) dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() source_rses = [tape1_rse_id, tape2_rse_id, disk1_rse_id, disk2_rse_id] all_rses = source_rses + [dst_rse_id] add_distance(disk1_rse_id, dst_rse_id, ranking=15) add_distance(disk2_rse_id, dst_rse_id, ranking=10) add_distance(tape1_rse_id, dst_rse_id, ranking=15) add_distance(tape2_rse_id, dst_rse_id, ranking=10) # add same file to all source RSEs file = { 'scope': mock_scope, 'name': 'lfn.' + generate_uuid(), 'type': 'FILE', 'bytes': 1, 'adler32': 'beefdead' } did = {'scope': file['scope'], 'name': file['name']} for rse_id in source_rses: add_replicas(rse_id=rse_id, files=[file], account=root_account) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) @transactional_session def __fake_source_ranking(source_rse_id, new_ranking, session=None): rowcount = session.query(models.Source).filter( models.Source.rse_id == source_rse_id).update( {'ranking': new_ranking}) if not rowcount: models.Source(request_id=request['id'], scope=request['scope'], name=request['name'], rse_id=source_rse_id, dest_rse_id=request['dest_rse_id'], ranking=new_ranking, bytes=request['bytes'], url=None, is_using=False). \ save(session=session, flush=False) # On equal priority and distance, disk should be preferred over tape. Both disk sources will be returned [[_, [transfer]]] = next_transfers_to_submit(rses=all_rses).items() assert len(transfer[0].legacy_sources) == 2 assert transfer[0].legacy_sources[0][0] in (disk1_rse_name, disk2_rse_name) # Change the rating of the disk RSEs. Disk still preferred, because it must fail twice before tape is tried __fake_source_ranking(disk1_rse_id, -1) __fake_source_ranking(disk2_rse_id, -1) [[_, [transfer]]] = next_transfers_to_submit(rses=all_rses).items() assert len(transfer[0].legacy_sources) == 2 assert transfer[0].legacy_sources[0][0] in (disk1_rse_name, disk2_rse_name) # Change the rating of the disk RSEs again. Tape RSEs must now be preferred. # Multiple tape sources are not allowed. Only one tape RSE source must be returned. __fake_source_ranking(disk1_rse_id, -2) __fake_source_ranking(disk2_rse_id, -2) [[_, transfers]] = next_transfers_to_submit(rses=all_rses).items() assert len(transfers) == 1 transfer = transfers[0] assert len(transfer[0].legacy_sources) == 1 assert transfer[0].legacy_sources[0][0] in (tape1_rse_name, tape2_rse_name) # On equal source ranking, but different distance; the smaller distance is preferred [[_, [transfer]]] = next_transfers_to_submit(rses=all_rses).items() assert len(transfer[0].legacy_sources) == 1 assert transfer[0].legacy_sources[0][0] == tape2_rse_name # On different source ranking, the bigger ranking is preferred __fake_source_ranking(tape2_rse_id, -1) [[_, [transfer]]] = next_transfers_to_submit(rses=all_rses).items() assert len(transfer[0].legacy_sources) == 1 assert transfer[0].legacy_sources[0][0] == tape1_rse_name
def test_tpc(containerized_rses, root_account, test_scope, did_factory, rse_client, rule_client, artifact): if len(containerized_rses) < 2: pytest.skip("TPC tests need at least 2 containerized rse's for execution}") rse1_name, rse1_id = containerized_rses[0] rse2_name, rse2_id = containerized_rses[1] base_file_name = generate_uuid() test_file = did_factory.upload_test_file(rse1_name, name=base_file_name + '.000', return_full_item=True) test_file_did_str = '%s:%s' % (test_file['did_scope'], test_file['did_name']) test_file_did = { 'scope': test_scope, 'name': test_file['did_name'] } test_file_name_hash = hashlib.md5(test_file_did_str.encode('utf-8')).hexdigest() test_file_expected_pfn = '%s/%s/%s/%s' % (test_file_did['scope'], test_file_name_hash[0:2], test_file_name_hash[2:4], test_file_did['name']) rse1_hostname = rse_client.get_protocols(rse1_name)[0]['hostname'] rse2_hostname = rse_client.get_protocols(rse2_name)[0]['hostname'] rule_id = add_rule(dids=[test_file_did], account=root_account, copies=1, rse_expression=rse2_name, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) rule = rule_client.get_replication_rule(rule_id[0]) re_evaluator(once=True) assert rule['locks_ok_cnt'] == 0 assert rule['locks_replicating_cnt'] == 1 [[_, [transfer_path]]] = next_transfers_to_submit(rses=[rse1_id, rse2_id]).items() assert transfer_path[0].rws.rule_id == rule_id[0] src_url = transfer_path[0].legacy_sources[0][1] dest_url = transfer_path[0].dest_url check_url(src_url, rse1_hostname, test_file_expected_pfn) check_url(dest_url, rse2_hostname, test_file_expected_pfn) # Run Submitter submitter.submitter(once=True) # Get FTS transfer job id request = get_request_by_did(rse_id=rse2_id, **test_file_did) fts_transfer_id = request['external_id'] # Check FTS transfer job assert fts_transfer_id is not None # Wait for the FTS transfer to finish fts_transfer_status = None for _ in range(MAX_POLL_WAIT_SECONDS): fts_transfer_status = poll_fts_transfer_status(fts_transfer_id) if fts_transfer_status not in ['SUBMITTED', 'ACTIVE']: break time.sleep(1) assert fts_transfer_status == 'FINISHED' poller.run(once=True, older_than=0) finisher.run(once=True) rule = rule_client.get_replication_rule(rule_id[0]) assert rule['locks_ok_cnt'] == 1 assert rule['locks_replicating_cnt'] == 0 if artifact is not None: date = datetime.date.today().strftime("%Y-%m-%d") with open(artifact, 'w') as artifact_file: artifact_file.write( f"/var/log/fts3/{date}/{rse1_name.lower()}__{rse2_name.lower()}/*__{fts_transfer_id}" )
def run_once(bulk, group_bulk, filter_transfertool, transfertool, ignore_availability, rse_ids, scheme, failover_scheme, partition_hash_var, timeout, transfertool_kwargs, heartbeat_handler, activity): worker_number, total_workers, logger = heartbeat_handler.live() start_time = time.time() transfers = next_transfers_to_submit( total_workers=total_workers, worker_number=worker_number, partition_hash_var=partition_hash_var, failover_schemes=failover_scheme, limit=bulk, activity=activity, rses=rse_ids, schemes=scheme, filter_transfertool=filter_transfertool, transfertools_by_name={ transfertool: TRANSFERTOOL_CLASSES_BY_NAME[transfertool] }, older_than=None, request_type=RequestType.TRANSFER, ignore_availability=ignore_availability, logger=logger, ) total_transfers = len( list(hop for paths in transfers.values() for path in paths for hop in path)) record_timer( 'daemons.conveyor.transfer_submitter.get_transfers.per_transfer', (time.time() - start_time) * 1000 / (total_transfers or 1)) GET_TRANSFERS_COUNTER.inc(total_transfers) record_timer('daemons.conveyor.transfer_submitter.get_transfers.transfers', total_transfers) logger(logging.INFO, 'Got %s transfers for %s in %s seconds', total_transfers, activity, time.time() - start_time) for builder, transfer_paths in transfers.items(): transfertool_obj = builder.make_transfertool( logger=logger, **transfertool_kwargs.get(builder.transfertool_class, {})) start_time = time.time() logger(logging.DEBUG, 'Starting to group transfers for %s (%s)', activity, transfertool_obj) grouped_jobs = transfertool_obj.group_into_submit_jobs(transfer_paths) record_timer('daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfer_paths) or 1)) logger(logging.DEBUG, 'Starting to submit transfers for %s (%s)', activity, transfertool_obj) for job in grouped_jobs: worker_number, total_workers, logger = heartbeat_handler.live() logger( logging.DEBUG, 'submitjob: transfers=%s, job_params=%s' % ([str(t) for t in job['transfers']], job['job_params'])) submit_transfer(transfertool_obj=transfertool_obj, transfers=job['transfers'], job_params=job['job_params'], submitter='transfer_submitter', timeout=timeout, logger=logger) queue_empty = False if total_transfers < group_bulk: queue_empty = True logger(logging.DEBUG, 'Only %s transfers for %s which is less than group bulk %s', total_transfers, activity, group_bulk) return queue_empty