def test_fts_recoverable_failures_handled_on_multihop( vo, did_factory, root_account, replica_client, file_factory, core_config_mock, caches_mock): """ Verify that the poller correctly handles recoverable FTS job failures """ src_rse = 'XRD1' src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo) jump_rse = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo) dst_rse = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) all_rses = [src_rse_id, jump_rse_id, dst_rse_id] # Create and upload a real file, but register it with wrong checksum. This will trigger # a FTS "Recoverable" failure on checksum validation local_file = file_factory.file_generator() did = did_factory.random_did() did_factory.upload_client.upload([{ 'path': local_file, 'rse': src_rse, 'did_scope': did['scope'].external, 'did_name': did['name'], 'no_register': True, }]) replica_client.add_replicas(rse=src_rse, files=[{ 'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa' }]) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None) request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did) assert request['state'] == RequestState.FAILED request = request_core.get_request_by_did(rse_id=jump_rse_id, **did) assert request['state'] == RequestState.FAILED
def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config_mock, caches_mock): """ Verify that the receiver correctly handles successful multihop jobs """ receiver_thread = threading.Thread(target=receiver, kwargs={ 'id': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1 }) receiver_thread.start() try: src_rse = 'XRD1' src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo) jump_rse = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo) dst_rse = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) all_rses = [src_rse_id, jump_rse_id, dst_rse_id] did = did_factory.upload_test_file(src_rse) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None) request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.DONE, run_poller=False, **did) assert request['state'] == RequestState.DONE request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.DONE, run_poller=False, **did) assert request['state'] == RequestState.DONE finally: receiver_graceful_stop.set() receiver_thread.join(timeout=5) receiver_graceful_stop.clear()
def test_multisource(vo, did_factory, root_account, replica_client, core_config_mock, caches_mock): src_rse1 = 'XRD4' src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo) src_rse2 = 'XRD1' src_rse2_id = rse_core.get_rse_id(rse=src_rse2, vo=vo) dst_rse = 'XRD3' dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) all_rses = [src_rse1_id, src_rse2_id, dst_rse_id] # Add a good replica on the RSE which has a higher distance ranking did = did_factory.upload_test_file(src_rse1) # Add non-existing replica which will fail during multisource transfers on the RSE with lower cost (will be the preferred source) replica_client.add_replicas(rse=src_rse2, files=[{ 'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa' }]) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None) @read_session def __source_exists(src_rse_id, scope, name, session=None): return session.query(models.Source) \ .filter(models.Source.rse_id == src_rse_id) \ .filter(models.Source.scope == scope) \ .filter(models.Source.name == name) \ .count() != 0 # Entries in the source table must be created for both sources of the multi-source transfer assert __source_exists(src_rse_id=src_rse1_id, **did) assert __source_exists(src_rse_id=src_rse2_id, **did) replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE assert not __source_exists(src_rse_id=src_rse1_id, **did) assert not __source_exists(src_rse_id=src_rse2_id, **did)
def test_request_submitted_in_order(rse_factory, did_factory, root_account): src_rses = [rse_factory.make_posix_rse() for _ in range(2)] dst_rses = [rse_factory.make_posix_rse() for _ in range(3)] for _, src_rse_id in src_rses: for _, dst_rse_id in dst_rses: distance_core.add_distance(src_rse_id=src_rse_id, dest_rse_id=dst_rse_id, ranking=10) distance_core.add_distance(src_rse_id=dst_rse_id, dest_rse_id=src_rse_id, ranking=10) # Create a certain number of files on source RSEs with replication rules towards random destination RSEs nb_files = 15 dids = [] requests = [] src_rses_iterator = itertools.cycle(src_rses) dst_rses_iterator = itertools.cycle(dst_rses) for _ in range(nb_files): src_rse_name, src_rse_id = next(src_rses_iterator) dst_rse_name, dst_rse_id = next(dst_rses_iterator) did = did_factory.upload_test_file(rse_name=src_rse_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) requests.append(request_core.get_request_by_did(rse_id=dst_rse_id, **did)) dids.append(did) # Forge request creation time to a random moment in the past hour @transactional_session def _forge_requests_creation_time(session=None): base_time = datetime.utcnow().replace(microsecond=0, minute=0) - timedelta(hours=1) assigned_times = set() for request in requests: request_creation_time = None while not request_creation_time or request_creation_time in assigned_times: # Ensure uniqueness to avoid multiple valid submission orders and make tests deterministic with simple sorting techniques request_creation_time = base_time + timedelta(minutes=randint(0, 3600)) assigned_times.add(request_creation_time) session.query(Request).filter(Request.id == request['id']).update({'created_at': request_creation_time}) request['created_at'] = request_creation_time _forge_requests_creation_time() requests = sorted(requests, key=lambda r: r['created_at']) for request in requests: assert request_core.get_request(request_id=request['id'])['state'] == RequestState.QUEUED requests_id_in_submission_order = [] with patch('rucio.transfertool.mock.MockTransfertool.submit') as mock_transfertool_submit: # Record the order of requests passed to MockTranfertool.submit() mock_transfertool_submit.side_effect = lambda jobs, _: requests_id_in_submission_order.extend([j['metadata']['request_id'] for j in jobs]) submitter(once=True, rses=[{'id': rse_id} for _, rse_id in dst_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None) for request in requests: assert request_core.get_request(request_id=request['id'])['state'] == RequestState.SUBMITTED # Requests must be submitted in the order of their creation assert requests_id_in_submission_order == [r['id'] for r in requests]
def test_fts_non_recoverable_failures_handled_on_multihop( vo, did_factory, root_account, replica_client, core_config_mock, caches_mock): """ Verify that the poller correctly handles non-recoverable FTS job failures """ src_rse = 'XRD1' src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo) jump_rse = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo) dst_rse = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) all_rses = [src_rse_id, jump_rse_id, dst_rse_id] # Register a did which doesn't exist. It will trigger an non-recoverable error during the FTS transfer. did = did_factory.random_did() replica_client.add_replicas(rse=src_rse, files=[{ 'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa' }]) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None) request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did) assert request['state'] == RequestState.FAILED request = request_core.get_request_by_did(rse_id=jump_rse_id, **did) assert request['state'] == RequestState.FAILED
def test_ignore_availability(rse_factory, did_factory, root_account, core_config_mock, caches_mock): def __setup_test(): src_rse, src_rse_id = rse_factory.make_posix_rse() dst_rse, dst_rse_id = rse_factory.make_posix_rse() distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10) did = did_factory.upload_test_file(src_rse) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) rse_core.update_rse(src_rse_id, {'availability_read': False}) return src_rse_id, dst_rse_id, did src_rse_id, dst_rse_id, did = __setup_test() submitter(once=True, rses=[{ 'id': rse_id } for rse_id in (src_rse_id, dst_rse_id)], partition_wait_time=None, transfertool='mock', transfertype='single') request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) assert request['state'] == RequestState.NO_SOURCES src_rse_id, dst_rse_id, did = __setup_test() submitter(once=True, rses=[{ 'id': rse_id } for rse_id in (src_rse_id, dst_rse_id)], partition_wait_time=None, transfertool='mock', transfertype='single', ignore_availability=True) request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) assert request['state'] == RequestState.SUBMITTED
def test_request_submitted(rse_factory, file_factory, root_account): """ Conveyor (DAEMON): Test the submitter""" src_rse_name, src_rse_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() distance_core.add_distance(src_rse_id=src_rse_id, dest_rse_id=dst_rse_id, ranking=10) distance_core.add_distance(src_rse_id=dst_rse_id, dest_rse_id=src_rse_id, ranking=10) did = file_factory.upload_test_file(rse_name=src_rse_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) assert request['state'] == RequestState.QUEUED # run submitter with a RSE filter which doesn't contain the needed one submitter(once=True, rses=[{'id': src_rse_id}], mock=True, transfertool='mock', transfertype='bulk', filter_transfertool=None, bulk=None) request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) assert request['state'] == RequestState.QUEUED submitter(once=True, rses=[{'id': dst_rse_id}], mock=True, transfertool='mock', transfertype='bulk', filter_transfertool=None, bulk=None) request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) assert request['state'] == RequestState.SUBMITTED
def test_multihop_sources_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that multihop transfers are handled and intermediate request correctly created """ src_rse_name, src_rse_id = rse_factory.make_posix_rse() _, jump_rse1_id = rse_factory.make_posix_rse() _, jump_rse2_id = rse_factory.make_posix_rse() _, jump_rse3_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id] all_rses = jump_rses + [src_rse_id, dst_rse_id] for rse_id in jump_rses: rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True) distance_core.add_distance(src_rse_id, jump_rse1_id, ranking=10) distance_core.add_distance(jump_rse1_id, jump_rse2_id, ranking=10) distance_core.add_distance(jump_rse2_id, jump_rse3_id, ranking=10) distance_core.add_distance(jump_rse3_id, dst_rse_id, ranking=10) did = did_factory.upload_test_file(src_rse_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None) # Ensure that each intermediate request was correctly created for rse_id in jump_rses: assert request_core.get_request_by_did(rse_id=rse_id, **did) @read_session def __ensure_source_exists(rse_id, scope, name, session=None): return session.query(Source). \ filter(Source.rse_id == rse_id). \ filter(Source.scope == scope). \ filter(Source.name == name). \ one() # Ensure that sources where created for transfers for rse_id in jump_rses + [src_rse_id]: __ensure_source_exists(rse_id, **did)
def test_tpc(containerized_rses, root_account, test_scope, did_factory, rse_client, rule_client, artifact): if len(containerized_rses) < 2: pytest.skip( "TPC tests need at least 2 containerized rse's for execution}") rse1_name, rse1_id = containerized_rses[0] rse2_name, rse2_id = containerized_rses[1] base_file_name = generate_uuid() test_file = did_factory.upload_test_file(rse1_name, name=base_file_name + '.000', return_full_item=True) test_file_did_str = '%s:%s' % (test_file['did_scope'], test_file['did_name']) test_file_did = {'scope': test_scope, 'name': test_file['did_name']} test_file_name_hash = hashlib.md5( test_file_did_str.encode('utf-8')).hexdigest() test_file_expected_pfn = '%s/%s/%s/%s' % ( test_file_did['scope'], test_file_name_hash[0:2], test_file_name_hash[2:4], test_file_did['name']) rse1_hostname = rse_client.get_protocols(rse1_name)[0]['hostname'] rse2_hostname = rse_client.get_protocols(rse2_name)[0]['hostname'] rule_id = add_rule(dids=[test_file_did], account=root_account, copies=1, rse_expression=rse2_name, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) rule = rule_client.get_replication_rule(rule_id[0]) re_evaluator(once=True) assert rule['locks_ok_cnt'] == 0 assert rule['locks_replicating_cnt'] == 1 [[_, [transfer_path]] ] = next_transfers_to_submit(rses=[rse1_id, rse2_id]).items() assert transfer_path[0].rws.rule_id == rule_id[0] src_url = transfer_path[0].legacy_sources[0][1] dest_url = transfer_path[0].dest_url check_url(src_url, rse1_hostname, test_file_expected_pfn) check_url(dest_url, rse2_hostname, test_file_expected_pfn) # Run Submitter submitter.submitter(once=True) # Get FTS transfer job id request = get_request_by_did(rse_id=rse2_id, **test_file_did) fts_transfer_id = request['external_id'] # Check FTS transfer job assert fts_transfer_id is not None # Wait for the FTS transfer to finish fts_transfer_status = None for _ in range(MAX_POLL_WAIT_SECONDS): fts_transfer_status = poll_fts_transfer_status(fts_transfer_id) if fts_transfer_status not in ['SUBMITTED', 'ACTIVE']: break time.sleep(1) assert fts_transfer_status == 'FINISHED' poller.run(once=True, older_than=0) finisher.run(once=True) rule = rule_client.get_replication_rule(rule_id[0]) assert rule['locks_ok_cnt'] == 1 assert rule['locks_replicating_cnt'] == 0 if artifact is not None: date = datetime.date.today().strftime("%Y-%m-%d") with open(artifact, 'w') as artifact_file: artifact_file.write( f"/var/log/fts3/{date}/{rse1_name.lower()}__{rse2_name.lower()}/*__{fts_transfer_id}" )
def test_hop_penalty(rse_factory, did_factory, root_account, file_config_mock, core_config_mock, caches_mock): """ Test that both global hop_penalty and the per-rse one are correctly taken into consideration """ # +------+ +------+ +------+ # | | | 5 | | | # | RSE1 +--->| RSE2 +--->| RSE3 | # | | | | | | # +------+ +------+ +--^---+ # | # +------+ +------+ | # | | | 20 | | # | RSE4 +--->| RSE5 +-------+ # | | | | # +------+ +------+ rse1, rse1_id = rse_factory.make_posix_rse() rse2, rse2_id = rse_factory.make_posix_rse() rse3, rse3_id = rse_factory.make_posix_rse() rse4, rse4_id = rse_factory.make_posix_rse() rse5, rse5_id = rse_factory.make_posix_rse() all_rses = [rse1_id, rse2_id, rse3_id, rse4_id, rse5_id] distance_core.add_distance(rse1_id, rse2_id, ranking=10) distance_core.add_distance(rse2_id, rse3_id, ranking=10) distance_core.add_distance(rse4_id, rse5_id, ranking=10) distance_core.add_distance(rse5_id, rse3_id, ranking=10) rse_core.add_rse_attribute(rse2_id, 'available_for_multihop', True) rse_core.add_rse_attribute(rse5_id, 'available_for_multihop', True) rse_core.add_rse_attribute(rse5_id, 'hop_penalty', 20) did = did_factory.random_did() replica_core.add_replica(rse_id=rse1_id, account=root_account, bytes_=1, **did) replica_core.add_replica(rse_id=rse4_id, account=root_account, bytes_=1, **did) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=rse3, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', ignore_availability=True) # Ensure the path was created through the correct middle hop request_core.get_request_by_did(rse_id=rse2_id, **did) with pytest.raises(RequestNotFound): request_core.get_request_by_did(rse_id=rse5_id, **did)
def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_account, core_config_mock, caches_mock): """ Verify that the receiver correctly handles multihop jobs which fail """ receiver_thread = threading.Thread(target=receiver, kwargs={ 'id': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1 }) receiver_thread.start() try: src_rse = 'XRD1' src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo) jump_rse = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo) dst_rse = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) all_rses = [src_rse_id, jump_rse_id, dst_rse_id] # Register a did which doesn't exist. It will trigger a failure error during the FTS transfer. did = did_factory.random_did() replica_client.add_replicas(rse=src_rse, files=[{ 'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa' }]) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None) request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.FAILED, run_poller=False, **did) assert request['state'] == RequestState.FAILED # We use FTS "Completion" messages in receiver. In case of multi-hops transfer failures, FTS doesn't start # next transfers; so it never sends a "completion" message for some hops. Rely on poller in such cases. # TODO: set the run_poller argument to False if we ever manage to make the receiver correctly handle multi-hop failures. request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, run_poller=True, **did) assert request['state'] == RequestState.FAILED finally: receiver_graceful_stop.set() receiver_thread.join(timeout=5) receiver_graceful_stop.clear()
def test_multisource_receiver(vo, did_factory, replica_client, root_account): """ Run receiver as a background thread to automatically handle fts notifications. Ensure that a multi-source job in which the first source fails is correctly handled by receiver. """ receiver_thread = threading.Thread(target=receiver, kwargs={ 'id': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1 }) receiver_thread.start() try: src_rse1 = 'XRD4' src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo) src_rse2 = 'XRD1' src_rse2_id = rse_core.get_rse_id(rse=src_rse2, vo=vo) dst_rse = 'XRD3' dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) all_rses = [src_rse1_id, src_rse2_id, dst_rse_id] # Add a good replica on the RSE which has a higher distance ranking did = did_factory.upload_test_file(src_rse1) # Add non-existing replica which will fail during multisource transfers on the RSE with lower cost (will be the preferred source) replica_client.add_replicas(rse=src_rse2, files=[{ 'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa' }]) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None) request = None for _ in range(MAX_POLL_WAIT_SECONDS): request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) # The request must not be marked as failed. Not even temporarily. It is a multi-source transfer and the # the first, failed, source must not change the replica state. We must wait for all sources to be tried. assert request['state'] != RequestState.FAILED if request['state'] == RequestState.DONE: break time.sleep(1) assert request['state'] == RequestState.DONE finally: receiver_graceful_stop.set() receiver_thread.join(timeout=5) receiver_graceful_stop.clear()
def test_source_avoid_deletion(vo, caches_mock, core_config_mock, rse_factory, did_factory, root_account, file_factory): """ Test that sources on a file block it from deletion """ _, reaper_region = caches_mock src_rse1, src_rse1_id = rse_factory.make_mock_rse() src_rse2, src_rse2_id = rse_factory.make_mock_rse() dst_rse, dst_rse_id = rse_factory.make_mock_rse() all_rses = [src_rse1_id, src_rse2_id, dst_rse_id] any_source = f'{src_rse1}|{src_rse2}' for rse_id in [src_rse1_id, src_rse2_id]: rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=1) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=1, free=0) distance_core.add_distance(src_rse1_id, dst_rse_id, ranking=20) distance_core.add_distance(src_rse2_id, dst_rse_id, ranking=10) # Upload a test file to both rses without registering did = did_factory.random_did() # Register replica on one source RSE replica_core.add_replica(rse_id=src_rse1_id, account=root_account, bytes_=1, tombstone=datetime(year=1970, month=1, day=1), **did) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Reaper will not delete a file which only has one replica if there is any pending transfer for it reaper_region.invalidate() reaper(once=True, rses=[], include_rses=any_source, exclude_rses=None) replica = next( iter(replica_core.list_replicas(dids=[did], rse_expression=any_source))) assert len(replica['pfns']) == 1 # Register replica on second source rse replica_core.add_replica(rse_id=src_rse2_id, account=root_account, bytes_=1, tombstone=datetime(year=1970, month=1, day=1), **did) replica = next( iter(replica_core.list_replicas(dids=[did], rse_expression=any_source))) assert len(replica['pfns']) == 2 # Submit the transfer. This will create the sources. submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None) # None of the replicas will be removed. They are protected by an entry in the sources table reaper_region.invalidate() reaper(once=True, rses=[], include_rses=any_source, exclude_rses=None) replica = next( iter(replica_core.list_replicas(dids=[did], rse_expression=any_source))) assert len(replica['pfns']) == 2 @transactional_session def __delete_sources(rse_id, scope, name, session=None): session.execute( delete(Source).where(Source.rse_id == rse_id, Source.scope == scope, Source.name == name)) # Deletion succeeds for one replica (second still protected by existing request) __delete_sources(src_rse1_id, **did) __delete_sources(src_rse2_id, **did) reaper_region.invalidate() reaper(once=True, rses=[], include_rses=any_source, exclude_rses=None) replica = next( iter(replica_core.list_replicas(dids=[did], rse_expression=any_source))) assert len(replica['pfns']) == 1
def test_globus(rse_factory, did_factory, root_account): """ Test bulk submissions with globus transfertool. Rely on mocks, because we don't contact a real globus server in tests """ # +------+ +------+ # | | | | # | RSE1 +--->| RSE2 | # | | | | # +------+ +------+ # # +------+ +------+ # | | | | # | RSE3 +--->| RSE4 | # | | | | # +------+ +------+ rse1, rse1_id = rse_factory.make_posix_rse() rse2, rse2_id = rse_factory.make_posix_rse() rse3, rse3_id = rse_factory.make_posix_rse() rse4, rse4_id = rse_factory.make_posix_rse() all_rses = [rse1_id, rse2_id, rse3_id, rse4_id] distance_core.add_distance(rse1_id, rse2_id, ranking=10) distance_core.add_distance(rse3_id, rse4_id, ranking=10) for rse_id in all_rses: rse_core.add_rse_attribute(rse_id, 'globus_endpoint_id', rse_id) # Single submission did1 = did_factory.upload_test_file(rse1) rule_core.add_rule(dids=[did1], account=root_account, copies=1, rse_expression=rse2, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) did2 = did_factory.upload_test_file(rse3) rule_core.add_rule(dids=[did2], account=root_account, copies=1, rse_expression=rse4, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) with patch( 'rucio.transfertool.globus.bulk_submit_xfer') as mock_bulk_submit: mock_bulk_submit.return_value = 0 submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertool='globus', transfertype='single', filter_transfertool=None) # Called separately for each job assert len(mock_bulk_submit.call_args_list) == 2 (submitjob, ), _kwargs = mock_bulk_submit.call_args_list[0] assert len(submitjob) == 1 # Bulk submission did1 = did_factory.upload_test_file(rse1) rule_core.add_rule(dids=[did1], account=root_account, copies=1, rse_expression=rse2, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) did2 = did_factory.upload_test_file(rse3) rule_core.add_rule(dids=[did2], account=root_account, copies=1, rse_expression=rse4, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) with patch( 'rucio.transfertool.globus.bulk_submit_xfer') as mock_bulk_submit: mock_bulk_submit.return_value = 0 submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertool='globus', transfertype='bulk', filter_transfertool=None) mock_bulk_submit.assert_called_once() (submitjob, ), _kwargs = mock_bulk_submit.call_args_list[0] # both jobs were grouped together and submitted in one call assert len(submitjob) == 2 job_did1 = next( iter( filter(lambda job: did1['name'] in job['sources'][0], submitjob))) assert len(job_did1['sources']) == 1 assert len(job_did1['destinations']) == 1 assert job_did1['metadata']['src_rse'] == rse1 assert job_did1['metadata']['dst_rse'] == rse2 assert job_did1['metadata']['name'] == did1['name'] assert job_did1['metadata']['source_globus_endpoint_id'] == rse1_id assert job_did1['metadata']['dest_globus_endpoint_id'] == rse2_id job_did2 = next( iter( filter(lambda job: did2['name'] in job['sources'][0], submitjob))) assert len(job_did2['sources']) == 1 assert len(job_did2['destinations']) == 1 assert job_did2['metadata']['src_rse'] == rse3 assert job_did2['metadata']['dst_rse'] == rse4 assert job_did2['metadata']['name'] == did2['name'] request = request_core.get_request_by_did(rse_id=rse2_id, **did1) assert request['state'] == RequestState.SUBMITTED request = request_core.get_request_by_did(rse_id=rse4_id, **did2) assert request['state'] == RequestState.SUBMITTED
def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that intermediate replicas created by the submitter are protected from deletion even if their tombstone is set to epoch. After successful transfers, intermediate replicas with default (epoch) tombstone must be removed. The others must be left intact. """ src_rse1_name = 'XRD1' src_rse1_id = rse_core.get_rse_id(rse=src_rse1_name, vo=vo) src_rse2_name = 'XRD2' src_rse2_id = rse_core.get_rse_id(rse=src_rse2_name, vo=vo) jump_rse_name = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse_name, vo=vo) dst_rse_name = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse_name, vo=vo) all_rses = [src_rse1_id, src_rse2_id, jump_rse_id, dst_rse_id] did = did_factory.upload_test_file(src_rse1_name) # Copy replica to a second source. To avoid the special case of having a unique last replica, which could be handled in a special (more careful) way rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=src_rse2_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) replica = __wait_for_replica_transfer(dst_rse_id=src_rse2_id, **did) assert replica['state'] == ReplicaState.AVAILABLE rse_core.set_rse_limits(rse_id=jump_rse_id, name='MinFreeSpace', value=1) rse_core.set_rse_usage(rse_id=jump_rse_id, source='storage', used=1, free=0) try: rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Submit transfers to FTS # Ensure a replica was created on the intermediary host with epoch tombstone submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) request = request_core.get_request_by_did(rse_id=jump_rse_id, **did) assert request['state'] == RequestState.SUBMITTED replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica['tombstone'] == datetime(year=1970, month=1, day=1) assert replica['state'] == ReplicaState.COPYING # The intermediate replica is protected by its state (Copying) rucio.daemons.reaper.reaper.REGION.invalidate() reaper(once=True, rses=[], include_rses=jump_rse_name, exclude_rses=None) replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.COPYING # Wait for the intermediate replica to become ready replica = __wait_for_replica_transfer(dst_rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE # The intermediate replica is protected by an entry in the sources table # Reaper must not remove this replica, even if it has an obsolete tombstone rucio.daemons.reaper.reaper.REGION.invalidate() reaper(once=True, rses=[], include_rses=jump_rse_name, exclude_rses=None) replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica # FTS fails the second transfer, so run submitter again to copy from jump rse to destination rse submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) # Wait for the destination replica to become ready replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE rucio.daemons.reaper.reaper.REGION.invalidate() reaper(once=True, rses=[], include_rses='test_container_xrd=True', exclude_rses=None) with pytest.raises(ReplicaNotFound): replica_core.get_replica(rse_id=jump_rse_id, **did) finally: @transactional_session def _cleanup_all_usage_and_limits(rse_id, session=None): session.query(models.RSELimit).filter_by(rse_id=rse_id).delete() session.query(models.RSEUsage).filter_by( rse_id=rse_id, source='storage').delete() _cleanup_all_usage_and_limits(rse_id=jump_rse_id)
def test_overwrite_on_tape(rse_factory, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that overwrite is not set for transfers towards TAPE RSEs Submit a real transfer to FTS and rely on the gfal "mock" plugin to trigger a failure. The failure is triggered when gfal_stat is called on the destination URL and it returns a result. To achieve this via the mock plugin, it's enough to have a mock:// protocol/scheme and add size_pre=<something> url parameter. https://gitlab.cern.ch/dmc/gfal2/-/blob/master/src/plugins/mock/README_PLUGIN_MOCK """ # +------+ +------+ +------+ # | | | | | | # | RSE1 +--->| RSE2 |--->| RSE3 | # | | | | |(tape)| # +------+ +------+ +------+ rse1, rse1_id = rse_factory.make_rse( scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default') rse2, rse2_id = rse_factory.make_rse( scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default') rse3, rse3_id = rse_factory.make_rse( scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default', rse_type=RSEType.TAPE) all_rses = [rse1_id, rse2_id, rse3_id] distance_core.add_distance(rse1_id, rse2_id, ranking=10) distance_core.add_distance(rse2_id, rse3_id, ranking=10) rse_core.add_rse_attribute(rse2_id, 'available_for_multihop', True) for rse_id in all_rses: rse_core.add_rse_attribute(rse_id, 'fts', 'https://fts:8446') # multihop transfer: did1 = did_factory.upload_test_file(rse1) # direct transfer: did2 = did_factory.upload_test_file(rse2) rule_core.add_rule(dids=[did1, did2], account=root_account, copies=1, rse_expression=rse3, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Wrap dest url generation to add size_pre=2 query parameter non_mocked_dest_url = transfer_core.DirectTransferDefinition._dest_url def mocked_dest_url(cls, *args): return set_query_parameters(non_mocked_dest_url(*args), {'size_pre': 2}) with patch('rucio.core.transfer.DirectTransferDefinition._dest_url', new=mocked_dest_url): submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None) request = __wait_for_request_state(dst_rse_id=rse3_id, state=RequestState.FAILED, **did1) assert request['state'] == RequestState.FAILED request = __wait_for_request_state(dst_rse_id=rse3_id, state=RequestState.FAILED, **did2) assert request['state'] == RequestState.FAILED assert 'Destination file exists and overwrite is not enabled' in request[ 'err_msg']
def test_multihop_sources_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that multihop transfers are handled and intermediate request correctly created """ src_rse_name, src_rse_id = rse_factory.make_posix_rse() _, jump_rse1_id = rse_factory.make_posix_rse() _, jump_rse2_id = rse_factory.make_posix_rse() _, jump_rse3_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id] all_rses = jump_rses + [src_rse_id, dst_rse_id] for rse_id in jump_rses: rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True) rse_tombstone_delay = 3600 rse_multihop_tombstone_delay = 12 * 3600 default_multihop_tombstone_delay = 24 * 3600 # if both attributes are set, the multihop one will take precedence rse_core.add_rse_attribute(jump_rse1_id, 'tombstone_delay', rse_tombstone_delay) rse_core.add_rse_attribute(jump_rse1_id, 'multihop_tombstone_delay', rse_multihop_tombstone_delay) # if multihop delay not set, it's the default multihop takes precedence. Not normal tombstone delay. rse_core.add_rse_attribute(jump_rse2_id, 'tombstone_delay', rse_tombstone_delay) core_config.set(section='transfers', option='multihop_tombstone_delay', value=default_multihop_tombstone_delay) # if multihop delay is set to 0, the replica will have no tombstone rse_core.add_rse_attribute(jump_rse3_id, 'multihop_tombstone_delay', 0) distance_core.add_distance(src_rse_id, jump_rse1_id, ranking=10) distance_core.add_distance(jump_rse1_id, jump_rse2_id, ranking=10) distance_core.add_distance(jump_rse2_id, jump_rse3_id, ranking=10) distance_core.add_distance(jump_rse3_id, dst_rse_id, ranking=10) did = did_factory.upload_test_file(src_rse_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None) # Ensure that each intermediate request was correctly created for rse_id in jump_rses: assert request_core.get_request_by_did(rse_id=rse_id, **did) @read_session def __ensure_source_exists(rse_id, scope, name, session=None): return session.query(Source). \ filter(Source.rse_id == rse_id). \ filter(Source.scope == scope). \ filter(Source.name == name). \ one() # Ensure that sources where created for transfers for rse_id in jump_rses + [src_rse_id]: __ensure_source_exists(rse_id, **did) # Ensure the tombstone is correctly set on intermediate replicas expected_tombstone = datetime.utcnow() + timedelta(seconds=rse_multihop_tombstone_delay) replica = replica_core.get_replica(jump_rse1_id, **did) assert expected_tombstone - timedelta(minutes=5) < replica['tombstone'] < expected_tombstone + timedelta(minutes=5) expected_tombstone = datetime.utcnow() + timedelta(seconds=default_multihop_tombstone_delay) replica = replica_core.get_replica(jump_rse2_id, **did) assert expected_tombstone - timedelta(minutes=5) < replica['tombstone'] < expected_tombstone + timedelta(minutes=5) replica = replica_core.get_replica(jump_rse3_id, **did) assert replica['tombstone'] is None