def test_fts_recoverable_failures_handled_on_multihop(
vo, did_factory, root_account, replica_client, file_factory,
core_config_mock, caches_mock):
"""
Verify that the poller correctly handles recoverable FTS job failures
"""
src_rse = 'XRD1'
src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
jump_rse = 'XRD3'
jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
dst_rse = 'XRD4'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
# Create and upload a real file, but register it with wrong checksum. This will trigger
# a FTS "Recoverable" failure on checksum validation
local_file = file_factory.file_generator()
did = did_factory.random_did()
did_factory.upload_client.upload([{
'path': local_file,
'rse': src_rse,
'did_scope': did['scope'].external,
'did_name': did['name'],
'no_register': True,
}])
replica_client.add_replicas(rse=src_rse,
files=[{
'scope': did['scope'].external,
'name': did['name'],
'bytes': 1,
'adler32': 'aaaaaaaa'
}])
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=2,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = __wait_for_request_state(dst_rse_id=dst_rse_id,
state=RequestState.FAILED,
**did)
assert request['state'] == RequestState.FAILED
request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
assert request['state'] == RequestState.FAILED
def test_multihop_receiver_on_success(vo, did_factory, root_account,
core_config_mock, caches_mock):
"""
Verify that the receiver correctly handles successful multihop jobs
"""
receiver_thread = threading.Thread(target=receiver,
kwargs={
'id': 0,
'full_mode': True,
'all_vos': True,
'total_threads': 1
})
receiver_thread.start()
try:
src_rse = 'XRD1'
src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
jump_rse = 'XRD3'
jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
dst_rse = 'XRD4'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
did = did_factory.upload_test_file(src_rse)
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=2,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = __wait_for_request_state(dst_rse_id=jump_rse_id,
state=RequestState.DONE,
run_poller=False,
**did)
assert request['state'] == RequestState.DONE
request = __wait_for_request_state(dst_rse_id=dst_rse_id,
state=RequestState.DONE,
run_poller=False,
**did)
assert request['state'] == RequestState.DONE
finally:
receiver_graceful_stop.set()
receiver_thread.join(timeout=5)
receiver_graceful_stop.clear()
def test_multisource(vo, did_factory, root_account, replica_client,
core_config_mock, caches_mock):
src_rse1 = 'XRD4'
src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo)
src_rse2 = 'XRD1'
src_rse2_id = rse_core.get_rse_id(rse=src_rse2, vo=vo)
dst_rse = 'XRD3'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
all_rses = [src_rse1_id, src_rse2_id, dst_rse_id]
# Add a good replica on the RSE which has a higher distance ranking
did = did_factory.upload_test_file(src_rse1)
# Add non-existing replica which will fail during multisource transfers on the RSE with lower cost (will be the preferred source)
replica_client.add_replicas(rse=src_rse2,
files=[{
'scope': did['scope'].external,
'name': did['name'],
'bytes': 1,
'adler32': 'aaaaaaaa'
}])
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=2,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
@read_session
def __source_exists(src_rse_id, scope, name, session=None):
return session.query(models.Source) \
.filter(models.Source.rse_id == src_rse_id) \
.filter(models.Source.scope == scope) \
.filter(models.Source.name == name) \
.count() != 0
# Entries in the source table must be created for both sources of the multi-source transfer
assert __source_exists(src_rse_id=src_rse1_id, **did)
assert __source_exists(src_rse_id=src_rse2_id, **did)
replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
assert not __source_exists(src_rse_id=src_rse1_id, **did)
assert not __source_exists(src_rse_id=src_rse2_id, **did)
def test_request_submitted_in_order(rse_factory, did_factory, root_account):
src_rses = [rse_factory.make_posix_rse() for _ in range(2)]
dst_rses = [rse_factory.make_posix_rse() for _ in range(3)]
for _, src_rse_id in src_rses:
for _, dst_rse_id in dst_rses:
distance_core.add_distance(src_rse_id=src_rse_id, dest_rse_id=dst_rse_id, ranking=10)
distance_core.add_distance(src_rse_id=dst_rse_id, dest_rse_id=src_rse_id, ranking=10)
# Create a certain number of files on source RSEs with replication rules towards random destination RSEs
nb_files = 15
dids = []
requests = []
src_rses_iterator = itertools.cycle(src_rses)
dst_rses_iterator = itertools.cycle(dst_rses)
for _ in range(nb_files):
src_rse_name, src_rse_id = next(src_rses_iterator)
dst_rse_name, dst_rse_id = next(dst_rses_iterator)
did = did_factory.upload_test_file(rse_name=src_rse_name)
rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
requests.append(request_core.get_request_by_did(rse_id=dst_rse_id, **did))
dids.append(did)
# Forge request creation time to a random moment in the past hour
@transactional_session
def _forge_requests_creation_time(session=None):
base_time = datetime.utcnow().replace(microsecond=0, minute=0) - timedelta(hours=1)
assigned_times = set()
for request in requests:
request_creation_time = None
while not request_creation_time or request_creation_time in assigned_times:
# Ensure uniqueness to avoid multiple valid submission orders and make tests deterministic with simple sorting techniques
request_creation_time = base_time + timedelta(minutes=randint(0, 3600))
assigned_times.add(request_creation_time)
session.query(Request).filter(Request.id == request['id']).update({'created_at': request_creation_time})
request['created_at'] = request_creation_time
_forge_requests_creation_time()
requests = sorted(requests, key=lambda r: r['created_at'])
for request in requests:
assert request_core.get_request(request_id=request['id'])['state'] == RequestState.QUEUED
requests_id_in_submission_order = []
with patch('rucio.transfertool.mock.MockTransfertool.submit') as mock_transfertool_submit:
# Record the order of requests passed to MockTranfertool.submit()
mock_transfertool_submit.side_effect = lambda jobs, _: requests_id_in_submission_order.extend([j['metadata']['request_id'] for j in jobs])
submitter(once=True, rses=[{'id': rse_id} for _, rse_id in dst_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None)
for request in requests:
assert request_core.get_request(request_id=request['id'])['state'] == RequestState.SUBMITTED
# Requests must be submitted in the order of their creation
assert requests_id_in_submission_order == [r['id'] for r in requests]
def test_fts_non_recoverable_failures_handled_on_multihop(
vo, did_factory, root_account, replica_client, core_config_mock,
caches_mock):
"""
Verify that the poller correctly handles non-recoverable FTS job failures
"""
src_rse = 'XRD1'
src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
jump_rse = 'XRD3'
jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
dst_rse = 'XRD4'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
# Register a did which doesn't exist. It will trigger an non-recoverable error during the FTS transfer.
did = did_factory.random_did()
replica_client.add_replicas(rse=src_rse,
files=[{
'scope': did['scope'].external,
'name': did['name'],
'bytes': 1,
'adler32': 'aaaaaaaa'
}])
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=2,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = __wait_for_request_state(dst_rse_id=dst_rse_id,
state=RequestState.FAILED,
**did)
assert request['state'] == RequestState.FAILED
request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
assert request['state'] == RequestState.FAILED
def test_ignore_availability(rse_factory, did_factory, root_account,
core_config_mock, caches_mock):
def __setup_test():
src_rse, src_rse_id = rse_factory.make_posix_rse()
dst_rse, dst_rse_id = rse_factory.make_posix_rse()
distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
did = did_factory.upload_test_file(src_rse)
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
rse_core.update_rse(src_rse_id, {'availability_read': False})
return src_rse_id, dst_rse_id, did
src_rse_id, dst_rse_id, did = __setup_test()
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in (src_rse_id, dst_rse_id)],
partition_wait_time=None,
transfertool='mock',
transfertype='single')
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
assert request['state'] == RequestState.NO_SOURCES
src_rse_id, dst_rse_id, did = __setup_test()
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in (src_rse_id, dst_rse_id)],
partition_wait_time=None,
transfertool='mock',
transfertype='single',
ignore_availability=True)
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
assert request['state'] == RequestState.SUBMITTED
def test_request_submitted(rse_factory, file_factory, root_account):
""" Conveyor (DAEMON): Test the submitter"""
src_rse_name, src_rse_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
distance_core.add_distance(src_rse_id=src_rse_id, dest_rse_id=dst_rse_id, ranking=10)
distance_core.add_distance(src_rse_id=dst_rse_id, dest_rse_id=src_rse_id, ranking=10)
did = file_factory.upload_test_file(rse_name=src_rse_name)
rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
assert request['state'] == RequestState.QUEUED
# run submitter with a RSE filter which doesn't contain the needed one
submitter(once=True, rses=[{'id': src_rse_id}], mock=True, transfertool='mock', transfertype='bulk', filter_transfertool=None, bulk=None)
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
assert request['state'] == RequestState.QUEUED
submitter(once=True, rses=[{'id': dst_rse_id}], mock=True, transfertool='mock', transfertype='bulk', filter_transfertool=None, bulk=None)
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
assert request['state'] == RequestState.SUBMITTED
def test_multihop_sources_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock):
"""
Ensure that multihop transfers are handled and intermediate request correctly created
"""
src_rse_name, src_rse_id = rse_factory.make_posix_rse()
_, jump_rse1_id = rse_factory.make_posix_rse()
_, jump_rse2_id = rse_factory.make_posix_rse()
_, jump_rse3_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id]
all_rses = jump_rses + [src_rse_id, dst_rse_id]
for rse_id in jump_rses:
rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True)
distance_core.add_distance(src_rse_id, jump_rse1_id, ranking=10)
distance_core.add_distance(jump_rse1_id, jump_rse2_id, ranking=10)
distance_core.add_distance(jump_rse2_id, jump_rse3_id, ranking=10)
distance_core.add_distance(jump_rse3_id, dst_rse_id, ranking=10)
did = did_factory.upload_test_file(src_rse_name)
rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None)
# Ensure that each intermediate request was correctly created
for rse_id in jump_rses:
assert request_core.get_request_by_did(rse_id=rse_id, **did)
@read_session
def __ensure_source_exists(rse_id, scope, name, session=None):
return session.query(Source). \
filter(Source.rse_id == rse_id). \
filter(Source.scope == scope). \
filter(Source.name == name). \
one()
# Ensure that sources where created for transfers
for rse_id in jump_rses + [src_rse_id]:
__ensure_source_exists(rse_id, **did)
def test_tpc(containerized_rses, root_account, test_scope, did_factory,
rse_client, rule_client, artifact):
if len(containerized_rses) < 2:
pytest.skip(
"TPC tests need at least 2 containerized rse's for execution}")
rse1_name, rse1_id = containerized_rses[0]
rse2_name, rse2_id = containerized_rses[1]
base_file_name = generate_uuid()
test_file = did_factory.upload_test_file(rse1_name,
name=base_file_name + '.000',
return_full_item=True)
test_file_did_str = '%s:%s' % (test_file['did_scope'],
test_file['did_name'])
test_file_did = {'scope': test_scope, 'name': test_file['did_name']}
test_file_name_hash = hashlib.md5(
test_file_did_str.encode('utf-8')).hexdigest()
test_file_expected_pfn = '%s/%s/%s/%s' % (
test_file_did['scope'], test_file_name_hash[0:2],
test_file_name_hash[2:4], test_file_did['name'])
rse1_hostname = rse_client.get_protocols(rse1_name)[0]['hostname']
rse2_hostname = rse_client.get_protocols(rse2_name)[0]['hostname']
rule_id = add_rule(dids=[test_file_did],
account=root_account,
copies=1,
rse_expression=rse2_name,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
rule = rule_client.get_replication_rule(rule_id[0])
re_evaluator(once=True)
assert rule['locks_ok_cnt'] == 0
assert rule['locks_replicating_cnt'] == 1
[[_, [transfer_path]]
] = next_transfers_to_submit(rses=[rse1_id, rse2_id]).items()
assert transfer_path[0].rws.rule_id == rule_id[0]
src_url = transfer_path[0].legacy_sources[0][1]
dest_url = transfer_path[0].dest_url
check_url(src_url, rse1_hostname, test_file_expected_pfn)
check_url(dest_url, rse2_hostname, test_file_expected_pfn)
# Run Submitter
submitter.submitter(once=True)
# Get FTS transfer job id
request = get_request_by_did(rse_id=rse2_id, **test_file_did)
fts_transfer_id = request['external_id']
# Check FTS transfer job
assert fts_transfer_id is not None
# Wait for the FTS transfer to finish
fts_transfer_status = None
for _ in range(MAX_POLL_WAIT_SECONDS):
fts_transfer_status = poll_fts_transfer_status(fts_transfer_id)
if fts_transfer_status not in ['SUBMITTED', 'ACTIVE']:
break
time.sleep(1)
assert fts_transfer_status == 'FINISHED'
poller.run(once=True, older_than=0)
finisher.run(once=True)
rule = rule_client.get_replication_rule(rule_id[0])
assert rule['locks_ok_cnt'] == 1
assert rule['locks_replicating_cnt'] == 0
if artifact is not None:
date = datetime.date.today().strftime("%Y-%m-%d")
with open(artifact, 'w') as artifact_file:
artifact_file.write(
f"/var/log/fts3/{date}/{rse1_name.lower()}__{rse2_name.lower()}/*__{fts_transfer_id}"
)
def test_hop_penalty(rse_factory, did_factory, root_account, file_config_mock,
core_config_mock, caches_mock):
"""
Test that both global hop_penalty and the per-rse one are correctly taken into consideration
"""
# +------+ +------+ +------+
# | | | 5 | | |
# | RSE1 +--->| RSE2 +--->| RSE3 |
# | | | | | |
# +------+ +------+ +--^---+
# |
# +------+ +------+ |
# | | | 20 | |
# | RSE4 +--->| RSE5 +-------+
# | | | |
# +------+ +------+
rse1, rse1_id = rse_factory.make_posix_rse()
rse2, rse2_id = rse_factory.make_posix_rse()
rse3, rse3_id = rse_factory.make_posix_rse()
rse4, rse4_id = rse_factory.make_posix_rse()
rse5, rse5_id = rse_factory.make_posix_rse()
all_rses = [rse1_id, rse2_id, rse3_id, rse4_id, rse5_id]
distance_core.add_distance(rse1_id, rse2_id, ranking=10)
distance_core.add_distance(rse2_id, rse3_id, ranking=10)
distance_core.add_distance(rse4_id, rse5_id, ranking=10)
distance_core.add_distance(rse5_id, rse3_id, ranking=10)
rse_core.add_rse_attribute(rse2_id, 'available_for_multihop', True)
rse_core.add_rse_attribute(rse5_id, 'available_for_multihop', True)
rse_core.add_rse_attribute(rse5_id, 'hop_penalty', 20)
did = did_factory.random_did()
replica_core.add_replica(rse_id=rse1_id,
account=root_account,
bytes_=1,
**did)
replica_core.add_replica(rse_id=rse4_id,
account=root_account,
bytes_=1,
**did)
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=rse3,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertool='mock',
transfertype='single',
ignore_availability=True)
# Ensure the path was created through the correct middle hop
request_core.get_request_by_did(rse_id=rse2_id, **did)
with pytest.raises(RequestNotFound):
request_core.get_request_by_did(rse_id=rse5_id, **did)
def test_multihop_receiver_on_failure(vo, did_factory, replica_client,
root_account, core_config_mock,
caches_mock):
"""
Verify that the receiver correctly handles multihop jobs which fail
"""
receiver_thread = threading.Thread(target=receiver,
kwargs={
'id': 0,
'full_mode': True,
'all_vos': True,
'total_threads': 1
})
receiver_thread.start()
try:
src_rse = 'XRD1'
src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
jump_rse = 'XRD3'
jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
dst_rse = 'XRD4'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
# Register a did which doesn't exist. It will trigger a failure error during the FTS transfer.
did = did_factory.random_did()
replica_client.add_replicas(rse=src_rse,
files=[{
'scope': did['scope'].external,
'name': did['name'],
'bytes': 1,
'adler32': 'aaaaaaaa'
}])
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=2,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = __wait_for_request_state(dst_rse_id=jump_rse_id,
state=RequestState.FAILED,
run_poller=False,
**did)
assert request['state'] == RequestState.FAILED
# We use FTS "Completion" messages in receiver. In case of multi-hops transfer failures, FTS doesn't start
# next transfers; so it never sends a "completion" message for some hops. Rely on poller in such cases.
# TODO: set the run_poller argument to False if we ever manage to make the receiver correctly handle multi-hop failures.
request = __wait_for_request_state(dst_rse_id=dst_rse_id,
state=RequestState.FAILED,
run_poller=True,
**did)
assert request['state'] == RequestState.FAILED
finally:
receiver_graceful_stop.set()
receiver_thread.join(timeout=5)
receiver_graceful_stop.clear()
def test_multisource_receiver(vo, did_factory, replica_client, root_account):
"""
Run receiver as a background thread to automatically handle fts notifications.
Ensure that a multi-source job in which the first source fails is correctly handled by receiver.
"""
receiver_thread = threading.Thread(target=receiver,
kwargs={
'id': 0,
'full_mode': True,
'all_vos': True,
'total_threads': 1
})
receiver_thread.start()
try:
src_rse1 = 'XRD4'
src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo)
src_rse2 = 'XRD1'
src_rse2_id = rse_core.get_rse_id(rse=src_rse2, vo=vo)
dst_rse = 'XRD3'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
all_rses = [src_rse1_id, src_rse2_id, dst_rse_id]
# Add a good replica on the RSE which has a higher distance ranking
did = did_factory.upload_test_file(src_rse1)
# Add non-existing replica which will fail during multisource transfers on the RSE with lower cost (will be the preferred source)
replica_client.add_replicas(rse=src_rse2,
files=[{
'scope': did['scope'].external,
'name': did['name'],
'bytes': 1,
'adler32': 'aaaaaaaa'
}])
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=2,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = None
for _ in range(MAX_POLL_WAIT_SECONDS):
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
# The request must not be marked as failed. Not even temporarily. It is a multi-source transfer and the
# the first, failed, source must not change the replica state. We must wait for all sources to be tried.
assert request['state'] != RequestState.FAILED
if request['state'] == RequestState.DONE:
break
time.sleep(1)
assert request['state'] == RequestState.DONE
finally:
receiver_graceful_stop.set()
receiver_thread.join(timeout=5)
receiver_graceful_stop.clear()
def test_source_avoid_deletion(vo, caches_mock, core_config_mock, rse_factory,
did_factory, root_account, file_factory):
""" Test that sources on a file block it from deletion """
_, reaper_region = caches_mock
src_rse1, src_rse1_id = rse_factory.make_mock_rse()
src_rse2, src_rse2_id = rse_factory.make_mock_rse()
dst_rse, dst_rse_id = rse_factory.make_mock_rse()
all_rses = [src_rse1_id, src_rse2_id, dst_rse_id]
any_source = f'{src_rse1}|{src_rse2}'
for rse_id in [src_rse1_id, src_rse2_id]:
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=1)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=1, free=0)
distance_core.add_distance(src_rse1_id, dst_rse_id, ranking=20)
distance_core.add_distance(src_rse2_id, dst_rse_id, ranking=10)
# Upload a test file to both rses without registering
did = did_factory.random_did()
# Register replica on one source RSE
replica_core.add_replica(rse_id=src_rse1_id,
account=root_account,
bytes_=1,
tombstone=datetime(year=1970, month=1, day=1),
**did)
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
# Reaper will not delete a file which only has one replica if there is any pending transfer for it
reaper_region.invalidate()
reaper(once=True, rses=[], include_rses=any_source, exclude_rses=None)
replica = next(
iter(replica_core.list_replicas(dids=[did],
rse_expression=any_source)))
assert len(replica['pfns']) == 1
# Register replica on second source rse
replica_core.add_replica(rse_id=src_rse2_id,
account=root_account,
bytes_=1,
tombstone=datetime(year=1970, month=1, day=1),
**did)
replica = next(
iter(replica_core.list_replicas(dids=[did],
rse_expression=any_source)))
assert len(replica['pfns']) == 2
# Submit the transfer. This will create the sources.
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertool='mock',
transfertype='single',
filter_transfertool=None)
# None of the replicas will be removed. They are protected by an entry in the sources table
reaper_region.invalidate()
reaper(once=True, rses=[], include_rses=any_source, exclude_rses=None)
replica = next(
iter(replica_core.list_replicas(dids=[did],
rse_expression=any_source)))
assert len(replica['pfns']) == 2
@transactional_session
def __delete_sources(rse_id, scope, name, session=None):
session.execute(
delete(Source).where(Source.rse_id == rse_id,
Source.scope == scope, Source.name == name))
# Deletion succeeds for one replica (second still protected by existing request)
__delete_sources(src_rse1_id, **did)
__delete_sources(src_rse2_id, **did)
reaper_region.invalidate()
reaper(once=True, rses=[], include_rses=any_source, exclude_rses=None)
replica = next(
iter(replica_core.list_replicas(dids=[did],
rse_expression=any_source)))
assert len(replica['pfns']) == 1
def test_globus(rse_factory, did_factory, root_account):
"""
Test bulk submissions with globus transfertool.
Rely on mocks, because we don't contact a real globus server in tests
"""
# +------+ +------+
# | | | |
# | RSE1 +--->| RSE2 |
# | | | |
# +------+ +------+
#
# +------+ +------+
# | | | |
# | RSE3 +--->| RSE4 |
# | | | |
# +------+ +------+
rse1, rse1_id = rse_factory.make_posix_rse()
rse2, rse2_id = rse_factory.make_posix_rse()
rse3, rse3_id = rse_factory.make_posix_rse()
rse4, rse4_id = rse_factory.make_posix_rse()
all_rses = [rse1_id, rse2_id, rse3_id, rse4_id]
distance_core.add_distance(rse1_id, rse2_id, ranking=10)
distance_core.add_distance(rse3_id, rse4_id, ranking=10)
for rse_id in all_rses:
rse_core.add_rse_attribute(rse_id, 'globus_endpoint_id', rse_id)
# Single submission
did1 = did_factory.upload_test_file(rse1)
rule_core.add_rule(dids=[did1],
account=root_account,
copies=1,
rse_expression=rse2,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
did2 = did_factory.upload_test_file(rse3)
rule_core.add_rule(dids=[did2],
account=root_account,
copies=1,
rse_expression=rse4,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
with patch(
'rucio.transfertool.globus.bulk_submit_xfer') as mock_bulk_submit:
mock_bulk_submit.return_value = 0
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=10,
partition_wait_time=None,
transfertool='globus',
transfertype='single',
filter_transfertool=None)
# Called separately for each job
assert len(mock_bulk_submit.call_args_list) == 2
(submitjob, ), _kwargs = mock_bulk_submit.call_args_list[0]
assert len(submitjob) == 1
# Bulk submission
did1 = did_factory.upload_test_file(rse1)
rule_core.add_rule(dids=[did1],
account=root_account,
copies=1,
rse_expression=rse2,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
did2 = did_factory.upload_test_file(rse3)
rule_core.add_rule(dids=[did2],
account=root_account,
copies=1,
rse_expression=rse4,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
with patch(
'rucio.transfertool.globus.bulk_submit_xfer') as mock_bulk_submit:
mock_bulk_submit.return_value = 0
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=10,
partition_wait_time=None,
transfertool='globus',
transfertype='bulk',
filter_transfertool=None)
mock_bulk_submit.assert_called_once()
(submitjob, ), _kwargs = mock_bulk_submit.call_args_list[0]
# both jobs were grouped together and submitted in one call
assert len(submitjob) == 2
job_did1 = next(
iter(
filter(lambda job: did1['name'] in job['sources'][0],
submitjob)))
assert len(job_did1['sources']) == 1
assert len(job_did1['destinations']) == 1
assert job_did1['metadata']['src_rse'] == rse1
assert job_did1['metadata']['dst_rse'] == rse2
assert job_did1['metadata']['name'] == did1['name']
assert job_did1['metadata']['source_globus_endpoint_id'] == rse1_id
assert job_did1['metadata']['dest_globus_endpoint_id'] == rse2_id
job_did2 = next(
iter(
filter(lambda job: did2['name'] in job['sources'][0],
submitjob)))
assert len(job_did2['sources']) == 1
assert len(job_did2['destinations']) == 1
assert job_did2['metadata']['src_rse'] == rse3
assert job_did2['metadata']['dst_rse'] == rse4
assert job_did2['metadata']['name'] == did2['name']
request = request_core.get_request_by_did(rse_id=rse2_id, **did1)
assert request['state'] == RequestState.SUBMITTED
request = request_core.get_request_by_did(rse_id=rse4_id, **did2)
assert request['state'] == RequestState.SUBMITTED
def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account,
core_config_mock,
caches_mock):
"""
Ensure that intermediate replicas created by the submitter are protected from deletion even if their tombstone is
set to epoch.
After successful transfers, intermediate replicas with default (epoch) tombstone must be removed. The others must
be left intact.
"""
src_rse1_name = 'XRD1'
src_rse1_id = rse_core.get_rse_id(rse=src_rse1_name, vo=vo)
src_rse2_name = 'XRD2'
src_rse2_id = rse_core.get_rse_id(rse=src_rse2_name, vo=vo)
jump_rse_name = 'XRD3'
jump_rse_id = rse_core.get_rse_id(rse=jump_rse_name, vo=vo)
dst_rse_name = 'XRD4'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse_name, vo=vo)
all_rses = [src_rse1_id, src_rse2_id, jump_rse_id, dst_rse_id]
did = did_factory.upload_test_file(src_rse1_name)
# Copy replica to a second source. To avoid the special case of having a unique last replica, which could be handled in a special (more careful) way
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=src_rse2_name,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
replica = __wait_for_replica_transfer(dst_rse_id=src_rse2_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
rse_core.set_rse_limits(rse_id=jump_rse_id, name='MinFreeSpace', value=1)
rse_core.set_rse_usage(rse_id=jump_rse_id,
source='storage',
used=1,
free=0)
try:
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse_name,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
# Submit transfers to FTS
# Ensure a replica was created on the intermediary host with epoch tombstone
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
assert request['state'] == RequestState.SUBMITTED
replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
assert replica['tombstone'] == datetime(year=1970, month=1, day=1)
assert replica['state'] == ReplicaState.COPYING
# The intermediate replica is protected by its state (Copying)
rucio.daemons.reaper.reaper.REGION.invalidate()
reaper(once=True,
rses=[],
include_rses=jump_rse_name,
exclude_rses=None)
replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
assert replica['state'] == ReplicaState.COPYING
# Wait for the intermediate replica to become ready
replica = __wait_for_replica_transfer(dst_rse_id=jump_rse_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
# The intermediate replica is protected by an entry in the sources table
# Reaper must not remove this replica, even if it has an obsolete tombstone
rucio.daemons.reaper.reaper.REGION.invalidate()
reaper(once=True,
rses=[],
include_rses=jump_rse_name,
exclude_rses=None)
replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
assert replica
# FTS fails the second transfer, so run submitter again to copy from jump rse to destination rse
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
# Wait for the destination replica to become ready
replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
rucio.daemons.reaper.reaper.REGION.invalidate()
reaper(once=True,
rses=[],
include_rses='test_container_xrd=True',
exclude_rses=None)
with pytest.raises(ReplicaNotFound):
replica_core.get_replica(rse_id=jump_rse_id, **did)
finally:
@transactional_session
def _cleanup_all_usage_and_limits(rse_id, session=None):
session.query(models.RSELimit).filter_by(rse_id=rse_id).delete()
session.query(models.RSEUsage).filter_by(
rse_id=rse_id, source='storage').delete()
_cleanup_all_usage_and_limits(rse_id=jump_rse_id)
def test_overwrite_on_tape(rse_factory, did_factory, root_account,
core_config_mock, caches_mock):
"""
Ensure that overwrite is not set for transfers towards TAPE RSEs
Submit a real transfer to FTS and rely on the gfal "mock" plugin to trigger a failure. The failure is triggered
when gfal_stat is called on the destination URL and it returns a result. To achieve this via the mock
plugin, it's enough to have a mock:// protocol/scheme and add size_pre=<something> url parameter.
https://gitlab.cern.ch/dmc/gfal2/-/blob/master/src/plugins/mock/README_PLUGIN_MOCK
"""
# +------+ +------+ +------+
# | | | | | |
# | RSE1 +--->| RSE2 |--->| RSE3 |
# | | | | |(tape)|
# +------+ +------+ +------+
rse1, rse1_id = rse_factory.make_rse(
scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
rse2, rse2_id = rse_factory.make_rse(
scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
rse3, rse3_id = rse_factory.make_rse(
scheme='mock',
protocol_impl='rucio.rse.protocols.posix.Default',
rse_type=RSEType.TAPE)
all_rses = [rse1_id, rse2_id, rse3_id]
distance_core.add_distance(rse1_id, rse2_id, ranking=10)
distance_core.add_distance(rse2_id, rse3_id, ranking=10)
rse_core.add_rse_attribute(rse2_id, 'available_for_multihop', True)
for rse_id in all_rses:
rse_core.add_rse_attribute(rse_id, 'fts', 'https://fts:8446')
# multihop transfer:
did1 = did_factory.upload_test_file(rse1)
# direct transfer:
did2 = did_factory.upload_test_file(rse2)
rule_core.add_rule(dids=[did1, did2],
account=root_account,
copies=1,
rse_expression=rse3,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
# Wrap dest url generation to add size_pre=2 query parameter
non_mocked_dest_url = transfer_core.DirectTransferDefinition._dest_url
def mocked_dest_url(cls, *args):
return set_query_parameters(non_mocked_dest_url(*args),
{'size_pre': 2})
with patch('rucio.core.transfer.DirectTransferDefinition._dest_url',
new=mocked_dest_url):
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
group_bulk=10,
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = __wait_for_request_state(dst_rse_id=rse3_id,
state=RequestState.FAILED,
**did1)
assert request['state'] == RequestState.FAILED
request = __wait_for_request_state(dst_rse_id=rse3_id,
state=RequestState.FAILED,
**did2)
assert request['state'] == RequestState.FAILED
assert 'Destination file exists and overwrite is not enabled' in request[
'err_msg']
def test_multihop_sources_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock):
"""
Ensure that multihop transfers are handled and intermediate request correctly created
"""
src_rse_name, src_rse_id = rse_factory.make_posix_rse()
_, jump_rse1_id = rse_factory.make_posix_rse()
_, jump_rse2_id = rse_factory.make_posix_rse()
_, jump_rse3_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id]
all_rses = jump_rses + [src_rse_id, dst_rse_id]
for rse_id in jump_rses:
rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True)
rse_tombstone_delay = 3600
rse_multihop_tombstone_delay = 12 * 3600
default_multihop_tombstone_delay = 24 * 3600
# if both attributes are set, the multihop one will take precedence
rse_core.add_rse_attribute(jump_rse1_id, 'tombstone_delay', rse_tombstone_delay)
rse_core.add_rse_attribute(jump_rse1_id, 'multihop_tombstone_delay', rse_multihop_tombstone_delay)
# if multihop delay not set, it's the default multihop takes precedence. Not normal tombstone delay.
rse_core.add_rse_attribute(jump_rse2_id, 'tombstone_delay', rse_tombstone_delay)
core_config.set(section='transfers', option='multihop_tombstone_delay', value=default_multihop_tombstone_delay)
# if multihop delay is set to 0, the replica will have no tombstone
rse_core.add_rse_attribute(jump_rse3_id, 'multihop_tombstone_delay', 0)
distance_core.add_distance(src_rse_id, jump_rse1_id, ranking=10)
distance_core.add_distance(jump_rse1_id, jump_rse2_id, ranking=10)
distance_core.add_distance(jump_rse2_id, jump_rse3_id, ranking=10)
distance_core.add_distance(jump_rse3_id, dst_rse_id, ranking=10)
did = did_factory.upload_test_file(src_rse_name)
rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None)
# Ensure that each intermediate request was correctly created
for rse_id in jump_rses:
assert request_core.get_request_by_did(rse_id=rse_id, **did)
@read_session
def __ensure_source_exists(rse_id, scope, name, session=None):
return session.query(Source). \
filter(Source.rse_id == rse_id). \
filter(Source.scope == scope). \
filter(Source.name == name). \
one()
# Ensure that sources where created for transfers
for rse_id in jump_rses + [src_rse_id]:
__ensure_source_exists(rse_id, **did)
# Ensure the tombstone is correctly set on intermediate replicas
expected_tombstone = datetime.utcnow() + timedelta(seconds=rse_multihop_tombstone_delay)
replica = replica_core.get_replica(jump_rse1_id, **did)
assert expected_tombstone - timedelta(minutes=5) < replica['tombstone'] < expected_tombstone + timedelta(minutes=5)
expected_tombstone = datetime.utcnow() + timedelta(seconds=default_multihop_tombstone_delay)
replica = replica_core.get_replica(jump_rse2_id, **did)
assert expected_tombstone - timedelta(minutes=5) < replica['tombstone'] < expected_tombstone + timedelta(minutes=5)
replica = replica_core.get_replica(jump_rse3_id, **did)
assert replica['tombstone'] is None
