def test_cache_add_replicas_file_not_exist(self):
""" RucioCache(Func): Test rucio cache add replica with not existed file(failed) """
ret = cache_add_replicas(self.rse_exist_volatile, self.files_noExist, self.account, self.lifetime)
assert_equal(ret, DID_NOT_FOUND)
for file in self.files_noExist:
try:
replica.get_replica(self.rse_exist_volatile, file['scope'], file['name'])
assert False
except Exception, e:
if "No row was found" in str(e):
assert True
else:
assert False
def test_cache_add_replicas_file_meta_wrong(self):
""" RucioCache(Func): Test rucio cache add replica with wrong meta data(failed) """
ret = cache_add_replicas(self.rse_exist_volatile, self.files_exist_wrong_meta, self.account, self.lifetime)
assert_equal(ret, META_MISMATCH)
for file in self.files_exist_wrong_meta:
try:
replica.get_replica(self.rse_exist_volatile, file['scope'], file['name'])
assert False
except Exception, e:
if "No row was found" in str(e):
assert True
else:
assert False
def test_cache_add_delete_replicas(self):
""" RucioCache(Func): Test rucio cache add and delete replicas(success) """
ret = cache_add_replicas(self.rse_exist_volatile, self.files_exist, self.account, self.lifetime)
assert_equal(ret, 0)
for file in self.files_exist:
reps = replica.get_replica(self.rse_exist_volatile, file['scope'], file['name'])
assert_equal(len(reps) > 0, True)
ret = cache_delete_replicas(self.rse_exist_volatile, self.files_exist, self.account)
assert_equal(ret, 0)
for file in self.files_exist:
try:
reps = replica.get_replica(self.rse_exist_volatile, file['scope'], file['name'])
assert False
except Exception, e:
if "No row was found" in str(e):
assert True
else:
assert False
def _get_path_nondeterministic_server(self, scope, name):
""" Provides the path of a replica for non-detemernisic sites. Will be assigned to get path by the __init__ method if neccessary. """
r = replica.get_replica(rse=self.rse['rse'], scope=scope, name=name, rse_id=self.rse['id'])
if 'path' in r and r['path'] is not None:
path = r['path']
elif 'state' in r and (r['state'] is None or r['state'] == 'UNAVAILABLE'):
raise exception.ReplicaUnAvailable('Missing path information and state is UNAVAILABLE for replica %s:%s on none-determinstic storage named %s' % (scope, name, self.rse['rse']))
else:
raise exception.ReplicaNotFound('Missing path information for replica %s:%s on none-determinstic storage named %s' % (scope, name, self.rse['rse']))
if path.startswith('/'):
path = path[1:]
if path.endswith('/'):
path = path[:-1]
return path
def test_add_rule_with_purge(self):
""" REPLICATION RULE (CORE): Add a replication rule with purge setting"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, purge_replicas=True)[0]
delete_rule(rule_id)
# Check if the Locks are created properly
for file in files:
replica = get_replica(rse=self.rse4, scope=file['scope'], name=file['name'])
assert(replica['tombstone'] == OBSOLETE)
def test_update_lock_counter(self):
""" RSE (CORE): Test the update of a replica lock counter """
rse = 'MOCK'
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
add_replica(rse=rse, scope=tmp_scope, name=tmp_file, bytes=1L, adler32='0cc737eb', account='jdoe')
values = (1, 1, 1, -1, -1, -1, 1, 1, -1)
tombstones = (True, True, True, True, True, False, True, True, True)
lock_counters = (1, 2, 3, 2, 1, 0, 1, 2, 1)
for value, tombstone, lock_counter in zip(values, tombstones, lock_counters):
status = update_replica_lock_counter(rse=rse, scope=tmp_scope, name=tmp_file, value=value)
assert_equal(status, True)
replica = get_replica(rse=rse, scope=tmp_scope, name=tmp_file)
assert_equal(replica['tombstone'] is None, tombstone)
assert_equal(lock_counter, replica['lock_cnt'])
def test_multihop_sources_created(rse_factory, did_factory, root_account,
core_config_mock, caches_mock):
"""
Ensure that multihop transfers are handled and intermediate request correctly created
"""
src_rse_name, src_rse_id = rse_factory.make_posix_rse()
_, jump_rse1_id = rse_factory.make_posix_rse()
_, jump_rse2_id = rse_factory.make_posix_rse()
_, jump_rse3_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id]
all_rses = jump_rses + [src_rse_id, dst_rse_id]
for rse_id in jump_rses:
rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True)
rse_tombstone_delay = 3600
rse_multihop_tombstone_delay = 12 * 3600
default_multihop_tombstone_delay = 24 * 3600
# if both attributes are set, the multihop one will take precedence
rse_core.add_rse_attribute(jump_rse1_id, 'tombstone_delay',
rse_tombstone_delay)
rse_core.add_rse_attribute(jump_rse1_id, 'multihop_tombstone_delay',
rse_multihop_tombstone_delay)
# if multihop delay not set, it's the default multihop takes precedence. Not normal tombstone delay.
rse_core.add_rse_attribute(jump_rse2_id, 'tombstone_delay',
rse_tombstone_delay)
core_config.set(section='transfers',
option='multihop_tombstone_delay',
value=default_multihop_tombstone_delay)
# if multihop delay is set to 0, the replica will have no tombstone
rse_core.add_rse_attribute(jump_rse3_id, 'multihop_tombstone_delay', 0)
distance_core.add_distance(src_rse_id, jump_rse1_id, ranking=10)
distance_core.add_distance(jump_rse1_id, jump_rse2_id, ranking=10)
distance_core.add_distance(jump_rse2_id, jump_rse3_id, ranking=10)
distance_core.add_distance(jump_rse3_id, dst_rse_id, ranking=10)
did = did_factory.upload_test_file(src_rse_name)
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse_name,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertool='mock',
transfertype='single',
filter_transfertool=None)
# Ensure that each intermediate request was correctly created
for rse_id in jump_rses:
assert request_core.get_request_by_did(rse_id=rse_id, **did)
@read_session
def __ensure_source_exists(rse_id, scope, name, session=None):
return session.query(Source). \
filter(Source.rse_id == rse_id). \
filter(Source.scope == scope). \
filter(Source.name == name). \
one()
# Ensure that sources where created for transfers
for rse_id in jump_rses + [src_rse_id]:
__ensure_source_exists(rse_id, **did)
# Ensure the tombstone is correctly set on intermediate replicas
expected_tombstone = datetime.utcnow() + timedelta(
seconds=rse_multihop_tombstone_delay)
replica = replica_core.get_replica(jump_rse1_id, **did)
assert expected_tombstone - timedelta(minutes=5) < replica[
'tombstone'] < expected_tombstone + timedelta(minutes=5)
expected_tombstone = datetime.utcnow() + timedelta(
seconds=default_multihop_tombstone_delay)
replica = replica_core.get_replica(jump_rse2_id, **did)
assert expected_tombstone - timedelta(minutes=5) < replica[
'tombstone'] < expected_tombstone + timedelta(minutes=5)
replica = replica_core.get_replica(jump_rse3_id, **did)
assert replica['tombstone'] is None
def test_archive_removal_impact_on_constituents(rse_factory, did_factory,
mock_scope, root_account,
caches_mock):
[cache_region] = caches_mock
rse_name, rse_id = rse_factory.make_mock_rse()
scope = mock_scope
account = root_account
# Create an 2 archives and 4 files:
# - One only exists in the first archive
# - One in both, plus another replica, which is not in an archive
# - One in both, plus another replica, which is not in an archive; and this replica has expired
# - One in both, plus another replica, which is not in an archive; and this replica has expired; but a replication rule exists on this second replica
# Also add these files to datasets, one of which will be removed at the end
nb_constituents = 4
nb_c_outside_archive = nb_constituents - 1
constituent_size = 2000
archive_size = 1000
uuid = str(generate_uuid())
constituents = [{
'scope': scope,
'name': 'lfn.%s.%d' % (uuid, i)
} for i in range(nb_constituents)]
did_factory.register_dids(constituents)
c_first_archive_only, c_with_replica, c_with_expired_replica, c_with_replica_and_rule = constituents
replica_core.add_replica(rse_id=rse_id,
account=account,
bytes_=constituent_size,
**c_with_replica)
replica_core.add_replica(rse_id=rse_id,
account=account,
bytes_=constituent_size,
tombstone=datetime.utcnow() - timedelta(days=1),
**c_with_expired_replica)
replica_core.add_replica(rse_id=rse_id,
account=account,
bytes_=constituent_size,
tombstone=datetime.utcnow() - timedelta(days=1),
**c_with_replica_and_rule)
rule_core.add_rule(dids=[c_with_replica_and_rule],
account=account,
copies=1,
rse_expression=rse_name,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
archive1, archive2 = [{
'scope': scope,
'name': 'archive_%s.%d.zip' % (uuid, i)
} for i in range(2)]
replica_core.add_replica(rse_id=rse_id,
bytes_=archive_size,
account=account,
**archive1)
replica_core.add_replica(rse_id=rse_id,
bytes_=archive_size,
account=account,
**archive2)
did_core.attach_dids(dids=[{
'scope': c['scope'],
'name': c['name'],
'bytes': constituent_size
} for c in constituents],
account=account,
**archive1)
did_core.attach_dids(dids=[
{
'scope': c['scope'],
'name': c['name'],
'bytes': constituent_size
} for c in
[c_with_replica, c_with_expired_replica, c_with_replica_and_rule]
],
account=account,
**archive2)
dataset1, dataset2 = [{
'scope': scope,
'name': 'dataset_%s.%i' % (uuid, i)
} for i in range(2)]
did_core.add_did(did_type='DATASET', account=account, **dataset1)
did_core.attach_dids(dids=constituents, account=account, **dataset1)
did_core.add_did(did_type='DATASET', account=account, **dataset2)
did_core.attach_dids(dids=[c_first_archive_only, c_with_expired_replica],
account=account,
**dataset2)
@read_session
def __get_archive_contents_history_count(archive, session=None):
return session.query(ConstituentAssociationHistory).filter_by(
**archive).count()
# Run reaper the first time.
# the expired non-archive replica of c_with_expired_replica must be removed,
# but the did must not be remove and it must still remain in the dataset because
# it still has the replica from inside the archive
assert replica_core.get_replica(rse_id=rse_id, **c_with_expired_replica)
cache_region.invalidate()
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=2 * archive_size +
nb_c_outside_archive * constituent_size)
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=2 * archive_size +
nb_c_outside_archive * constituent_size,
free=1)
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None)
for did in constituents + [archive1, archive2]:
assert did_core.get_did(**did)
for did in [archive1, archive2, c_with_replica, c_with_replica_and_rule]:
assert replica_core.get_replica(rse_id=rse_id, **did)
with pytest.raises(ReplicaNotFound):
# The replica is only on the archive, not on the constituent
replica_core.get_replica(rse_id=rse_id, **c_first_archive_only)
with pytest.raises(ReplicaNotFound):
# The replica outside the archive was removed by reaper
nb_c_outside_archive -= 1
replica_core.get_replica(rse_id=rse_id, **c_with_expired_replica)
# Compared to get_replica, list_replicas resolves archives, must return replicas for all files
assert len(list(replica_core.list_replicas(dids=constituents))) == 4
assert len(list(did_core.list_content(**dataset1))) == 4
assert len(list(did_core.list_archive_content(**archive1))) == 4
assert len(list(did_core.list_archive_content(**archive2))) == 3
assert __get_archive_contents_history_count(archive1) == 0
assert __get_archive_contents_history_count(archive2) == 0
# Expire the first archive and run reaper again
# the archive will be removed; and c_first_archive_only must be removed from datasets
# and from the did table.
replica_core.set_tombstone(rse_id=rse_id,
tombstone=datetime.utcnow() - timedelta(days=1),
**archive1)
cache_region.invalidate()
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=2 * archive_size +
nb_c_outside_archive * constituent_size)
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=2 * archive_size +
nb_c_outside_archive * constituent_size,
free=1)
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None)
with pytest.raises(DataIdentifierNotFound):
assert did_core.get_did(**archive1)
with pytest.raises(DataIdentifierNotFound):
assert did_core.get_did(**c_first_archive_only)
assert len(list(replica_core.list_replicas(dids=constituents))) == 3
assert len(list(did_core.list_content(**dataset1))) == 3
assert len(list(did_core.list_archive_content(**archive1))) == 0
assert len(list(did_core.list_archive_content(**archive2))) == 3
assert __get_archive_contents_history_count(archive1) == 4
assert __get_archive_contents_history_count(archive2) == 0
# Expire the second archive replica and run reaper another time
# c_with_expired_replica is removed because its external replica got removed at previous step
# and it exist only inside the archive now.
# If not open, Dataset2 will be removed because it will be empty.
did_core.set_status(open=False, **dataset2)
replica_core.set_tombstone(rse_id=rse_id,
tombstone=datetime.utcnow() - timedelta(days=1),
**archive2)
cache_region.invalidate()
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=archive_size +
nb_c_outside_archive * constituent_size)
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=archive_size +
nb_c_outside_archive * constituent_size,
free=1)
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None)
# The archive must be removed
with pytest.raises(DataIdentifierNotFound):
assert did_core.get_did(**archive2)
# The DIDs which only existed in the archive are also removed
with pytest.raises(DataIdentifierNotFound):
assert did_core.get_did(**c_first_archive_only)
with pytest.raises(DataIdentifierNotFound):
assert did_core.get_did(**c_with_expired_replica)
# If the DID has a non-expired replica outside the archive without rules on it, the DID is not removed
assert did_core.get_did(**c_with_replica)
# If the DID has an expired replica outside the archive, but has rules on that replica, the DID is not removed
assert did_core.get_did(**c_with_replica_and_rule)
assert len(list(replica_core.list_replicas(dids=constituents))) == 2
assert len(list(did_core.list_content(**dataset1))) == 2
with pytest.raises(DataIdentifierNotFound):
did_core.get_did(**dataset2)
assert len(list(did_core.list_content(**dataset2))) == 0
assert len(list(did_core.list_archive_content(**archive2))) == 0
assert __get_archive_contents_history_count(archive1) == 4
assert __get_archive_contents_history_count(archive2) == 3
def test_to_repair_a_rule_with_NONE_grouping_whose_transfer_failed(self):
""" JUDGE REPAIRER: Test to repair a rule with 1 failed transfer (lock)"""
rule_repairer(once=True) # Clean out the repairer
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse4_id, bytes_=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=1,
rse_expression=self.T1,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)[0]
failed_rse_id = get_replica_locks(scope=files[2]['scope'],
name=files[2]['name'])[0].rse_id
assert (get_replica(
scope=files[2]['scope'],
name=files[2]['name'],
rse_id=failed_rse_id)['state'] == ReplicaState.COPYING)
assert (get_replica(scope=files[2]['scope'],
name=files[2]['name'],
rse_id=failed_rse_id)['lock_cnt'] == 1)
successful_transfer(
scope=scope,
name=files[0]['name'],
rse_id=get_replica_locks(scope=files[0]['scope'],
name=files[2]['name'])[0].rse_id,
nowait=False)
successful_transfer(
scope=scope,
name=files[1]['name'],
rse_id=get_replica_locks(scope=files[1]['scope'],
name=files[2]['name'])[0].rse_id,
nowait=False)
failed_transfer(
scope=scope,
name=files[2]['name'],
rse_id=get_replica_locks(scope=files[2]['scope'],
name=files[2]['name'])[0].rse_id)
assert (rule_id == get_rule(rule_id)['id'].replace('-', '').lower())
assert (RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
assert (RuleState.REPLICATING == get_rule(rule_id)['state'])
assert (get_replica(
scope=files[2]['scope'],
name=files[2]['name'],
rse_id=failed_rse_id)['state'] == ReplicaState.UNAVAILABLE)
assert (get_replica(scope=files[2]['scope'],
name=files[2]['name'],
rse_id=failed_rse_id)['lock_cnt'] == 0)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = InternalScope('mock', **self.vo)
jdoe = InternalAccount('jdoe', **self.vo)
root = InternalAccount('root', **self.vo)
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK', **self.vo)
set_local_account_limit(jdoe, rse_id, -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for _ in range(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for _ in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account=root)
# arbitrary keys do not work without JSON support (sqlite, Oracle < 12)
if json_implemented():
# Add generic metadata on did
set_metadata(tmp_scope, dsns1[0]['name'], "test_key", "test_value")
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'] is not None
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
if get_policy() != 'atlas':
LOG.info("Skipping atlas-specific test")
return
tmp_scope = InternalScope('mock', **self.vo)
jdoe = InternalAccount('jdoe', **self.vo)
root = InternalAccount('root', **self.vo)
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
rse_id = add_rse(rse, **self.vo)
set_local_account_limit(jdoe, rse_id, -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for _ in range(nbdatasets)]
add_dids(dids=dsns2, account=root)
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert (get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'] is None)
for dsn in dsns2:
assert (get_did(scope=InternalScope('archive', **self.vo),
name=dsn['name'])['name'] == dsn['name'])
assert (len([
x for x in list_rules(
filters={
'scope': InternalScope('archive', **self.vo),
'name': dsn['name']
})
]) == 1)
session = get_session()
lost_requests = get_lost_requests(input_file_name)
for scope, name, rse_name in lost_requests:
# print rse_name, scope, name
if rse_name not in rse_ids:
rse_id = rse_core.get_rse_id(rse_name, session=session)
rse_ids[rse_name] = rse_id
else:
rse_id = rse_ids[rse_name]
try:
replica = replica_core.get_replica(rse=rse_name,
scope=scope,
name=name,
rse_id=rse_id,
session=session)
except NoResultFound:
continue
if replica['state'] != ReplicaState.AVAILABLE:
continue
if rse_name not in protocols:
rse_info = rsemgr.get_rse_info(rse_name, session=session)
protocols[rse_name] = rsemgr.create_protocol(rse_info, 'write',
'srm,gsiftp')
lfn = {
'scope': replica['scope'],
'name': replica['name'],
'path': replica['path']
add_rules(dids=dsns1,
rules=[{
'account': 'jdoe',
'copies': 1,
'rse_expression': 'MOCK',
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(
get_replica(scope=replica['scope'],
name=replica['name'],
rse='MOCK')['tombstone'], None)
def test_list_expired_dids_with_locked_rules(self):
""" UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules"""
tmp_scope = 'mock'
# Add quota
set_account_limit('jdoe', get_rse_id('MOCK'), -1)
dsn = {
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account,
core_config_mock,
caches_mock):
"""
Ensure that intermediate replicas created by the submitter are protected from deletion even if their tombstone is
set to epoch.
After successful transfers, intermediate replicas with default (epoch) tombstone must be removed. The others must
be left intact.
"""
src_rse1_name = 'XRD1'
src_rse1_id = rse_core.get_rse_id(rse=src_rse1_name, vo=vo)
src_rse2_name = 'XRD2'
src_rse2_id = rse_core.get_rse_id(rse=src_rse2_name, vo=vo)
jump_rse_name = 'XRD3'
jump_rse_id = rse_core.get_rse_id(rse=jump_rse_name, vo=vo)
dst_rse_name = 'XRD4'
dst_rse_id = rse_core.get_rse_id(rse=dst_rse_name, vo=vo)
all_rses = [src_rse1_id, src_rse2_id, jump_rse_id, dst_rse_id]
did = did_factory.upload_test_file(src_rse1_name)
# Copy replica to a second source. To avoid the special case of having a unique last replica, which could be handled in a special (more careful) way
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=src_rse2_name,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
replica = __wait_for_replica_transfer(dst_rse_id=src_rse2_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
rse_core.set_rse_limits(rse_id=jump_rse_id, name='MinFreeSpace', value=1)
rse_core.set_rse_usage(rse_id=jump_rse_id,
source='storage',
used=1,
free=0)
try:
rule_core.add_rule(dids=[did],
account=root_account,
copies=1,
rse_expression=dst_rse_name,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
# Submit transfers to FTS
# Ensure a replica was created on the intermediary host with epoch tombstone
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
assert request['state'] == RequestState.SUBMITTED
replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
assert replica['tombstone'] == datetime(year=1970, month=1, day=1)
assert replica['state'] == ReplicaState.COPYING
# The intermediate replica is protected by its state (Copying)
rucio.daemons.reaper.reaper.REGION.invalidate()
reaper(once=True,
rses=[],
include_rses=jump_rse_name,
exclude_rses=None)
replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
assert replica['state'] == ReplicaState.COPYING
# Wait for the intermediate replica to become ready
replica = __wait_for_replica_transfer(dst_rse_id=jump_rse_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
# The intermediate replica is protected by an entry in the sources table
# Reaper must not remove this replica, even if it has an obsolete tombstone
rucio.daemons.reaper.reaper.REGION.invalidate()
reaper(once=True,
rses=[],
include_rses=jump_rse_name,
exclude_rses=None)
replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
assert replica
# FTS fails the second transfer, so run submitter again to copy from jump rse to destination rse
submitter(once=True,
rses=[{
'id': rse_id
} for rse_id in all_rses],
partition_wait_time=None,
transfertype='single',
filter_transfertool=None)
# Wait for the destination replica to become ready
replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
assert replica['state'] == ReplicaState.AVAILABLE
rucio.daemons.reaper.reaper.REGION.invalidate()
reaper(once=True,
rses=[],
include_rses='test_container_xrd=True',
exclude_rses=None)
with pytest.raises(ReplicaNotFound):
replica_core.get_replica(rse_id=jump_rse_id, **did)
finally:
@transactional_session
def _cleanup_all_usage_and_limits(rse_id, session=None):
session.query(models.RSELimit).filter_by(rse_id=rse_id).delete()
session.query(models.RSEUsage).filter_by(
rse_id=rse_id, source='storage').delete()
_cleanup_all_usage_and_limits(rse_id=jump_rse_id)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = InternalScope('mock')
jdoe = InternalAccount('jdoe')
root = InternalAccount('root')
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK')
set_account_limit(jdoe, rse_id, -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for i in range(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account=root)
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(
get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'], None)
def test_removal_all_replicas2(rse_factory, root_account, mock_scope,
core_config_mock, caches_mock):
""" UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """
rse1, rse1_id = rse_factory.make_posix_rse()
rse2, rse2_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
set_local_account_limit(root_account, rse1_id, -1)
set_local_account_limit(root_account, rse2_id, -1)
nbdatasets = 1
nbfiles = 5
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': mock_scope,
'type': 'DATASET',
'lifetime': -1
} for _ in range(nbdatasets)]
add_dids(dids=dsns1, account=root_account)
replicas = list()
for dsn in dsns1:
files = [{
'scope': mock_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb'
} for _ in range(nbfiles)]
attach_dids(scope=mock_scope,
name=dsn['name'],
rse_id=rse1_id,
dids=files,
account=root_account)
add_replicas(rse_id=rse2_id,
files=files,
account=root_account,
ignore_availability=True)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': root_account,
'copies': 1,
'rse_expression': rse1,
'grouping': 'DATASET'
}])
add_rules(dids=dsns1,
rules=[{
'account': root_account,
'copies': 1,
'rse_expression': rse2,
'grouping': 'DATASET',
'lifetime': -86400
}])
# Clean the rules on MOCK2. Replicas are tombstoned with non Epoch
rule_cleaner(once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse2_id)['tombstone'] is not None
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse1_id)['tombstone'] == datetime(year=1970,
month=1,
day=1)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse2_id)['tombstone'] == datetime(year=1970,
month=1,
day=1)
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in range(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse=rse,
dids=files,
account='root')
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert (get_replica(scope=replica['scope'],
name=replica['name'],
rse=rse)['tombstone'] is None)
for dsn in dsns2:
assert (get_did(scope='archive',
name=dsn['name'])['name'] == dsn['name'])
assert (len([
x for x in list_rules(filters={
'scope': 'archive',
'name': dsn['name']
})
]) == 1)
