def test_add_rules_datasets_none(self):
""" REPLICATION RULE (CORE): Add replication rules to multiple datasets, NONE Grouping"""
scope = 'mock'
files1 = create_files(3, scope, self.rse4)
dataset1 = 'dataset_' + str(uuid())
add_did(scope, dataset1, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset1, files1, 'jdoe')
files2 = create_files(3, scope, self.rse4)
dataset2 = 'dataset_' + str(uuid())
add_did(scope, dataset2, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset2, files2, 'jdoe')
# Add the rules to both DS
add_rules(dids=[{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}],
rules=[{'account': 'jdoe',
'copies': 1,
'rse_expression': self.T1,
'grouping': 'NONE',
'weight': None,
'lifetime': None,
'locked': False,
'subscription_id': None},
{'account': 'root',
'copies': 1,
'rse_expression': self.T1,
'grouping': 'NONE',
'weight': 'fakeweight',
'lifetime': None,
'locked': False,
'subscription_id': None}])
# Check if the Locks are created properly
for file in files1:
rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]
assert(rse_locks[0] == rse_locks[1])
for file in files2:
rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]
assert(rse_locks[0] == rse_locks[1])
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK')
set_account_limit('jdoe', rse_id, -1)
dsns1 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1} for i in range(nbdatasets)]
dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1,
'rules': [{'account': 'jdoe', 'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'}]} for i in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account='root')
replicas = list()
for dsn in dsns1 + dsns2:
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(),
'bytes': 1, 'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account='root')
replicas += files
add_rules(dids=dsns1, rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = InternalScope('mock')
jdoe = InternalAccount('jdoe')
root = InternalAccount('root')
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK')
set_local_account_limit(jdoe, rse_id, -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for i in range(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account=root)
# Add generic metadata on did
test_metadata = {"test_key": "test_value"}
try:
add_did_meta(tmp_scope, dsns1[0]['name'], test_metadata)
except NotImplementedError:
# add_did_meta is not Implemented for Oracle < 12
pass
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(
get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'], None)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = InternalScope('mock', **self.vo)
jdoe = InternalAccount('jdoe', **self.vo)
root = InternalAccount('root', **self.vo)
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK', **self.vo)
set_local_account_limit(jdoe, rse_id, -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for _ in range(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for _ in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account=root)
# arbitrary keys do not work without JSON support (sqlite, Oracle < 12)
if json_implemented():
# Add generic metadata on did
set_metadata(tmp_scope, dsns1[0]['name'], "test_key", "test_value")
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'] is not None
def test_removal_all_replicas2(rse_factory, root_account, mock_scope,
core_config_mock, caches_mock):
""" UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """
rse1, rse1_id = rse_factory.make_posix_rse()
rse2, rse2_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
set_local_account_limit(root_account, rse1_id, -1)
set_local_account_limit(root_account, rse2_id, -1)
nbdatasets = 1
nbfiles = 5
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': mock_scope,
'type': 'DATASET',
'lifetime': -1
} for _ in range(nbdatasets)]
add_dids(dids=dsns1, account=root_account)
replicas = list()
for dsn in dsns1:
files = [{
'scope': mock_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb'
} for _ in range(nbfiles)]
attach_dids(scope=mock_scope,
name=dsn['name'],
rse_id=rse1_id,
dids=files,
account=root_account)
add_replicas(rse_id=rse2_id,
files=files,
account=root_account,
ignore_availability=True)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': root_account,
'copies': 1,
'rse_expression': rse1,
'grouping': 'DATASET'
}])
add_rules(dids=dsns1,
rules=[{
'account': root_account,
'copies': 1,
'rse_expression': rse2,
'grouping': 'DATASET',
'lifetime': -86400
}])
# Clean the rules on MOCK2. Replicas are tombstoned with non Epoch
rule_cleaner(once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse2_id)['tombstone'] is not None
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse1_id)['tombstone'] == datetime(year=1970,
month=1,
day=1)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse2_id)['tombstone'] == datetime(year=1970,
month=1,
day=1)
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse='MOCK',
dids=files,
account='root')
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': 'jdoe',
'copies': 1,
'rse_expression': 'MOCK',
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(
get_replica(scope=replica['scope'],
name=replica['name'],
rse='MOCK')['tombstone'], None)
def test_list_expired_dids_with_locked_rules(self):
""" UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules"""