def test_to_repair_a_rule_with_DATASET_grouping_whose_transfer_failed(
self):
""" JUDGE REPAIRER: Test to repair a rule with 1 failed transfer (lock)"""
rule_repairer(once=True) # Clean out the repairer
scope = InternalScope('mock')
files = create_files(4, scope, self.rse4_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=1,
rse_expression=self.T1,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
activity='DebugJudge')[0]
successful_transfer(
scope=scope,
name=files[0]['name'],
rse_id=get_replica_locks(scope=files[0]['scope'],
name=files[2]['name'])[0].rse_id,
nowait=False)
successful_transfer(
scope=scope,
name=files[1]['name'],
rse_id=get_replica_locks(scope=files[1]['scope'],
name=files[2]['name'])[0].rse_id,
nowait=False)
failed_transfer(
scope=scope,
name=files[2]['name'],
rse_id=get_replica_locks(scope=files[2]['scope'],
name=files[2]['name'])[0].rse_id)
failed_transfer(
scope=scope,
name=files[3]['name'],
rse_id=get_replica_locks(scope=files[3]['scope'],
name=files[3]['name'])[0].rse_id)
assert (rule_id == get_rule(rule_id)['id'].replace('-', '').lower())
assert (RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
assert (RuleState.REPLICATING == get_rule(rule_id)['state'])
assert (get_replica_locks(
scope=files[2]['scope'],
name=files[2]['name'])[0].rse_id == get_replica_locks(
scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
assert (get_replica_locks(
scope=files[1]['scope'],
name=files[1]['name'])[0].rse_id == get_replica_locks(
scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
def _create_tmp_DID(self, type='DATASET'):
did_name = 'fe_test_did_%s' % generate_uuid()
add_did(scope=self.tmp_scope,
name=did_name,
did_type='DATASET',
account=self.root)
return did_name
def test_judge_evaluate_detach_datasetlock(self):
""" JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container"""
re_evaluator(once=True)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), self.jdoe)
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe)
# Add a rule to the Container
add_rule(dids=[{'scope': scope, 'name': container}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)
# Check if the datasetlock is there
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) > 0)
detach_dids(scope, container, [{'scope': scope, 'name': dataset}])
# Fake judge
re_evaluator(once=True)
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) == 0)
def test_delete_dids(self):
""" DATA IDENTIFIERS (CORE): Delete dids """
tmp_scope = 'mock'
dsns = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'did_type': DIDType.DATASET} for i in xrange(5)]
for dsn in dsns:
add_did(scope=tmp_scope, name=dsn['name'], type='DATASET', account='root')
delete_dids(dids=dsns, account='root')
def test_judge_evaluate_detach_datasetlock(self):
""" JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container"""
re_evaluator(once=True)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe')
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe')
# Add a rule to the Container
add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)
# Check if the datasetlock is there
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) > 0)
detach_dids(scope, container, [{'scope': scope, 'name': dataset}])
# Fake judge
re_evaluator(once=True)
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) == 0)
def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
account_update(once=True)
account_counter_before = get_counter(self.rse1_id, 'jdoe')
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_counter(self.rse1_id, 'jdoe')
assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes'])
assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
for r in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(r[3], 2)
def test_judge_expire_rule_with_child_rule(self):
""" JUDGE CLEANER: Test the judge when deleting expired rules with child rules"""
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=1,
rse_expression=self.rse1,
grouping='NONE',
weight='fakeweight',
lifetime=None,
locked=False,
subscription_id=None)[0]
child_rule = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=1,
rse_expression=self.rse3,
grouping='NONE',
weight='fakeweight',
lifetime=-3,
locked=False,
subscription_id=None)[0]
update_rule(rule_id, {'child_rule_id': child_rule})
rule_cleaner(once=True)
def test_bb8_rebalance_rule(self):
""" BB8: Test the rebalance rule method"""
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
rule = {}
try:
rule = get_rule(rule_id)
except:
assert_raises(RuleNotFound, get_rule, rule_id)
child_rule = rebalance_rule(rule, 'Rebalance', self.rse3, priority=3)
rule_cleaner(once=True)
assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow())
assert(get_rule(rule_id)['child_rule_id'] == child_rule)
rule_cleaner(once=True)
assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow())
successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse3_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse3_id, nowait=False)
with assert_raises(UnsupportedOperation):
delete_rule(rule_id)
successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse3_id, nowait=False)
rule_cleaner(once=True)
assert(get_rule(child_rule)['state'] == RuleState.OK)
def test_judge_add_files_to_dataset(self):
""" JUDGE EVALUATOR: Test the judge when adding files to dataset"""
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
# Add a first rule to the DS
add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=2,
rse_expression=self.T1,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
attach_dids(scope, dataset, files, self.jdoe)
re_evaluator(once=True)
files = create_files(3, scope, self.rse1_id)
attach_dids(scope, dataset, files, self.jdoe)
# Fake judge
re_evaluator(once=True)
# Check if the Locks are created properly
for file in files:
assert (len(
get_replica_locks(scope=file['scope'],
name=file['name'])) == 2)
def test_add_rule_container_dataset_with_weights(self):
""" REPLICATION RULE (CORE): Add a replication rule on a container, DATASET Grouping, WEIGHTS"""
scope = 'mock'
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe')
all_files = []
dataset_files = []
for i in xrange(3):
files = create_files(3, scope, self.rse1)
all_files.extend(files)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe')
dataset_files.append({'scope': scope, 'name': dataset, 'files': files})
add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)
t1 = set([self.rse1_id, self.rse3_id, self.rse5_id])
for dataset in dataset_files:
first_locks = None
for file in dataset['files']:
if first_locks is None:
first_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])])
rse_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])])
assert(len(t1.intersection(rse_locks)) == 2)
assert(len(first_locks.intersection(rse_locks)) == 2)
assert_in(self.rse1_id, rse_locks)
def test_judge_deny_rule(self):
""" JUDGE INJECTOR: Test the judge when asking approval for a rule and denying it"""
scope = 'mock'
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
rule_id = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account='jdoe',
copies=1,
rse_expression=self.rse4,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
ask_approval=True)[0]
assert (get_rule(rule_id)['state'] == RuleState.WAITING_APPROVAL)
deny_rule(rule_id=rule_id, approver='root')
assert_raises(RuleNotFound, get_rule, rule_id)
def test_judge_inject_rule(self):
""" JUDGE INJECTOR: Test the judge when injecting a rule"""
scope = 'mock'
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
rule_id = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account='jdoe',
copies=2,
rse_expression=self.T1,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
asynchronous=True)[0]
assert (get_rule(rule_id)['state'] == RuleState.INJECT)
rule_injector(once=True)
# Check if the Locks are created properly
for file in files:
assert (len(
get_replica_locks(scope=file['scope'],
name=file['name'])) == 2)
assert (get_rule(rule_id)['state'] == RuleState.REPLICATING)
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# Add quota
set_account_limit('root', get_rse_id(site_a), -1)
set_account_limit('root', get_rse_id(site_b), -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root')
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
for rule in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(rule[3], 2)
def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
account_update(once=True)
account_counter_before = get_usage(self.rse1_id, self.jdoe)
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_usage(self.rse1_id, self.jdoe)
assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes'])
assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_run_transmogrifier(self):
""" SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
rse1, _ = self.rse_factory.make_mock_rse()
rse2, _ = self.rse_factory.make_mock_rse()
rse3, _ = self.rse_factory.make_mock_rse()
rse_expression = '%s|%s|%s' % (rse1, rse2, rse3)
tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
root = InternalAccount('root', **self.vo)
add_scope(tmp_scope, root)
subscription_name = uuid()
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope,
name=dsn,
did_type=DIDType.DATASET,
account=root)
subid = self.sub_client.add_subscription(name=subscription_name,
account='root',
filter_={
'scope': [
tmp_scope.external,
],
'pattern': 'dataset-.*',
'split_rule': True
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=None,
retroactive=0,
dry_run=0,
comments='Ni ! Ni!',
priority=1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(
scope=tmp_scope.external, name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert len(rules) == 2
set_new_dids([
{
'scope': tmp_scope,
'name': dsn
},
], 1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(
scope=tmp_scope.external, name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert len(rules) == 2
def test_judge_inject_delayed_rule(self):
""" JUDGE INJECTOR: Test the judge when injecting a delayed rule"""
scope = InternalScope('mock', **self.vo)
files = create_files(1, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
[file] = files
# Add a delayed rule
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, delay_injection=3600)[0]
rule = get_rule(rule_id)
assert rule['state'] == RuleState.INJECT
assert rule['updated_at'] < rule['created_at']
assert datetime.utcnow() + timedelta(seconds=3550) < rule['created_at'] < datetime.utcnow() + timedelta(seconds=3650)
# The time to create the rule has not yet arrived. The injector must skip this rule, no locks must be created
rule_injector(once=True)
assert get_rule(rule_id)['state'] == RuleState.INJECT
assert not get_replica_locks(scope=file['scope'], name=file['name'])
# simulate that time to inject the rule has arrived
@transactional_session
def __update_created_at(session=None):
session.query(ReplicationRule).filter_by(id=rule_id).one().created_at = datetime.utcnow()
__update_created_at()
# The injector must create the locks now
rule_injector(once=True)
assert get_rule(rule_id)['state'] == RuleState.REPLICATING
assert len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2
def test_touch_dids(self):
""" DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope,
name=tmp_dsn1,
type=DIDType.DATASET,
account='root')
add_did(scope=tmp_scope,
name=tmp_dsn2,
type=DIDType.DATASET,
account='root')
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
touch_dids(dids=[{
'scope': tmp_scope,
'name': tmp_dsn1,
'type': DIDType.DATASET,
'accessed_at': now
}])
assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_to_repair_a_rule_with_only_1_rse_whose_transfers_failed(self):
""" JUDGE REPAIRER: Test to repair a rule with only 1 rse whose transfers failed (lock)"""
rule_repairer(once=True) # Clean out the repairer
scope = 'mock'
files = create_files(4, scope, self.rse4, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
successful_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False)
failed_transfer(scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id)
failed_transfer(scope=scope, name=files[3]['name'], rse_id=get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
cancel_request_did(scope=scope, name=files[2]['name'], dest_rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id)
cancel_request_did(scope=scope, name=files[3]['name'], dest_rse_id=get_replica_locks(scope=files[3]['scope'], name=files[2]['name'])[0].rse_id)
assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower())
assert(RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
# Stil assert STUCK because of delays:
assert(RuleState.STUCK == get_rule(rule_id)['state'])
assert(get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id == get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
def test_repair_a_rule_with_source_replica_expression(self):
""" JUDGE EVALUATOR: Test the judge when a with two rules with source_replica_expression"""
scope = 'mock'
files = create_files(3, scope, self.rse4)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
rule_id1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
rule_id2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, source_replica_expression=self.rse1)[0]
assert(RuleState.REPLICATING == get_rule(rule_id1)['state'])
assert(RuleState.STUCK == get_rule(rule_id2)['state'])
successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse1_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse1_id, nowait=False)
successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse1_id, nowait=False)
# Also make replicas AVAILABLE
session = get_session()
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[0]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[1]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[2]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
session.commit()
rule_repairer(once=True)
assert(RuleState.OK == get_rule(rule_id1)['state'])
assert(RuleState.REPLICATING == get_rule(rule_id2)['state'])
def test_to_repair_a_rule_with_NONE_grouping_whose_transfer_failed(self):
""" JUDGE REPAIRER: Test to repair a rule with 1 failed transfer (lock)"""
rule_repairer(once=True) # Clean out the repairer
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse4_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
failed_rse_id = get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id
assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['state'] == ReplicaState.COPYING)
assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['lock_cnt'] == 1)
successful_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False)
failed_transfer(scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id)
assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower())
assert(RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['state'] == ReplicaState.UNAVAILABLE)
assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['lock_cnt'] == 0)
def test_repair_a_rule_with_source_replica_expression(self):
""" JUDGE EVALUATOR: Test the judge when a with two rules with source_replica_expression"""
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse4_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
# Add a first rule to the DS
rule_id1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
rule_id2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, source_replica_expression=self.rse1)[0]
assert(RuleState.REPLICATING == get_rule(rule_id1)['state'])
assert(RuleState.STUCK == get_rule(rule_id2)['state'])
successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse1_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse1_id, nowait=False)
successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse1_id, nowait=False)
# Also make replicas AVAILABLE
session = get_session()
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[0]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[1]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[2]['name'], rse_id=self.rse1_id).one()
replica.state = ReplicaState.AVAILABLE
session.commit()
rule_repairer(once=True)
assert(RuleState.OK == get_rule(rule_id1)['state'])
assert(RuleState.REPLICATING == get_rule(rule_id2)['state'])
def test_to_repair_a_rule_with_only_1_rse_whose_transfers_failed(self):
""" JUDGE REPAIRER: Test to repair a rule with only 1 rse whose transfers failed (lock)"""
rule_repairer(once=True) # Clean out the repairer
scope = InternalScope('mock', **self.vo)
files = create_files(4, scope, self.rse4_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
successful_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False)
failed_transfer(scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id)
failed_transfer(scope=scope, name=files[3]['name'], rse_id=get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
cancel_request_did(scope=scope, name=files[2]['name'], dest_rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id)
cancel_request_did(scope=scope, name=files[3]['name'], dest_rse_id=get_replica_locks(scope=files[3]['scope'], name=files[2]['name'])[0].rse_id)
assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower())
assert(RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
# Stil assert STUCK because of delays:
assert(RuleState.STUCK == get_rule(rule_id)['state'])
assert(get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id == get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blocklisted(self):
""" JUDGE REPAIRER: Test to repair a rule with only 1 rse whose site is blocklisted"""
rse = rse_name_generator()
rse_id = add_rse(rse, **self.vo)
update_rse(rse_id, {'availability_write': False})
set_local_account_limit(self.jdoe, rse_id, -1)
rule_repairer(once=True) # Clean out the repairer
scope = InternalScope('mock', **self.vo)
files = create_files(4, scope, self.rse4_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True, activity='DebugJudge')[0]
assert(RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
# Stil assert STUCK because of ignore_availability:
assert(RuleState.STUCK == get_rule(rule_id)['state'])
region = make_region().configure('dogpile.cache.memcached',
expiration_time=3600,
arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True})
region.delete(sha256(rse.encode()).hexdigest())
update_rse(rse_id, {'availability_write': True})
rule_repairer(once=True)
assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
def testdid(vo):
did_name = 'testdid_%s' % generate_uuid()
mock_scope = InternalScope('mock', vo=vo)
didtype = 'DATASET'
account = InternalAccount('root', vo=vo)
add_did(scope=mock_scope, name=did_name, type=didtype, account=account)
yield {'name': did_name, 'scope': mock_scope}
delete_dids(dids=[{'name': did_name, 'scope': mock_scope, 'did_type': didtype, 'purge_replicas': True}], account=account)
def test_add_rule(self):
""" REPLICATION RULE (CLIENT): Add a replication rule """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE')
assert_is_instance(ret, list)
def test_get_metadata(self):
""" DID Meta (JSON): Get did meta """
skip_without_json()
did_name = 'mock_did_%s' % generate_uuid()
meta_key = 'my_key_%s' % generate_uuid()
meta_value = 'my_value_%s' % generate_uuid()
add_did(scope=self.tmp_scope, name=did_name, type='DATASET', account=self.root)
set_metadata(scope=self.tmp_scope, name=did_name, key=meta_key, value=meta_value)
assert get_metadata(scope=self.tmp_scope, name=did_name, plugin='JSON')[meta_key] == meta_value
def test_delete_dids(self):
""" DATA IDENTIFIERS (CORE): Delete dids """
tmp_scope = 'mock'
dsns = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'purge_replicas': False,
'did_type': DIDType.DATASET} for i in range(5)]
for dsn in dsns:
add_did(scope=tmp_scope, name=dsn['name'], type='DATASET', account='root')
delete_dids(dids=dsns, account='root')
def test_update_dids(self):
""" DATA IDENTIFIERS (CORE): Update file size and checksum"""
tmp_scope = 'mock'
dsn = 'dsn_%s' % generate_uuid()
lfn = 'lfn.%s' % str(generate_uuid())
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root')
files = [{'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}}]
def test_update_dids(self):
""" DATA IDENTIFIERS (CORE): Update file size and checksum"""
tmp_scope = 'mock'
dsn = 'dsn_%s' % generate_uuid()
lfn = 'lfn.%s' % str(generate_uuid())
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root')
files = [{'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}}]
def test_run_transmogrifier(self):
""" SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
subscription_name = uuid()
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope,
name=dsn,
type=DIDType.DATASET,
account='root')
subid = self.sub_client.add_subscription(name=subscription_name,
account='root',
filter={
'scope': [
tmp_scope,
],
'pattern': 'dataset-.*',
'split_rule': True
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
'MOCK-POSIX|MOCK2|MOCK3',
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=None,
retroactive=0,
dry_run=0,
comments='Ni ! Ni!',
priority=1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(scope=tmp_scope,
name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert_equal(len(rules), 2)
set_new_dids([
{
'scope': tmp_scope,
'name': dsn
},
], 1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(scope=tmp_scope,
name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert_equal(len(rules), 2)
def test_get_did_meta(self):
""" DID Meta (Hardcoded): Get did meta """
did_name = 'mock_did_%s' % generate_uuid()
dataset_meta = {'project': 'data12_8TeV'}
add_did(scope=self.tmp_scope,
name=did_name,
type='DATASET',
meta=dataset_meta,
account=self.root)
assert get_metadata(scope=self.tmp_scope,
name=did_name)['project'] == 'data12_8TeV'
def test_get_rule(self):
""" REPLICATION RULE (CORE): Test to get a previously created rule"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower())
assert_raises(RuleNotFound, get_rule, uuid())
def test_add_rule_with_r2d2_container_treating_and_duplicate_rule(self):
""" JUDGE INJECTOR (CORE): Add a replication rule with an r2d2 container treatment and duplicate rule"""
scope = 'mock'
container = 'asdf.r2d2_request.2016-04-01-15-00-00.ads.' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe')
datasets = []
for i in range(3):
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
datasets.append(dataset)
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
attach_dids(scope, container, [{
'scope': scope,
'name': dataset
}], 'jdoe')
add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account='jdoe',
copies=1,
rse_expression=self.rse1,
grouping='DATASET',
weight=None,
lifetime=900,
locked=False,
subscription_id=None,
ask_approval=False)
rule_id = add_rule(dids=[{
'scope': scope,
'name': container
}],
account='jdoe',
copies=1,
rse_expression=self.rse1,
grouping='DATASET',
weight=None,
lifetime=900,
locked=False,
subscription_id=None,
ask_approval=True)[0]
approve_rule(rule_id, approver='root')
assert (get_rule(rule_id)['state'] == RuleState.INJECT)
rule_injector(once=True)
# Check if there is a rule for each file
with assert_raises(RuleNotFound):
get_rule(rule_id)
for dataset in datasets:
assert (len(
[r for r in list_rules({
'scope': scope,
'name': dataset
})]) > 0)
def test_dataset_lock(self):
""" DATASETLOCK (CLIENT): Get a datasetlock for a specific dataset"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=True, subscription_id=None)[0]
rule_ids = [lock['rule_id'] for lock in self.lock_client.get_dataset_locks(scope=scope, name=dataset)]
assert_in(rule_id_1, rule_ids)
def test_abacus_collection_replica_cleanup(self):
""" ABACUS (COLLECTION REPLICA): Test if the cleanup procedure works correctly. """
collection_replica.run(once=True)
db_session = session.get_session()
rse1 = rse_name_generator()
rse_id1 = add_rse(rse1, **self.vo)
rse2 = rse_name_generator()
rse_id2 = add_rse(rse2, **self.vo)
scope = InternalScope('mock', **self.vo)
dataset = 'dataset_%s' % generate_uuid()
jdoe = InternalAccount('jdoe', **self.vo)
add_did(scope, dataset, DIDType.DATASET, jdoe)
models.CollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1,
state=ReplicaState.AVAILABLE,
bytes=1).save(session=db_session, flush=False)
models.CollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2,
state=ReplicaState.AVAILABLE,
bytes=1).save(session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1,
did_type=DIDType.DATASET).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1,
did_type=DIDType.DATASET).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2,
did_type=DIDType.DATASET).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2,
did_type=DIDType.DATASET).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=None,
did_type=DIDType.DATASET).save(
session=db_session, flush=False)
db_session.commit()
assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3
self.did_client.set_metadata(scope.external, dataset, 'lifetime', -1)
def test_delete_rule(self):
""" REPLICATION RULE (CLIENT): Delete a replication rule """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
ret = self.rule_client.delete_replication_rule(rule_id=rule_id)
assert(ret is True)
assert_raises(RuleNotFound, self.rule_client.delete_replication_rule, rule_id)
def test_get_rule_by_account(self):
""" ACCOUNT (CLIENT): Get Replication Rule by account """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE')
get = self.account_client.list_account_rules('jdoe')
rules = [rule['id'] for rule in get]
assert_in(ret[0], rules)
def test_locked_rule(self):
""" REPLICATION RULE (CLIENT): Delete a locked replication rule"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=True, subscription_id=None)[0]
assert_raises(AccessDenied, delete_rule, rule_id_1)
self.rule_client.update_replication_rule(rule_id=rule_id_1, options={'locked': False})
delete_rule(rule_id=rule_id_1)
def test_add_rule_duplicate(self):
""" REPLICATION RULE (CORE): Add a replication rule duplicate"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)
# Add a second rule and check if the right locks are created
assert_raises(DuplicateRule, add_rule, dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)
def test_add_did_meta(self):
""" DID Meta (Hardcoded): Add did meta """
did_name = 'mock_did_%s' % generate_uuid()
add_did(scope=self.tmp_scope,
name=did_name,
type='DATASET',
account=self.root)
set_metadata(scope=self.tmp_scope,
name=did_name,
key='project',
value='data12_8TeV')
assert get_metadata(scope=self.tmp_scope,
name=did_name)['project'] == 'data12_8TeV'
def test_judge_expire_rule(self):
""" JUDGE CLEANER: Test the judge when deleting expired rules"""
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.DATASET, self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=1,
rse_expression=self.rse1,
grouping='NONE',
weight='fakeweight',
lifetime=-3,
locked=False,
subscription_id=None)[0]
add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=2,
rse_expression=self.T1,
grouping='NONE',
weight='fakeweight',
lifetime=None,
locked=False,
subscription_id=None)[0]
add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=3,
rse_expression=self.T1,
grouping='NONE',
weight='fakeweight',
lifetime=None,
locked=False,
subscription_id=None)[0]
rule_cleaner(once=True)
for file in files:
rse_locks = get_replica_locks(scope=file['scope'],
name=file['name'])
assert (len(rse_locks) == 5)
def test_add_did_to_followed(self):
""" DATA IDENTIFIERS (CORE): Mark a did as followed """
tmp_scope = InternalScope('mock')
dsn = 'dsn_%s' % generate_uuid()
root = InternalAccount('root')
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root)
add_did_to_followed(scope=tmp_scope, name=dsn, account=root)
users = get_users_following_did(scope=tmp_scope, name=dsn)
rows = 0
for user in users:
rows += 1
assert_equal(rows, 1)
def test_delete_rule(self):
""" REPLICATION RULE (CORE): Test to delete a previously created rule"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
delete_rule(rule_id)
for file in files:
rse_locks = get_replica_locks(scope=file['scope'], name=file['name'])
assert(len(rse_locks) == 0)
assert_raises(RuleNotFound, delete_rule, uuid())
def test_rule_add_fails_account_limit(self):
""" REPLICATION RULE (CORE): Test if a rule fails correctly when account limit conflict"""
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
set_account_limit(account='jdoe', rse_id=self.rse1_id, bytes=5)
assert_raises(InsufficientAccountLimit, add_rule, dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
delete_account_limit(account='jdoe', rse_id=self.rse1_id)
def test_get_did_with_dynamic(self):
""" DATA IDENTIFIERS (CORE): Get did with dynamic resolve of size"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
tmp_dsn3 = 'dsn_%s' % generate_uuid()
tmp_dsn4 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope,
name=tmp_dsn1,
type=DIDType.DATASET,
account='root')
add_replica(rse='MOCK',
scope=tmp_scope,
name=tmp_dsn2,
bytes=10,
account='root')
add_replica(rse='MOCK',
scope=tmp_scope,
name=tmp_dsn3,
bytes=10,
account='root')
attach_dids(scope=tmp_scope,
name=tmp_dsn1,
dids=[{
'scope': tmp_scope,
'name': tmp_dsn2
}, {
'scope': tmp_scope,
'name': tmp_dsn3
}],
account='root')
add_did(scope=tmp_scope,
name=tmp_dsn4,
type=DIDType.CONTAINER,
account='root')
attach_dids(scope=tmp_scope,
name=tmp_dsn4,
dids=[{
'scope': tmp_scope,
'name': tmp_dsn1
}],
account='root')
assert_equal(
get_did(scope=tmp_scope, name=tmp_dsn1, dynamic=True)['bytes'], 20)
assert_equal(
get_did(scope=tmp_scope, name=tmp_dsn4, dynamic=True)['bytes'], 20)
def test_get_users_following_did(self):
""" DATA IDENTIFIERS (CORE): Get the list of users following a did """
tmp_scope = InternalScope('mock', **self.vo)
dsn = 'dsn_%s' % generate_uuid()
root = InternalAccount('root', **self.vo)
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root)
add_did_to_followed(scope=tmp_scope, name=dsn, account=root)
users = get_users_following_did(scope=tmp_scope, name=dsn)
rows = 0
for user in users:
rows += 1
assert rows == 1
def test_add_rule_with_ignore_availability(self):
""" REPLICATION RULE (CORE): Add a replication rule with ignore_availability setting"""
rse = rse_name_generator()
add_rse(rse)
update_rse(rse, {'availability_write': False})
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
with assert_raises(InvalidRSEExpression):
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True)[0]
def test_add_rule_with_purge(self):
""" REPLICATION RULE (CORE): Add a replication rule with purge setting"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, purge_replicas=True)[0]
delete_rule(rule_id)
# Check if the Locks are created properly
for file in files:
replica = get_replica(rse=self.rse4, scope=file['scope'], name=file['name'])
assert(replica['tombstone'] == OBSOLETE)
def test_add_rule_dataset_none_with_weights(self):
""" REPLICATION RULE (CORE): Add a replication rule on a dataset, NONE Grouping, WEIGHTS"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight="fakeweight", lifetime=None, locked=False, subscription_id=None)
# Check if the Locks are created properly
t1 = set([self.rse1_id, self.rse3_id, self.rse5_id])
for file in files:
rse_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])])
assert(len(t1.intersection(rse_locks)) == 2)
assert_in(self.rse1_id, rse_locks)
def test_change_rule_lifetime(self):
""" REPLICATION RULE (CLIENT): Change rule lifetime"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight='fakeweight', lifetime=150, locked=True, subscription_id=None)[0]
get = self.rule_client.get_replication_rule(rule_id_1)
self.rule_client.update_replication_rule(rule_id_1, options={'lifetime': 10000})
get2 = self.rule_client.get_replication_rule(rule_id_1)
assert(get['expires_at'] != get2['expires_at'])
def test_touch_dids(self):
""" DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}])
assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)
for line in r2.body.split('\n'):
print line
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def test_list_rules_by_did(self):
""" DID (CLIENT): List Replication Rules per DID """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
rule_id_2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
ret = self.did_client.list_did_rules(scope=scope, name=dataset)
ids = [rule['id'] for rule in ret]
assert_in(rule_id_1, ids)
assert_in(rule_id_2, ids)
def test_dataset_callback_no(self):
""" REPLICATION RULE (CORE): Test dataset callback should not be sent"""
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
set_status(scope=scope, name=dataset, open=False)
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, notify='C')[0]
successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse3_id, nowait=False)
successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse3_id, nowait=False)
# Check if rule exists
assert(False == check_dataset_ok_callback(scope, dataset, self.rse3, rule_id))
def test_add_rule_container_none(self):
""" REPLICATION RULE (CORE): Add a replication rule on a container, NONE Grouping"""
scope = 'mock'
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe')
all_files = []
for i in xrange(3):
files = create_files(3, scope, self.rse1)
all_files.extend(files)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe')
add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.T2, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)
for file in all_files:
rse_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])])
assert_in(self.rse4_id, rse_locks)
assert_not_in(self.rse5_id, rse_locks)
def test_rse_counter_unavailable_replicas(self):
""" REPLICATION RULE (CORE): Test if creating UNAVAILABLE replicas updates the RSE Counter correctly"""
rse_update(once=True)
rse_counter_before = get_rse_counter(self.rse3_id)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
# Check if the rse has been updated correctly
rse_update(once=True)
rse_counter_after = get_rse_counter(self.rse3_id)
assert(rse_counter_before['bytes'] + 3*100 == rse_counter_after['bytes'])
assert(rse_counter_before['files'] + 3 == rse_counter_after['files'])
def test_delete_rule_and_cancel_transfers(self):
""" REPLICATION RULE (CORE): Test to delete a previously created rule and do not cancel overlapping transfers"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=3, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
delete_rule(rule_id_1)
for file in files:
rse_locks = get_replica_locks(scope=file['scope'], name=file['name'])
assert(len(rse_locks) == 5)
# TODO Need to check transfer queue here, this is actually not the check of this test case
assert_raises(RuleNotFound, delete_rule, uuid())
def test_account_counter_rule_create(self):
""" REPLICATION RULE (CORE): Test if the account counter is updated correctly when new rule is created"""
account_update(once=True)
account_counter_before = get_account_counter(self.rse1_id, 'jdoe')
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
# Check if the counter has been updated correctly
account_update(once=True)
account_counter_after = get_account_counter(self.rse1_id, 'jdoe')
assert(account_counter_before['bytes'] + 3*100 == account_counter_after['bytes'])
assert(account_counter_before['files'] + 3 == account_counter_after['files'])
def test_add_rules_datasets_none(self):
""" REPLICATION RULE (CORE): Add replication rules to multiple datasets, NONE Grouping"""
scope = 'mock'
files1 = create_files(3, scope, self.rse4)
dataset1 = 'dataset_' + str(uuid())
add_did(scope, dataset1, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset1, files1, 'jdoe')
files2 = create_files(3, scope, self.rse4)
dataset2 = 'dataset_' + str(uuid())
add_did(scope, dataset2, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset2, files2, 'jdoe')
# Add the rules to both DS
add_rules(dids=[{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}],
rules=[{'account': 'jdoe',
'copies': 1,
'rse_expression': self.T1,
'grouping': 'NONE',
'weight': None,
'lifetime': None,
'locked': False,
'subscription_id': None},
{'account': 'root',
'copies': 1,
'rse_expression': self.T1,
'grouping': 'NONE',
'weight': 'fakeweight',
'lifetime': None,
'locked': False,
'subscription_id': None}])
# Check if the Locks are created properly
for file in files1:
rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]
assert(rse_locks[0] == rse_locks[1])
for file in files2:
rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]
assert(rse_locks[0] == rse_locks[1])
