def test_judge_evaluate_detach_datasetlock(self):
""" JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container"""
re_evaluator(once=True)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe')
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe')
# Add a rule to the Container
add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)
# Check if the datasetlock is there
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) > 0)
detach_dids(scope, container, [{'scope': scope, 'name': dataset}])
# Fake judge
re_evaluator(once=True)
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) == 0)
def test_judge_evaluate_detach_datasetlock(self):
""" JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container"""
re_evaluator(once=True)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), self.jdoe)
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe)
# Add a rule to the Container
add_rule(dids=[{'scope': scope, 'name': container}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)
# Check if the datasetlock is there
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) > 0)
detach_dids(scope, container, [{'scope': scope, 'name': dataset}])
# Fake judge
re_evaluator(once=True)
locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)]
assert(len(locks) == 0)
def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
account_update(once=True)
account_counter_before = get_counter(self.rse1_id, 'jdoe')
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_counter(self.rse1_id, 'jdoe')
assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes'])
assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
account_update(once=True)
account_counter_before = get_usage(self.rse1_id, self.jdoe)
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_usage(self.rse1_id, self.jdoe)
assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes'])
assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def detach_dids(scope, name, dids, issuer, vo='def', session=None):
"""
Detach data identifier
:param scope: The scope name.
:param name: The data identifier name.
:param dids: The content.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
"""
kwargs = {'scope': scope, 'name': name, 'dids': dids, 'issuer': issuer}
if not rucio.api.permission.has_permission(issuer=issuer,
vo=vo,
action='detach_dids',
kwargs=kwargs,
session=session):
raise rucio.common.exception.AccessDenied(
'Account %s can not detach data identifiers from %s:%s' %
(issuer, scope, name))
scope = InternalScope(scope, vo=vo)
for d in dids:
d['scope'] = InternalScope(d['scope'], vo=vo)
return did.detach_dids(scope=scope, name=name, dids=dids, session=session)
def test_reattach_dids(self):
""" DATA IDENTIFIERS (CORE): Repeatedly attach and detach DIDs """
tmp_scope = InternalScope('mock')
root = InternalAccount('root')
parent_name = 'parent_%s' % generate_uuid()
add_did(scope=tmp_scope, name=parent_name, type=DIDType.DATASET, account=root)
child_name = 'child_%s' % generate_uuid()
files = [{'scope': tmp_scope, 'name': child_name,
'bytes': 12345, 'adler32': '0cc737eb'}]
rse_id = get_rse_id('MOCK')
attach_dids(scope=tmp_scope, name=parent_name, rse_id=rse_id, dids=files, account=root)
detach_dids(scope=tmp_scope, name=parent_name, dids=files)
attach_dids(scope=tmp_scope, name=parent_name, rse_id=rse_id, dids=files, account=root)
detach_dids(scope=tmp_scope, name=parent_name, dids=files)
def test_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the detach is done correctly"""
re_evaluator(once=True)
scope = InternalScope('mock', **self.vo)
container = 'container_' + str(uuid())
add_did(scope, container, DIDType.from_sym('CONTAINER'), self.jdoe)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe)
# Add a first rule to the Container
rule_id = add_rule(dids=[{'scope': scope, 'name': container}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
# Fake judge
re_evaluator(once=True)
assert(9 == get_rule(rule_id)['locks_ok_cnt'])
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
assert(8 == get_rule(rule_id)['locks_ok_cnt'])
def detach_dids(scope, name, dids, issuer):
"""
Detach data identifier
:param scope: The scope name.
:param name: The data identifier name.
:param dids: The content.
:param issuer: The issuer account.
"""
kwargs = {'scope': scope, 'name': name, 'dids': dids, 'issuer': issuer}
if not rucio.api.permission.has_permission(issuer=issuer, action='detach_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not detach data identifiers from %s:%s' % (issuer, scope, name))
return did.detach_dids(scope=scope, name=name, dids=dids)
def detach_dids(scope, name, dids, issuer):
"""
Detach data identifier
:param scope: The scope name.
:param name: The data identifier name.
:param dids: The content.
:param issuer: The issuer account.
"""
kwargs = {'scope': scope, 'name': name, 'dids': dids, 'issuer': issuer}
if not rucio.api.permission.has_permission(issuer=issuer, action='detach_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not detach data identifiers from %s:%s' % (issuer, scope, name))
return did.detach_dids(scope=scope, name=name, dids=dids)