def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# Add quota
set_account_limit('root', get_rse_id(site_a), -1)
set_account_limit('root', get_rse_id(site_b), -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root')
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
for rule in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(rule[3], 2)
def test_run_transmogrifier(self):
""" SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
rse1, _ = self.rse_factory.make_mock_rse()
rse2, _ = self.rse_factory.make_mock_rse()
rse3, _ = self.rse_factory.make_mock_rse()
rse_expression = '%s|%s|%s' % (rse1, rse2, rse3)
tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
root = InternalAccount('root', **self.vo)
add_scope(tmp_scope, root)
subscription_name = uuid()
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope,
name=dsn,
did_type=DIDType.DATASET,
account=root)
subid = self.sub_client.add_subscription(name=subscription_name,
account='root',
filter_={
'scope': [
tmp_scope.external,
],
'pattern': 'dataset-.*',
'split_rule': True
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=None,
retroactive=0,
dry_run=0,
comments='Ni ! Ni!',
priority=1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(
scope=tmp_scope.external, name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert len(rules) == 2
set_new_dids([
{
'scope': tmp_scope,
'name': dsn
},
], 1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(
scope=tmp_scope.external, name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert len(rules) == 2
def test_list_scopes(self):
""" SCOPE (CORE): List scopes """
for scope in self.scopes:
add_scope(scope=scope, account='jdoe')
scopes = get_scopes(account='jdoe')
for s in scopes:
assert_in(s, scopes)
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
for r in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(r[3], 2)
def test_list_scopes(self):
""" SCOPE (CORE): List scopes """
for scope in self.scopes:
add_scope(scope=scope, account=self.jdoe)
scopes = get_scopes(account=self.jdoe)
for scope in scopes:
assert scope in scopes
def __setup_scopes_for_vos(*vos):
scope_uuid = str(generate_uuid()).lower()[:16]
scope_name = 'shr_%s' % scope_uuid
created_scopes = []
for vo in vos:
scope = InternalScope(scope_name, vo=vo)
scope_core.add_scope(scope, InternalAccount('root', vo=vo))
created_scopes.append(scope)
return scope_name, created_scopes
def create_scopes(vos, account_name=None):
scope_uuid = str(generate_uuid()).lower()[:16]
scope_name = 'shr_%s' % scope_uuid
created_scopes = []
for vo in vos:
scope = InternalScope(scope_name, vo=vo)
add_scope(scope, InternalAccount(account_name if account_name else 'root', vo=vo))
created_scopes.append(scope)
return scope_name, created_scopes
def test_run_transmogrifier(self):
""" SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
subscription_name = uuid()
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope,
name=dsn,
type=DIDType.DATASET,
account='root')
subid = self.sub_client.add_subscription(name=subscription_name,
account='root',
filter={
'scope': [
tmp_scope,
],
'pattern': 'dataset-.*',
'split_rule': True
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
'MOCK-POSIX|MOCK2|MOCK3',
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=None,
retroactive=0,
dry_run=0,
comments='Ni ! Ni!',
priority=1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(scope=tmp_scope,
name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert_equal(len(rules), 2)
set_new_dids([
{
'scope': tmp_scope,
'name': dsn
},
], 1)
run(threads=1, bulk=1000000, once=True)
rules = [
rule for rule in self.did_client.list_did_rules(scope=tmp_scope,
name=dsn)
if str(rule['subscription_id']) == str(subid)
]
assert_equal(len(rules), 2)
def test_permission_add_did(self):
""" PERMISSION(CORE): Check permission to add a did"""
scope = scope_name_generator()
add_scope(scope=InternalScope(scope), account=InternalAccount('root'))
assert_true(
has_permission(issuer='panda',
action='add_did',
kwargs={'scope': scope}))
assert_false(
has_permission(issuer='spock',
action='add_did',
kwargs={'scope': scope}))
def add_scope(scope, account, issuer):
"""
Creates a scope for an account.
:param account: The account name.
:param scope: The scope identifier.
:param issuer: The issuer account.
"""
validate_schema(name='scope', obj=scope)
kwargs = {'scope': scope, 'account': account}
if not rucio.api.permission.has_permission(issuer=issuer, action='add_scope', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not add scope' % (issuer))
core_scope.add_scope(scope, account)
def test_list_rules_states(vo, rest_client, auth_token):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
root = InternalAccount('root', vo=vo)
add_scope(tmp_scope, root)
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a, vo=vo)
site_b_id = add_rse(site_b, vo=vo)
# Add quota
set_local_account_limit(root, site_a_id, -1)
set_local_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
did_type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name,
account='root',
filter_={'account': ['root', ], 'scope': [tmp_scope.external, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='We want a shrubbery',
issuer='root',
vo=vo)
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
response = rest_client.get('/subscriptions/%s/%s/Rules/States' % ('root', subscription_name), headers=headers(auth(auth_token)))
assert response.status_code == 200
rulestates = None
for line in response.get_data(as_text=True).split('\n'):
if line:
rulestates = loads(line)
if rulestates[1] == subscription_name:
break
assert rulestates is not None
assert rulestates[3] == 2
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)
for line in r2.body.split('\n'):
print line
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def add_scope(scope, account, issuer):
"""
Creates a scope for an account.
:param account: The account name.
:param scope: The scope identifier.
:param issuer: The issuer account.
"""
validate_schema(name='scope', obj=scope)
kwargs = {'scope': scope, 'account': account}
if not rucio.api.permission.has_permission(
issuer=issuer, action='add_scope', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not add scope' % (issuer))
core_scope.add_scope(scope, account)
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8])
root = InternalAccount('root')
add_scope(tmp_scope, root)
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a)
site_b_id = add_rse(site_b)
# Add quota
set_account_limit(root, site_a_id, -1)
set_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope.external, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery', issuer='root')
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(res1.status, 200)
token = str(res1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
res2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)
for line in res2.body.split('\n'):
print(line)
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def add_scope(scope, account, issuer, vo='def', session=None):
"""
Creates a scope for an account.
:param account: The account name.
:param scope: The scope identifier.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
"""
validate_schema(name='scope', obj=scope, vo=vo)
kwargs = {'scope': scope, 'account': account}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='add_scope', kwargs=kwargs, session=session):
raise rucio.common.exception.AccessDenied('Account %s can not add scope' % (issuer))
scope = InternalScope(scope, vo=vo)
account = InternalAccount(account, vo=vo)
core_scope.add_scope(scope, account, session=session)
def test_overlapping_did_names(rse_factory, did_factory, download_client,
root_account, mock_scope, vo):
"""
Downloading two different did with different scope but same name to the same directory must fail
"""
rse, _ = rse_factory.make_posix_rse()
scope1 = mock_scope
scope2 = InternalScope(scope_name_generator(), vo=vo)
scope_core.add_scope(scope2, root_account)
did1 = did_factory.upload_test_file(rse, scope=scope1)
did2 = did_factory.upload_test_file(rse, scope=scope2, name=did1['name'])
dataset = did_factory.make_dataset()
did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset)
did1_str = '%s:%s' % (did1['scope'], did1['name'])
did2_str = '%s:%s' % (did2['scope'], did2['name'])
dataset1_did_str = '%s:%s' % (dataset['scope'], dataset['name'])
with TemporaryDirectory() as tmp_dir:
with pytest.raises(RucioException):
download_client.download_dids([{
'did': dataset1_did_str,
'base_dir': tmp_dir,
'no_subdir': True
}])
with TemporaryDirectory() as tmp_dir:
with pytest.raises(RucioException):
download_client.download_dids([{
'did': did1_str,
'base_dir': tmp_dir,
'no_subdir': True
}, {
'did': did2_str,
'base_dir': tmp_dir,
'no_subdir': True
}])
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {
'vo': config_get('client',
'vo',
raise_exception=False,
default='tst')
}
new_vo = {'vo': 'new'}
if not vo_core.vo_exists(**new_vo):
vo_core.add_vo(description='Test',
email='*****@*****.**',
**new_vo)
if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
scope_core.add_scope(InternalScope('data13_hip', **new_vo),
InternalAccount('root', **new_vo))
nb_rses = 2
else:
vo = {}
new_vo = {}
nb_rses = 1
mock_protocol = {
'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
}
nb_files = 30
file_size = 2147483648 # 2G
rse_names = []
all_file_names = []
for j in range(nb_rses):
rse_name = rse_name_generator()
rse_names.append(rse_name)
rse_id = rse_core.add_rse(rse_name, **vo)
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
if new_vo:
rse_id_new = rse_core.add_rse(rse_name, **new_vo)
rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id,
scope=InternalScope('data13_hip', **vo),
name=file_name,
bytes=file_size,
tombstone=datetime.utcnow() -
timedelta(days=1),
account=InternalAccount('root', **vo),
adler32=None,
md5=None)
if new_vo:
replica_core.add_replica(
rse_id=rse_id_new,
scope=InternalScope('data13_hip', **new_vo),
name=file_name,
bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **new_vo),
adler32=None,
md5=None)
all_file_names.append(file_names)
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id,
name='MaxBeingDeletedFiles',
value=10)
if new_vo:
rse_core.set_rse_usage(rse_id=rse_id_new,
source='storage',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id_new,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id_new,
name='MaxBeingDeletedFiles',
value=10)
if not vo:
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files - 5
else:
# Check we reap all VOs by default
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == nb_files - 5
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **new_vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == nb_files - 5
# Check we don't affect a second VO that isn't specified
reaper(once=True,
rses=[],
include_rses=rse_names[1],
exclude_rses=[],
vos=['new'])
reaper(once=True,
rses=[],
include_rses=rse_names[1],
exclude_rses=[],
vos=['new'])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[1]],
rse_expression=rse_names[1]))), nb_files
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **new_vo),
'name': n
} for n in all_file_names[1]],
rse_expression=rse_names[1]))), nb_files - 5
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
new_vo = {'vo': 'new'}
if not vo_core.vo_exists(**new_vo):
vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo)
if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo))
nb_rses = 2
else:
vo = {}
new_vo = {}
nb_rses = 1
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
nb_files = 250
file_size = 200 # 2G
rse_names = []
all_file_names = []
for j in range(nb_rses):
rse_name = rse_name_generator()
rse_names.append(rse_name)
rse_id = rse_core.add_rse(rse_name, **vo)
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
if new_vo:
rse_id_new = rse_core.add_rse(rse_name, **new_vo)
rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
if new_vo:
replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **new_vo), adler32=None, md5=None)
all_file_names.append(file_names)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
# rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
if new_vo:
rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1)
rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size)
# rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10)
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
if not vo:
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files
# Check first if the reaper does not delete anything if no space is needed
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000)
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files
# Now put it over threshold and delete
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == 200
else:
# Check we reap all VOs by default
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == 200
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == 200
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
root = InternalAccount('root', **self.vo)
add_scope(tmp_scope, root)
rse1, _ = self.rse_factory.make_mock_rse()
rse2, _ = self.rse_factory.make_mock_rse()
rse_expression = '%s|%s' % (rse1, rse2)
rse3, _ = self.rse_factory.make_mock_rse()
rse4, _ = self.rse_factory.make_mock_rse()
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope,
name=dsn,
did_type=DIDType.DATASET,
account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name,
account='root',
filter_={
'account': [
'root',
],
'scope': [
tmp_scope.external,
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='This is a comment',
issuer='root',
**self.vo)
# Add two rules
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=rse3,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=rse4,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
for rule in list_subscription_rule_states(account='root',
name=subscription_name,
**self.vo):
assert rule[3] == 2
def test_permission_add_did(self):
""" PERMISSION(CORE): Check permission to add a did"""
scope = scope_name_generator()
add_scope(scope=scope, account='root')
assert_true(has_permission(issuer='panda', action='add_did', kwargs={'scope': scope}))
assert_false(has_permission(issuer='spock', action='add_did', kwargs={'scope': scope}))
def test_is_scope_owner(self):
""" SCOPE (CORE): Is scope owner """
scope = scope_name_generator()
add_scope(scope=scope, account='jdoe')
anwser = is_scope_owner(scope=scope, account='jdoe')
assert_equal(anwser, True)
def test_is_scope_owner(self):
""" SCOPE (CORE): Is scope owner """
scope = InternalScope(scope_name_generator(), **self.vo)
add_scope(scope=scope, account=self.jdoe)
anwser = is_scope_owner(scope=scope, account=self.jdoe)
assert anwser is True
def test_is_scope_owner(self):
""" SCOPE (CORE): Is scope owner """
scope = scope_name_generator()
add_scope(scope=scope, account='jdoe')
anwser = is_scope_owner(scope=scope, account='jdoe')
assert_equal(anwser, True)