def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.new_vo = {'vo': 'new'}
cls.def_header = {'X-Rucio-VO': 'def'}
cls.vo_header = {'X-Rucio-VO': cls.vo['vo']}
cls.new_header = {'X-Rucio-VO': 'new'}
if not vo_exists(**cls.new_vo):
add_vo(description='Test', email='*****@*****.**', **cls.new_vo)
# Setup accounts at two VOs so we can determine which VO we authenticated against
usr_uuid = str(generate_uuid()).lower()[:16]
cls.account_tst = 'tst-%s' % usr_uuid
cls.account_new = 'new-%s' % usr_uuid
add_account(cls.account_tst, 'USER', '*****@*****.**', 'root', **cls.vo)
add_account(cls.account_new, 'USER', '*****@*****.**', 'root', **cls.new_vo)
else:
LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.')
cls.vo = {}
cls.new_vo = {}
cls.def_header = {}
cls.vo_header = {}
cls.new_header = {}
cls.account_tst = ''
cls.account_new = ''
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.new_vo = {'vo': 'new'}
cls.multi_vo = True
if not vo_exists(**cls.new_vo):
add_vo(description='Test', email='*****@*****.**', **cls.new_vo)
else:
cls.vo = {}
cls.new_vo = {}
cls.multi_vo = False
# Add test account
cls.account_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_account(account=cls.account_name, type='user', email='*****@*****.**', issuer='root', **cls.vo)
cls.account = InternalAccount(cls.account_name, **cls.vo)
# Add test scope
cls.scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_scope(scope=cls.scope_name, account=cls.account_name, issuer='root', **cls.vo)
cls.scope = InternalScope(cls.scope_name, **cls.vo)
# Get test RSEs
cls.rse_name = 'MOCK'
cls.rse_id = get_rse_id(rse=cls.rse_name, **cls.vo)
cls.rse2_name = 'MOCK2'
cls.rse2_id = get_rse_id(rse=cls.rse2_name, **cls.vo)
cls.rse3_name = rse_name_generator()
cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', **cls.new_vo)
cls.rse4_name = rse_name_generator()
cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', **cls.new_vo)
api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, **cls.new_vo)
def add_vo(new_vo,
issuer,
description=None,
email=None,
vo='def',
session=None):
'''
Add a new VO.
:param new_vo: The name/tag of the VO to add (3 characters).
:param description: A description of the VO. e.g the full name or a brief description
:param email: A contact for the VO.
:param issuer: The user issuing the command.
:param vo: The vo of the user issuing the command.
:param session: The database session in use.
'''
new_vo = vo_core.map_vo(new_vo)
validate_schema('vo', new_vo, vo=vo)
kwargs = {}
if not has_permission(
issuer=issuer, action='add_vo', kwargs=kwargs, vo=vo,
session=session):
raise exception.AccessDenied(
'Account {} cannot add a VO'.format(issuer))
vo_core.add_vo(vo=new_vo,
description=description,
email=email,
session=session)
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.new_vo = {'vo': 'new'}
if not vo_exists(**cls.new_vo):
add_vo(description='Test', email='*****@*****.**', **cls.new_vo)
else:
LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.')
cls.vo = {}
cls.new_vo = {}
def second_vo():
from rucio.common.config import config_get_bool
from rucio.core.vo import vo_exists, add_vo
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
pytest.skip('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode would result in failures.')
new_vo = 'new'
if not vo_exists(vo=new_vo):
add_vo(vo=new_vo, description='Test', email='*****@*****.**')
return new_vo
def add_vo(new_vo, issuer, password, description=None, email=None, vo='def'):
'''
Add a new VO.
:param new_vo: The name/tag of the VO to add (3 characters).
:param description: A description of the VO. e.g the full name or a brief description
:param email: A contact for the VO.
:param issuer: The user issuing the command.
:param vo: The vo of the user issuing the command.
:param password: The password to set for the root user of the new VO
'''
validate_schema('vo', new_vo)
kwargs = {}
if not has_permission(issuer=issuer, action='add_vo', kwargs=kwargs,
vo=vo):
raise exception.AccessDenied(
'Account {} cannot add a VO'.format(issuer))
vo_core.add_vo(vo=new_vo,
password=password,
description=description,
email=email)
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {
'vo': config_get('client',
'vo',
raise_exception=False,
default='tst')
}
new_vo = {'vo': 'new'}
if not vo_core.vo_exists(**new_vo):
vo_core.add_vo(description='Test',
email='*****@*****.**',
**new_vo)
if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
scope_core.add_scope(InternalScope('data13_hip', **new_vo),
InternalAccount('root', **new_vo))
nb_rses = 2
else:
vo = {}
new_vo = {}
nb_rses = 1
mock_protocol = {
'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
}
nb_files = 30
file_size = 2147483648 # 2G
rse_names = []
all_file_names = []
for j in range(nb_rses):
rse_name = rse_name_generator()
rse_names.append(rse_name)
rse_id = rse_core.add_rse(rse_name, **vo)
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
if new_vo:
rse_id_new = rse_core.add_rse(rse_name, **new_vo)
rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id,
scope=InternalScope('data13_hip', **vo),
name=file_name,
bytes=file_size,
tombstone=datetime.utcnow() -
timedelta(days=1),
account=InternalAccount('root', **vo),
adler32=None,
md5=None)
if new_vo:
replica_core.add_replica(
rse_id=rse_id_new,
scope=InternalScope('data13_hip', **new_vo),
name=file_name,
bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **new_vo),
adler32=None,
md5=None)
all_file_names.append(file_names)
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id,
name='MaxBeingDeletedFiles',
value=10)
if new_vo:
rse_core.set_rse_usage(rse_id=rse_id_new,
source='storage',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id_new,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id_new,
name='MaxBeingDeletedFiles',
value=10)
if not vo:
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files - 5
else:
# Check we reap all VOs by default
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == nb_files - 5
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **new_vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == nb_files - 5
# Check we don't affect a second VO that isn't specified
reaper(once=True,
rses=[],
include_rses=rse_names[1],
exclude_rses=[],
vos=['new'])
reaper(once=True,
rses=[],
include_rses=rse_names[1],
exclude_rses=[],
vos=['new'])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[1]],
rse_expression=rse_names[1]))), nb_files
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **new_vo),
'name': n
} for n in all_file_names[1]],
rse_expression=rse_names[1]))), nb_files - 5
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
new_vo = {'vo': 'new'}
if not vo_core.vo_exists(**new_vo):
vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo)
if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo))
nb_rses = 2
else:
vo = {}
new_vo = {}
nb_rses = 1
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
nb_files = 250
file_size = 200 # 2G
rse_names = []
all_file_names = []
for j in range(nb_rses):
rse_name = rse_name_generator()
rse_names.append(rse_name)
rse_id = rse_core.add_rse(rse_name, **vo)
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
if new_vo:
rse_id_new = rse_core.add_rse(rse_name, **new_vo)
rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
if new_vo:
replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **new_vo), adler32=None, md5=None)
all_file_names.append(file_names)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
# rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
if new_vo:
rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1)
rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size)
# rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10)
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
if not vo:
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files
# Check first if the reaper does not delete anything if no space is needed
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000)
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files
# Now put it over threshold and delete
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == 200
else:
# Check we reap all VOs by default
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == 200
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == 200
