def add_account(account, type, email, session=None):
""" Add an account with the given account name and type.
:param account: the name of the new account.
:param type: the type of the new account.
:param email: The Email address associated with the account.
:param session: the database session in use.
"""
vo = account.vo
if not vo_exists(vo=vo, session=session):
raise exception.VONotFound('VO {} not found'.format(vo))
# Reserve the name 'super_root' for multi_vo admins
if account.external == 'super_root':
if not (vo == 'def' and config_get_bool(
'common', 'multi_vo', raise_exception=False, default=False)):
raise exception.UnsupportedAccountName(
'The name "%s" cannot be used.' % account.external)
new_account = models.Account(account=account,
account_type=type,
email=email,
status=AccountStatus.ACTIVE)
try:
new_account.save(session=session)
except IntegrityError:
raise exception.Duplicate('Account ID \'%s\' already exists!' %
account)
# Create the account counters for this account
rucio.core.account_counter.create_counters_for_new_account(account=account,
session=session)
def add_scope(scope, account, session=None):
""" add a scope for the given account name.
:param scope: the name for the new scope.
:param account: the account to add the scope to.
:param session: The database session in use.
"""
if not vo_exists(vo=scope.vo, session=session):
raise exception.RucioException('VO {} not found'.format(scope.vo))
result = session.query(models.Account).filter_by(
account=account, status=AccountStatus.ACTIVE).first()
if result is None:
raise AccountNotFound('Account ID \'%s\' does not exist' % account)
new_scope = models.Scope(scope=scope,
account=account,
status=ScopeStatus.OPEN)
try:
new_scope.save(session=session)
except IntegrityError as e:
if match('.*IntegrityError.*ORA-00001: unique constraint.*SCOPES_PK.*violated.*', e.args[0]) \
or match('.*IntegrityError.*1062, "Duplicate entry.*for key.*', e.args[0]) \
or match('.*IntegrityError.*UNIQUE constraint failed: scopes.scope.*', e.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', e.args[0])\
or match('.*sqlite3.IntegrityError.*is not unique.*', e.args[0]):
raise Duplicate('Scope \'%s\' already exists!' % scope)
except:
raise RucioException(str(format_exc()))
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.new_vo = {'vo': 'new'}
cls.def_header = {'X-Rucio-VO': 'def'}
cls.vo_header = {'X-Rucio-VO': cls.vo['vo']}
cls.new_header = {'X-Rucio-VO': 'new'}
if not vo_exists(**cls.new_vo):
add_vo(description='Test', email='*****@*****.**', **cls.new_vo)
# Setup accounts at two VOs so we can determine which VO we authenticated against
usr_uuid = str(generate_uuid()).lower()[:16]
cls.account_tst = 'tst-%s' % usr_uuid
cls.account_new = 'new-%s' % usr_uuid
add_account(cls.account_tst, 'USER', '*****@*****.**', 'root', **cls.vo)
add_account(cls.account_new, 'USER', '*****@*****.**', 'root', **cls.new_vo)
else:
LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.')
cls.vo = {}
cls.new_vo = {}
cls.def_header = {}
cls.vo_header = {}
cls.new_header = {}
cls.account_tst = ''
cls.account_new = ''
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.new_vo = {'vo': 'new'}
cls.multi_vo = True
if not vo_exists(**cls.new_vo):
add_vo(description='Test', email='*****@*****.**', **cls.new_vo)
else:
cls.vo = {}
cls.new_vo = {}
cls.multi_vo = False
# Add test account
cls.account_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_account(account=cls.account_name, type='user', email='*****@*****.**', issuer='root', **cls.vo)
cls.account = InternalAccount(cls.account_name, **cls.vo)
# Add test scope
cls.scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_scope(scope=cls.scope_name, account=cls.account_name, issuer='root', **cls.vo)
cls.scope = InternalScope(cls.scope_name, **cls.vo)
# Get test RSEs
cls.rse_name = 'MOCK'
cls.rse_id = get_rse_id(rse=cls.rse_name, **cls.vo)
cls.rse2_name = 'MOCK2'
cls.rse2_id = get_rse_id(rse=cls.rse2_name, **cls.vo)
cls.rse3_name = rse_name_generator()
cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', **cls.new_vo)
cls.rse4_name = rse_name_generator()
cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', **cls.new_vo)
api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, **cls.new_vo)
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
cls.new_vo = {'vo': 'new'}
if not vo_exists(**cls.new_vo):
add_vo(description='Test', email='*****@*****.**', **cls.new_vo)
else:
LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.')
cls.vo = {}
cls.new_vo = {}
def second_vo():
from rucio.common.config import config_get_bool
from rucio.core.vo import vo_exists, add_vo
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
pytest.skip('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode would result in failures.')
new_vo = 'new'
if not vo_exists(vo=new_vo):
add_vo(vo=new_vo, description='Test', email='*****@*****.**')
return new_vo
def add_account(account, type, email, session=None):
""" Add an account with the given account name and type.
:param account: the name of the new account.
:param type: the type of the new account.
:param email: The Email address associated with the account.
:param session: the database session in use.
"""
if not vo_exists(vo=account.vo, session=session):
raise exception.RucioException('VO {} not found'.format(account.vo))
new_account = models.Account(account=account, account_type=type, email=email,
status=AccountStatus.ACTIVE)
try:
new_account.save(session=session)
except IntegrityError:
raise exception.Duplicate('Account ID \'%s\' already exists!' % account)
# Create the account counters for this account
rucio.core.account_counter.create_counters_for_new_account(account=account, session=session)
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {
'vo': config_get('client',
'vo',
raise_exception=False,
default='tst')
}
new_vo = {'vo': 'new'}
if not vo_core.vo_exists(**new_vo):
vo_core.add_vo(description='Test',
email='*****@*****.**',
**new_vo)
if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
scope_core.add_scope(InternalScope('data13_hip', **new_vo),
InternalAccount('root', **new_vo))
nb_rses = 2
else:
vo = {}
new_vo = {}
nb_rses = 1
mock_protocol = {
'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
}
nb_files = 30
file_size = 2147483648 # 2G
rse_names = []
all_file_names = []
for j in range(nb_rses):
rse_name = rse_name_generator()
rse_names.append(rse_name)
rse_id = rse_core.add_rse(rse_name, **vo)
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
if new_vo:
rse_id_new = rse_core.add_rse(rse_name, **new_vo)
rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id,
scope=InternalScope('data13_hip', **vo),
name=file_name,
bytes=file_size,
tombstone=datetime.utcnow() -
timedelta(days=1),
account=InternalAccount('root', **vo),
adler32=None,
md5=None)
if new_vo:
replica_core.add_replica(
rse_id=rse_id_new,
scope=InternalScope('data13_hip', **new_vo),
name=file_name,
bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **new_vo),
adler32=None,
md5=None)
all_file_names.append(file_names)
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id,
name='MaxBeingDeletedFiles',
value=10)
if new_vo:
rse_core.set_rse_usage(rse_id=rse_id_new,
source='storage',
used=nb_files * file_size,
free=800)
rse_core.set_rse_limits(rse_id=rse_id_new,
name='MinFreeSpace',
value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id_new,
name='MaxBeingDeletedFiles',
value=10)
if not vo:
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files - 5
else:
# Check we reap all VOs by default
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == nb_files - 5
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **new_vo),
'name': n
} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == nb_files - 5
# Check we don't affect a second VO that isn't specified
reaper(once=True,
rses=[],
include_rses=rse_names[1],
exclude_rses=[],
vos=['new'])
reaper(once=True,
rses=[],
include_rses=rse_names[1],
exclude_rses=[],
vos=['new'])
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **vo),
'name': n
} for n in all_file_names[1]],
rse_expression=rse_names[1]))), nb_files
assert len(
list(
replica_core.list_replicas(
dids=[{
'scope': InternalScope('data13_hip', **new_vo),
'name': n
} for n in all_file_names[1]],
rse_expression=rse_names[1]))), nb_files - 5
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
new_vo = {'vo': 'new'}
if not vo_core.vo_exists(**new_vo):
vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo)
if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo))
nb_rses = 2
else:
vo = {}
new_vo = {}
nb_rses = 1
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
nb_files = 250
file_size = 200 # 2G
rse_names = []
all_file_names = []
for j in range(nb_rses):
rse_name = rse_name_generator()
rse_names.append(rse_name)
rse_id = rse_core.add_rse(rse_name, **vo)
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
if new_vo:
rse_id_new = rse_core.add_rse(rse_name, **new_vo)
rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
if new_vo:
replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **new_vo), adler32=None, md5=None)
all_file_names.append(file_names)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
# rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
if new_vo:
rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1)
rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size)
# rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10)
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
if not vo:
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files
# Check first if the reaper does not delete anything if no space is needed
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000)
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == nb_files
# Now put it over threshold and delete
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_name))) == 200
else:
# Check we reap all VOs by default
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == 200
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]],
rse_expression=rse_names[0]))) == 200
