def add_account(account, type, email, session=None): """ Add an account with the given account name and type. :param account: the name of the new account. :param type: the type of the new account. :param email: The Email address associated with the account. :param session: the database session in use. """ vo = account.vo if not vo_exists(vo=vo, session=session): raise exception.VONotFound('VO {} not found'.format(vo)) # Reserve the name 'super_root' for multi_vo admins if account.external == 'super_root': if not (vo == 'def' and config_get_bool( 'common', 'multi_vo', raise_exception=False, default=False)): raise exception.UnsupportedAccountName( 'The name "%s" cannot be used.' % account.external) new_account = models.Account(account=account, account_type=type, email=email, status=AccountStatus.ACTIVE) try: new_account.save(session=session) except IntegrityError: raise exception.Duplicate('Account ID \'%s\' already exists!' % account) # Create the account counters for this account rucio.core.account_counter.create_counters_for_new_account(account=account, session=session)
def add_scope(scope, account, session=None): """ add a scope for the given account name. :param scope: the name for the new scope. :param account: the account to add the scope to. :param session: The database session in use. """ if not vo_exists(vo=scope.vo, session=session): raise exception.RucioException('VO {} not found'.format(scope.vo)) result = session.query(models.Account).filter_by( account=account, status=AccountStatus.ACTIVE).first() if result is None: raise AccountNotFound('Account ID \'%s\' does not exist' % account) new_scope = models.Scope(scope=scope, account=account, status=ScopeStatus.OPEN) try: new_scope.save(session=session) except IntegrityError as e: if match('.*IntegrityError.*ORA-00001: unique constraint.*SCOPES_PK.*violated.*', e.args[0]) \ or match('.*IntegrityError.*1062, "Duplicate entry.*for key.*', e.args[0]) \ or match('.*IntegrityError.*UNIQUE constraint failed: scopes.scope.*', e.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', e.args[0])\ or match('.*sqlite3.IntegrityError.*is not unique.*', e.args[0]): raise Duplicate('Scope \'%s\' already exists!' % scope) except: raise RucioException(str(format_exc()))
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.new_vo = {'vo': 'new'} cls.def_header = {'X-Rucio-VO': 'def'} cls.vo_header = {'X-Rucio-VO': cls.vo['vo']} cls.new_header = {'X-Rucio-VO': 'new'} if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) # Setup accounts at two VOs so we can determine which VO we authenticated against usr_uuid = str(generate_uuid()).lower()[:16] cls.account_tst = 'tst-%s' % usr_uuid cls.account_new = 'new-%s' % usr_uuid add_account(cls.account_tst, 'USER', '*****@*****.**', 'root', **cls.vo) add_account(cls.account_new, 'USER', '*****@*****.**', 'root', **cls.new_vo) else: LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.') cls.vo = {} cls.new_vo = {} cls.def_header = {} cls.vo_header = {} cls.new_header = {} cls.account_tst = '' cls.account_new = ''
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.new_vo = {'vo': 'new'} cls.multi_vo = True if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) else: cls.vo = {} cls.new_vo = {} cls.multi_vo = False # Add test account cls.account_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10)) add_account(account=cls.account_name, type='user', email='*****@*****.**', issuer='root', **cls.vo) cls.account = InternalAccount(cls.account_name, **cls.vo) # Add test scope cls.scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10)) add_scope(scope=cls.scope_name, account=cls.account_name, issuer='root', **cls.vo) cls.scope = InternalScope(cls.scope_name, **cls.vo) # Get test RSEs cls.rse_name = 'MOCK' cls.rse_id = get_rse_id(rse=cls.rse_name, **cls.vo) cls.rse2_name = 'MOCK2' cls.rse2_id = get_rse_id(rse=cls.rse2_name, **cls.vo) cls.rse3_name = rse_name_generator() cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', **cls.new_vo) cls.rse4_name = rse_name_generator() cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', **cls.new_vo) api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, **cls.new_vo)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.new_vo = {'vo': 'new'} if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) else: LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.') cls.vo = {} cls.new_vo = {}
def second_vo(): from rucio.common.config import config_get_bool from rucio.core.vo import vo_exists, add_vo multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: pytest.skip('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode would result in failures.') new_vo = 'new' if not vo_exists(vo=new_vo): add_vo(vo=new_vo, description='Test', email='*****@*****.**') return new_vo
def add_account(account, type, email, session=None): """ Add an account with the given account name and type. :param account: the name of the new account. :param type: the type of the new account. :param email: The Email address associated with the account. :param session: the database session in use. """ if not vo_exists(vo=account.vo, session=session): raise exception.RucioException('VO {} not found'.format(account.vo)) new_account = models.Account(account=account, account_type=type, email=email, status=AccountStatus.ACTIVE) try: new_account.save(session=session) except IntegrityError: raise exception.Duplicate('Account ID \'%s\' already exists!' % account) # Create the account counters for this account rucio.core.account_counter.create_counters_for_new_account(account=account, session=session)
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = { 'vo': config_get('client', 'vo', raise_exception=False, default='tst') } new_vo = {'vo': 'new'} if not vo_core.vo_exists(**new_vo): vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo) if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)): scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo)) nb_rses = 2 else: vo = {} new_vo = {} nb_rses = 1 mock_protocol = { 'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } } nb_files = 30 file_size = 2147483648 # 2G rse_names = [] all_file_names = [] for j in range(nb_rses): rse_name = rse_name_generator() rse_names.append(rse_name) rse_id = rse_core.add_rse(rse_name, **vo) rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) if new_vo: rse_id_new = rse_core.add_rse(rse_name, **new_vo) rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol) file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) if new_vo: replica_core.add_replica( rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **new_vo), adler32=None, md5=None) all_file_names.append(file_names) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if new_vo: rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10) if not vo: reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_name))) == nb_files - 5 else: # Check we reap all VOs by default reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_names[0]))) == nb_files - 5 assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **new_vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_names[0]))) == nb_files - 5 # Check we don't affect a second VO that isn't specified reaper(once=True, rses=[], include_rses=rse_names[1], exclude_rses=[], vos=['new']) reaper(once=True, rses=[], include_rses=rse_names[1], exclude_rses=[], vos=['new']) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[1]], rse_expression=rse_names[1]))), nb_files assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **new_vo), 'name': n } for n in all_file_names[1]], rse_expression=rse_names[1]))), nb_files - 5
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} new_vo = {'vo': 'new'} if not vo_core.vo_exists(**new_vo): vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo) if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)): scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo)) nb_rses = 2 else: vo = {} new_vo = {} nb_rses = 1 mock_protocol = {'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}} nb_files = 250 file_size = 200 # 2G rse_names = [] all_file_names = [] for j in range(nb_rses): rse_name = rse_name_generator() rse_names.append(rse_name) rse_id = rse_core.add_rse(rse_name, **vo) rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) if new_vo: rse_id_new = rse_core.add_rse(rse_name, **new_vo) rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol) file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) if new_vo: replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **new_vo), adler32=None, md5=None) all_file_names.append(file_names) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size) # rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if new_vo: rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size) # rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10) from rucio.daemons.reaper.reaper2 import REGION REGION.invalidate() if not vo: assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == nb_files # Check first if the reaper does not delete anything if no space is needed rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == nb_files # Now put it over threshold and delete rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) from rucio.daemons.reaper.reaper2 import REGION REGION.invalidate() reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == 200 else: # Check we reap all VOs by default reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_names[0]))) == 200 assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_names[0]))) == 200