def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.new_vo = {'vo': 'new'} cls.def_header = {'X-Rucio-VO': 'def'} cls.vo_header = {'X-Rucio-VO': cls.vo['vo']} cls.new_header = {'X-Rucio-VO': 'new'} if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) # Setup accounts at two VOs so we can determine which VO we authenticated against usr_uuid = str(generate_uuid()).lower()[:16] cls.account_tst = 'tst-%s' % usr_uuid cls.account_new = 'new-%s' % usr_uuid add_account(cls.account_tst, 'USER', '*****@*****.**', 'root', **cls.vo) add_account(cls.account_new, 'USER', '*****@*****.**', 'root', **cls.new_vo) else: LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.') cls.vo = {} cls.new_vo = {} cls.def_header = {} cls.vo_header = {} cls.new_header = {} cls.account_tst = '' cls.account_new = ''
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.new_vo = {'vo': 'new'} cls.multi_vo = True if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) else: cls.vo = {} cls.new_vo = {} cls.multi_vo = False # Add test account cls.account_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10)) add_account(account=cls.account_name, type='user', email='*****@*****.**', issuer='root', **cls.vo) cls.account = InternalAccount(cls.account_name, **cls.vo) # Add test scope cls.scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10)) add_scope(scope=cls.scope_name, account=cls.account_name, issuer='root', **cls.vo) cls.scope = InternalScope(cls.scope_name, **cls.vo) # Get test RSEs cls.rse_name = 'MOCK' cls.rse_id = get_rse_id(rse=cls.rse_name, **cls.vo) cls.rse2_name = 'MOCK2' cls.rse2_id = get_rse_id(rse=cls.rse2_name, **cls.vo) cls.rse3_name = rse_name_generator() cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', **cls.new_vo) cls.rse4_name = rse_name_generator() cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', **cls.new_vo) api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, **cls.new_vo)
def add_vo(new_vo, issuer, description=None, email=None, vo='def', session=None): ''' Add a new VO. :param new_vo: The name/tag of the VO to add (3 characters). :param description: A description of the VO. e.g the full name or a brief description :param email: A contact for the VO. :param issuer: The user issuing the command. :param vo: The vo of the user issuing the command. :param session: The database session in use. ''' new_vo = vo_core.map_vo(new_vo) validate_schema('vo', new_vo, vo=vo) kwargs = {} if not has_permission( issuer=issuer, action='add_vo', kwargs=kwargs, vo=vo, session=session): raise exception.AccessDenied( 'Account {} cannot add a VO'.format(issuer)) vo_core.add_vo(vo=new_vo, description=description, email=email, session=session)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} cls.new_vo = {'vo': 'new'} if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) else: LOG.warning('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode will result in failures.') cls.vo = {} cls.new_vo = {}
def second_vo(): from rucio.common.config import config_get_bool from rucio.core.vo import vo_exists, add_vo multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: pytest.skip('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode would result in failures.') new_vo = 'new' if not vo_exists(vo=new_vo): add_vo(vo=new_vo, description='Test', email='*****@*****.**') return new_vo
def add_vo(new_vo, issuer, password, description=None, email=None, vo='def'): ''' Add a new VO. :param new_vo: The name/tag of the VO to add (3 characters). :param description: A description of the VO. e.g the full name or a brief description :param email: A contact for the VO. :param issuer: The user issuing the command. :param vo: The vo of the user issuing the command. :param password: The password to set for the root user of the new VO ''' validate_schema('vo', new_vo) kwargs = {} if not has_permission(issuer=issuer, action='add_vo', kwargs=kwargs, vo=vo): raise exception.AccessDenied( 'Account {} cannot add a VO'.format(issuer)) vo_core.add_vo(vo=new_vo, password=password, description=description, email=email)
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = { 'vo': config_get('client', 'vo', raise_exception=False, default='tst') } new_vo = {'vo': 'new'} if not vo_core.vo_exists(**new_vo): vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo) if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)): scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo)) nb_rses = 2 else: vo = {} new_vo = {} nb_rses = 1 mock_protocol = { 'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } } nb_files = 30 file_size = 2147483648 # 2G rse_names = [] all_file_names = [] for j in range(nb_rses): rse_name = rse_name_generator() rse_names.append(rse_name) rse_id = rse_core.add_rse(rse_name, **vo) rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) if new_vo: rse_id_new = rse_core.add_rse(rse_name, **new_vo) rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol) file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) if new_vo: replica_core.add_replica( rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **new_vo), adler32=None, md5=None) all_file_names.append(file_names) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if new_vo: rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10) if not vo: reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_name))) == nb_files - 5 else: # Check we reap all VOs by default reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_names[0]))) == nb_files - 5 assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **new_vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_names[0]))) == nb_files - 5 # Check we don't affect a second VO that isn't specified reaper(once=True, rses=[], include_rses=rse_names[1], exclude_rses=[], vos=['new']) reaper(once=True, rses=[], include_rses=rse_names[1], exclude_rses=[], vos=['new']) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[1]], rse_expression=rse_names[1]))), nb_files assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **new_vo), 'name': n } for n in all_file_names[1]], rse_expression=rse_names[1]))), nb_files - 5
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} new_vo = {'vo': 'new'} if not vo_core.vo_exists(**new_vo): vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo) if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)): scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo)) nb_rses = 2 else: vo = {} new_vo = {} nb_rses = 1 mock_protocol = {'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}} nb_files = 250 file_size = 200 # 2G rse_names = [] all_file_names = [] for j in range(nb_rses): rse_name = rse_name_generator() rse_names.append(rse_name) rse_id = rse_core.add_rse(rse_name, **vo) rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) if new_vo: rse_id_new = rse_core.add_rse(rse_name, **new_vo) rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol) file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) if new_vo: replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **new_vo), adler32=None, md5=None) all_file_names.append(file_names) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size) # rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if new_vo: rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size) # rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10) from rucio.daemons.reaper.reaper2 import REGION REGION.invalidate() if not vo: assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == nb_files # Check first if the reaper does not delete anything if no space is needed rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == nb_files # Now put it over threshold and delete rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) from rucio.daemons.reaper.reaper2 import REGION REGION.invalidate() reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == 200 else: # Check we reap all VOs by default reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_names[0]))) == 200 assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_names[0]))) == 200