def test_update_vo_unsupported(self): """ MULTI VO (REST): Test updating VO through REST layer raises UnsupportedOperation """ mw = [] headers1 = {'X-Rucio-Account': 'super_root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} headers1.update(self.def_header) res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(res1.status, 200) token = str(res1.header('X-Rucio-Auth-Token')) params = {'email': '*****@*****.**', 'decription': 'Try updating in single vo mode'} try: config_set('common', 'multi_vo', 'False') headers2 = {'X-Rucio-Auth-Token': str(token)} res2 = TestApp(vo_app.wsgifunc(*mw)).put('/' + self.vo['vo'], headers=headers2, expect_errors=True, params=dumps(params)) assert_equal(res2.status, 409) config_remove_option('common', 'multi_vo') headers2 = {'X-Rucio-Auth-Token': str(token)} res2 = TestApp(vo_app.wsgifunc(*mw)).put('/' + self.vo['vo'], headers=headers2, expect_errors=True, params=dumps(params)) assert_equal(res2.status, 409) finally: # Make sure we don't leave the config changed due to a test failure if self.vo: config_set('common', 'multi_vo', 'True') else: config_remove_option('common', 'multi_vo')
def test_config_default_override(self): """LFN2PFN: Test override of default LFN2PFN algorithm via config (Success)""" if not config.config_has_section('policy'): config.config_add_section('policy') try: orig_value = config.config_get('policy', 'lfn2pfn_algorithm_default') except (NoOptionError, NoSectionError): orig_value = None def static_test(scope, name, rse, rse_attrs, proto_attrs): """Static test function for config override.""" del scope del name del rse del rse_attrs del proto_attrs return "static_test_value" RSEDeterministicTranslation.register(static_test) try: config.config_set('policy', 'lfn2pfn_algorithm_default', 'static_test') RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access assert_equal(self.translator.path("foo", "bar"), "static_test_value") finally: if orig_value is None: config.config_remove_option('policy', 'lfn2pfn_algorithm_default') else: config.config_set('policy', 'lfn2pfn_algorithm_default', orig_value) RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
def file_config_mock(request): """ Fixture which allows to have an isolated in-memory configuration file instance which is not persisted after exiting the fixture. This override works only in tests which use config calls directly, not in the ones working via the API, as the server config is not changed. """ from unittest import mock from rucio.common.config import Config, config_set, config_has_section, config_add_section # Get the fixture parameters overrides = [] params = __get_fixture_param(request) if params: overrides = params.get("overrides", overrides) parser = Config().parser with mock.patch('rucio.common.config.get_config', side_effect=lambda: parser): for section, option, value in (overrides or []): if not config_has_section(section): config_add_section(section) config_set(section, option, value) yield
def test_upload_file_with_supported_protocol_from_config(rse_factory, upload_client, mock_scope, file_factory): """ Upload (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config read from rucio.cfg is called """ rse_name, rse_id = rse_factory.make_rse() # FIXME: # The correct order to test should actually be ssh,xrootd,posix # However the preferred_impl is not working correctly. # Once preferred_impl is fixed, this should be changed back add_protocol(rse_id, {'scheme': 'scp', 'hostname': '%s.cern.ch' % rse_id, 'port': 0, 'prefix': '/test/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}}) add_protocol(rse_id, {'scheme': 'file', 'hostname': '%s.cern.ch' % rse_id, 'port': 0, 'prefix': '/test/', 'impl': 'rucio.rse.protocols.posix.Default', 'domains': { 'lan': {'read': 2, 'write': 2, 'delete': 2}, 'wan': {'read': 2, 'write': 2, 'delete': 2}}}) add_protocol(rse_id, {'scheme': 'root', 'hostname': '%s.cern.ch' % rse_id, 'port': 0, 'prefix': '/test/', 'impl': 'rucio.rse.protocols.ssh.Default', 'domains': { 'lan': {'read': 3, 'write': 3, 'delete': 3}, 'wan': {'read': 3, 'write': 3, 'delete': 3}}}) config_add_section('upload') config_set('upload', 'preferred_impl', 'rclone, xrootd') supported_impl = 'xrootd' path = file_factory.file_generator() name = os.path.basename(path) item = { 'path': path, 'rse': rse_name, 'did_scope': str(mock_scope), 'did_name': name, 'guid': generate_uuid() } with TemporaryDirectory() as tmp_dir: with patch('rucio.rse.protocols.%s.Default.put' % supported_impl, side_effect=lambda pfn, dest, dir, **kw: shutil.copy(path, tmp_dir)) as mock_put, \ patch('rucio.rse.protocols.%s.Default.connect' % supported_impl),\ patch('rucio.rse.protocols.%s.Default.exists' % supported_impl, side_effect=lambda pfn, **kw: False),\ patch('rucio.rse.protocols.%s.Default.delete' % supported_impl),\ patch('rucio.rse.protocols.%s.Default.rename' % supported_impl),\ patch('rucio.rse.protocols.%s.Default.stat' % supported_impl, side_effect=lambda pfn: {'filesize': os.stat(path)[os.path.stat.ST_SIZE], 'adler32': adler32(path)}),\ patch('rucio.rse.protocols.%s.Default.close' % supported_impl): mock_put.__name__ = "mock_put" upload_client.upload([item]) mock_put.assert_called()
def test_module_load(self): """LFN2PFN: Test ability to provide LFN2PFN functions via module (Success)""" if not config.config_has_section('policy'): config.config_add_section('policy') config.config_set('policy', 'lfn2pfn_module', 'rucio.tests.lfn2pfn_module_test') RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access self.rse_attributes['lfn2pfn_algorithm'] = 'lfn2pfn_module_algorithm' self.create_translator() assert self.translator.path("foo", "bar") == "lfn2pfn_module_algorithm_value"
def test_multi_vo_flag(self): """ MULTI VO (CORE): Test operations fail in single_vo mode """ try: config_set('common', 'multi_vo', 'False') with assert_raises(UnsupportedOperation): vo_api.list_vos(issuer='super_root', vo='def') config_remove_option('common', 'multi_vo') with assert_raises(UnsupportedOperation): vo_api.list_vos(issuer='super_root', vo='def') finally: # Make sure we don't leave the config changed due to a test failure if self.vo: config_set('common', 'multi_vo', 'True') else: config_remove_option('common', 'multi_vo')
def test_super_root_naming(self): """ MULTI VO (CORE): Test we can only name accounts super_root when appropriate """ with assert_raises(Duplicate): # Ensure we fail from duplication rather than the choice of name add_account('super_root', 'USER', '*****@*****.**', 'root', vo='def') with assert_raises(UnsupportedAccountName): add_account('super_root', 'USER', '*****@*****.**', 'root', **self.vo) try: config_remove_option('common', 'multi_vo') with assert_raises(UnsupportedAccountName): add_account('super_root', 'USER', '*****@*****.**', 'root', **self.vo) with assert_raises(UnsupportedAccountName): add_account('super_root', 'USER', '*****@*****.**', 'root', vo='def') finally: # Make sure we don't leave the config changed due to a test failure if self.vo: config_set('common', 'multi_vo', 'True') else: config_remove_option('common', 'multi_vo')
def test_download_file_with_supported_protocol_from_config( rse_factory, did_factory, download_client, mock_scope): """ Download (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config read from rucio.cfg is called """ rse, rse_id = rse_factory.make_rse() # FIXME: # The correct order to test should actually be scp,file,root # However the preferred_impl is not working correctly. # Once preferred_impl is fixed, this should be changed back add_protocol( rse_id, { 'scheme': 'scp', 'hostname': '%s.cern.ch' % rse_id, 'port': 0, 'prefix': '/test/', 'impl': 'rucio.rse.protocols.posix.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) add_protocol( rse_id, { 'scheme': 'file', 'hostname': '%s.cern.ch' % rse_id, 'port': 0, 'prefix': '/test/', 'impl': 'rucio.rse.protocols.scp.Default', 'domains': { 'lan': { 'read': 2, 'write': 2, 'delete': 2 }, 'wan': { 'read': 2, 'write': 2, 'delete': 2 } } }) add_protocol( rse_id, { 'scheme': 'root', 'hostname': '%s.cern.ch' % rse_id, 'port': 0, 'prefix': '/test/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 3, 'write': 3, 'delete': 3 }, 'wan': { 'read': 3, 'write': 3, 'delete': 3 } } }) config_add_section('download') config_set('download', 'preferred_impl', 'rclone, xrootd') supported_impl = 'xrootd' path = file_generator() name = os.path.basename(path) item = { 'path': path, 'rse': rse, 'did_scope': str(mock_scope), 'did_name': name, 'guid': generate_uuid(), } did_factory.upload_client.upload([item]) did_str = '%s:%s' % (mock_scope, name) with patch('rucio.rse.protocols.%s.Default.get' % supported_impl, side_effect=lambda pfn, dest, **kw: shutil.copy(path, dest)) as mock_get, \ patch('rucio.rse.protocols.%s.Default.connect' % supported_impl),\ patch('rucio.rse.protocols.%s.Default.close' % supported_impl): download_client.download_dids([{ 'did': did_str, 'impl': supported_impl }]) mock_get.assert_called()
def test_queue_requests_state(vo, use_preparer): """ REQUEST (CORE): test queuing requests """ if use_preparer == 'preparer enabled': use_preparer = True elif use_preparer == 'preparer disabled': use_preparer = False else: return pytest.xfail(reason=f'unknown test parameter use_preparer={use_preparer}') db_session = session.get_session() dest_rse = 'MOCK' dest_rse2 = 'MOCK2' source_rse = 'MOCK4' source_rse2 = 'MOCK5' dest_rse_id = get_rse_id(dest_rse, vo=vo) dest_rse_id2 = get_rse_id(dest_rse2, vo=vo) source_rse_id = get_rse_id(source_rse, vo=vo) source_rse_id2 = get_rse_id(source_rse2, vo=vo) scope = InternalScope('mock', vo=vo) account = InternalAccount('root', vo=vo) user_activity = 'User Subscription' config_set('conveyor', 'use_preparer', str(use_preparer)) target_state = RequestState.PREPARING if use_preparer else RequestState.QUEUED name = generate_uuid() name2 = generate_uuid() name3 = generate_uuid() add_replica(source_rse_id, scope, name, 1, account, session=db_session) add_replica(source_rse_id2, scope, name2, 1, account, session=db_session) add_replica(source_rse_id, scope, name3, 1, account, session=db_session) set_rse_transfer_limits(dest_rse_id, user_activity, max_transfers=1, session=db_session) set_rse_transfer_limits(dest_rse_id2, user_activity, max_transfers=1, session=db_session) set_rse_transfer_limits(source_rse_id, user_activity, max_transfers=1, session=db_session) set_rse_transfer_limits(source_rse_id2, user_activity, max_transfers=1, session=db_session) requests = [{ 'dest_rse_id': dest_rse_id, 'src_rse_id': source_rse_id, 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name, 'scope': scope, 'rule_id': generate_uuid(), 'retry_count': 1, 'requested_at': datetime.now().replace(year=2015), 'attributes': { 'activity': user_activity, 'bytes': 10, 'md5': '', 'adler32': '' } }, { 'dest_rse_id': dest_rse_id, 'src_rse_id': source_rse_id2, 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name2, 'scope': scope, 'rule_id': generate_uuid(), 'retry_count': 1, 'requested_at': datetime.now().replace(year=2015), 'attributes': { 'activity': 'unknown', 'bytes': 10, 'md5': '', 'adler32': '' } }, { 'dest_rse_id': dest_rse_id2, 'src_rse_id': source_rse_id, 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name3, 'scope': scope, 'rule_id': generate_uuid(), 'retry_count': 1, 'requested_at': datetime.now().replace(year=2015), 'attributes': { 'activity': user_activity, 'bytes': 10, 'md5': '', 'adler32': '' } }] try: queue_requests(requests, session=db_session) request = get_request_by_did(scope, name, dest_rse_id, session=db_session) assert request['state'] == target_state request = get_request_by_did(scope, name2, dest_rse_id, session=db_session) assert request['state'] == target_state request = get_request_by_did(scope, name3, dest_rse_id2, session=db_session) assert request['state'] == target_state finally: config_remove_option('conveyor', 'use_preparer') db_session.query(models.Source).delete() db_session.query(models.Request).delete() db_session.query(models.RSETransferLimit).delete() db_session.query(models.Distance).delete() db_session.commit() reset_config_table()