def test_update_vo_unsupported(self):
""" MULTI VO (REST): Test updating VO through REST layer raises UnsupportedOperation """
mw = []
headers1 = {'X-Rucio-Account': 'super_root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
headers1.update(self.def_header)
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(res1.status, 200)
token = str(res1.header('X-Rucio-Auth-Token'))
params = {'email': '*****@*****.**', 'decription': 'Try updating in single vo mode'}
try:
config_set('common', 'multi_vo', 'False')
headers2 = {'X-Rucio-Auth-Token': str(token)}
res2 = TestApp(vo_app.wsgifunc(*mw)).put('/' + self.vo['vo'], headers=headers2, expect_errors=True, params=dumps(params))
assert_equal(res2.status, 409)
config_remove_option('common', 'multi_vo')
headers2 = {'X-Rucio-Auth-Token': str(token)}
res2 = TestApp(vo_app.wsgifunc(*mw)).put('/' + self.vo['vo'], headers=headers2, expect_errors=True, params=dumps(params))
assert_equal(res2.status, 409)
finally:
# Make sure we don't leave the config changed due to a test failure
if self.vo:
config_set('common', 'multi_vo', 'True')
else:
config_remove_option('common', 'multi_vo')
def test_config_default_override(self):
"""LFN2PFN: Test override of default LFN2PFN algorithm via config (Success)"""
if not config.config_has_section('policy'):
config.config_add_section('policy')
try:
orig_value = config.config_get('policy',
'lfn2pfn_algorithm_default')
except (NoOptionError, NoSectionError):
orig_value = None
def static_test(scope, name, rse, rse_attrs, proto_attrs):
"""Static test function for config override."""
del scope
del name
del rse
del rse_attrs
del proto_attrs
return "static_test_value"
RSEDeterministicTranslation.register(static_test)
try:
config.config_set('policy', 'lfn2pfn_algorithm_default',
'static_test')
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
assert_equal(self.translator.path("foo", "bar"),
"static_test_value")
finally:
if orig_value is None:
config.config_remove_option('policy',
'lfn2pfn_algorithm_default')
else:
config.config_set('policy', 'lfn2pfn_algorithm_default',
orig_value)
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
def file_config_mock(request):
"""
Fixture which allows to have an isolated in-memory configuration file instance which
is not persisted after exiting the fixture.
This override works only in tests which use config calls directly, not in the ones working
via the API, as the server config is not changed.
"""
from unittest import mock
from rucio.common.config import Config, config_set, config_has_section, config_add_section
# Get the fixture parameters
overrides = []
params = __get_fixture_param(request)
if params:
overrides = params.get("overrides", overrides)
parser = Config().parser
with mock.patch('rucio.common.config.get_config',
side_effect=lambda: parser):
for section, option, value in (overrides or []):
if not config_has_section(section):
config_add_section(section)
config_set(section, option, value)
yield
def test_upload_file_with_supported_protocol_from_config(rse_factory, upload_client, mock_scope, file_factory):
""" Upload (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config read from rucio.cfg is called """
rse_name, rse_id = rse_factory.make_rse()
# FIXME:
# The correct order to test should actually be ssh,xrootd,posix
# However the preferred_impl is not working correctly.
# Once preferred_impl is fixed, this should be changed back
add_protocol(rse_id, {'scheme': 'scp',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
add_protocol(rse_id, {'scheme': 'file',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
'lan': {'read': 2, 'write': 2, 'delete': 2},
'wan': {'read': 2, 'write': 2, 'delete': 2}}})
add_protocol(rse_id, {'scheme': 'root',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.ssh.Default',
'domains': {
'lan': {'read': 3, 'write': 3, 'delete': 3},
'wan': {'read': 3, 'write': 3, 'delete': 3}}})
config_add_section('upload')
config_set('upload', 'preferred_impl', 'rclone, xrootd')
supported_impl = 'xrootd'
path = file_factory.file_generator()
name = os.path.basename(path)
item = {
'path': path,
'rse': rse_name,
'did_scope': str(mock_scope),
'did_name': name,
'guid': generate_uuid()
}
with TemporaryDirectory() as tmp_dir:
with patch('rucio.rse.protocols.%s.Default.put' % supported_impl, side_effect=lambda pfn, dest, dir, **kw: shutil.copy(path, tmp_dir)) as mock_put, \
patch('rucio.rse.protocols.%s.Default.connect' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.exists' % supported_impl, side_effect=lambda pfn, **kw: False),\
patch('rucio.rse.protocols.%s.Default.delete' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.rename' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.stat' % supported_impl, side_effect=lambda pfn: {'filesize': os.stat(path)[os.path.stat.ST_SIZE], 'adler32': adler32(path)}),\
patch('rucio.rse.protocols.%s.Default.close' % supported_impl):
mock_put.__name__ = "mock_put"
upload_client.upload([item])
mock_put.assert_called()
def test_module_load(self):
"""LFN2PFN: Test ability to provide LFN2PFN functions via module (Success)"""
if not config.config_has_section('policy'):
config.config_add_section('policy')
config.config_set('policy', 'lfn2pfn_module', 'rucio.tests.lfn2pfn_module_test')
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
self.rse_attributes['lfn2pfn_algorithm'] = 'lfn2pfn_module_algorithm'
self.create_translator()
assert self.translator.path("foo", "bar") == "lfn2pfn_module_algorithm_value"
def test_multi_vo_flag(self):
""" MULTI VO (CORE): Test operations fail in single_vo mode """
try:
config_set('common', 'multi_vo', 'False')
with assert_raises(UnsupportedOperation):
vo_api.list_vos(issuer='super_root', vo='def')
config_remove_option('common', 'multi_vo')
with assert_raises(UnsupportedOperation):
vo_api.list_vos(issuer='super_root', vo='def')
finally:
# Make sure we don't leave the config changed due to a test failure
if self.vo:
config_set('common', 'multi_vo', 'True')
else:
config_remove_option('common', 'multi_vo')
def test_super_root_naming(self):
""" MULTI VO (CORE): Test we can only name accounts super_root when appropriate """
with assert_raises(Duplicate): # Ensure we fail from duplication rather than the choice of name
add_account('super_root', 'USER', '*****@*****.**', 'root', vo='def')
with assert_raises(UnsupportedAccountName):
add_account('super_root', 'USER', '*****@*****.**', 'root', **self.vo)
try:
config_remove_option('common', 'multi_vo')
with assert_raises(UnsupportedAccountName):
add_account('super_root', 'USER', '*****@*****.**', 'root', **self.vo)
with assert_raises(UnsupportedAccountName):
add_account('super_root', 'USER', '*****@*****.**', 'root', vo='def')
finally:
# Make sure we don't leave the config changed due to a test failure
if self.vo:
config_set('common', 'multi_vo', 'True')
else:
config_remove_option('common', 'multi_vo')
def test_download_file_with_supported_protocol_from_config(
rse_factory, did_factory, download_client, mock_scope):
""" Download (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config read from rucio.cfg is called """
rse, rse_id = rse_factory.make_rse()
# FIXME:
# The correct order to test should actually be scp,file,root
# However the preferred_impl is not working correctly.
# Once preferred_impl is fixed, this should be changed back
add_protocol(
rse_id, {
'scheme': 'scp',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
rse_id, {
'scheme': 'file',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.scp.Default',
'domains': {
'lan': {
'read': 2,
'write': 2,
'delete': 2
},
'wan': {
'read': 2,
'write': 2,
'delete': 2
}
}
})
add_protocol(
rse_id, {
'scheme': 'root',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 3,
'write': 3,
'delete': 3
},
'wan': {
'read': 3,
'write': 3,
'delete': 3
}
}
})
config_add_section('download')
config_set('download', 'preferred_impl', 'rclone, xrootd')
supported_impl = 'xrootd'
path = file_generator()
name = os.path.basename(path)
item = {
'path': path,
'rse': rse,
'did_scope': str(mock_scope),
'did_name': name,
'guid': generate_uuid(),
}
did_factory.upload_client.upload([item])
did_str = '%s:%s' % (mock_scope, name)
with patch('rucio.rse.protocols.%s.Default.get' % supported_impl, side_effect=lambda pfn, dest, **kw: shutil.copy(path, dest)) as mock_get, \
patch('rucio.rse.protocols.%s.Default.connect' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.close' % supported_impl):
download_client.download_dids([{
'did': did_str,
'impl': supported_impl
}])
mock_get.assert_called()
def test_queue_requests_state(vo, use_preparer):
""" REQUEST (CORE): test queuing requests """
if use_preparer == 'preparer enabled':
use_preparer = True
elif use_preparer == 'preparer disabled':
use_preparer = False
else:
return pytest.xfail(reason=f'unknown test parameter use_preparer={use_preparer}')
db_session = session.get_session()
dest_rse = 'MOCK'
dest_rse2 = 'MOCK2'
source_rse = 'MOCK4'
source_rse2 = 'MOCK5'
dest_rse_id = get_rse_id(dest_rse, vo=vo)
dest_rse_id2 = get_rse_id(dest_rse2, vo=vo)
source_rse_id = get_rse_id(source_rse, vo=vo)
source_rse_id2 = get_rse_id(source_rse2, vo=vo)
scope = InternalScope('mock', vo=vo)
account = InternalAccount('root', vo=vo)
user_activity = 'User Subscription'
config_set('conveyor', 'use_preparer', str(use_preparer))
target_state = RequestState.PREPARING if use_preparer else RequestState.QUEUED
name = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
add_replica(source_rse_id, scope, name, 1, account, session=db_session)
add_replica(source_rse_id2, scope, name2, 1, account, session=db_session)
add_replica(source_rse_id, scope, name3, 1, account, session=db_session)
set_rse_transfer_limits(dest_rse_id, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(dest_rse_id2, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(source_rse_id, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(source_rse_id2, user_activity, max_transfers=1, session=db_session)
requests = [{
'dest_rse_id': dest_rse_id,
'src_rse_id': source_rse_id,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': user_activity,
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': dest_rse_id,
'src_rse_id': source_rse_id2,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'unknown',
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': dest_rse_id2,
'src_rse_id': source_rse_id,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': user_activity,
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
try:
queue_requests(requests, session=db_session)
request = get_request_by_did(scope, name, dest_rse_id, session=db_session)
assert request['state'] == target_state
request = get_request_by_did(scope, name2, dest_rse_id, session=db_session)
assert request['state'] == target_state
request = get_request_by_did(scope, name3, dest_rse_id2, session=db_session)
assert request['state'] == target_state
finally:
config_remove_option('conveyor', 'use_preparer')
db_session.query(models.Source).delete()
db_session.query(models.Request).delete()
db_session.query(models.RSETransferLimit).delete()
db_session.query(models.Distance).delete()
db_session.commit()
reset_config_table()
