def test_config_default_override(self):
"""LFN2PFN: Test override of default LFN2PFN algorithm via config (Success)"""
if not config.config_has_section('policy'):
config.config_add_section('policy')
try:
orig_value = config.config_get('policy',
'lfn2pfn_algorithm_default')
except (NoOptionError, NoSectionError):
orig_value = None
def static_test(scope, name, rse, rse_attrs, proto_attrs):
"""Static test function for config override."""
del scope
del name
del rse
del rse_attrs
del proto_attrs
return "static_test_value"
RSEDeterministicTranslation.register(static_test)
try:
config.config_set('policy', 'lfn2pfn_algorithm_default',
'static_test')
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
assert_equal(self.translator.path("foo", "bar"),
"static_test_value")
finally:
if orig_value is None:
config.config_remove_option('policy',
'lfn2pfn_algorithm_default')
else:
config.config_set('policy', 'lfn2pfn_algorithm_default',
orig_value)
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
def file_config_mock(request):
"""
Fixture which allows to have an isolated in-memory configuration file instance which
is not persisted after exiting the fixture.
This override works only in tests which use config calls directly, not in the ones working
via the API, as the server config is not changed.
"""
from unittest import mock
from rucio.common.config import Config, config_set, config_has_section, config_add_section
# Get the fixture parameters
overrides = []
params = __get_fixture_param(request)
if params:
overrides = params.get("overrides", overrides)
parser = Config().parser
with mock.patch('rucio.common.config.get_config',
side_effect=lambda: parser):
for section, option, value in (overrides or []):
if not config_has_section(section):
config_add_section(section)
config_set(section, option, value)
yield
def test_upload_file_with_supported_protocol_from_config(rse_factory, upload_client, mock_scope, file_factory):
""" Upload (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config read from rucio.cfg is called """
rse_name, rse_id = rse_factory.make_rse()
# FIXME:
# The correct order to test should actually be ssh,xrootd,posix
# However the preferred_impl is not working correctly.
# Once preferred_impl is fixed, this should be changed back
add_protocol(rse_id, {'scheme': 'scp',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
add_protocol(rse_id, {'scheme': 'file',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
'lan': {'read': 2, 'write': 2, 'delete': 2},
'wan': {'read': 2, 'write': 2, 'delete': 2}}})
add_protocol(rse_id, {'scheme': 'root',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.ssh.Default',
'domains': {
'lan': {'read': 3, 'write': 3, 'delete': 3},
'wan': {'read': 3, 'write': 3, 'delete': 3}}})
config_add_section('upload')
config_set('upload', 'preferred_impl', 'rclone, xrootd')
supported_impl = 'xrootd'
path = file_factory.file_generator()
name = os.path.basename(path)
item = {
'path': path,
'rse': rse_name,
'did_scope': str(mock_scope),
'did_name': name,
'guid': generate_uuid()
}
with TemporaryDirectory() as tmp_dir:
with patch('rucio.rse.protocols.%s.Default.put' % supported_impl, side_effect=lambda pfn, dest, dir, **kw: shutil.copy(path, tmp_dir)) as mock_put, \
patch('rucio.rse.protocols.%s.Default.connect' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.exists' % supported_impl, side_effect=lambda pfn, **kw: False),\
patch('rucio.rse.protocols.%s.Default.delete' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.rename' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.stat' % supported_impl, side_effect=lambda pfn: {'filesize': os.stat(path)[os.path.stat.ST_SIZE], 'adler32': adler32(path)}),\
patch('rucio.rse.protocols.%s.Default.close' % supported_impl):
mock_put.__name__ = "mock_put"
upload_client.upload([item])
mock_put.assert_called()
def test_module_load(self):
"""LFN2PFN: Test ability to provide LFN2PFN functions via module (Success)"""
if not config.config_has_section('policy'):
config.config_add_section('policy')
config.config_set('policy', 'lfn2pfn_module', 'rucio.tests.lfn2pfn_module_test')
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
self.rse_attributes['lfn2pfn_algorithm'] = 'lfn2pfn_module_algorithm'
self.create_translator()
assert self.translator.path("foo", "bar") == "lfn2pfn_module_algorithm_value"
def test_download_file_with_supported_protocol_from_config(
rse_factory, did_factory, download_client, mock_scope):
""" Download (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config read from rucio.cfg is called """
rse, rse_id = rse_factory.make_rse()
# FIXME:
# The correct order to test should actually be scp,file,root
# However the preferred_impl is not working correctly.
# Once preferred_impl is fixed, this should be changed back
add_protocol(
rse_id, {
'scheme': 'scp',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
rse_id, {
'scheme': 'file',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.scp.Default',
'domains': {
'lan': {
'read': 2,
'write': 2,
'delete': 2
},
'wan': {
'read': 2,
'write': 2,
'delete': 2
}
}
})
add_protocol(
rse_id, {
'scheme': 'root',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 3,
'write': 3,
'delete': 3
},
'wan': {
'read': 3,
'write': 3,
'delete': 3
}
}
})
config_add_section('download')
config_set('download', 'preferred_impl', 'rclone, xrootd')
supported_impl = 'xrootd'
path = file_generator()
name = os.path.basename(path)
item = {
'path': path,
'rse': rse,
'did_scope': str(mock_scope),
'did_name': name,
'guid': generate_uuid(),
}
did_factory.upload_client.upload([item])
did_str = '%s:%s' % (mock_scope, name)
with patch('rucio.rse.protocols.%s.Default.get' % supported_impl, side_effect=lambda pfn, dest, **kw: shutil.copy(path, dest)) as mock_get, \
patch('rucio.rse.protocols.%s.Default.connect' % supported_impl),\
patch('rucio.rse.protocols.%s.Default.close' % supported_impl):
download_client.download_dids([{
'did': did_str,
'impl': supported_impl
}])
mock_get.assert_called()