def test_sort_geoip_wan(vo, rest_client, auth_token, protocols_setup,
content_type):
"""Replicas: test sorting a few WANs via geoip."""
n = 10
nmap = {}
def fake_get_distance(se1, se2, *args, **kwargs):
nonlocal n, nmap
n = n - 1
print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
assert se1, 'pfn host must be se1 for this test'
nmap[se1] = n
return n
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'schemes':
schemes,
'sort':
'geoip',
}
with mock.patch('rucio.core.replica_sorter.__get_distance',
side_effect=fake_get_distance):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
# because urlparse hostname result is lower case
sorted_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
assert len(sources_list) == 6
sorted_replica_hosts = list(
sorted(sources_list, key=lambda source: source['priority']))
sorted_replica_hosts = list(
map(lambda source: urlparse(source['pfn']).hostname,
sorted_replica_hosts))
assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
assert len(sources_dict) == 6
sorted_replica_hosts = list(
sorted(sources_dict,
key=lambda pfn: sources_dict[pfn]['priority']))
sorted_replica_hosts = list(
map(lambda source: urlparse(source).hostname,
sorted_replica_hosts))
assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
def test_sort_geoip_lan_before_wan(vo, rest_client, auth_token,
protocols_setup, content_type, info_id):
"""Replicas: test sorting LAN sites before WANs via geoip."""
n = 2
nmap = {}
def fake_get_distance(se1, se2, *args, **kwargs):
nonlocal n, nmap
n = n - 1
print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
assert se1, 'pfn host must be se1 for this test'
nmap[se1] = n
return n
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'client_location': {
'site': protocols_setup['rse_info'][info_id]['site']
},
'schemes':
schemes,
'sort':
'geoip',
}
with mock.patch('rucio.core.replica_sorter.__get_distance',
side_effect=fake_get_distance):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
# because urlparse hostname result is lower case
sorted_wan_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
# 3 for wan and 2 for lan, since one is blocked for lan for each site
assert len(sources_list) == 5
sorted_replica_hosts = list(
sorted(sources_list, key=lambda source: source['priority']))
print(sorted_replica_hosts)
lan_pfns = list(
filter(lambda source: source['domain'] == 'lan',
sorted_replica_hosts))
assert len(lan_pfns) == 2
for lanpfn in lan_pfns:
assert protocols_setup['rse_info'][info_id]['name'] == lanpfn[
'rse']
sorted_replica_wan_hosts = list(
map(
lambda source: urlparse(source['pfn']).hostname,
filter(lambda source: source['domain'] != 'lan',
sorted_replica_hosts)))
assert sorted_wan_hosts == sorted_replica_wan_hosts
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
# 3 for wan and 2 for lan, since one is blocked for lan for each site
assert len(sources_dict) == 5
sorted_replica_hosts = list(
sorted(sources_dict,
key=lambda pfn: sources_dict[pfn]['priority']))
lan_pfns = list(
filter(lambda pfn: sources_dict[pfn]['domain'] == 'lan',
sorted_replica_hosts))
assert len(lan_pfns) == 2
for lanpfn in lan_pfns:
assert protocols_setup['rse_info'][info_id]['id'] == sources_dict[
lanpfn]['rse_id']
wan_pfns = filter(lambda pfn: sources_dict[pfn]['domain'] != 'lan',
sorted_replica_hosts)
sorted_replica_wan_hosts = list(
map(lambda pfn: urlparse(pfn).hostname, wan_pfns))
assert sorted_wan_hosts == sorted_replica_wan_hosts
def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
content_type):
"""Replicas: test not sorting only LANs."""
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
# yes, this is rather a hack (but works on the API as well). I would like to have an rse_expression parameter instead.
'client_location': {
'site':
'|site='.join(
map(lambda info: info['site'], protocols_setup['rse_info']))
},
'schemes':
schemes,
}
rest_backend = os.environ.get('REST_BACKEND', 'webpy')
if rest_backend == 'webpy':
sort_replicas_mock_target = 'rucio.web.rest.replica.sort_replicas'
elif rest_backend == 'flask':
sort_replicas_mock_target = 'rucio.web.rest.flaskapi.v1.replicas.sort_replicas'
else:
return pytest.xfail('unknown REST_BACKEND: ' + rest_backend)
def fake_sort_replicas(dictreplica, *args, **kwargs):
# test that nothing is passed to sort_replicas
assert not dictreplica
return []
# invalidate cache for parse_expression('site=…')
rse_expression_parser.REGION.invalidate()
with mock.patch(sort_replicas_mock_target, side_effect=fake_sort_replicas):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
# 4 for lan, since one is blocked for lan for each site
assert len(sources_list) == 4
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
# 4 for lan, since one is blocked for lan for each site
assert len(sources_dict) == 4