def get_replicas():
return parse_replicas_from_string(
rest_client.get(
'/replicas/%s/%s?select=geoip' %
(mock_scope.external, protocols_setup['files'][0]['name']),
headers=headers(auth(auth_token), vohdr(vo),
accept(
Mime.JSON_STREAM))).get_data(as_text=True))
def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
content_type):
"""Replicas: test not sorting only LANs."""
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
# yes, this is rather a hack (but works on the API as well). I would like to have an rse_expression parameter instead.
'client_location': {
'site':
'|site='.join(
map(lambda info: info['site'], protocols_setup['rse_info']))
},
'schemes':
schemes,
}
def fake_sort_replicas(dictreplica, *args, **kwargs):
# test that nothing is passed to sort_replicas
assert not dictreplica
return []
# invalidate cache for parse_expression('site=…')
rse_expression_parser.REGION.invalidate()
with mock.patch('rucio.web.rest.flaskapi.v1.replicas.sort_replicas',
side_effect=fake_sort_replicas):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
# 4 for lan, since one is blocked for lan for each site
assert len(sources_list) == 4
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
# 4 for lan, since one is blocked for lan for each site
assert len(sources_dict) == 4
def test_sort_geoip_lan_before_wan(vo, rest_client, auth_token,
protocols_setup, content_type, info_id):
"""Replicas: test sorting LAN sites before WANs via geoip."""
n = 2
nmap = {}
def fake_get_distance(se1, se2, *args, **kwargs):
nonlocal n, nmap
n = n - 1
print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
assert se1, 'pfn host must be se1 for this test'
nmap[se1] = n
return n
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'client_location': {
'site': protocols_setup['rse_info'][info_id]['site']
},
'schemes':
schemes,
'sort':
'geoip',
}
# invalidate cache for parse_expression('site=…')
rse_expression_parser.REGION.invalidate()
with mock.patch('rucio.core.replica_sorter.__get_distance',
side_effect=fake_get_distance):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
# because urlparse hostname result is lower case
sorted_wan_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
# 3 for wan and 2 for lan, since one is blocked for lan for each site
assert len(sources_list) == 5
sorted_replica_hosts = list(
sorted(sources_list, key=lambda source: source['priority']))
print(sorted_replica_hosts)
lan_pfns = list(
filter(lambda source: source['domain'] == 'lan',
sorted_replica_hosts))
assert len(lan_pfns) == 2
for lanpfn in lan_pfns:
assert protocols_setup['rse_info'][info_id]['name'] == lanpfn[
'rse']
sorted_replica_wan_hosts = list(
map(
lambda source: urlparse(source['pfn']).hostname,
filter(lambda source: source['domain'] != 'lan',
sorted_replica_hosts)))
assert sorted_wan_hosts == sorted_replica_wan_hosts
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
# 3 for wan and 2 for lan, since one is blocked for lan for each site
assert len(sources_dict) == 5
sorted_replica_hosts = list(
sorted(sources_dict,
key=lambda pfn: sources_dict[pfn]['priority']))
lan_pfns = list(
filter(lambda pfn: sources_dict[pfn]['domain'] == 'lan',
sorted_replica_hosts))
assert len(lan_pfns) == 2
for lanpfn in lan_pfns:
assert protocols_setup['rse_info'][info_id]['id'] == sources_dict[
lanpfn]['rse_id']
wan_pfns = filter(lambda pfn: sources_dict[pfn]['domain'] != 'lan',
sorted_replica_hosts)
sorted_replica_wan_hosts = list(
map(lambda pfn: urlparse(pfn).hostname, wan_pfns))
assert sorted_wan_hosts == sorted_replica_wan_hosts
def test_sort_geoip_wan(vo, rest_client, auth_token, protocols_setup,
content_type):
"""Replicas: test sorting a few WANs via geoip."""
n = 10
nmap = {}
def fake_get_distance(se1, se2, *args, **kwargs):
nonlocal n, nmap
n = n - 1
print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
assert se1, 'pfn host must be se1 for this test'
nmap[se1] = n
return n
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'schemes':
schemes,
'sort':
'geoip',
}
with mock.patch('rucio.core.replica_sorter.__get_distance',
side_effect=fake_get_distance):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
# because urlparse hostname result is lower case
sorted_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
assert len(sources_list) == 6
sorted_replica_hosts = list(
sorted(sources_list, key=lambda source: source['priority']))
sorted_replica_hosts = list(
map(lambda source: urlparse(source['pfn']).hostname,
sorted_replica_hosts))
assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
assert len(sources_dict) == 6
sorted_replica_hosts = list(
sorted(sources_dict,
key=lambda pfn: sources_dict[pfn]['priority']))
sorted_replica_hosts = list(
map(lambda source: urlparse(source).hostname,
sorted_replica_hosts))
assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
def test_sort_geoip_wan_client_location(vo, rest_client, auth_token,
protocols_setup, content_type,
mock_geoip_db, mock_get_lat_long):
"""Replicas: test sorting a few WANs via geoip."""
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'schemes':
schemes,
'sort':
'geoip',
}
first_aut_then_jpn = [
'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com',
'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com'
]
first_jpn_then_aut = [
'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com',
'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com'
]
for client_location, expected_order in (
('Switzerland', first_aut_then_jpn),
('Romania', first_aut_then_jpn),
('Austria', first_aut_then_jpn),
('United Kingdom', first_aut_then_jpn),
('Libya', first_aut_then_jpn),
('China', first_jpn_then_aut),
('United States', first_jpn_then_aut),
('Japan', first_jpn_then_aut),
('Taiwan', first_jpn_then_aut),
('Israel', first_aut_then_jpn),
('Finland', first_aut_then_jpn),
('United Arab Emirates', first_aut_then_jpn),
):
response = rest_client.post(
'/replicas/list',
headers=headers(
auth(auth_token), vohdr(vo), accept(content_type),
[('X-Forwarded-For', LOCATION_TO_IP[client_location])]),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
replicas = []
pfns = []
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
pfns = [s['pfn'] for s in replicas[0]['sources']]
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
pfns = list(replicas[0]['pfns'])
print(client_location, pfns)
assert len(replicas) == 1
assert [urlparse(pfn).hostname for pfn in pfns] == expected_order
