def test_ssh_success(vo, rest_client):
"""AUTHENTICATION (REST): SSH RSA public key exchange (correct credentials)."""
root = InternalAccount('root', vo=vo)
try:
add_account_identity(PUBLIC_KEY,
IdentityType.SSH,
root,
email='*****@*****.**')
except Duplicate:
pass # might already exist, can skip
headers_dict = {'X-Rucio-Account': 'root'}
response = rest_client.get('/auth/ssh_challenge_token',
headers=headers(hdrdict(headers_dict),
vohdr(vo)))
assert response.status_code == 200
assert 'challenge-' in response.headers.get('X-Rucio-SSH-Challenge-Token')
signature = ssh_sign(PRIVATE_KEY,
response.headers.get('X-Rucio-SSH-Challenge-Token'))
headers_dict = {
'X-Rucio-Account': 'root',
'X-Rucio-SSH-Signature': signature
}
response = rest_client.get('/auth/ssh',
headers=headers(hdrdict(headers_dict),
vohdr(vo)))
assert response.status_code == 200
assert len(response.headers.get('X-Rucio-Auth-Token')) > 32
del_account_identity(PUBLIC_KEY, IdentityType.SSH, root)
def test_userpass_fail(vo, rest_client):
"""AUTHENTICATION (REST): Username and password (wrong credentials)."""
response = rest_client.get('/auth/userpass',
headers=headers(
loginhdr('wrong', 'wrong', 'wrong'),
vohdr(vo)))
assert response.status_code == 401
def test_delete_identity_of_account(vo, rest_client):
""" ACCOUNT (REST): send a DELETE to remove an identity of an account."""
account = account_name_generator()
identity = uuid()
password = '******'
add_account(account, 'USER', '*****@*****.**', 'root', vo=vo)
add_identity(identity, IdentityType.USERPASS, '*****@*****.**', password)
add_account_identity(identity, IdentityType.USERPASS,
InternalAccount(account, vo=vo), '*****@*****.**')
auth_response = rest_client.get('/auth/userpass',
headers=headers(
loginhdr(account, identity, password),
vohdr(vo)))
assert auth_response.status_code == 200
assert 'X-Rucio-Auth-Token' in auth_response.headers
token = str(auth_response.headers.get('X-Rucio-Auth-Token'))
assert len(token) != 0
# normal deletion
data = {'authtype': 'USERPASS', 'identity': identity}
response = rest_client.delete('/accounts/' + account + '/identities',
headers=headers(auth(token)),
json=data)
assert response.status_code == 200
# unauthorized deletion
other_account = account_name_generator()
data = {'authtype': 'USERPASS', 'identity': identity}
response = rest_client.delete('/accounts/' + other_account + '/identities',
headers=headers(auth(token)),
json=data)
assert response.status_code == 401
def test_userpass_success(vo, rest_client):
"""AUTHENTICATION (REST): Username and password (correct credentials)."""
response = rest_client.get('/auth/userpass',
headers=headers(
loginhdr('root', 'ddmlab', 'secret'),
vohdr(vo)))
assert response.status_code == 200
assert len(response.headers.get('X-Rucio-Auth-Token')) > 32
def get_replicas():
return parse_replicas_from_string(
rest_client.get(
'/replicas/%s/%s?select=geoip' %
(mock_scope.external, protocols_setup['files'][0]['name']),
headers=headers(auth(auth_token), vohdr(vo),
accept(
Mime.JSON_STREAM))).get_data(as_text=True))
def check_error_api(params, exception_class, exception_message, code):
headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
response = rest_client.get('/requests/list',
query_string=params,
headers=headers(auth(auth_token), vohdr(vo),
hdrdict(headers_dict)))
assert response.status_code == code
body = parse_response(response.get_data(as_text=True))
assert body['ExceptionClass'] == exception_class
assert body['ExceptionMessage'] == exception_message
def auth_token(rest_client, long_vo):
from rucio.tests.common import vohdr, headers, loginhdr
auth_response = rest_client.get('/auth/userpass',
headers=headers(
loginhdr('root', 'ddmlab', 'secret'),
vohdr(long_vo)))
assert auth_response.status_code == 200
token = auth_response.headers.get('X-Rucio-Auth-Token')
assert token
return str(token)
def check_correct_api(params, expected_requests):
headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
response = rest_client.get('/requests/list',
query_string=params,
headers=headers(auth(auth_token), vohdr(vo),
hdrdict(headers_dict)))
assert response.status_code == 200
requests = set()
for request in response.get_data(as_text=True).split('\n')[:-1]:
request = parse_response(request)
requests.add((request['state'], request['source_rse_id'],
request['dest_rse_id'], request['name']))
assert requests == expected_requests
def test_saml_fail(vo, rest_client):
"""AUTHENTICATION (REST): SAML Username and password (wrong credentials)."""
headers_dict = {'X-Rucio-Account': 'root'}
userpass = {'username': '******', 'password': '******'}
response = rest_client.get('/auth/saml',
headers=headers(hdrdict(headers_dict),
vohdr(vo)))
if not response.headers.get('X-Rucio-Auth-Token'):
SAML_auth_url = response.headers.get('X-Rucio-SAML-Auth-URL')
response = session().post(SAML_auth_url,
data=userpass,
verify=False,
allow_redirects=True)
response = rest_client.get('/auth/saml',
headers=headers(hdrdict(headers_dict)))
assert response.status_code == 401
def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token,
protocols_setup, content_type):
"""Replicas: test sorting via geoip with ignoring geoip errors."""
class MockedGeoIPError(Exception):
def __init__(self, *args):
super(MockedGeoIPError, self).__init__(*args)
def fake_get_geoip_db(*args, **kwargs):
raise MockedGeoIPError()
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'schemes':
schemes,
'sort':
'geoip',
}
# invalidate cache for __get_distance so that __get_geoip_db is called
replica_sorter.REGION.invalidate()
with mock.patch('rucio.core.replica_sorter.__get_geoip_db',
side_effect=fake_get_geoip_db) as get_geoip_db_mock:
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
get_geoip_db_mock.assert_called()
def test_ssh_fail(vo, rest_client):
"""AUTHENTICATION (REST): SSH RSA public key exchange (wrong credentials)."""
root = InternalAccount('root', vo=vo)
try:
add_account_identity(PUBLIC_KEY,
IdentityType.SSH,
root,
email='*****@*****.**')
except Duplicate:
pass # might already exist, can skip
signature = ssh_sign(PRIVATE_KEY, 'sign_something_else')
headers_dict = {
'X-Rucio-Account': 'root',
'X-Rucio-SSH-Signature': signature
}
response = rest_client.get('/auth/ssh',
headers=headers(hdrdict(headers_dict),
vohdr(vo)))
assert response.status_code == 401
del_account_identity(PUBLIC_KEY, IdentityType.SSH, root)
def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
content_type):
"""Replicas: test not sorting only LANs."""
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
# yes, this is rather a hack (but works on the API as well). I would like to have an rse_expression parameter instead.
'client_location': {
'site':
'|site='.join(
map(lambda info: info['site'], protocols_setup['rse_info']))
},
'schemes':
schemes,
}
def fake_sort_replicas(dictreplica, *args, **kwargs):
# test that nothing is passed to sort_replicas
assert not dictreplica
return []
# invalidate cache for parse_expression('site=…')
rse_expression_parser.REGION.invalidate()
with mock.patch('rucio.web.rest.flaskapi.v1.replicas.sort_replicas',
side_effect=fake_sort_replicas):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
# 4 for lan, since one is blocked for lan for each site
assert len(sources_list) == 4
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
# 4 for lan, since one is blocked for lan for each site
assert len(sources_dict) == 4
def test_sort_geoip_lan_before_wan(vo, rest_client, auth_token,
protocols_setup, content_type, info_id):
"""Replicas: test sorting LAN sites before WANs via geoip."""
n = 2
nmap = {}
def fake_get_distance(se1, se2, *args, **kwargs):
nonlocal n, nmap
n = n - 1
print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
assert se1, 'pfn host must be se1 for this test'
nmap[se1] = n
return n
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'client_location': {
'site': protocols_setup['rse_info'][info_id]['site']
},
'schemes':
schemes,
'sort':
'geoip',
}
# invalidate cache for parse_expression('site=…')
rse_expression_parser.REGION.invalidate()
with mock.patch('rucio.core.replica_sorter.__get_distance',
side_effect=fake_get_distance):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
# because urlparse hostname result is lower case
sorted_wan_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
# 3 for wan and 2 for lan, since one is blocked for lan for each site
assert len(sources_list) == 5
sorted_replica_hosts = list(
sorted(sources_list, key=lambda source: source['priority']))
print(sorted_replica_hosts)
lan_pfns = list(
filter(lambda source: source['domain'] == 'lan',
sorted_replica_hosts))
assert len(lan_pfns) == 2
for lanpfn in lan_pfns:
assert protocols_setup['rse_info'][info_id]['name'] == lanpfn[
'rse']
sorted_replica_wan_hosts = list(
map(
lambda source: urlparse(source['pfn']).hostname,
filter(lambda source: source['domain'] != 'lan',
sorted_replica_hosts)))
assert sorted_wan_hosts == sorted_replica_wan_hosts
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
# 3 for wan and 2 for lan, since one is blocked for lan for each site
assert len(sources_dict) == 5
sorted_replica_hosts = list(
sorted(sources_dict,
key=lambda pfn: sources_dict[pfn]['priority']))
lan_pfns = list(
filter(lambda pfn: sources_dict[pfn]['domain'] == 'lan',
sorted_replica_hosts))
assert len(lan_pfns) == 2
for lanpfn in lan_pfns:
assert protocols_setup['rse_info'][info_id]['id'] == sources_dict[
lanpfn]['rse_id']
wan_pfns = filter(lambda pfn: sources_dict[pfn]['domain'] != 'lan',
sorted_replica_hosts)
sorted_replica_wan_hosts = list(
map(lambda pfn: urlparse(pfn).hostname, wan_pfns))
assert sorted_wan_hosts == sorted_replica_wan_hosts
def test_sort_geoip_wan(vo, rest_client, auth_token, protocols_setup,
content_type):
"""Replicas: test sorting a few WANs via geoip."""
n = 10
nmap = {}
def fake_get_distance(se1, se2, *args, **kwargs):
nonlocal n, nmap
n = n - 1
print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
assert se1, 'pfn host must be se1 for this test'
nmap[se1] = n
return n
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'schemes':
schemes,
'sort':
'geoip',
}
with mock.patch('rucio.core.replica_sorter.__get_distance',
side_effect=fake_get_distance):
response = rest_client.post('/replicas/list',
headers=headers(auth(auth_token),
vohdr(vo),
accept(content_type)),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
# because urlparse hostname result is lower case
sorted_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
print(replicas)
assert len(replicas) == 1
sources_list = replicas[0]['sources']
print(sources_list)
assert len(sources_list) == 6
sorted_replica_hosts = list(
sorted(sources_list, key=lambda source: source['priority']))
sorted_replica_hosts = list(
map(lambda source: urlparse(source['pfn']).hostname,
sorted_replica_hosts))
assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
print(replicas)
assert len(replicas) == 1
sources_dict = replicas[0]['pfns']
assert len(sources_dict) == 6
sorted_replica_hosts = list(
sorted(sources_dict,
key=lambda pfn: sources_dict[pfn]['priority']))
sorted_replica_hosts = list(
map(lambda source: urlparse(source).hostname,
sorted_replica_hosts))
assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
def test_redirect_metalink_list_replicas(vo, rest_client):
""" ROOT (REDIRECT REST): Test internal proxy prepend with metalink"""
# default behaviour - no location -> no proxy
response = rest_client.get('/redirect/mock/half-life_1/metalink',
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1' in body
assert 'proxy' not in body
response = rest_client.get('/redirect/mock/half-life_2/metalink',
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2' in body
assert 'proxy' not in body
response = rest_client.get('/redirect/mock/half-life_3/metalink',
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3' in body
assert 'proxy' not in body
# site without proxy
response = rest_client.get('/redirect/mock/half-life_1/metalink?' +
urlencode(client_location_without_proxy),
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1' in body
assert 'proxy' not in body
response = rest_client.get('/redirect/mock/half-life_2/metalink?' +
urlencode(client_location_without_proxy),
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2' in body
assert 'proxy' not in body
response = rest_client.get('/redirect/mock/half-life_3/metalink?' +
urlencode(client_location_without_proxy),
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3' in body
assert 'proxy' not in body
# at location with outgoing proxy, prepend for wan replica
response = rest_client.get('/redirect/mock/half-life_1/metalink?' +
urlencode(client_location_with_proxy),
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1' in body
response = rest_client.get('/redirect/mock/half-life_2/metalink?' +
urlencode(client_location_with_proxy),
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2' in body
response = rest_client.get('/redirect/mock/half-life_3/metalink?' +
urlencode(client_location_with_proxy),
headers=headers(vohdr(vo)))
assert response.status_code == 200
body = response.get_data(as_text=True)
assert 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3' in body
assert 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3' in body
def test_sort_geoip_wan_client_location(vo, rest_client, auth_token,
protocols_setup, content_type,
mock_geoip_db, mock_get_lat_long):
"""Replicas: test sorting a few WANs via geoip."""
data = {
'dids': [{
'scope': f['scope'].external,
'name': f['name'],
'type': 'FILE'
} for f in protocols_setup['files']],
'schemes':
schemes,
'sort':
'geoip',
}
first_aut_then_jpn = [
'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com',
'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com'
]
first_jpn_then_aut = [
'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com',
'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com'
]
for client_location, expected_order in (
('Switzerland', first_aut_then_jpn),
('Romania', first_aut_then_jpn),
('Austria', first_aut_then_jpn),
('United Kingdom', first_aut_then_jpn),
('Libya', first_aut_then_jpn),
('China', first_jpn_then_aut),
('United States', first_jpn_then_aut),
('Japan', first_jpn_then_aut),
('Taiwan', first_jpn_then_aut),
('Israel', first_aut_then_jpn),
('Finland', first_aut_then_jpn),
('United Arab Emirates', first_aut_then_jpn),
):
response = rest_client.post(
'/replicas/list',
headers=headers(
auth(auth_token), vohdr(vo), accept(content_type),
[('X-Forwarded-For', LOCATION_TO_IP[client_location])]),
json=data)
assert response.status_code == 200
replicas_response = response.get_data(as_text=True)
assert replicas_response
replicas = []
pfns = []
if content_type == Mime.METALINK:
replicas = parse_replicas_from_string(replicas_response)
pfns = [s['pfn'] for s in replicas[0]['sources']]
elif content_type == Mime.JSON_STREAM:
replicas = list(
map(
json.loads,
filter(
bool,
map(str.strip,
replicas_response.splitlines(keepends=False)))))
pfns = list(replicas[0]['pfns'])
print(client_location, pfns)
assert len(replicas) == 1
assert [urlparse(pfn).hostname for pfn in pfns] == expected_order