예제 #1
0
def test_ssh_success(vo, rest_client):
    """AUTHENTICATION (REST): SSH RSA public key exchange (correct credentials)."""

    root = InternalAccount('root', vo=vo)
    try:
        add_account_identity(PUBLIC_KEY,
                             IdentityType.SSH,
                             root,
                             email='*****@*****.**')
    except Duplicate:
        pass  # might already exist, can skip

    headers_dict = {'X-Rucio-Account': 'root'}
    response = rest_client.get('/auth/ssh_challenge_token',
                               headers=headers(hdrdict(headers_dict),
                                               vohdr(vo)))
    assert response.status_code == 200
    assert 'challenge-' in response.headers.get('X-Rucio-SSH-Challenge-Token')

    signature = ssh_sign(PRIVATE_KEY,
                         response.headers.get('X-Rucio-SSH-Challenge-Token'))

    headers_dict = {
        'X-Rucio-Account': 'root',
        'X-Rucio-SSH-Signature': signature
    }
    response = rest_client.get('/auth/ssh',
                               headers=headers(hdrdict(headers_dict),
                                               vohdr(vo)))
    assert response.status_code == 200
    assert len(response.headers.get('X-Rucio-Auth-Token')) > 32

    del_account_identity(PUBLIC_KEY, IdentityType.SSH, root)
예제 #2
0
def test_userpass_fail(vo, rest_client):
    """AUTHENTICATION (REST): Username and password (wrong credentials)."""
    response = rest_client.get('/auth/userpass',
                               headers=headers(
                                   loginhdr('wrong', 'wrong', 'wrong'),
                                   vohdr(vo)))
    assert response.status_code == 401
예제 #3
0
파일: test_account.py 프로젝트: rob-c/rucio
def test_delete_identity_of_account(vo, rest_client):
    """ ACCOUNT (REST): send a DELETE to remove an identity of an account."""
    account = account_name_generator()
    identity = uuid()
    password = '******'
    add_account(account, 'USER', '*****@*****.**', 'root', vo=vo)
    add_identity(identity, IdentityType.USERPASS, '*****@*****.**', password)
    add_account_identity(identity, IdentityType.USERPASS,
                         InternalAccount(account, vo=vo), '*****@*****.**')
    auth_response = rest_client.get('/auth/userpass',
                                    headers=headers(
                                        loginhdr(account, identity, password),
                                        vohdr(vo)))
    assert auth_response.status_code == 200
    assert 'X-Rucio-Auth-Token' in auth_response.headers
    token = str(auth_response.headers.get('X-Rucio-Auth-Token'))
    assert len(token) != 0

    # normal deletion
    data = {'authtype': 'USERPASS', 'identity': identity}
    response = rest_client.delete('/accounts/' + account + '/identities',
                                  headers=headers(auth(token)),
                                  json=data)
    assert response.status_code == 200

    # unauthorized deletion
    other_account = account_name_generator()
    data = {'authtype': 'USERPASS', 'identity': identity}
    response = rest_client.delete('/accounts/' + other_account + '/identities',
                                  headers=headers(auth(token)),
                                  json=data)
    assert response.status_code == 401
예제 #4
0
def test_userpass_success(vo, rest_client):
    """AUTHENTICATION (REST): Username and password (correct credentials)."""
    response = rest_client.get('/auth/userpass',
                               headers=headers(
                                   loginhdr('root', 'ddmlab', 'secret'),
                                   vohdr(vo)))
    assert response.status_code == 200
    assert len(response.headers.get('X-Rucio-Auth-Token')) > 32
예제 #5
0
 def get_replicas():
     return parse_replicas_from_string(
         rest_client.get(
             '/replicas/%s/%s?select=geoip' %
             (mock_scope.external, protocols_setup['files'][0]['name']),
             headers=headers(auth(auth_token), vohdr(vo),
                             accept(
                                 Mime.JSON_STREAM))).get_data(as_text=True))
예제 #6
0
 def check_error_api(params, exception_class, exception_message, code):
     headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
     response = rest_client.get('/requests/list',
                                query_string=params,
                                headers=headers(auth(auth_token), vohdr(vo),
                                                hdrdict(headers_dict)))
     assert response.status_code == code
     body = parse_response(response.get_data(as_text=True))
     assert body['ExceptionClass'] == exception_class
     assert body['ExceptionMessage'] == exception_message
예제 #7
0
def auth_token(rest_client, long_vo):
    from rucio.tests.common import vohdr, headers, loginhdr

    auth_response = rest_client.get('/auth/userpass',
                                    headers=headers(
                                        loginhdr('root', 'ddmlab', 'secret'),
                                        vohdr(long_vo)))
    assert auth_response.status_code == 200
    token = auth_response.headers.get('X-Rucio-Auth-Token')
    assert token
    return str(token)
예제 #8
0
 def check_correct_api(params, expected_requests):
     headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
     response = rest_client.get('/requests/list',
                                query_string=params,
                                headers=headers(auth(auth_token), vohdr(vo),
                                                hdrdict(headers_dict)))
     assert response.status_code == 200
     requests = set()
     for request in response.get_data(as_text=True).split('\n')[:-1]:
         request = parse_response(request)
         requests.add((request['state'], request['source_rse_id'],
                       request['dest_rse_id'], request['name']))
     assert requests == expected_requests
예제 #9
0
def test_saml_fail(vo, rest_client):
    """AUTHENTICATION (REST): SAML Username and password (wrong credentials)."""
    headers_dict = {'X-Rucio-Account': 'root'}
    userpass = {'username': '******', 'password': '******'}

    response = rest_client.get('/auth/saml',
                               headers=headers(hdrdict(headers_dict),
                                               vohdr(vo)))
    if not response.headers.get('X-Rucio-Auth-Token'):
        SAML_auth_url = response.headers.get('X-Rucio-SAML-Auth-URL')
        response = session().post(SAML_auth_url,
                                  data=userpass,
                                  verify=False,
                                  allow_redirects=True)
        response = rest_client.get('/auth/saml',
                                   headers=headers(hdrdict(headers_dict)))

    assert response.status_code == 401
예제 #10
0
def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token,
                                            protocols_setup, content_type):
    """Replicas: test sorting via geoip with ignoring geoip errors."""
    class MockedGeoIPError(Exception):
        def __init__(self, *args):
            super(MockedGeoIPError, self).__init__(*args)

    def fake_get_geoip_db(*args, **kwargs):
        raise MockedGeoIPError()

    data = {
        'dids': [{
            'scope': f['scope'].external,
            'name': f['name'],
            'type': 'FILE'
        } for f in protocols_setup['files']],
        'schemes':
        schemes,
        'sort':
        'geoip',
    }

    # invalidate cache for __get_distance so that __get_geoip_db is called
    replica_sorter.REGION.invalidate()

    with mock.patch('rucio.core.replica_sorter.__get_geoip_db',
                    side_effect=fake_get_geoip_db) as get_geoip_db_mock:
        response = rest_client.post('/replicas/list',
                                    headers=headers(auth(auth_token),
                                                    vohdr(vo),
                                                    accept(content_type)),
                                    json=data)
        assert response.status_code == 200

        replicas_response = response.get_data(as_text=True)
        assert replicas_response

        get_geoip_db_mock.assert_called()
예제 #11
0
def test_ssh_fail(vo, rest_client):
    """AUTHENTICATION (REST): SSH RSA public key exchange (wrong credentials)."""

    root = InternalAccount('root', vo=vo)
    try:
        add_account_identity(PUBLIC_KEY,
                             IdentityType.SSH,
                             root,
                             email='*****@*****.**')
    except Duplicate:
        pass  # might already exist, can skip

    signature = ssh_sign(PRIVATE_KEY, 'sign_something_else')

    headers_dict = {
        'X-Rucio-Account': 'root',
        'X-Rucio-SSH-Signature': signature
    }
    response = rest_client.get('/auth/ssh',
                               headers=headers(hdrdict(headers_dict),
                                               vohdr(vo)))
    assert response.status_code == 401

    del_account_identity(PUBLIC_KEY, IdentityType.SSH, root)
예제 #12
0
def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
                                  content_type):
    """Replicas: test not sorting only LANs."""

    data = {
        'dids': [{
            'scope': f['scope'].external,
            'name': f['name'],
            'type': 'FILE'
        } for f in protocols_setup['files']],
        # yes, this is rather a hack (but works on the API as well). I would like to have an rse_expression parameter instead.
        'client_location': {
            'site':
            '|site='.join(
                map(lambda info: info['site'], protocols_setup['rse_info']))
        },
        'schemes':
        schemes,
    }

    def fake_sort_replicas(dictreplica, *args, **kwargs):
        # test that nothing is passed to sort_replicas
        assert not dictreplica
        return []

    # invalidate cache for parse_expression('site=…')
    rse_expression_parser.REGION.invalidate()

    with mock.patch('rucio.web.rest.flaskapi.v1.replicas.sort_replicas',
                    side_effect=fake_sort_replicas):
        response = rest_client.post('/replicas/list',
                                    headers=headers(auth(auth_token),
                                                    vohdr(vo),
                                                    accept(content_type)),
                                    json=data)
    assert response.status_code == 200

    replicas_response = response.get_data(as_text=True)
    assert replicas_response

    if content_type == Mime.METALINK:
        replicas = parse_replicas_from_string(replicas_response)
        print(replicas)
        assert len(replicas) == 1
        sources_list = replicas[0]['sources']
        print(sources_list)
        # 4 for lan, since one is blocked for lan for each site
        assert len(sources_list) == 4

    elif content_type == Mime.JSON_STREAM:
        replicas = list(
            map(
                json.loads,
                filter(
                    bool,
                    map(str.strip,
                        replicas_response.splitlines(keepends=False)))))
        print(replicas)
        assert len(replicas) == 1
        sources_dict = replicas[0]['pfns']
        # 4 for lan, since one is blocked for lan for each site
        assert len(sources_dict) == 4
예제 #13
0
def test_sort_geoip_lan_before_wan(vo, rest_client, auth_token,
                                   protocols_setup, content_type, info_id):
    """Replicas: test sorting LAN sites before WANs via geoip."""
    n = 2
    nmap = {}

    def fake_get_distance(se1, se2, *args, **kwargs):
        nonlocal n, nmap
        n = n - 1
        print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
        assert se1, 'pfn host must be se1 for this test'
        nmap[se1] = n
        return n

    data = {
        'dids': [{
            'scope': f['scope'].external,
            'name': f['name'],
            'type': 'FILE'
        } for f in protocols_setup['files']],
        'client_location': {
            'site': protocols_setup['rse_info'][info_id]['site']
        },
        'schemes':
        schemes,
        'sort':
        'geoip',
    }

    # invalidate cache for parse_expression('site=…')
    rse_expression_parser.REGION.invalidate()

    with mock.patch('rucio.core.replica_sorter.__get_distance',
                    side_effect=fake_get_distance):
        response = rest_client.post('/replicas/list',
                                    headers=headers(auth(auth_token),
                                                    vohdr(vo),
                                                    accept(content_type)),
                                    json=data)
    assert response.status_code == 200

    replicas_response = response.get_data(as_text=True)
    assert replicas_response

    # because urlparse hostname result is lower case
    sorted_wan_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))

    if content_type == Mime.METALINK:
        replicas = parse_replicas_from_string(replicas_response)
        print(replicas)
        assert len(replicas) == 1
        sources_list = replicas[0]['sources']
        print(sources_list)
        # 3 for wan and 2 for lan, since one is blocked for lan for each site
        assert len(sources_list) == 5

        sorted_replica_hosts = list(
            sorted(sources_list, key=lambda source: source['priority']))
        print(sorted_replica_hosts)
        lan_pfns = list(
            filter(lambda source: source['domain'] == 'lan',
                   sorted_replica_hosts))
        assert len(lan_pfns) == 2
        for lanpfn in lan_pfns:
            assert protocols_setup['rse_info'][info_id]['name'] == lanpfn[
                'rse']

        sorted_replica_wan_hosts = list(
            map(
                lambda source: urlparse(source['pfn']).hostname,
                filter(lambda source: source['domain'] != 'lan',
                       sorted_replica_hosts)))
        assert sorted_wan_hosts == sorted_replica_wan_hosts

    elif content_type == Mime.JSON_STREAM:
        replicas = list(
            map(
                json.loads,
                filter(
                    bool,
                    map(str.strip,
                        replicas_response.splitlines(keepends=False)))))
        print(replicas)
        assert len(replicas) == 1
        sources_dict = replicas[0]['pfns']
        # 3 for wan and 2 for lan, since one is blocked for lan for each site
        assert len(sources_dict) == 5

        sorted_replica_hosts = list(
            sorted(sources_dict,
                   key=lambda pfn: sources_dict[pfn]['priority']))
        lan_pfns = list(
            filter(lambda pfn: sources_dict[pfn]['domain'] == 'lan',
                   sorted_replica_hosts))
        assert len(lan_pfns) == 2
        for lanpfn in lan_pfns:
            assert protocols_setup['rse_info'][info_id]['id'] == sources_dict[
                lanpfn]['rse_id']

        wan_pfns = filter(lambda pfn: sources_dict[pfn]['domain'] != 'lan',
                          sorted_replica_hosts)
        sorted_replica_wan_hosts = list(
            map(lambda pfn: urlparse(pfn).hostname, wan_pfns))
        assert sorted_wan_hosts == sorted_replica_wan_hosts
예제 #14
0
def test_sort_geoip_wan(vo, rest_client, auth_token, protocols_setup,
                        content_type):
    """Replicas: test sorting a few WANs via geoip."""
    n = 10
    nmap = {}

    def fake_get_distance(se1, se2, *args, **kwargs):
        nonlocal n, nmap
        n = n - 1
        print("fake_get_distance", {'se1': se1, 'se2': se2, 'n': n})
        assert se1, 'pfn host must be se1 for this test'
        nmap[se1] = n
        return n

    data = {
        'dids': [{
            'scope': f['scope'].external,
            'name': f['name'],
            'type': 'FILE'
        } for f in protocols_setup['files']],
        'schemes':
        schemes,
        'sort':
        'geoip',
    }

    with mock.patch('rucio.core.replica_sorter.__get_distance',
                    side_effect=fake_get_distance):
        response = rest_client.post('/replicas/list',
                                    headers=headers(auth(auth_token),
                                                    vohdr(vo),
                                                    accept(content_type)),
                                    json=data)
    assert response.status_code == 200

    replicas_response = response.get_data(as_text=True)
    assert replicas_response

    # because urlparse hostname result is lower case
    sorted_hosts = list(map(str.lower, sorted(nmap, key=nmap.get)))

    if content_type == Mime.METALINK:
        replicas = parse_replicas_from_string(replicas_response)
        print(replicas)
        assert len(replicas) == 1
        sources_list = replicas[0]['sources']
        print(sources_list)
        assert len(sources_list) == 6

        sorted_replica_hosts = list(
            sorted(sources_list, key=lambda source: source['priority']))
        sorted_replica_hosts = list(
            map(lambda source: urlparse(source['pfn']).hostname,
                sorted_replica_hosts))
        assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'

    elif content_type == Mime.JSON_STREAM:
        replicas = list(
            map(
                json.loads,
                filter(
                    bool,
                    map(str.strip,
                        replicas_response.splitlines(keepends=False)))))
        print(replicas)
        assert len(replicas) == 1
        sources_dict = replicas[0]['pfns']
        assert len(sources_dict) == 6

        sorted_replica_hosts = list(
            sorted(sources_dict,
                   key=lambda pfn: sources_dict[pfn]['priority']))
        sorted_replica_hosts = list(
            map(lambda source: urlparse(source).hostname,
                sorted_replica_hosts))
        assert sorted_hosts == sorted_replica_hosts, 'assert sorting of result as distance suggested'
예제 #15
0
def test_redirect_metalink_list_replicas(vo, rest_client):
    """ ROOT (REDIRECT REST): Test internal proxy prepend with metalink"""
    # default behaviour - no location -> no proxy
    response = rest_client.get('/redirect/mock/half-life_1/metalink',
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1' in body
    assert 'proxy' not in body
    response = rest_client.get('/redirect/mock/half-life_2/metalink',
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2' in body
    assert 'proxy' not in body
    response = rest_client.get('/redirect/mock/half-life_3/metalink',
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3' in body
    assert 'proxy' not in body

    # site without proxy
    response = rest_client.get('/redirect/mock/half-life_1/metalink?' +
                               urlencode(client_location_without_proxy),
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1' in body
    assert 'proxy' not in body
    response = rest_client.get('/redirect/mock/half-life_2/metalink?' +
                               urlencode(client_location_without_proxy),
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2' in body
    assert 'proxy' not in body
    response = rest_client.get('/redirect/mock/half-life_3/metalink?' +
                               urlencode(client_location_without_proxy),
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3' in body
    assert 'proxy' not in body

    # at location with outgoing proxy, prepend for wan replica
    response = rest_client.get('/redirect/mock/half-life_1/metalink?' +
                               urlencode(client_location_with_proxy),
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c9/df/half-life_1' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/c9/df/half-life_1' in body
    response = rest_client.get('/redirect/mock/half-life_2/metalink?' +
                               urlencode(client_location_with_proxy),
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/c1/8d/half-life_2' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/c1/8d/half-life_2' in body
    response = rest_client.get('/redirect/mock/half-life_3/metalink?' +
                               urlencode(client_location_with_proxy),
                               headers=headers(vohdr(vo)))
    assert response.status_code == 200
    body = response.get_data(as_text=True)
    assert 'root://proxy.aperture.com:1094//root://root.blackmesa.com:1409//training/facility/mock/16/30/half-life_3' in body
    assert 'root://root.aperture.com:1409//test/chamber/mock/16/30/half-life_3' in body
예제 #16
0
def test_sort_geoip_wan_client_location(vo, rest_client, auth_token,
                                        protocols_setup, content_type,
                                        mock_geoip_db, mock_get_lat_long):
    """Replicas: test sorting a few WANs via geoip."""

    data = {
        'dids': [{
            'scope': f['scope'].external,
            'name': f['name'],
            'type': 'FILE'
        } for f in protocols_setup['files']],
        'schemes':
        schemes,
        'sort':
        'geoip',
    }

    first_aut_then_jpn = [
        'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com',
        'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com'
    ]
    first_jpn_then_aut = [
        'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com',
        'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com'
    ]
    for client_location, expected_order in (
        ('Switzerland', first_aut_then_jpn),
        ('Romania', first_aut_then_jpn),
        ('Austria', first_aut_then_jpn),
        ('United Kingdom', first_aut_then_jpn),
        ('Libya', first_aut_then_jpn),
        ('China', first_jpn_then_aut),
        ('United States', first_jpn_then_aut),
        ('Japan', first_jpn_then_aut),
        ('Taiwan', first_jpn_then_aut),
        ('Israel', first_aut_then_jpn),
        ('Finland', first_aut_then_jpn),
        ('United Arab Emirates', first_aut_then_jpn),
    ):
        response = rest_client.post(
            '/replicas/list',
            headers=headers(
                auth(auth_token), vohdr(vo), accept(content_type),
                [('X-Forwarded-For', LOCATION_TO_IP[client_location])]),
            json=data)
        assert response.status_code == 200
        replicas_response = response.get_data(as_text=True)
        assert replicas_response

        replicas = []
        pfns = []
        if content_type == Mime.METALINK:
            replicas = parse_replicas_from_string(replicas_response)
            pfns = [s['pfn'] for s in replicas[0]['sources']]
        elif content_type == Mime.JSON_STREAM:
            replicas = list(
                map(
                    json.loads,
                    filter(
                        bool,
                        map(str.strip,
                            replicas_response.splitlines(keepends=False)))))
            pfns = list(replicas[0]['pfns'])

        print(client_location, pfns)
        assert len(replicas) == 1
        assert [urlparse(pfn).hostname for pfn in pfns] == expected_order