def test_when_the_url_returns_a_eula_error_it_returns_a_human_readable_message( faker): access_token = faker.password(length=42, special_chars=False) url = 'https://example.com/file.txt' config = config_fixture() responses.add(responses.POST, 'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user', status=200, match_querystring=False) responses.add( responses.GET, url, status=403, body=( '{"status_code":403,"error_description":"EULA Acceptance Failure",' '"resolution_url":"https://example.com/approve_app?client_id=foo"}' )) with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(ForbiddenException) as e: util.download(url, '/tmp', access_token=access_token, cfg=config) assert e.value.message == ( f'Request could not be completed because you need to agree to the EULA ' f'at https://example.com/approve_app?client_id=foo') assert len(responses.calls) == 2
def test_download_validates_token_once( mocker, faker, validate_access_token_url, resource_server_granule_url): client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=40, special_chars=False) cfg = config_fixture(oauth_client_id=client_id) url = validate_access_token_url.format( token=access_token, client_id=client_id ) responses.add(responses.POST, url, status=200) responses.add(responses.GET, resource_server_granule_url, status=200) responses.add(responses.GET, resource_server_granule_url, status=200) destination_file = mocker.Mock() response = download(cfg, resource_server_granule_url, access_token, None, destination_file) response = download(cfg, resource_server_granule_url, access_token, None, destination_file) assert response.status_code == 200 assert responses.assert_call_count(url, 1) is True assert responses.assert_call_count(resource_server_granule_url, 2) is True
def test_when_given_an_access_token_and_error_occurs_it_falls_back_to_basic_auth_if_enabled( monkeypatch, mocker, faker, resource_server_granule_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=42, special_chars=False) cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True) responses.add( responses.GET, resource_server_granule_url, status=401 ) responses.add( responses.GET, resource_server_granule_url, status=200 ) destination_file = mocker.Mock() response = download(cfg, resource_server_granule_url, access_token, None, destination_file) assert response.status_code == 200 assert len(responses.calls) == 2 assert 'Authorization' in responses.calls[1].request.headers assert 'Basic' in responses.calls[1].request.headers['Authorization'] destination_file.write.assert_called()
def test_resource_server_redirects_to_granule_url( monkeypatch, mocker, access_token, resource_server_redirect_url, resource_server_granule_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) responses.add( responses.GET, resource_server_redirect_url, status=301, headers=[('Location', resource_server_granule_url)] ) responses.add( responses.GET, resource_server_granule_url, status=303 ) destination_file = mocker.Mock() cfg = config_fixture() response = download(cfg, resource_server_redirect_url, access_token, None, destination_file) assert response.status_code == 303 assert len(responses.calls) == 2 rs_headers = responses.calls[0].request.headers assert 'Authorization' not in rs_headers
def test_download_follows_redirect_to_resource_server_with_code( monkeypatch, mocker, access_token, edl_redirect_url, resource_server_redirect_url): responses.add( responses.GET, edl_redirect_url, status=302, headers=[('Location', resource_server_redirect_url)] ) monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) responses.add( responses.GET, resource_server_redirect_url, status=302 ) destination_file = mocker.Mock() cfg = config_fixture() response = download(cfg, edl_redirect_url, access_token, None, destination_file) assert response.status_code == 302 assert len(responses.calls) == 2 edl_headers = responses.calls[0].request.headers assert 'Authorization' in edl_headers rs_headers = responses.calls[1].request.headers assert 'Authorization' not in rs_headers
def setUp(self): self.workdir = mkdtemp() self.inputdir = mkdtemp() self.catalog = Catalog('test-id', 'test catalog') self.catalog.normalize_and_save(self.inputdir, CatalogType.SELF_CONTAINED) self.config = config_fixture() print(self.config)
def test_when_given_unknown_url_it_raises_exception(faker): access_token = faker.password(length=40, special_chars=False) config = config_fixture() with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(Exception): util.download('msdos:choplifter.bas', 'tmp', access_token=access_token, cfg=config)
def test_when_given_an_http_url_it_downloads_the_url(monkeypatch, mocker, faker): access_token = faker.password(length=40, special_chars=False) http_download = mocker.Mock() monkeypatch.setattr(util.http, 'download', http_download) config = config_fixture() with mock.patch('builtins.open', mock.mock_open()): util.download('https://example.com/file.txt', 'tmp', access_token=access_token, cfg=config) http_download.assert_called()
def test_when_the_url_returns_a_403_it_throws_a_forbidden_exception(faker): access_token = faker.password(length=41, special_chars=False) url = 'https://example.com/file.txt' config = config_fixture() responses.add(responses.POST, 'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user', status=200, match_querystring=False) responses.add(responses.GET, url, status=403) with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(ForbiddenException) as e: util.download(url, '/tmp', access_token=access_token, cfg=config) assert e.value.message.startswith('Forbidden') assert len(responses.calls) == 2
def test_when_given_a_file_path_it_returns_the_file_path( monkeypatch, mocker, faker): access_token = faker.password(length=40, special_chars=False) http_download = mocker.Mock() monkeypatch.setattr(util.http, 'download', http_download) config = config_fixture() with mock.patch('builtins.open', mock.mock_open()): destination_path = util.download('file:///var/logs/example/file.txt', '/put/file/here/', access_token=access_token, cfg=config) assert destination_path.startswith('/var/logs/example/') assert destination_path.endswith('.txt')
def test_when_the_url_returns_a_500_it_does_not_raise_a_forbidden_exception_and_does_not_return_details_to_user( faker): access_token = faker.password(length=43, special_chars=False) url = 'https://example.com/file.txt' config = config_fixture() responses.add(responses.POST, 'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user', status=200, match_querystring=False) responses.add(responses.GET, url, status=500) with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(Exception) as e: util.download(url, '/tmp', access_token=access_token, cfg=config) assert e.type != ForbiddenException and e.type == Exception assert len(responses.calls) == 2
def test_uploads_to_s3_and_returns_its_s3_url(self, client): # Sets a non-test ENV environment variable to force things through the (mocked) download path s3 = MagicMock() s3.generate_presigned_url.return_value = 'https://example.com/presigned.txt' client.return_value = s3 cfg = config_fixture(use_localstack=True, staging_bucket='example', staging_path='staging/path') result = util.stage('file.txt', 'remote.txt', 'text/plain', cfg=cfg) s3.upload_file.assert_called_with( 'file.txt', 'example', 'staging/path/remote.txt', ExtraArgs={'ContentType': 'text/plain'}) self.assertEqual(result, 's3://example/staging/path/remote.txt')
def test_when_authn_succeeds_it_writes_to_provided_file( monkeypatch, mocker, access_token, resource_server_granule_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) responses.add( responses.GET, resource_server_granule_url, status=200 ) destination_file = mocker.Mock() cfg = config_fixture() response = download(cfg, resource_server_granule_url, access_token, None, destination_file) assert response.status_code == 200 assert len(responses.calls) == 1 destination_file.write.assert_called()
def test_when_given_a_url_and_data_it_downloads_with_query_string( monkeypatch, mocker, access_token, resource_server_granule_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) responses.add( responses.POST, resource_server_granule_url, status=200 ) destination_file = mocker.Mock() cfg = config_fixture() data = {'param': 'value'} response = download(cfg, resource_server_granule_url, access_token, data, destination_file) assert response.status_code == 200 assert len(responses.calls) == 1 assert responses.calls[0].request.body == b'param=value'
def test_download_validates_token_and_raises_exception( mocker, faker, validate_access_token_url): client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=42, special_chars=False) cfg = config_fixture(oauth_client_id=client_id) url = validate_access_token_url.format( token=access_token, client_id=client_id ) responses.add(responses.POST, url, status=403, json={ "error": "invalid_token", "error_description": "The token is either malformed or does not exist" }) destination_file = mocker.Mock() with pytest.raises(Exception): download(cfg, 'https://xyzzy.com/foo/bar', access_token, None, destination_file)
def test_when_no_access_token_is_provided_it_uses_basic_auth_and_downloads_when_enabled( mocker, faker, resource_server_granule_url): client_id = faker.password(length=22, special_chars=False) cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True) responses.add( responses.GET, resource_server_granule_url, status=200 ) destination_file = mocker.Mock() response = download(cfg, resource_server_granule_url, None, None, destination_file) assert response.status_code == 200 assert len(responses.calls) == 1 assert 'Authorization' in responses.calls[0].request.headers assert 'Basic' in responses.calls[0].request.headers['Authorization'] destination_file.write.assert_called()
def test_uses_location_prefix_when_provided(self, client): # Sets a non-test ENV environment variable to force things through the (mocked) download path s3 = MagicMock() s3.generate_presigned_url.return_value = 'https://example.com/presigned.txt' client.return_value = s3 cfg = config_fixture(use_localstack=True, staging_bucket='example', staging_path='staging/path') result = util.stage('file.txt', 'remote.txt', 'text/plain', location="s3://different-example/public/location/", cfg=cfg) s3.upload_file.assert_called_with( 'file.txt', 'different-example', 'public/location/remote.txt', ExtraArgs={'ContentType': 'text/plain'}) self.assertEqual(result, 's3://different-example/public/location/remote.txt')
def test_download_unknown_error_exception_if_all_else_fails( monkeypatch, mocker, faker, resource_server_granule_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=42, special_chars=False) cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False) responses.add( responses.GET, resource_server_granule_url, status=599 ) destination_file = mocker.Mock() with pytest.raises(Exception): download(cfg, resource_server_granule_url, access_token, None, destination_file) assert len(responses.calls) == 1
def test_when_given_an_access_token_and_error_occurs_it_does_not_fall_back_to_basic_auth( monkeypatch, mocker, faker, resource_server_granule_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=42, special_chars=False) cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False) responses.add( responses.GET, resource_server_granule_url, status=401 ) destination_file = mocker.Mock() with pytest.raises(Exception): download(cfg, resource_server_granule_url, access_token, None, destination_file) assert len(responses.calls) == 1 assert 'Authorization' not in responses.calls[0].request.headers
def test_download_follows_redirect_to_edl_and_adds_auth_headers( monkeypatch, mocker, access_token, resource_server_granule_url, edl_redirect_url): monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True) responses.add( responses.GET, resource_server_granule_url, status=302, headers=[('Location', edl_redirect_url)] ) responses.add( responses.GET, edl_redirect_url, status=302 ) destination_file = mocker.Mock() cfg = config_fixture() response = download(cfg, resource_server_granule_url, access_token, None, destination_file) # We should get redirected to EDL assert response.status_code == 302 assert len(responses.calls) == 2 # We shouldn't have Auth headers in the request, but they should # be added on the redirect to EDL request_headers = responses.calls[0].request.headers redirect_headers = responses.calls[1].request.headers assert 'Authorization' not in request_headers assert 'Authorization' in redirect_headers assert 'Basic' in redirect_headers['Authorization'] assert 'Bearer' in redirect_headers['Authorization']
def setUp(self): util._s3 = None self.config = config_fixture()