def test_when_the_url_returns_a_eula_error_it_returns_a_human_readable_message(
faker):
access_token = faker.password(length=42, special_chars=False)
url = 'https://example.com/file.txt'
config = config_fixture()
responses.add(responses.POST,
'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user',
status=200,
match_querystring=False)
responses.add(
responses.GET,
url,
status=403,
body=(
'{"status_code":403,"error_description":"EULA Acceptance Failure",'
'"resolution_url":"https://example.com/approve_app?client_id=foo"}'
))
with mock.patch('builtins.open', mock.mock_open()):
with pytest.raises(ForbiddenException) as e:
util.download(url, '/tmp', access_token=access_token, cfg=config)
assert e.value.message == (
f'Request could not be completed because you need to agree to the EULA '
f'at https://example.com/approve_app?client_id=foo')
assert len(responses.calls) == 2
def test_download_validates_token_once(
mocker,
faker,
validate_access_token_url,
resource_server_granule_url):
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=40, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id)
url = validate_access_token_url.format(
token=access_token,
client_id=client_id
)
responses.add(responses.POST, url, status=200)
responses.add(responses.GET, resource_server_granule_url, status=200)
responses.add(responses.GET, resource_server_granule_url, status=200)
destination_file = mocker.Mock()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert response.status_code == 200
assert responses.assert_call_count(url, 1) is True
assert responses.assert_call_count(resource_server_granule_url, 2) is True
def test_when_given_an_access_token_and_error_occurs_it_falls_back_to_basic_auth_if_enabled(
monkeypatch,
mocker,
faker,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True)
responses.add(
responses.GET,
resource_server_granule_url,
status=401
)
responses.add(
responses.GET,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 2
assert 'Authorization' in responses.calls[1].request.headers
assert 'Basic' in responses.calls[1].request.headers['Authorization']
destination_file.write.assert_called()
def test_resource_server_redirects_to_granule_url(
monkeypatch,
mocker,
access_token,
resource_server_redirect_url,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_redirect_url,
status=301,
headers=[('Location', resource_server_granule_url)]
)
responses.add(
responses.GET,
resource_server_granule_url,
status=303
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, resource_server_redirect_url, access_token, None, destination_file)
assert response.status_code == 303
assert len(responses.calls) == 2
rs_headers = responses.calls[0].request.headers
assert 'Authorization' not in rs_headers
def test_download_follows_redirect_to_resource_server_with_code(
monkeypatch,
mocker,
access_token,
edl_redirect_url,
resource_server_redirect_url):
responses.add(
responses.GET,
edl_redirect_url,
status=302,
headers=[('Location', resource_server_redirect_url)]
)
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_redirect_url,
status=302
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, edl_redirect_url, access_token, None, destination_file)
assert response.status_code == 302
assert len(responses.calls) == 2
edl_headers = responses.calls[0].request.headers
assert 'Authorization' in edl_headers
rs_headers = responses.calls[1].request.headers
assert 'Authorization' not in rs_headers
def setUp(self):
self.workdir = mkdtemp()
self.inputdir = mkdtemp()
self.catalog = Catalog('test-id', 'test catalog')
self.catalog.normalize_and_save(self.inputdir,
CatalogType.SELF_CONTAINED)
self.config = config_fixture()
print(self.config)
def test_when_given_unknown_url_it_raises_exception(faker):
access_token = faker.password(length=40, special_chars=False)
config = config_fixture()
with mock.patch('builtins.open', mock.mock_open()):
with pytest.raises(Exception):
util.download('msdos:choplifter.bas',
'tmp',
access_token=access_token,
cfg=config)
def test_when_given_an_http_url_it_downloads_the_url(monkeypatch, mocker,
faker):
access_token = faker.password(length=40, special_chars=False)
http_download = mocker.Mock()
monkeypatch.setattr(util.http, 'download', http_download)
config = config_fixture()
with mock.patch('builtins.open', mock.mock_open()):
util.download('https://example.com/file.txt',
'tmp',
access_token=access_token,
cfg=config)
http_download.assert_called()
def test_when_the_url_returns_a_403_it_throws_a_forbidden_exception(faker):
access_token = faker.password(length=41, special_chars=False)
url = 'https://example.com/file.txt'
config = config_fixture()
responses.add(responses.POST,
'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user',
status=200,
match_querystring=False)
responses.add(responses.GET, url, status=403)
with mock.patch('builtins.open', mock.mock_open()):
with pytest.raises(ForbiddenException) as e:
util.download(url, '/tmp', access_token=access_token, cfg=config)
assert e.value.message.startswith('Forbidden')
assert len(responses.calls) == 2
def test_when_given_a_file_path_it_returns_the_file_path(
monkeypatch, mocker, faker):
access_token = faker.password(length=40, special_chars=False)
http_download = mocker.Mock()
monkeypatch.setattr(util.http, 'download', http_download)
config = config_fixture()
with mock.patch('builtins.open', mock.mock_open()):
destination_path = util.download('file:///var/logs/example/file.txt',
'/put/file/here/',
access_token=access_token,
cfg=config)
assert destination_path.startswith('/var/logs/example/')
assert destination_path.endswith('.txt')
def test_when_the_url_returns_a_500_it_does_not_raise_a_forbidden_exception_and_does_not_return_details_to_user(
faker):
access_token = faker.password(length=43, special_chars=False)
url = 'https://example.com/file.txt'
config = config_fixture()
responses.add(responses.POST,
'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user',
status=200,
match_querystring=False)
responses.add(responses.GET, url, status=500)
with mock.patch('builtins.open', mock.mock_open()):
with pytest.raises(Exception) as e:
util.download(url, '/tmp', access_token=access_token, cfg=config)
assert e.type != ForbiddenException and e.type == Exception
assert len(responses.calls) == 2
def test_uploads_to_s3_and_returns_its_s3_url(self, client):
# Sets a non-test ENV environment variable to force things through the (mocked) download path
s3 = MagicMock()
s3.generate_presigned_url.return_value = 'https://example.com/presigned.txt'
client.return_value = s3
cfg = config_fixture(use_localstack=True,
staging_bucket='example',
staging_path='staging/path')
result = util.stage('file.txt', 'remote.txt', 'text/plain', cfg=cfg)
s3.upload_file.assert_called_with(
'file.txt',
'example',
'staging/path/remote.txt',
ExtraArgs={'ContentType': 'text/plain'})
self.assertEqual(result, 's3://example/staging/path/remote.txt')
def test_when_authn_succeeds_it_writes_to_provided_file(
monkeypatch,
mocker,
access_token,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 1
destination_file.write.assert_called()
def test_when_given_a_url_and_data_it_downloads_with_query_string(
monkeypatch,
mocker,
access_token,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.POST,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
cfg = config_fixture()
data = {'param': 'value'}
response = download(cfg, resource_server_granule_url, access_token, data, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 1
assert responses.calls[0].request.body == b'param=value'
def test_download_validates_token_and_raises_exception(
mocker,
faker,
validate_access_token_url):
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id)
url = validate_access_token_url.format(
token=access_token,
client_id=client_id
)
responses.add(responses.POST, url, status=403, json={
"error": "invalid_token",
"error_description": "The token is either malformed or does not exist"
})
destination_file = mocker.Mock()
with pytest.raises(Exception):
download(cfg, 'https://xyzzy.com/foo/bar', access_token, None, destination_file)
def test_when_no_access_token_is_provided_it_uses_basic_auth_and_downloads_when_enabled(
mocker,
faker,
resource_server_granule_url):
client_id = faker.password(length=22, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True)
responses.add(
responses.GET,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
response = download(cfg, resource_server_granule_url, None, None, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 1
assert 'Authorization' in responses.calls[0].request.headers
assert 'Basic' in responses.calls[0].request.headers['Authorization']
destination_file.write.assert_called()
def test_uses_location_prefix_when_provided(self, client):
# Sets a non-test ENV environment variable to force things through the (mocked) download path
s3 = MagicMock()
s3.generate_presigned_url.return_value = 'https://example.com/presigned.txt'
client.return_value = s3
cfg = config_fixture(use_localstack=True,
staging_bucket='example',
staging_path='staging/path')
result = util.stage('file.txt',
'remote.txt',
'text/plain',
location="s3://different-example/public/location/",
cfg=cfg)
s3.upload_file.assert_called_with(
'file.txt',
'different-example',
'public/location/remote.txt',
ExtraArgs={'ContentType': 'text/plain'})
self.assertEqual(result,
's3://different-example/public/location/remote.txt')
def test_download_unknown_error_exception_if_all_else_fails(
monkeypatch,
mocker,
faker,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False)
responses.add(
responses.GET,
resource_server_granule_url,
status=599
)
destination_file = mocker.Mock()
with pytest.raises(Exception):
download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert len(responses.calls) == 1
def test_when_given_an_access_token_and_error_occurs_it_does_not_fall_back_to_basic_auth(
monkeypatch,
mocker,
faker,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False)
responses.add(
responses.GET,
resource_server_granule_url,
status=401
)
destination_file = mocker.Mock()
with pytest.raises(Exception):
download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert len(responses.calls) == 1
assert 'Authorization' not in responses.calls[0].request.headers
def test_download_follows_redirect_to_edl_and_adds_auth_headers(
monkeypatch,
mocker,
access_token,
resource_server_granule_url,
edl_redirect_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_granule_url,
status=302,
headers=[('Location', edl_redirect_url)]
)
responses.add(
responses.GET,
edl_redirect_url,
status=302
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
# We should get redirected to EDL
assert response.status_code == 302
assert len(responses.calls) == 2
# We shouldn't have Auth headers in the request, but they should
# be added on the redirect to EDL
request_headers = responses.calls[0].request.headers
redirect_headers = responses.calls[1].request.headers
assert 'Authorization' not in request_headers
assert 'Authorization' in redirect_headers
assert 'Basic' in redirect_headers['Authorization']
assert 'Bearer' in redirect_headers['Authorization']
def setUp(self):
util._s3 = None
self.config = config_fixture()
