def test_download_validates_token_once(
mocker,
faker,
validate_access_token_url,
resource_server_granule_url):
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=40, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id)
url = validate_access_token_url.format(
token=access_token,
client_id=client_id
)
responses.add(responses.POST, url, status=200)
responses.add(responses.GET, resource_server_granule_url, status=200)
responses.add(responses.GET, resource_server_granule_url, status=200)
destination_file = mocker.Mock()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert response.status_code == 200
assert responses.assert_call_count(url, 1) is True
assert responses.assert_call_count(resource_server_granule_url, 2) is True
def test_when_given_an_access_token_and_error_occurs_it_falls_back_to_basic_auth_if_enabled(
monkeypatch,
mocker,
faker,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True)
responses.add(
responses.GET,
resource_server_granule_url,
status=401
)
responses.add(
responses.GET,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 2
assert 'Authorization' in responses.calls[1].request.headers
assert 'Basic' in responses.calls[1].request.headers['Authorization']
destination_file.write.assert_called()
def test_resource_server_redirects_to_granule_url(
monkeypatch,
mocker,
access_token,
resource_server_redirect_url,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_redirect_url,
status=301,
headers=[('Location', resource_server_granule_url)]
)
responses.add(
responses.GET,
resource_server_granule_url,
status=303
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, resource_server_redirect_url, access_token, None, destination_file)
assert response.status_code == 303
assert len(responses.calls) == 2
rs_headers = responses.calls[0].request.headers
assert 'Authorization' not in rs_headers
def test_download_follows_redirect_to_resource_server_with_code(
monkeypatch,
mocker,
access_token,
edl_redirect_url,
resource_server_redirect_url):
responses.add(
responses.GET,
edl_redirect_url,
status=302,
headers=[('Location', resource_server_redirect_url)]
)
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_redirect_url,
status=302
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, edl_redirect_url, access_token, None, destination_file)
assert response.status_code == 302
assert len(responses.calls) == 2
edl_headers = responses.calls[0].request.headers
assert 'Authorization' in edl_headers
rs_headers = responses.calls[1].request.headers
assert 'Authorization' not in rs_headers
def test_download_validates_token_and_raises_exception(
mocker,
faker,
validate_access_token_url):
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id)
url = validate_access_token_url.format(
token=access_token,
client_id=client_id
)
responses.add(responses.POST, url, status=403, json={
"error": "invalid_token",
"error_description": "The token is either malformed or does not exist"
})
destination_file = mocker.Mock()
with pytest.raises(Exception):
download(cfg, 'https://xyzzy.com/foo/bar', access_token, None, destination_file)
def test_download_unknown_error_exception_if_all_else_fails(
monkeypatch,
mocker,
faker,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False)
responses.add(
responses.GET,
resource_server_granule_url,
status=599
)
destination_file = mocker.Mock()
with pytest.raises(Exception):
download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert len(responses.calls) == 1
def test_when_given_an_access_token_and_error_occurs_it_does_not_fall_back_to_basic_auth(
monkeypatch,
mocker,
faker,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
client_id = faker.password(length=22, special_chars=False)
access_token = faker.password(length=42, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False)
responses.add(
responses.GET,
resource_server_granule_url,
status=401
)
destination_file = mocker.Mock()
with pytest.raises(Exception):
download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert len(responses.calls) == 1
assert 'Authorization' not in responses.calls[0].request.headers
def test_when_authn_succeeds_it_writes_to_provided_file(
monkeypatch,
mocker,
access_token,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 1
destination_file.write.assert_called()
def test_when_given_a_url_and_data_it_downloads_with_query_string(
monkeypatch,
mocker,
access_token,
resource_server_granule_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.POST,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
cfg = config_fixture()
data = {'param': 'value'}
response = download(cfg, resource_server_granule_url, access_token, data, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 1
assert responses.calls[0].request.body == b'param=value'
def test_when_no_access_token_is_provided_it_uses_basic_auth_and_downloads_when_enabled(
mocker,
faker,
resource_server_granule_url):
client_id = faker.password(length=22, special_chars=False)
cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True)
responses.add(
responses.GET,
resource_server_granule_url,
status=200
)
destination_file = mocker.Mock()
response = download(cfg, resource_server_granule_url, None, None, destination_file)
assert response.status_code == 200
assert len(responses.calls) == 1
assert 'Authorization' in responses.calls[0].request.headers
assert 'Basic' in responses.calls[0].request.headers['Authorization']
destination_file.write.assert_called()
def test_download_follows_redirect_to_edl_and_adds_auth_headers(
monkeypatch,
mocker,
access_token,
resource_server_granule_url,
edl_redirect_url):
monkeypatch.setattr(harmony.http, '_valid', lambda a, b, c: True)
responses.add(
responses.GET,
resource_server_granule_url,
status=302,
headers=[('Location', edl_redirect_url)]
)
responses.add(
responses.GET,
edl_redirect_url,
status=302
)
destination_file = mocker.Mock()
cfg = config_fixture()
response = download(cfg, resource_server_granule_url, access_token, None, destination_file)
# We should get redirected to EDL
assert response.status_code == 302
assert len(responses.calls) == 2
# We shouldn't have Auth headers in the request, but they should
# be added on the redirect to EDL
request_headers = responses.calls[0].request.headers
redirect_headers = responses.calls[1].request.headers
assert 'Authorization' not in request_headers
assert 'Authorization' in redirect_headers
assert 'Basic' in redirect_headers['Authorization']
assert 'Bearer' in redirect_headers['Authorization']
def download(url,
destination_dir,
logger=None,
access_token=None,
data=None,
cfg=None):
"""
Downloads the given URL to the given destination directory, using the basename of the URL
as the filename in the destination directory. Supports http://, https:// and s3:// schemes.
When using the s3:// scheme, will run against us-west-2 unless the "AWS_DEFAULT_REGION"
environment variable is set.
When using http:// or https:// schemes, the access_token will be used for authentication.
Parameters
----------
url : string
The URL to fetch
destination_dir : string
The directory in which to place the downloaded file
logger : Logger
A logger to which the function will write, if provided
access_token :
The Earthdata Login token of the caller to use for downloads
data : dict or Tuple[str, str]
Optional parameter for additional data to
send to the server when making a HTTP POST request through
urllib.get.urlopen. These data will be URL encoded to a query string
containing a series of `key=value` pairs, separated by ampersands. If
None (the default), urllib.get.urlopen will use the GET
method.
cfg : harmony.util.Config
The configuration values for this runtime environment.
Returns
-------
destination : string
The filename, including directory, of the downloaded file
"""
if cfg is None:
cfg = config()
if logger is None:
logger = build_logger(cfg)
if _is_file_url(url):
return _url_as_filename(url)
source = http.localhost_url(url, cfg.localstack_host)
destination_path = _filename(destination_dir, url)
if destination_path.exists():
return str(destination_path)
destination_path = str(destination_path)
with open(destination_path, 'wb') as destination_file:
if aws.is_s3(source):
aws.download(cfg, source, destination_file)
elif http.is_http(source):
http.download(cfg, source, access_token, data, destination_file)
else:
msg = f'Unable to download a url of unknown type: {url}'
logger.error(msg)
raise Exception(msg)
return destination_path
