def test_generate_bucket_deletes_old_index_files(aws_creds):
simulate_s3_folder(
prefix="",
subdirectories=["folder-that-is-basically-empty/"],
files=[{"name": "foo.html"}],
)
simulate_s3_folder(
prefix="folder-that-is-basically-empty/",
subdirectories=[],
files=[{"name": "index.html"}],
mock_upload=False,
mock_delete=True,
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
assert_index_created_correctly(
items=[
{
"name": "foo.html",
"last_modified": "22-Feb-2021 10:23",
"size": "1.2 kB",
},
],
path="/",
site_name="foo-bucket",
root_index=True,
)
assert_index_deleted(path="/folder-that-is-basically-empty/")
def test_generate_bucket_dir_with_site_name(aws_creds):
simulate_s3_folder(
prefix="",
subdirectories=[],
files=[
{
"name": "a-file",
"last_modified": "22-Feb-2021 10:23"
},
],
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
assert_index_created_correctly(
items=[
{
"name": "a-file",
"last_modified": "22-Feb-2021 10:23",
"size": "1.2 kB",
},
],
path="/",
site_name="test-site-name",
root_index=True,
)
def test_generate_bucket_dir_with_excluded_objects(aws_creds):
simulate_s3_folder(
prefix="",
subdirectories=[],
files=[
{"name": "the-not-excluded-thing", "last_modified": "22-Feb-2021 10:23"},
{"name": "root-one"},
{"name": "object-two.bar"},
{"name": "favicon.ico"},
{"name": "index.html"},
],
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
assert_index_created_correctly(
items=[
{
"name": "the-not-excluded-thing",
"last_modified": "22-Feb-2021 10:23",
"size": "1.2 kB",
},
],
path="/",
site_name="foo-bucket",
root_index=True,
)
def test_generate_bucket_dir_no_creds(delete_aws_creds):
httpretty.register_uri(
httpretty.PUT,
"http://169.254.169.254/latest/api/token",
status=404,
)
httpretty.register_uri(
httpretty.GET,
"http://169.254.169.254/latest/meta-data/iam/security-credentials/",
status=404,
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 1
def test_generate_bucket_dir_does_not_list_empty_folders(aws_creds):
simulate_s3_folder(
prefix="",
subdirectories=["folder-with-only-excluded/", "folder-we-like/"],
files=[],
)
simulate_s3_folder(
prefix="folder-with-only-excluded/",
subdirectories=[],
files=[{
"name": "excluded-object.zip"
}],
mock_upload=False,
)
simulate_s3_folder(
prefix="folder-we-like/",
subdirectories=[],
files=[{
"name": "the-not-excluded-thing"
}, {
"name": "excluded-object.zip"
}],
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
assert_index_created_correctly(
items=[
{
"name": "folder-we-like/",
"last_modified": "-",
"size": "-",
},
],
path="/",
site_name="foo-bucket",
root_index=True,
)
assert_index_created_correctly(
items=[
{
"name": "the-not-excluded-thing",
"last_modified": "22-Feb-2021 10:23",
"size": "1.2 kB",
},
],
path="/folder-we-like/",
site_name="foo-bucket",
)
def test_generate_bucket_dir_multithreaded_unhandled_client_error(aws_creds, caplog):
httpretty.register_uri(
httpretty.GET,
"https://foo-bucket.s3.eu-west-1.amazonaws.com/?list-type=2&delimiter=%2F&prefix=&encoding-type=url",
status=403,
body="""<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>WeirdError</Code>
<Message>Something funky happened</Message>
<RequestId>foo-request-id</RequestId>
<HostId>foo-host-id</HostId>
</Error>""",
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 1
assert (
"An unhandled ClientError occured when interacting with AWS: 'An error occurred (WeirdError) when calling the ListObjectsV2 operation: Something funky happened'."
in caplog.messages
)
def test_generate_bucket_dir_multithreaded_no_list_permissions(aws_creds, caplog):
httpretty.register_uri(
httpretty.GET,
"https://foo-bucket.s3.eu-west-1.amazonaws.com/?list-type=2&delimiter=%2F&prefix=&encoding-type=url",
status=403,
body="""<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>Access Denied</Message>
<RequestId>foo-request-id</RequestId>
<HostId>foo-host-id</HostId>
</Error>""",
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 1
assert (
"Access denied when making a ListObjectsV2 call. Please ensure appropriate AWS permissions are set."
in caplog.messages
)
def test_generate_bucket_dir_multithreaded_no_put_permissions(
aws_creds, caplog):
# This is a workaround for https://github.com/gabrielfalcao/HTTPretty/issues/416
def put_object_failed_request_callback(request, uri, response_headers):
status = 100
body = ""
if len(request.body) > 0:
status = 403
body = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>Access Denied</Message>
<RequestId>GEW5C7J1JCY4GVPC</RequestId>
<HostId>NaxEi4Nqz+XdWRxTZf0ww+oHsNl11xyauaaJaIy6SUhhF8waL4B/5vxRDGEWwcCQPW6UIb0yuHk=</HostId>
</Error>"""
return [status, {}, body.encode()]
simulate_s3_folder(
prefix="",
subdirectories=[],
files=[
{
"name": "an-object",
"last_modified": "22-Feb-2021 10:23",
"size": "30100"
},
],
mock_upload=False,
)
httpretty.register_uri(
httpretty.PUT,
"https://foo-bucket.s3.eu-west-1.amazonaws.com/index.html",
body=put_object_failed_request_callback,
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 1
assert (
"Access denied when making a PutObject call. Please ensure appropriate AWS permissions are set."
in caplog.messages)
def test_generate_bucket_dir_big_bucket(aws_creds):
simulate_s3_big_bucket()
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
def test_generate_bucket_dir_multithreaded_smoke(aws_creds):
"""We cannot use httpretty when multithreading, see:
https://github.com/gabrielfalcao/HTTPretty/issues/186
https://github.com/gabrielfalcao/HTTPretty/issues/209
Most of the testing value is thus performed with --single-threaded.
This test ensures that mulithreading itself basically works.
"""
simulate_s3_folder(
prefix="",
subdirectories=[
"deep-folder/",
"empty-folder/",
"folder with spaces/",
"regular-folder/",
"FOLDER_With_UnUsUaL_n4m3/",
],
files=[
{"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30100"},
{"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10800"},
],
)
simulate_s3_folder(
prefix="empty-folder/",
subdirectories=[],
files=[],
)
simulate_s3_folder(
prefix="folder with spaces/", subdirectories=[], files=[{"name": "an+object+with+spaces"}]
)
simulate_s3_folder(
prefix="regular-folder/",
subdirectories=[],
files=[
{
"name": "object-one.foo",
"last_modified": "2021-02-22T10:22:36.000Z",
"size": 16524288,
},
{"name": "object-two.bar", "last_modified": "2021-02-22T10:23:11.000Z", "size": 26921},
],
)
simulate_s3_folder(
prefix="folder+with+spaces/",
subdirectories=[],
files=[
{"name": "an+object+with+spaces"},
],
)
simulate_s3_folder(
prefix="deep-folder/",
subdirectories=[
"deep-folder/i/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/",
subdirectories=[
"deep-folder/i/ii/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/ii/",
subdirectories=[
"deep-folder/i/ii/iii/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/ii/iii/",
subdirectories=[],
files=[
{"name": "index.html", "etag": "164b668c016a3b64086d3326850209b9"},
{"name": "deep-object"},
],
mock_upload=False,
)
simulate_s3_folder(
prefix="FOLDER_With_UnUsUaL_n4m3/",
subdirectories=["FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/"],
files=[
{"name": "index.html", "last_modified": "22-Feb-2021 10:23"},
],
)
simulate_s3_folder(
prefix="FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/",
subdirectories=[],
files=[
{"name": "see!", "last_modified": "22-Feb-2021 10:23"},
],
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
def test_generate_bucket_dir_with_target_path(aws_creds, mocker, target_path):
mocker.patch.object(
sys,
"argv",
[
"bucket-dir",
"foo-bucket",
"--target-path",
target_path,
"--single-threaded",
],
)
simulate_s3_folder(
prefix="",
subdirectories=[
"deep-folder/",
"empty-folder/",
],
files=[
{"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30100"},
{"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10800"},
],
)
simulate_s3_folder(
prefix="deep-folder/",
subdirectories=[
"deep-folder/i/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/",
subdirectories=[
"deep-folder/i/ii/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/ii/",
subdirectories=[
"deep-folder/i/ii/iii/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/ii/folderthatisnotthere/",
subdirectories=[],
files=[],
mock_upload=False,
)
simulate_s3_folder(
prefix="deep-folder/i/ii/iii/",
subdirectories=[],
files=[
{"name": "deep-object"},
],
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
assert_index_created_correctly(
items=[
{"name": "deep-folder/", "last_modified": "-", "size": "-"},
{"name": "empty-folder/", "last_modified": "-", "size": "-"},
{"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30.1 kB"},
{"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10.8 kB"},
],
path="/",
site_name="foo-bucket",
root_index=True,
)
assert_index_created_correctly(
items=[{"name": "i/", "last_modified": "-", "size": "-"}],
path="/deep-folder/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[{"name": "ii/", "last_modified": "-", "size": "-"}],
path="/deep-folder/i/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[{"name": "iii/", "last_modified": "-", "size": "-"}],
path="/deep-folder/i/ii/",
site_name="foo-bucket",
)
def test_generate_bucket_dir(aws_creds):
simulate_s3_folder(
prefix="",
subdirectories=[
"deep-folder/",
"empty-folder/",
"folder with spaces/",
"regular-folder/",
"FOLDER_With_UnUsUaL_n4m3/",
],
files=[
{"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30100"},
{"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10800"},
],
)
simulate_s3_folder(
prefix="empty-folder/",
subdirectories=[],
files=[],
mock_upload=False,
)
simulate_s3_folder(
prefix="folder with spaces/", subdirectories=[], files=[{"name": "an+object+with+spaces"}]
)
simulate_s3_folder(
prefix="regular-folder/",
subdirectories=[],
files=[
{
"name": "object-one.foo",
"last_modified": "2021-02-22T10:22:36.000Z",
"size": 16524288,
},
{"name": "object-two.bar", "last_modified": "2021-02-22T10:23:11.000Z", "size": 26921},
],
)
simulate_s3_folder(
prefix="deep-folder/",
subdirectories=[
"deep-folder/i/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/",
subdirectories=[
"deep-folder/i/ii/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/ii/",
subdirectories=[
"deep-folder/i/ii/iii/",
],
files=[],
)
simulate_s3_folder(
prefix="deep-folder/i/ii/iii/",
subdirectories=[],
files=[
{"name": "index.html", "etag": "164b668c016a3b64086d3326850209b9"},
{"name": "deep-object"},
],
mock_upload=False,
)
simulate_s3_folder(
prefix="FOLDER_With_UnUsUaL_n4m3/",
subdirectories=["FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/"],
files=[
{"name": "index.html", "last_modified": "22-Feb-2021 10:23"},
],
)
simulate_s3_folder(
prefix="FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/",
subdirectories=[],
files=[
{"name": "see!", "last_modified": "22-Feb-2021 10:23"},
],
)
with pytest.raises(SystemExit) as system_exit:
bucket_dir.run_cli()
assert system_exit.value.code == 0
assert_index_created_correctly(
items=[
{"name": "deep-folder/", "last_modified": "-", "size": "-"},
{
"name": "folder with spaces/",
"last_modified": "-",
"size": "-",
"encoded_name": "folder%20with%20spaces/",
},
{"name": "FOLDER_With_UnUsUaL_n4m3/", "last_modified": "-", "size": "-"},
{"name": "regular-folder/", "last_modified": "-", "size": "-"},
{"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30.1 kB"},
{"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10.8 kB"},
],
path="/",
site_name="foo-bucket",
root_index=True,
)
assert_index_created_correctly(
items=[
{"name": "object-one.foo", "last_modified": "22-Feb-2021 10:22", "size": "16.5 MB"},
{"name": "object-two.bar", "last_modified": "22-Feb-2021 10:23", "size": "26.9 kB"},
],
path="/regular-folder/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[{"name": "i/", "last_modified": "-", "size": "-"}],
path="/deep-folder/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[{"name": "ii/", "last_modified": "-", "size": "-"}],
path="/deep-folder/i/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[{"name": "iii/", "last_modified": "-", "size": "-"}],
path="/deep-folder/i/ii/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[
{
"name": "an object with spaces",
"last_modified": "22-Feb-2021 10:23",
"size": "1.2 kB",
"encoded_name": "an%20object%20with%20spaces",
},
],
path="/folder%20with%20spaces/",
title="/folder with spaces/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[
{
"name": "it\\'gets*even.(weirder)/",
"last_modified": "-",
"size": "-",
"encoded_name": "it%5C%27gets%2Aeven.%28weirder%29/",
},
],
path="/FOLDER_With_UnUsUaL_n4m3/",
site_name="foo-bucket",
)
assert_index_created_correctly(
items=[
{
"name": "see!",
"last_modified": "22-Feb-2021 10:23",
"size": "1.2 kB",
"encoded_name": "see%21",
},
],
path="/FOLDER_With_UnUsUaL_n4m3/it%5C%27gets%2Aeven.%28weirder%29/",
title="/FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/",
site_name="foo-bucket",
)
