def test_generate_bucket_deletes_old_index_files(aws_creds): simulate_s3_folder( prefix="", subdirectories=["folder-that-is-basically-empty/"], files=[{"name": "foo.html"}], ) simulate_s3_folder( prefix="folder-that-is-basically-empty/", subdirectories=[], files=[{"name": "index.html"}], mock_upload=False, mock_delete=True, ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0 assert_index_created_correctly( items=[ { "name": "foo.html", "last_modified": "22-Feb-2021 10:23", "size": "1.2 kB", }, ], path="/", site_name="foo-bucket", root_index=True, ) assert_index_deleted(path="/folder-that-is-basically-empty/")
def test_generate_bucket_dir_with_site_name(aws_creds): simulate_s3_folder( prefix="", subdirectories=[], files=[ { "name": "a-file", "last_modified": "22-Feb-2021 10:23" }, ], ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0 assert_index_created_correctly( items=[ { "name": "a-file", "last_modified": "22-Feb-2021 10:23", "size": "1.2 kB", }, ], path="/", site_name="test-site-name", root_index=True, )
def test_generate_bucket_dir_with_excluded_objects(aws_creds): simulate_s3_folder( prefix="", subdirectories=[], files=[ {"name": "the-not-excluded-thing", "last_modified": "22-Feb-2021 10:23"}, {"name": "root-one"}, {"name": "object-two.bar"}, {"name": "favicon.ico"}, {"name": "index.html"}, ], ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0 assert_index_created_correctly( items=[ { "name": "the-not-excluded-thing", "last_modified": "22-Feb-2021 10:23", "size": "1.2 kB", }, ], path="/", site_name="foo-bucket", root_index=True, )
def test_generate_bucket_dir_no_creds(delete_aws_creds): httpretty.register_uri( httpretty.PUT, "http://169.254.169.254/latest/api/token", status=404, ) httpretty.register_uri( httpretty.GET, "http://169.254.169.254/latest/meta-data/iam/security-credentials/", status=404, ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 1
def test_generate_bucket_dir_does_not_list_empty_folders(aws_creds): simulate_s3_folder( prefix="", subdirectories=["folder-with-only-excluded/", "folder-we-like/"], files=[], ) simulate_s3_folder( prefix="folder-with-only-excluded/", subdirectories=[], files=[{ "name": "excluded-object.zip" }], mock_upload=False, ) simulate_s3_folder( prefix="folder-we-like/", subdirectories=[], files=[{ "name": "the-not-excluded-thing" }, { "name": "excluded-object.zip" }], ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0 assert_index_created_correctly( items=[ { "name": "folder-we-like/", "last_modified": "-", "size": "-", }, ], path="/", site_name="foo-bucket", root_index=True, ) assert_index_created_correctly( items=[ { "name": "the-not-excluded-thing", "last_modified": "22-Feb-2021 10:23", "size": "1.2 kB", }, ], path="/folder-we-like/", site_name="foo-bucket", )
def test_generate_bucket_dir_multithreaded_unhandled_client_error(aws_creds, caplog): httpretty.register_uri( httpretty.GET, "https://foo-bucket.s3.eu-west-1.amazonaws.com/?list-type=2&delimiter=%2F&prefix=&encoding-type=url", status=403, body="""<?xml version="1.0" encoding="UTF-8"?> <Error> <Code>WeirdError</Code> <Message>Something funky happened</Message> <RequestId>foo-request-id</RequestId> <HostId>foo-host-id</HostId> </Error>""", ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 1 assert ( "An unhandled ClientError occured when interacting with AWS: 'An error occurred (WeirdError) when calling the ListObjectsV2 operation: Something funky happened'." in caplog.messages )
def test_generate_bucket_dir_multithreaded_no_list_permissions(aws_creds, caplog): httpretty.register_uri( httpretty.GET, "https://foo-bucket.s3.eu-west-1.amazonaws.com/?list-type=2&delimiter=%2F&prefix=&encoding-type=url", status=403, body="""<?xml version="1.0" encoding="UTF-8"?> <Error> <Code>AccessDenied</Code> <Message>Access Denied</Message> <RequestId>foo-request-id</RequestId> <HostId>foo-host-id</HostId> </Error>""", ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 1 assert ( "Access denied when making a ListObjectsV2 call. Please ensure appropriate AWS permissions are set." in caplog.messages )
def test_generate_bucket_dir_multithreaded_no_put_permissions( aws_creds, caplog): # This is a workaround for https://github.com/gabrielfalcao/HTTPretty/issues/416 def put_object_failed_request_callback(request, uri, response_headers): status = 100 body = "" if len(request.body) > 0: status = 403 body = """<?xml version="1.0" encoding="UTF-8"?> <Error> <Code>AccessDenied</Code> <Message>Access Denied</Message> <RequestId>GEW5C7J1JCY4GVPC</RequestId> <HostId>NaxEi4Nqz+XdWRxTZf0ww+oHsNl11xyauaaJaIy6SUhhF8waL4B/5vxRDGEWwcCQPW6UIb0yuHk=</HostId> </Error>""" return [status, {}, body.encode()] simulate_s3_folder( prefix="", subdirectories=[], files=[ { "name": "an-object", "last_modified": "22-Feb-2021 10:23", "size": "30100" }, ], mock_upload=False, ) httpretty.register_uri( httpretty.PUT, "https://foo-bucket.s3.eu-west-1.amazonaws.com/index.html", body=put_object_failed_request_callback, ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 1 assert ( "Access denied when making a PutObject call. Please ensure appropriate AWS permissions are set." in caplog.messages)
def test_generate_bucket_dir_big_bucket(aws_creds): simulate_s3_big_bucket() with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0
def test_generate_bucket_dir_multithreaded_smoke(aws_creds): """We cannot use httpretty when multithreading, see: https://github.com/gabrielfalcao/HTTPretty/issues/186 https://github.com/gabrielfalcao/HTTPretty/issues/209 Most of the testing value is thus performed with --single-threaded. This test ensures that mulithreading itself basically works. """ simulate_s3_folder( prefix="", subdirectories=[ "deep-folder/", "empty-folder/", "folder with spaces/", "regular-folder/", "FOLDER_With_UnUsUaL_n4m3/", ], files=[ {"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30100"}, {"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10800"}, ], ) simulate_s3_folder( prefix="empty-folder/", subdirectories=[], files=[], ) simulate_s3_folder( prefix="folder with spaces/", subdirectories=[], files=[{"name": "an+object+with+spaces"}] ) simulate_s3_folder( prefix="regular-folder/", subdirectories=[], files=[ { "name": "object-one.foo", "last_modified": "2021-02-22T10:22:36.000Z", "size": 16524288, }, {"name": "object-two.bar", "last_modified": "2021-02-22T10:23:11.000Z", "size": 26921}, ], ) simulate_s3_folder( prefix="folder+with+spaces/", subdirectories=[], files=[ {"name": "an+object+with+spaces"}, ], ) simulate_s3_folder( prefix="deep-folder/", subdirectories=[ "deep-folder/i/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/", subdirectories=[ "deep-folder/i/ii/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/ii/", subdirectories=[ "deep-folder/i/ii/iii/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/ii/iii/", subdirectories=[], files=[ {"name": "index.html", "etag": "164b668c016a3b64086d3326850209b9"}, {"name": "deep-object"}, ], mock_upload=False, ) simulate_s3_folder( prefix="FOLDER_With_UnUsUaL_n4m3/", subdirectories=["FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/"], files=[ {"name": "index.html", "last_modified": "22-Feb-2021 10:23"}, ], ) simulate_s3_folder( prefix="FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/", subdirectories=[], files=[ {"name": "see!", "last_modified": "22-Feb-2021 10:23"}, ], ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0
def test_generate_bucket_dir_with_target_path(aws_creds, mocker, target_path): mocker.patch.object( sys, "argv", [ "bucket-dir", "foo-bucket", "--target-path", target_path, "--single-threaded", ], ) simulate_s3_folder( prefix="", subdirectories=[ "deep-folder/", "empty-folder/", ], files=[ {"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30100"}, {"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10800"}, ], ) simulate_s3_folder( prefix="deep-folder/", subdirectories=[ "deep-folder/i/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/", subdirectories=[ "deep-folder/i/ii/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/ii/", subdirectories=[ "deep-folder/i/ii/iii/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/ii/folderthatisnotthere/", subdirectories=[], files=[], mock_upload=False, ) simulate_s3_folder( prefix="deep-folder/i/ii/iii/", subdirectories=[], files=[ {"name": "deep-object"}, ], ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0 assert_index_created_correctly( items=[ {"name": "deep-folder/", "last_modified": "-", "size": "-"}, {"name": "empty-folder/", "last_modified": "-", "size": "-"}, {"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30.1 kB"}, {"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10.8 kB"}, ], path="/", site_name="foo-bucket", root_index=True, ) assert_index_created_correctly( items=[{"name": "i/", "last_modified": "-", "size": "-"}], path="/deep-folder/", site_name="foo-bucket", ) assert_index_created_correctly( items=[{"name": "ii/", "last_modified": "-", "size": "-"}], path="/deep-folder/i/", site_name="foo-bucket", ) assert_index_created_correctly( items=[{"name": "iii/", "last_modified": "-", "size": "-"}], path="/deep-folder/i/ii/", site_name="foo-bucket", )
def test_generate_bucket_dir(aws_creds): simulate_s3_folder( prefix="", subdirectories=[ "deep-folder/", "empty-folder/", "folder with spaces/", "regular-folder/", "FOLDER_With_UnUsUaL_n4m3/", ], files=[ {"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30100"}, {"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10800"}, ], ) simulate_s3_folder( prefix="empty-folder/", subdirectories=[], files=[], mock_upload=False, ) simulate_s3_folder( prefix="folder with spaces/", subdirectories=[], files=[{"name": "an+object+with+spaces"}] ) simulate_s3_folder( prefix="regular-folder/", subdirectories=[], files=[ { "name": "object-one.foo", "last_modified": "2021-02-22T10:22:36.000Z", "size": 16524288, }, {"name": "object-two.bar", "last_modified": "2021-02-22T10:23:11.000Z", "size": 26921}, ], ) simulate_s3_folder( prefix="deep-folder/", subdirectories=[ "deep-folder/i/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/", subdirectories=[ "deep-folder/i/ii/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/ii/", subdirectories=[ "deep-folder/i/ii/iii/", ], files=[], ) simulate_s3_folder( prefix="deep-folder/i/ii/iii/", subdirectories=[], files=[ {"name": "index.html", "etag": "164b668c016a3b64086d3326850209b9"}, {"name": "deep-object"}, ], mock_upload=False, ) simulate_s3_folder( prefix="FOLDER_With_UnUsUaL_n4m3/", subdirectories=["FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/"], files=[ {"name": "index.html", "last_modified": "22-Feb-2021 10:23"}, ], ) simulate_s3_folder( prefix="FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/", subdirectories=[], files=[ {"name": "see!", "last_modified": "22-Feb-2021 10:23"}, ], ) with pytest.raises(SystemExit) as system_exit: bucket_dir.run_cli() assert system_exit.value.code == 0 assert_index_created_correctly( items=[ {"name": "deep-folder/", "last_modified": "-", "size": "-"}, { "name": "folder with spaces/", "last_modified": "-", "size": "-", "encoded_name": "folder%20with%20spaces/", }, {"name": "FOLDER_With_UnUsUaL_n4m3/", "last_modified": "-", "size": "-"}, {"name": "regular-folder/", "last_modified": "-", "size": "-"}, {"name": "root-one", "last_modified": "22-Feb-2021 10:23", "size": "30.1 kB"}, {"name": "root-two", "last_modified": "22-Feb-2021 10:24", "size": "10.8 kB"}, ], path="/", site_name="foo-bucket", root_index=True, ) assert_index_created_correctly( items=[ {"name": "object-one.foo", "last_modified": "22-Feb-2021 10:22", "size": "16.5 MB"}, {"name": "object-two.bar", "last_modified": "22-Feb-2021 10:23", "size": "26.9 kB"}, ], path="/regular-folder/", site_name="foo-bucket", ) assert_index_created_correctly( items=[{"name": "i/", "last_modified": "-", "size": "-"}], path="/deep-folder/", site_name="foo-bucket", ) assert_index_created_correctly( items=[{"name": "ii/", "last_modified": "-", "size": "-"}], path="/deep-folder/i/", site_name="foo-bucket", ) assert_index_created_correctly( items=[{"name": "iii/", "last_modified": "-", "size": "-"}], path="/deep-folder/i/ii/", site_name="foo-bucket", ) assert_index_created_correctly( items=[ { "name": "an object with spaces", "last_modified": "22-Feb-2021 10:23", "size": "1.2 kB", "encoded_name": "an%20object%20with%20spaces", }, ], path="/folder%20with%20spaces/", title="/folder with spaces/", site_name="foo-bucket", ) assert_index_created_correctly( items=[ { "name": "it\\'gets*even.(weirder)/", "last_modified": "-", "size": "-", "encoded_name": "it%5C%27gets%2Aeven.%28weirder%29/", }, ], path="/FOLDER_With_UnUsUaL_n4m3/", site_name="foo-bucket", ) assert_index_created_correctly( items=[ { "name": "see!", "last_modified": "22-Feb-2021 10:23", "size": "1.2 kB", "encoded_name": "see%21", }, ], path="/FOLDER_With_UnUsUaL_n4m3/it%5C%27gets%2Aeven.%28weirder%29/", title="/FOLDER_With_UnUsUaL_n4m3/it\\'gets*even.(weirder)/", site_name="foo-bucket", )