def test_pass_tmp_dir(
mock_TemporaryFile,
mock_calculate_md5_and_etag,
mock_update_s3_hash_tagging,
mock_upload_to_s3,
mock_check_s3_for_md5,
mock_check_s3_for_etag,
mock_temp_file,
mock_successful_download,
):
transfer_to_s3(mock_successful_download, SRC, DST, temp_dir=TEMP_DIR)
mock_TemporaryFile.assert_called_with("r+b", dir=TEMP_DIR)
def test_success_with_wrong_md5(
mock_calculate_md5_and_etag,
mock_update_s3_hash_tagging,
mock_upload_to_s3,
mock_check_s3_for_md5,
mock_check_s3_for_etag,
mock_temp_file,
mock_successful_download,
):
with pytest.raises(InvalidDigestError):
transfer_to_s3(mock_successful_download, SRC, DST, WRONG_MD5)
mock_upload_to_s3.assert_not_called()
mock_update_s3_hash_tagging.assert_not_called()
def test_success_with_md5(
mock_calculate_md5_and_etag,
mock_update_s3_hash_tagging,
mock_upload_to_s3,
mock_check_s3_for_md5,
mock_check_s3_for_etag,
mock_temp_file,
mock_successful_download,
):
transfer_to_s3(mock_successful_download, SRC, DST, MD5)
mock_check_s3_for_md5.assert_called_with(S3DST, MD5)
mock_upload_to_s3.assert_called_with(mock_temp_file, S3DST)
mock_calculate_md5_and_etag.assert_called_with(mock_temp_file)
mock_update_s3_hash_tagging.assert_called_with(S3DST, MD5)
def s3_to_s3(src: str, dst: str, md5: Optional[str] = None, temp_dir: Optional[str] = None) -> FetchedType:
s3src = S3Object.parse(src)
s3dst = S3Object.parse(dst)
if is_s3_file(s3src):
if md5:
# This version does the validation just for a single s3-file
transfer_to_s3(download_from_s3, src, dst, md5, temp_dir)
else:
s3_to_s3_single(s3src, s3dst)
return FetchedType.FILE
else:
# For all other cases we just transfer
s3_to_s3_deep(s3src, s3dst)
return FetchedType.DIRECTORY
def test_success_with_etag_cache_hit(
mock_calculate_md5_and_etag,
mock_update_s3_hash_tagging,
mock_upload_to_s3,
mock_check_s3_for_md5,
mock_check_s3_for_etag,
mock_temp_file,
mock_successful_download,
):
mock_check_s3_for_etag.return_value = True
transfer_to_s3(mock_successful_download, SRC, DST, MD5)
mock_successful_download.assert_called_once()
mock_upload_to_s3.assert_not_called()
mock_update_s3_hash_tagging.assert_called_with(S3DST, MD5)
def test_client_error(mock_client_error_download):
with pytest.raises(HttpClientError):
transfer_to_s3(mock_client_error_download, SRC, DST)
def test_server_error(mock_server_error_download):
with pytest.raises(HttpServerError):
transfer_to_s3(mock_server_error_download, SRC, DST)
def http_to_s3(src: str,
dst: str,
md5: Optional[str] = None,
temp_dir: Optional[str] = None):
transfer_to_s3(http_download, src, dst, md5, temp_dir)