def test_upload_put_timeout(tmp_path, caplog):
"""Tests whether timeout error is handled correctly when uploading."""
caplog.set_level(logging.DEBUG, 'snowflake.connector')
f_name = str(tmp_path / 'some_file.txt')
resp = requests.Response()
meta = SnowflakeFileMeta(
name=f_name,
src_file_name=f_name,
stage_location_type='GCS',
presigned_url='some_url',
sha256_digest='asd',
)
with open(f_name, 'w') as f:
f.write(random_string(15))
with mock.patch('snowflake.connector.vendored.requests.put'
if vendored_request else 'requests.put',
side_effect=requests.exceptions.Timeout(response=resp)):
SnowflakeGCSUtil.upload_file(f_name, meta, None, 99, 64000)
assert isinstance(meta.last_error, requests.exceptions.Timeout)
assert meta.result_status == ResultStatus.NEED_RETRY
assert all([
log in caplog.record_tuples
for log in [('snowflake.connector.gcs_util', logging.DEBUG,
'GCS file upload Timeout Error: ')]
])
def test_native_upload_access_token(caplog):
"""Tests that GCS access token error is correctly logged when uploading."""
meta = {}
SnowflakeGCSUtil.upload_file(None, meta, None, 99)
assert meta['result_status'] == ResultStatus.ERROR
assert (('snowflake.connector.gcs_util', logging.ERROR, "GCS upload operation with an access token is "
"currently unsupported") in caplog.record_tuples)
def test_download_uncaught_exception(tmpdir):
"""Tests whether non-retryable errors are handled correctly when downloading."""
resp = Response()
resp.status_code = 501
meta = {'presigned_url': ['some_url'], 'sha256_digest': 'asd'}
with mock.patch('requests.get', side_effect=requests.exceptions.HTTPError(response=resp)):
with pytest.raises(requests.exceptions.HTTPError):
SnowflakeGCSUtil._native_download_file(meta, str(tmpdir), 99)
def test_download_retry_errors(errno, tmpdir):
"""Tests whether retryable errors are handled correctly when downloading."""
resp = Response()
resp.status_code = errno
meta = {'presigned_url': ['some_url'], 'sha256_digest': 'asd'}
with mock.patch('requests.get', side_effect=requests.exceptions.HTTPError(response=resp)):
SnowflakeGCSUtil._native_download_file(meta, str(tmpdir), 99)
assert isinstance(meta['last_error'], requests.exceptions.HTTPError)
assert meta['result_status'] == ResultStatus.NEED_RETRY
def test_upload_get_timeout(tmpdir, caplog):
"""Tests whether timeout error is handled correctly when downloading."""
resp = Response()
meta = {'presigned_url': ['some_url'], 'sha256_digest': 'asd'}
with mock.patch('requests.get', side_effect=requests.exceptions.Timeout(response=resp)):
SnowflakeGCSUtil._native_download_file(meta, str(tmpdir), 99)
assert isinstance(meta['last_error'], requests.exceptions.Timeout)
assert meta['result_status'] == ResultStatus.NEED_RETRY
assert ('snowflake.connector.gcs_util', logging.DEBUG, 'GCS file download Timeout Error: ') in caplog.record_tuples
def test_upload_uncaught_exception(tmpdir):
"""Tests whether non-retryable errors are handled correctly when uploading."""
f_name = str(tmpdir.join('some_file.txt'))
resp = Response()
resp.status_code = 501
meta = {'presigned_url': ['some_url'], 'sha256_digest': 'asd'}
with open(f_name, 'w') as f:
f.write(random_string(15))
with mock.patch('requests.put', side_effect=requests.exceptions.HTTPError(response=resp)):
with pytest.raises(requests.exceptions.HTTPError):
SnowflakeGCSUtil.upload_file(f_name, meta, None, 99)
def test_upload_retry_errors(errno, tmpdir):
"""Tests whether retryable errors are handled correctly when upploading."""
f_name = str(tmpdir.join('some_file.txt'))
resp = Response()
resp.status_code = errno
meta = {'presigned_url': ['some_url'], 'sha256_digest': 'asd'}
with open(f_name, 'w') as f:
f.write(random_string(15))
with mock.patch('requests.put', side_effect=requests.exceptions.HTTPError(response=resp)):
SnowflakeGCSUtil.upload_file(f_name, meta, None, 99)
assert isinstance(meta['last_error'], requests.exceptions.HTTPError)
assert meta['result_status'] == ResultStatus.NEED_RETRY
def test_upload_put_timeout(tmpdir, caplog):
"""Tests whether timeout error is handled correctly when uploading."""
f_name = str(tmpdir.join('some_file.txt'))
resp = Response()
meta = {'presigned_url': ['some_url'], 'sha256_digest': 'asd'}
with open(f_name, 'w') as f:
f.write(random_string(15))
with mock.patch('requests.put', side_effect=requests.exceptions.Timeout(response=resp)):
SnowflakeGCSUtil.upload_file(f_name, meta, None, 99)
assert isinstance(meta['last_error'], requests.exceptions.Timeout)
assert meta['result_status'] == ResultStatus.NEED_RETRY
assert all([log in caplog.record_tuples for log in [
('snowflake.connector.gcs_util', logging.DEBUG, 'GCS file upload Timeout Error: ')
]])
def test_download_uncaught_exception(tmp_path):
"""Tests whether non-retryable errors are handled correctly when downloading."""
resp = requests.Response()
resp.status_code = 501
meta = SnowflakeFileMeta(
name=str(tmp_path / 'some_file'),
src_file_name=str(tmp_path / 'some_file'),
stage_location_type='GCS',
presigned_url='some_url',
sha256_digest='asd',
)
with mock.patch('snowflake.connector.vendored.requests.get'
if vendored_request else 'requests.get',
side_effect=requests.exceptions.HTTPError(response=resp)):
with pytest.raises(requests.exceptions.HTTPError):
SnowflakeGCSUtil._native_download_file(meta, str(tmp_path), 99)
def test_get_file_header_none_with_presigned_url():
"""Tests whether default file handle created by get_file_header is as expected."""
file_header = SnowflakeGCSUtil.get_file_header(
{"presigned_url": "www.example.com"}, 'file')
assert file_header.digest is None
assert file_header.content_length is None
assert file_header.encryption_metadata is None
def test_download_retry_errors(errno, tmp_path):
"""Tests whether retryable errors are handled correctly when downloading."""
resp = requests.Response()
resp.status_code = errno
meta = SnowflakeFileMeta(
name=str(tmp_path / 'some_file'),
src_file_name=str(tmp_path / 'some_file'),
stage_location_type='GCS',
presigned_url='some_url',
sha256_digest='asd',
)
with mock.patch('snowflake.connector.vendored.requests.get'
if vendored_request else 'requests.get',
side_effect=requests.exceptions.HTTPError(response=resp)):
SnowflakeGCSUtil._native_download_file(meta, str(tmp_path), 99)
assert isinstance(meta.last_error, requests.exceptions.HTTPError)
assert meta.result_status == ResultStatus.NEED_RETRY
def test_create_client(caplog):
"""Creates a GCSUtil with an access token."""
caplog.set_level(logging.DEBUG, 'snowflake.connector')
client = SnowflakeGCSUtil.create_client(
{'creds': {
'GCS_ACCESS_TOKEN': 'fake_token'
}})
assert client is not None
assert client == 'fake_token'
def test_create_client(caplog):
"""Creates a GCSUtil with an access token."""
caplog.set_level(logging.DEBUG, "snowflake.connector")
client = SnowflakeGCSUtil.create_client(
{"creds": {
"GCS_ACCESS_TOKEN": "fake_token"
}})
assert client is not None
assert client == "fake_token"
def test_upload_get_timeout(tmp_path, caplog):
"""Tests whether timeout error is handled correctly when downloading."""
caplog.set_level(logging.DEBUG, 'snowflake.connector')
resp = requests.Response()
meta = SnowflakeFileMeta(
name=str(tmp_path / 'some_file'),
src_file_name=str(tmp_path / 'some_file'),
stage_location_type='GCS',
presigned_url='some_url',
sha256_digest='asd',
)
with mock.patch('snowflake.connector.vendored.requests.get'
if vendored_request else 'requests.get',
side_effect=requests.exceptions.Timeout(response=resp)):
SnowflakeGCSUtil._native_download_file(meta, str(tmp_path), 99)
assert isinstance(meta.last_error, requests.exceptions.Timeout)
assert meta.result_status == ResultStatus.NEED_RETRY
assert ('snowflake.connector.gcs_util', logging.DEBUG,
'GCS file download Timeout Error: ') in caplog.record_tuples
def test_upload_uncaught_exception(tmpdir):
"""Tests whether non-retryable errors are handled correctly when uploading."""
f_name = str(tmpdir.join('some_file.txt'))
resp = requests.Response()
resp.status_code = 501
meta = SnowflakeFileMeta(
name=f_name,
src_file_name=f_name,
stage_location_type='GCS',
presigned_url='some_url',
sha256_digest='asd',
)
with open(f_name, 'w') as f:
f.write(random_string(15))
with mock.patch('snowflake.connector.vendored.requests.put'
if vendored_request else 'requests.put',
side_effect=requests.exceptions.HTTPError(response=resp)):
with pytest.raises(requests.exceptions.HTTPError):
SnowflakeGCSUtil.upload_file(f_name, meta, None, 99, 64000)
def test_upload_retry_errors(errno, tmpdir):
"""Tests whether retryable errors are handled correctly when upploading."""
f_name = str(tmpdir.join('some_file.txt'))
resp = requests.Response()
resp.status_code = errno
meta = SnowflakeFileMeta(
name=f_name,
src_file_name=f_name,
stage_location_type='GCS',
presigned_url='some_url',
sha256_digest='asd',
)
with open(f_name, 'w') as f:
f.write(random_string(15))
with mock.patch('snowflake.connector.vendored.requests.put'
if vendored_request else 'requests.put',
side_effect=requests.exceptions.HTTPError(response=resp)):
SnowflakeGCSUtil.upload_file(f_name, meta, None, 99, 64000)
assert isinstance(meta.last_error, requests.exceptions.HTTPError)
assert meta.result_status == ResultStatus.NEED_RETRY
def test_get_file_header_none_with_presigned_url(tmp_path):
"""Tests whether default file handle created by get_file_header is as expected."""
meta = SnowflakeFileMeta(
name=str(tmp_path / 'some_file'),
src_file_name=str(tmp_path / 'some_file'),
stage_location_type='GCS',
presigned_url='www.example.com',
)
file_header = SnowflakeGCSUtil.get_file_header(meta, 'file')
assert file_header.digest is None
assert file_header.content_length is None
assert file_header.encryption_metadata is None
def test_create_client(caplog):
"""Creates a GCSUtil with an access token."""
client = SnowflakeGCSUtil.create_client(
{'creds': {
'GCS_ACCESS_TOKEN': 'fake_token'
}})
assert client is None
assert all([
log in caplog.record_tuples for log in
[('snowflake.connector.gcs_util', logging.DEBUG,
"len(GCS_ACCESS_TOKEN): 10"),
('snowflake.connector.gcs_util', logging.DEBUG,
"GCS operations with an access token are currently unsupported")]
])
def test_create_client(caplog):
"""Creates a GCSUtil with an access token."""
client = SnowflakeGCSUtil.create_client({'creds': {'GCS_ACCESS_TOKEN': 'fake_token'}})
assert client is not None
assert client == 'fake_token'
def test_get_file_header_none():
"""Tests whether default file handle created by get_file_header is as expected."""
file_header = SnowflakeGCSUtil.get_file_header({}, 'file')
assert file_header.digest is None
assert file_header.content_length is None
assert file_header.encryption_metadata is None
