def test_extra_headers(authorized_transport, secret_file):
blob_name, data, headers = secret_file
# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
download = resumable_requests.Download(media_url, headers=headers)
# Consume the resource.
response = download.consume(authorized_transport)
assert response.status_code == http_client.OK
assert response.content == data
check_tombstoned(download, authorized_transport)
# Attempt to consume the resource **without** the headers.
download_wo = resumable_requests.Download(media_url)
with pytest.raises(resumable_media.InvalidResponse) as exc_info:
download_wo.consume(authorized_transport)
check_error_response(exc_info, http_client.BAD_REQUEST, ENCRYPTED_ERR)
check_tombstoned(download_wo, authorized_transport)
def _download_slice(media_url, slice_):
assert slice_.step is None
end = None
if slice_.stop is not None:
end = slice_.stop - 1
return resumable_requests.Download(media_url, start=slice_.start, end=end)
def test_non_existent_file(authorized_transport, bucket):
blob_name = u"does-not-exist.txt"
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
download = resumable_requests.Download(media_url)
# Try to consume the resource and fail.
with pytest.raises(resumable_media.InvalidResponse) as exc_info:
download.consume(authorized_transport)
check_error_response(exc_info, http_client.NOT_FOUND, NOT_FOUND_ERR)
check_tombstoned(download, authorized_transport)
def test_download_full(add_files, authorized_transport):
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)
# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
download = resumable_requests.Download(media_url)
# Consume the resource.
response = download.consume(authorized_transport)
assert response.status_code == http_client.OK
assert response.content == actual_contents
check_tombstoned(download, authorized_transport)
def _fetch_data(self, report_details: Dict[str, Any],
buffer: BytesIO) -> int:
try:
report_url = report_details['url']
request = requests.Download(report_url, stream=buffer)
request.consume(transport=self.transport)
return self._stream_size(buffer)
except Exception as e:
logging.error(e)
return -1
def upload_report(self,
bucket: str,
report_details: Dict[str, Any],
input_buffer: BytesIO = None):
output_buffer = StringIO() #BytesIO()
try:
if not input_buffer:
input_buffer = BytesIO()
request = requests.Download(report_details['url'],
stream=input_buffer)
request.consume(transport=self.transport)
logging.info('Report data size: {bytes}'.format(bytes=0))
input_buffer.seek(0)
soup = self._soupify(input_buffer)
# del input_buffer
headers = soup.find('thead').find_all('th')
fieldnames = []
for header in headers:
fieldnames.append(CSVHelpers.sanitize_string(header.string))
rows = soup.find('tbody').find_all('tr')
report_data = []
for row in rows:
data = []
for col in row.contents:
data.append(col.string)
report_data.append(dict(zip(fieldnames, data)))
writer = csv.DictWriter(output_buffer, fieldnames=fieldnames)
writer.writeheader()
for row in report_data:
writer.writerow(row)
output_buffer.seek(0)
Cloud_Storage.write_file(bucket=bucket,
file=f"{report_details['id']}.csv",
data=output_buffer.getvalue())
report_details['schema'] = CSVHelpers.create_table_schema(
fieldnames)
except Exception as e:
logging.error(e)
def test_bad_range(simple_file, authorized_transport):
blob_name, data = simple_file
# Make sure we have an invalid range.
start = 32
end = 63
assert len(data) < start < end
# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
download = resumable_requests.Download(media_url, start=start, end=end)
# Try to consume the resource and fail.
with pytest.raises(resumable_media.InvalidResponse) as exc_info:
download.consume(authorized_transport)
check_error_response(exc_info, http_client.REQUESTED_RANGE_NOT_SATISFIABLE,
b'Request range not satisfiable')
check_tombstoned(download, authorized_transport)
def test_corrupt_download(add_files, corrupting_transport):
for info in ALL_FILES:
blob_name = _get_blob_name(info)
# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
stream = io.BytesIO()
download = resumable_requests.Download(media_url, stream=stream)
# Consume the resource.
with pytest.raises(common.DataCorruption) as exc_info:
download.consume(corrupting_transport)
assert download.finished
msg = download_mod._CHECKSUM_MISMATCH.format(
download.media_url, CorruptingAuthorizedSession.EMPTY_HASH,
info[u'checksum'])
assert exc_info.value.args == (msg, )
def test_download_to_stream(add_files, authorized_transport):
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)
# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
stream = io.BytesIO()
download = resumable_requests.Download(media_url, stream=stream)
# Consume the resource.
response = download.consume(authorized_transport)
assert response.status_code == http_client.OK
with pytest.raises(RuntimeError) as exc_info:
getattr(response, u"content")
assert exc_info.value.args == (NO_BODY_ERR, )
assert response._content is False
assert response._content_consumed is True
assert stream.getvalue() == actual_contents
check_tombstoned(download, authorized_transport)
def check_content(blob_name, expected_content, transport, headers=None):
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
download = resumable_requests.Download(media_url, headers=headers)
response = download.consume(transport)
assert response.status_code == http_client.OK
assert response.content == expected_content
