def test_extra_headers(authorized_transport, secret_file): blob_name, data, headers = secret_file # Create the actual download object. media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) download = resumable_requests.Download(media_url, headers=headers) # Consume the resource. response = download.consume(authorized_transport) assert response.status_code == http_client.OK assert response.content == data check_tombstoned(download, authorized_transport) # Attempt to consume the resource **without** the headers. download_wo = resumable_requests.Download(media_url) with pytest.raises(resumable_media.InvalidResponse) as exc_info: download_wo.consume(authorized_transport) check_error_response(exc_info, http_client.BAD_REQUEST, ENCRYPTED_ERR) check_tombstoned(download_wo, authorized_transport)
def _download_slice(media_url, slice_): assert slice_.step is None end = None if slice_.stop is not None: end = slice_.stop - 1 return resumable_requests.Download(media_url, start=slice_.start, end=end)
def test_non_existent_file(authorized_transport, bucket): blob_name = u"does-not-exist.txt" media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) download = resumable_requests.Download(media_url) # Try to consume the resource and fail. with pytest.raises(resumable_media.InvalidResponse) as exc_info: download.consume(authorized_transport) check_error_response(exc_info, http_client.NOT_FOUND, NOT_FOUND_ERR) check_tombstoned(download, authorized_transport)
def test_download_full(add_files, authorized_transport): for info in ALL_FILES: actual_contents = _get_contents(info) blob_name = _get_blob_name(info) # Create the actual download object. media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) download = resumable_requests.Download(media_url) # Consume the resource. response = download.consume(authorized_transport) assert response.status_code == http_client.OK assert response.content == actual_contents check_tombstoned(download, authorized_transport)
def _fetch_data(self, report_details: Dict[str, Any], buffer: BytesIO) -> int: try: report_url = report_details['url'] request = requests.Download(report_url, stream=buffer) request.consume(transport=self.transport) return self._stream_size(buffer) except Exception as e: logging.error(e) return -1
def upload_report(self, bucket: str, report_details: Dict[str, Any], input_buffer: BytesIO = None): output_buffer = StringIO() #BytesIO() try: if not input_buffer: input_buffer = BytesIO() request = requests.Download(report_details['url'], stream=input_buffer) request.consume(transport=self.transport) logging.info('Report data size: {bytes}'.format(bytes=0)) input_buffer.seek(0) soup = self._soupify(input_buffer) # del input_buffer headers = soup.find('thead').find_all('th') fieldnames = [] for header in headers: fieldnames.append(CSVHelpers.sanitize_string(header.string)) rows = soup.find('tbody').find_all('tr') report_data = [] for row in rows: data = [] for col in row.contents: data.append(col.string) report_data.append(dict(zip(fieldnames, data))) writer = csv.DictWriter(output_buffer, fieldnames=fieldnames) writer.writeheader() for row in report_data: writer.writerow(row) output_buffer.seek(0) Cloud_Storage.write_file(bucket=bucket, file=f"{report_details['id']}.csv", data=output_buffer.getvalue()) report_details['schema'] = CSVHelpers.create_table_schema( fieldnames) except Exception as e: logging.error(e)
def test_bad_range(simple_file, authorized_transport): blob_name, data = simple_file # Make sure we have an invalid range. start = 32 end = 63 assert len(data) < start < end # Create the actual download object. media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) download = resumable_requests.Download(media_url, start=start, end=end) # Try to consume the resource and fail. with pytest.raises(resumable_media.InvalidResponse) as exc_info: download.consume(authorized_transport) check_error_response(exc_info, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, b'Request range not satisfiable') check_tombstoned(download, authorized_transport)
def test_corrupt_download(add_files, corrupting_transport): for info in ALL_FILES: blob_name = _get_blob_name(info) # Create the actual download object. media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) stream = io.BytesIO() download = resumable_requests.Download(media_url, stream=stream) # Consume the resource. with pytest.raises(common.DataCorruption) as exc_info: download.consume(corrupting_transport) assert download.finished msg = download_mod._CHECKSUM_MISMATCH.format( download.media_url, CorruptingAuthorizedSession.EMPTY_HASH, info[u'checksum']) assert exc_info.value.args == (msg, )
def test_download_to_stream(add_files, authorized_transport): for info in ALL_FILES: actual_contents = _get_contents(info) blob_name = _get_blob_name(info) # Create the actual download object. media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) stream = io.BytesIO() download = resumable_requests.Download(media_url, stream=stream) # Consume the resource. response = download.consume(authorized_transport) assert response.status_code == http_client.OK with pytest.raises(RuntimeError) as exc_info: getattr(response, u"content") assert exc_info.value.args == (NO_BODY_ERR, ) assert response._content is False assert response._content_consumed is True assert stream.getvalue() == actual_contents check_tombstoned(download, authorized_transport)
def check_content(blob_name, expected_content, transport, headers=None): media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) download = resumable_requests.Download(media_url, headers=headers) response = download.consume(transport) assert response.status_code == http_client.OK assert response.content == expected_content