def put_blob(
self,
image_reference: str,
digest: str,
octets_count: int,
data: requests.models.Response,
max_chunk=1024 * 1024 * 1, # 1 MiB
mimetype: str = 'application/data',
):
head_res = self.head_blob(
image_reference=image_reference,
digest=digest,
)
if head_res.ok:
logger.info(f'skipping blob upload {digest=} - already exists')
return
data_is_requests_resp = isinstance(data, requests.models.Response)
data_is_generator = isinstance(data, typing.Generator)
data_is_filelike = hasattr(data, 'read')
data_is_bytes = isinstance(data, bytes)
if octets_count < max_chunk or data_is_filelike or data_is_requests_resp or data_is_bytes:
if data_is_requests_resp:
data = data.content
elif data_is_generator:
# at least GCR does not like chunked-uploads; if small enough, workaround this
# and create one (not-that-big) bytes-obj
_data = bytes()
for chunk in data:
_data += chunk
elif data_is_filelike:
pass # if filelike, http.client will handle streaming for us
return self._put_blob_single_post(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data=data,
)
else:
if data_is_requests_resp:
with data:
return self._put_blob_chunked(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data_iterator=data.iter_content(chunk_size=max_chunk),
chunk_size=max_chunk,
)
elif data_is_generator:
return self._put_blob_chunked(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data_iterator=data.iter_content(chunk_size=max_chunk),
chunk_size=max_chunk,
)
else:
raise NotImplementedError
def put_blob(
self,
image_reference: str,
digest: str,
octets_count: int,
data: requests.models.Response,
max_chunk=1024 * 1024 * 1, # 1 MiB
):
data_is_requests_response = isinstance(data, requests.models.Response)
data_is_generator = isinstance(data, typing.Generator)
data_is_filelike = hasattr(data, 'read')
if octets_count < max_chunk or data_is_filelike:
if data_is_requests_response:
data = data.content
elif data_is_generator:
# at least GCR does not like chunked-uploads; if small enough, workaround this
# and create one (not-that-big) bytes-obj
_data = bytes()
for chunk in data:
_data += chunk
elif data_is_filelike:
pass # if filelike, http.client will handle streaming for us
return self._put_blob_single_post(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data=data,
)
else:
if data_is_requests_response:
with data:
return self._put_blob_chunked(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data_iterator=data.iter_content(chunk_size=max_chunk),
chunk_size=max_chunk,
)
elif data_is_generator:
return self._put_blob_chunked(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data_iterator=data.iter_content(chunk_size=max_chunk),
chunk_size=max_chunk,
)
else:
raise NotImplementedError
def save_response_content(response: requests.models.Response,
destination: str,
chunk_size: int = 32768) -> None:
with open(destination, "wb") as f:
for chunk in response.iter_content(chunk_size):
if chunk: # filter out keep-alive new chunksqs
f.write(chunk)
def save_file(filename: str, data: requests.models.Response):
"""Saves a file to the current directory."""
filename = filter_filename(filename)
file_size = int(data.headers.get('content-length', 0))
try:
with open(filename, 'wb') as f:
with tqdm(total=file_size, unit='B',
unit_scale=True, unit_divisor=1024,
desc=filename, initial=0,
ascii=True, miniters=1) as pbar:
pbar.format_meter(n=file_size,total=file_size,elapsed=pbar.format_dict['elapsed'],initial=0,ascii=True,unit_divisor=1024,unit_scale=True, unit='B',miniters=1)
for chunk in data.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
pbar.update(len(chunk))
print(f"Saved file as '{filename}'")
except OSError as exc:
if exc.errno == 36: # filename too long
(_, extension) = os.path.splitext(filename) # this can fail
# 'extension' already contains the leading '.', hence
# there is no need for a '.' in between "{}{}"
random_filename = f"{random_string(15)}{extension}"
save_file(random_filename, data)
else:
raise # re-raise if .errno is different then 36
except Exception:
raise
def _download_video(self, fp: str, resp: requests.models.Response) -> None:
"""Write the media to file at given fp from the response"""
with open(fp, "wb") as outfile:
for chunk in resp.iter_content(chunk_size=1024):
if chunk:
outfile.write(chunk)
outfile.flush()
def save_page_to_file(response: requests.models.Response, filename: str):
"""Save content from response to given file."""
with open(filename, 'wb') as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
print("Image saved to file:", filename, "\n")
def save_file(self, filename: str, data: requests.models.Response):
"""Saves a file to the current directory."""
def filter_filename(filename: str):
"""Filters a filename non alphabetic and non delimiters charaters."""
valid_chars = "-_.() "
return "".join(c for c in filename
if c.isalnum() or c in valid_chars)
filename = filter_filename(filename)
try:
with open(filename, "wb") as f:
for chunk in data.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
self.logger.info(f'Saved file as "{filename}".')
except OSError as exc:
if (platform.system() == "Linux" and exc.errno
== 36) or (platform.system() == "Darwin"
and exc.errno == 63): # filename too long
(_, extension) = os.path.splitext(filename) # this can fail
# 'extension' already contains the leading '.', hence
# there is no need for a '.' in between "{}{}"
random_filename = f"{random_string(15)}{extension}"
self.save_file(random_filename, data)
else:
raise # re-raise if .errno is different than 36 or 63
except Exception:
raise
def download(filename: str, request: requests.models.Response):
"""
Takes a filename and get or post request, then downloads the file
while outputting a download status bar.
Preconditions: filename must be valid
"""
assert type(filename) == str, 'Error: filename must be a string'
assert type(request) == requests.models.Response, 'Error: request must be a class<requests.models.Response>'
with open(filename, 'wb') as f:
start = time.perf_counter()
if request is None:
f.write(request.content)
else:
total_length = int(request.headers.get('content-length'))
dl = 0
for chunk in request.iter_content(chunk_size=1024*1024):
dl += len(chunk)
if chunk:
f.write(chunk)
f.flush()
done = int(50 * dl / int(total_length))
stars = '=' * done
spaces = ' ' * (50-done)
bps = dl//(time.perf_counter() - start)
percent = int((100*dl)/total_length)
print(f"\r[{stars}{spaces}] {bps} bps, {percent}% ", end='\r', flush=True)
def save_document(response: requests.models.Response, doc_control_num: str,
date: datetime.date, doc_type: str):
str_date = date.strftime("%Y-%m-%d")
filepath = os.path.join(OUTPUT_DIR, str_date, doc_type,
f"{doc_control_num}.zip")
with open(filepath, 'wb') as file:
for chunk in response.iter_content(chunk_size=1024):
file.write(chunk)
def save_file(filename: str, data: requests.models.Response):
"""Saves a file to the current directory."""
try:
with open(filename, 'wb') as f:
for chunk in data.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
print(f"Saved file as '{filename}'")
except OSError as exc:
if exc.errno == 36: # filename too long
(_, extension) = os.path.splitext(filename) # this can fail
# 'extension' already contains the leading '.', hence
# there is no need for a '.' in between "{}{}"
random_filename = f"{random_string(15)}{extension}"
save_file(random_filename, data)
else:
raise # re-raise if .errno is different then 36
except Exception:
raise
def save_file(filename: str, data: requests.models.Response):
"""Saves a file to the current directory."""
filename = filter_filename(filename)
try:
with open(filename, 'wb') as f:
for chunk in data.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
print(f"Saved file as '{filename}'")
except OSError as exc:
if exc.errno == 36: # filename too long
(_, extension) = os.path.splitext(filename) # this can fail
# 'extension' already contains the leading '.', hence
# there is no need for a '.' in between "{}{}"
random_filename = f"{random_string(15)}{extension}"
save_file(random_filename, data)
else:
raise # re-raise if .errno is different then 36
except Exception:
raise
def save_file(filename: str, data: requests.models.Response):
"""Saves a file to the current directory."""
valid_chars = '-_.() abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
filename = ''.join(c for c in filename if c in valid_chars)
try:
with open(filename, 'wb') as f:
for chunk in data.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
print(f"Saved file as '{filename}'")
except OSError as exc:
if exc.errno == 36: # filename too long
(_, extension) = os.path.splitext(filename) # this can fail
# 'extension' already contains the leading '.', hence
# there is no need for a '.' in between "{}{}"
random_filename = f"{random_string(15)}{extension}"
save_file(random_filename, data)
else:
raise # re-raise if .errno is different then 36
except Exception:
raise
def put_blob(
self,
image_reference: typing.Union[str, om.OciImageReference],
digest: str,
octets_count: int,
data: requests.models.Response,
max_chunk=1024 * 1024 * 1, # 1 MiB
mimetype: str='application/data',
):
image_reference = om.OciImageReference(image_reference)
head_res = self.head_blob(
image_reference=image_reference,
digest=digest,
)
if head_res.ok:
logger.info(f'skipping blob upload {digest=} - already exists')
return
data_is_requests_resp = isinstance(data, requests.models.Response)
data_is_generator = isinstance(data, typing.Generator)
data_is_filelike = hasattr(data, 'read')
data_is_bytes = isinstance(data, bytes)
if octets_count < max_chunk or data_is_filelike or data_is_requests_resp or data_is_bytes:
if data_is_requests_resp:
data = data.content
elif data_is_generator:
# at least GCR does not like chunked-uploads; if small enough, workaround this
# and create one (not-that-big) bytes-obj
_data = bytes()
for chunk in data:
_data += chunk
data = _data
elif data_is_filelike:
pass # if filelike, http.client will handle streaming for us
return self._put_blob_single_post(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data=data,
)
elif octets_count >= max_chunk and data_is_generator:
# workaround: write into temporary file, as at least GCR does not implement
# chunked-upload, and requests will not properly work w/ all generators
# (in particular, it will not work w/ our "fake" on)
with tempfile.TemporaryFile() as tf:
for chunk in data:
tf.write(chunk)
tf.seek(0)
return self._put_blob_single_post(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data=tf,
)
else:
if data_is_requests_resp:
with data:
return self._put_blob_chunked(
image_reference=image_reference,
digest=digest,
octets_count=octets_count,
data_iterator=data.iter_content(chunk_size=max_chunk),
chunk_size=max_chunk,
)
else:
raise NotImplementedError