def f(m: MultipartEncoderMonitor):
# update every 100KB
m.buf_bytes_read += m.bytes_read - m.prev_bytes_read
m.prev_bytes_read = m.bytes_read
if m.buf_bytes_read >= 1e5:
# print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}")
bar.update(m.buf_bytes_read)
m.buf_bytes_read = 0
def post_files(self, files: FileList, **data: JSON) -> JSON:
# upload files using custom json-data protocol
# build the fields
file_header = {"Content-Encoding": "gzip"}
def mk_file_fields(field_name: str, f: Path):
# compress the file, in-place
# TODO - disable compression where unneeded, e.g. .gz, .zip, .png, etc
with compress_file(f) as f_gz:
return (
field_name,
(f.name, open(f_gz, "rb"), guess_type(f), file_header),
)
fields = [mk_file_fields(k, x) for (k, v) in files.items() for x in v]
fields.append(("json_data", json.dumps(data)))
e = MultipartEncoder(fields=fields)
extra_headers = {"Content-Type": f"{e.content_type}; dp-files=True"}
max_size = 25 if c.config.is_public else 100
if e.len > max_size * SIZE_1_MB:
raise ReportTooLargeError(
f"Report and attachments over f{max_size} MB after compression (~{e.len/SIZE_1_MB:.1f} MB) - please reduce the size of your charts/plots"
)
elif e.len > SIZE_1_MB:
log.debug("Using upload monitor")
fill_char = click.style("=", fg="yellow")
with click.progressbar(
length=e.len,
width=0,
show_eta=True,
label="Uploading files",
fill_char=fill_char,
) as bar:
def f(m: MultipartEncoderMonitor):
# update every 100KB
m.buf_bytes_read += m.bytes_read - m.prev_bytes_read
m.prev_bytes_read = m.bytes_read
if m.buf_bytes_read >= 1e5:
# print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}")
bar.update(m.buf_bytes_read)
m.buf_bytes_read = 0
m = MultipartEncoderMonitor(e, callback=f)
m.buf_bytes_read = 0
m.prev_bytes_read = 0
r = self.session.post(self.url,
data=m,
headers=extra_headers,
timeout=self.timeout)
else:
r = self.session.post(self.url,
data=e,
headers=extra_headers,
timeout=self.timeout)
return _process_res(r)
def post_files(self, files: FileList, **data: JSON) -> JSON:
# upload files using custom json-data protocol
# build the fields
file_header = {"Content-Encoding": "gzip"}
def mk_file_fields(field_name: str, f: Path):
# compress the file, in-place
# TODO - disable compression where unneeded, e.g. .gz, .zip, .png, etc
with compress_file(f) as f_gz:
return (field_name, (f.name, open(f_gz, "rb"), guess_type(f), file_header))
fields = [mk_file_fields(k, x) for (k, v) in files.items() for x in v]
fields.append(("json_data", json.dumps(data)))
e = MultipartEncoder(fields=fields)
extra_headers = {"Content-Type": f"{e.content_type}; dp-files=True"}
if e.len > 1e6: # 1 MB
log.debug("Using upload monitor")
fill_char = click.style("=", fg="yellow")
with click.progressbar(
length=e.len, width=0, show_eta=True, label="Uploading files", fill_char=fill_char
) as bar:
def f(m: MultipartEncoderMonitor):
# update every 100KB
m.buf_bytes_read += m.bytes_read - m.prev_bytes_read
m.prev_bytes_read = m.bytes_read
if m.buf_bytes_read >= 1e5:
# print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}")
bar.update(m.buf_bytes_read)
m.buf_bytes_read = 0
m = MultipartEncoderMonitor(e, callback=f)
m.buf_bytes_read = 0
m.prev_bytes_read = 0
r = self.session.post(self.url, data=m, headers=extra_headers, timeout=self.timeout)
else:
r = self.session.post(self.url, data=e, headers=extra_headers, timeout=self.timeout)
return self._process_res(r)
