def nest_endpoint(self, endpoint: str) -> t.ContextManager["Resource"]:
"""Returns a context manager allowing recursive nesting in endpoints"""
a = copy(self)
a.url = up.urljoin(a.url, endpoint)
log.debug(f"Nesting endpoint {endpoint}")
yield a
log.debug(f"Unnesting endpoint {endpoint}")
def build_bundle(dp_config: DatapaneCfg,
use_git: bool = False) -> t.ContextManager[Path]:
"""
Build a local sdist-bundle on the client for uploading
currently requires version and docstring
"""
try:
import flit_core # noqa
except ImportError:
raise MissingCloudPackagesError()
proj_dir = dp_config.proj_dir
# TODO - add git support
incs = _check_glob_patterns(dp_config.include, "include")
excs = _check_glob_patterns(dp_config.exclude, "exclude")
with temp_fname(suffix=".tar.gz",
prefix="datapane-temp-bundle-") as sdist_file:
sdist_file_p = Path(sdist_file)
temp_mod = preprocess_src_dir(dp_config)
try:
sb = Bundler(proj_dir, Module(dp_config.script), incs, excs)
sb.build(sdist_file_p)
finally:
if temp_mod:
temp_mod.unlink()
log.debug(f"Generated sdist {sdist_file_p}")
yield sdist_file_p
def cleanup_tmp():
"""Ensure we cleanup the tmp_dir on Python VM exit"""
log.debug(f"Removing current session DP tmp work dir {tmp_dir}")
shutil.rmtree(tmp_dir, ignore_errors=True)
# try remove cache_dir if empty
with suppress(OSError):
cache_dir.rmdir()
log.debug("Removed empty dp-cache dir")
def post_files(self, files: FileList, **data: JSON) -> JSON:
# upload files using custom json-data protocol
# build the fields
file_header = {"Content-Encoding": "gzip"}
def mk_file_fields(field_name: str, f: Path):
# compress the file, in-place
# TODO - disable compression where unneeded, e.g. .gz, .zip, .png, etc
with compress_file(f) as f_gz:
return (
field_name,
(f.name, open(f_gz, "rb"), guess_type(f), file_header),
)
fields = [mk_file_fields(k, x) for (k, v) in files.items() for x in v]
fields.append(("json_data", json.dumps(data)))
e = MultipartEncoder(fields=fields)
extra_headers = {"Content-Type": f"{e.content_type}; dp-files=True"}
max_size = 25 if c.config.is_public else 100
if e.len > max_size * SIZE_1_MB:
raise ReportTooLargeError(
f"Report and attachments over f{max_size} MB after compression (~{e.len/SIZE_1_MB:.1f} MB) - please reduce the size of your charts/plots"
)
elif e.len > SIZE_1_MB:
log.debug("Using upload monitor")
fill_char = click.style("=", fg="yellow")
with click.progressbar(
length=e.len,
width=0,
show_eta=True,
label="Uploading files",
fill_char=fill_char,
) as bar:
def f(m: MultipartEncoderMonitor):
# update every 100KB
m.buf_bytes_read += m.bytes_read - m.prev_bytes_read
m.prev_bytes_read = m.bytes_read
if m.buf_bytes_read >= 1e5:
# print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}")
bar.update(m.buf_bytes_read)
m.buf_bytes_read = 0
m = MultipartEncoderMonitor(e, callback=f)
m.buf_bytes_read = 0
m.prev_bytes_read = 0
r = self.session.post(self.url,
data=m,
headers=extra_headers,
timeout=self.timeout)
else:
r = self.session.post(self.url,
data=e,
headers=extra_headers,
timeout=self.timeout)
return _process_res(r)
def check_pip_version() -> None:
cli_version = Version(__version__)
url = "https://pypi.org/pypi/datapane/json"
r = requests.get(url=url)
r.raise_for_status()
pip_version = Version(r.json()["info"]["version"])
log.debug(f"CLI version {cli_version}, latest pip version {pip_version}")
if pip_version > cli_version:
error_msg = (
f"Your client is out-of-date (version {cli_version}) and may be causing errors, "
+ f"please upgrade to version {pip_version}")
else: # no newer pip - perhaps local dev?
error_msg = f"Your client is out-of-date (version {cli_version}) with the server and may be causing errors"
raise IncompatibleVersionException(error_msg)
def preprocess_src_dir(dp_config: DatapaneCfg) -> Path:
"""Preprocess source-dir as needed"""
old_mod = dp_config.proj_dir / dp_config.script
# TODO - pass mod_code via classvar in dp_config
if old_mod.suffix == ".ipynb":
log.debug(f"Converting notebook {dp_config.script}")
mod_code = extract_py_notebook(old_mod)
# write the code to the new python module, avoiding basic name-clash
new_mod = old_mod.with_suffix(".py")
if new_mod.exists():
new_mod = old_mod.with_name(f"_{old_mod.stem}.py")
new_mod.write_text(mod_code, encoding="utf-8")
dp_config.script = Path(new_mod.name)
return new_mod
def post_files(self, files: FileList, **data: JSON) -> JSON:
# upload files using custom json-data protocol
# build the fields
file_header = {"Content-Encoding": "gzip"}
def mk_file_fields(field_name: str, f: Path):
# compress the file, in-place
# TODO - disable compression where unneeded, e.g. .gz, .zip, .png, etc
with compress_file(f) as f_gz:
return (field_name, (f.name, open(f_gz, "rb"), guess_type(f), file_header))
fields = [mk_file_fields(k, x) for (k, v) in files.items() for x in v]
fields.append(("json_data", json.dumps(data)))
e = MultipartEncoder(fields=fields)
extra_headers = {"Content-Type": f"{e.content_type}; dp-files=True"}
if e.len > 1e6: # 1 MB
log.debug("Using upload monitor")
fill_char = click.style("=", fg="yellow")
with click.progressbar(
length=e.len, width=0, show_eta=True, label="Uploading files", fill_char=fill_char
) as bar:
def f(m: MultipartEncoderMonitor):
# update every 100KB
m.buf_bytes_read += m.bytes_read - m.prev_bytes_read
m.prev_bytes_read = m.bytes_read
if m.buf_bytes_read >= 1e5:
# print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}")
bar.update(m.buf_bytes_read)
m.buf_bytes_read = 0
m = MultipartEncoderMonitor(e, callback=f)
m.buf_bytes_read = 0
m.prev_bytes_read = 0
r = self.session.post(self.url, data=m, headers=extra_headers, timeout=self.timeout)
else:
r = self.session.post(self.url, data=e, headers=extra_headers, timeout=self.timeout)
return self._process_res(r)
def __exit__(self, exc_type, exc_value, exc_traceback):
log.debug(f"Removing {self.name}")
if self.file.exists():
self.file.unlink() # (missing_ok=True)
################################################################################
# Tmpfile handling
# We create a tmp-dir per Python execution that stores all working files,
# we attempt to delete where possible, but where not, we allow the atexit handler
# to cleanup for us on shutdown
# This tmp-dir needs to be in the cwd rather than /tmp so can be previewed in Jupyter
# To avoid cluttering up the user's cwd, we nest these inside a `dp-cache` intermediate dir
cache_dir = Path("dp-cache").absolute()
cache_dir.mkdir(parents=True, exist_ok=True)
# Remove any old ./dp-tmp-* dirs over 24hrs old which might not have been cleaned up due to unexpected exit
one_day_ago = time.time() - timedelta(days=1).total_seconds()
prev_tmp_dirs = (p for p in cache_dir.glob("dp-tmp-*")
if p.is_dir() and p.stat().st_mtime < one_day_ago)
for p in prev_tmp_dirs:
log.debug(f"Removing stale temp dir {p}")
shutil.rmtree(p, ignore_errors=True)
# create new dp-tmp for this session, nested inside `dp-cache`
tmp_dir = Path(mkdtemp(prefix="dp-tmp-", dir=cache_dir)).absolute()
class DPTmpFile:
"""
Generate a tempfile in dp temp dir
when used as a contextmanager, deleted on removing scope
otherwise, removed by atexit hook
"""
def __init__(self, ext: str):
fd, name = mkstemp(suffix=ext, prefix="dp-tmp-", dir=tmp_dir)
os.close(fd)
def check_login(config=None, cli_login: bool = False) -> SDict:
r = Resource(endpoint="/settings/details/",
config=config).get(cli_login=cli_login)
log.debug(f"Connected successfully to DP Server as {r.username}")
return t.cast(SDict, r)
