def nest_endpoint(self, endpoint: str) -> t.ContextManager["Resource"]: """Returns a context manager allowing recursive nesting in endpoints""" a = copy(self) a.url = up.urljoin(a.url, endpoint) log.debug(f"Nesting endpoint {endpoint}") yield a log.debug(f"Unnesting endpoint {endpoint}")
def build_bundle(dp_config: DatapaneCfg, use_git: bool = False) -> t.ContextManager[Path]: """ Build a local sdist-bundle on the client for uploading currently requires version and docstring """ try: import flit_core # noqa except ImportError: raise MissingCloudPackagesError() proj_dir = dp_config.proj_dir # TODO - add git support incs = _check_glob_patterns(dp_config.include, "include") excs = _check_glob_patterns(dp_config.exclude, "exclude") with temp_fname(suffix=".tar.gz", prefix="datapane-temp-bundle-") as sdist_file: sdist_file_p = Path(sdist_file) temp_mod = preprocess_src_dir(dp_config) try: sb = Bundler(proj_dir, Module(dp_config.script), incs, excs) sb.build(sdist_file_p) finally: if temp_mod: temp_mod.unlink() log.debug(f"Generated sdist {sdist_file_p}") yield sdist_file_p
def cleanup_tmp(): """Ensure we cleanup the tmp_dir on Python VM exit""" log.debug(f"Removing current session DP tmp work dir {tmp_dir}") shutil.rmtree(tmp_dir, ignore_errors=True) # try remove cache_dir if empty with suppress(OSError): cache_dir.rmdir() log.debug("Removed empty dp-cache dir")
def post_files(self, files: FileList, **data: JSON) -> JSON: # upload files using custom json-data protocol # build the fields file_header = {"Content-Encoding": "gzip"} def mk_file_fields(field_name: str, f: Path): # compress the file, in-place # TODO - disable compression where unneeded, e.g. .gz, .zip, .png, etc with compress_file(f) as f_gz: return ( field_name, (f.name, open(f_gz, "rb"), guess_type(f), file_header), ) fields = [mk_file_fields(k, x) for (k, v) in files.items() for x in v] fields.append(("json_data", json.dumps(data))) e = MultipartEncoder(fields=fields) extra_headers = {"Content-Type": f"{e.content_type}; dp-files=True"} max_size = 25 if c.config.is_public else 100 if e.len > max_size * SIZE_1_MB: raise ReportTooLargeError( f"Report and attachments over f{max_size} MB after compression (~{e.len/SIZE_1_MB:.1f} MB) - please reduce the size of your charts/plots" ) elif e.len > SIZE_1_MB: log.debug("Using upload monitor") fill_char = click.style("=", fg="yellow") with click.progressbar( length=e.len, width=0, show_eta=True, label="Uploading files", fill_char=fill_char, ) as bar: def f(m: MultipartEncoderMonitor): # update every 100KB m.buf_bytes_read += m.bytes_read - m.prev_bytes_read m.prev_bytes_read = m.bytes_read if m.buf_bytes_read >= 1e5: # print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}") bar.update(m.buf_bytes_read) m.buf_bytes_read = 0 m = MultipartEncoderMonitor(e, callback=f) m.buf_bytes_read = 0 m.prev_bytes_read = 0 r = self.session.post(self.url, data=m, headers=extra_headers, timeout=self.timeout) else: r = self.session.post(self.url, data=e, headers=extra_headers, timeout=self.timeout) return _process_res(r)
def check_pip_version() -> None: cli_version = Version(__version__) url = "https://pypi.org/pypi/datapane/json" r = requests.get(url=url) r.raise_for_status() pip_version = Version(r.json()["info"]["version"]) log.debug(f"CLI version {cli_version}, latest pip version {pip_version}") if pip_version > cli_version: error_msg = ( f"Your client is out-of-date (version {cli_version}) and may be causing errors, " + f"please upgrade to version {pip_version}") else: # no newer pip - perhaps local dev? error_msg = f"Your client is out-of-date (version {cli_version}) with the server and may be causing errors" raise IncompatibleVersionException(error_msg)
def preprocess_src_dir(dp_config: DatapaneCfg) -> Path: """Preprocess source-dir as needed""" old_mod = dp_config.proj_dir / dp_config.script # TODO - pass mod_code via classvar in dp_config if old_mod.suffix == ".ipynb": log.debug(f"Converting notebook {dp_config.script}") mod_code = extract_py_notebook(old_mod) # write the code to the new python module, avoiding basic name-clash new_mod = old_mod.with_suffix(".py") if new_mod.exists(): new_mod = old_mod.with_name(f"_{old_mod.stem}.py") new_mod.write_text(mod_code, encoding="utf-8") dp_config.script = Path(new_mod.name) return new_mod
def post_files(self, files: FileList, **data: JSON) -> JSON: # upload files using custom json-data protocol # build the fields file_header = {"Content-Encoding": "gzip"} def mk_file_fields(field_name: str, f: Path): # compress the file, in-place # TODO - disable compression where unneeded, e.g. .gz, .zip, .png, etc with compress_file(f) as f_gz: return (field_name, (f.name, open(f_gz, "rb"), guess_type(f), file_header)) fields = [mk_file_fields(k, x) for (k, v) in files.items() for x in v] fields.append(("json_data", json.dumps(data))) e = MultipartEncoder(fields=fields) extra_headers = {"Content-Type": f"{e.content_type}; dp-files=True"} if e.len > 1e6: # 1 MB log.debug("Using upload monitor") fill_char = click.style("=", fg="yellow") with click.progressbar( length=e.len, width=0, show_eta=True, label="Uploading files", fill_char=fill_char ) as bar: def f(m: MultipartEncoderMonitor): # update every 100KB m.buf_bytes_read += m.bytes_read - m.prev_bytes_read m.prev_bytes_read = m.bytes_read if m.buf_bytes_read >= 1e5: # print(f"{m.buf_bytes_read=}, {m.prev_bytes_read=}") bar.update(m.buf_bytes_read) m.buf_bytes_read = 0 m = MultipartEncoderMonitor(e, callback=f) m.buf_bytes_read = 0 m.prev_bytes_read = 0 r = self.session.post(self.url, data=m, headers=extra_headers, timeout=self.timeout) else: r = self.session.post(self.url, data=e, headers=extra_headers, timeout=self.timeout) return self._process_res(r)
def __exit__(self, exc_type, exc_value, exc_traceback): log.debug(f"Removing {self.name}") if self.file.exists(): self.file.unlink() # (missing_ok=True)
################################################################################ # Tmpfile handling # We create a tmp-dir per Python execution that stores all working files, # we attempt to delete where possible, but where not, we allow the atexit handler # to cleanup for us on shutdown # This tmp-dir needs to be in the cwd rather than /tmp so can be previewed in Jupyter # To avoid cluttering up the user's cwd, we nest these inside a `dp-cache` intermediate dir cache_dir = Path("dp-cache").absolute() cache_dir.mkdir(parents=True, exist_ok=True) # Remove any old ./dp-tmp-* dirs over 24hrs old which might not have been cleaned up due to unexpected exit one_day_ago = time.time() - timedelta(days=1).total_seconds() prev_tmp_dirs = (p for p in cache_dir.glob("dp-tmp-*") if p.is_dir() and p.stat().st_mtime < one_day_ago) for p in prev_tmp_dirs: log.debug(f"Removing stale temp dir {p}") shutil.rmtree(p, ignore_errors=True) # create new dp-tmp for this session, nested inside `dp-cache` tmp_dir = Path(mkdtemp(prefix="dp-tmp-", dir=cache_dir)).absolute() class DPTmpFile: """ Generate a tempfile in dp temp dir when used as a contextmanager, deleted on removing scope otherwise, removed by atexit hook """ def __init__(self, ext: str): fd, name = mkstemp(suffix=ext, prefix="dp-tmp-", dir=tmp_dir) os.close(fd)
def check_login(config=None, cli_login: bool = False) -> SDict: r = Resource(endpoint="/settings/details/", config=config).get(cli_login=cli_login) log.debug(f"Connected successfully to DP Server as {r.username}") return t.cast(SDict, r)