def drop_privileges(out_dir: Path,
uid_name: str = "nobody",
gid_name: str = "nogroup") -> None:
"""Drop to non-root user if required. TODO - do we still need this??"""
import grp
import subprocess
# from: https://stackoverflow.com/questions/2699907/dropping-root-permissions-in-python
if os.getuid() != 0:
return
try:
# Get the uid/gid from the name
running_uid = pwd.getpwnam(uid_name).pw_uid
running_gid = grp.getgrnam(gid_name).gr_gid
# chown the output dir
subprocess.run(["chown", "-R", f"{uid_name}:{gid_name}",
str(out_dir)],
check=True)
# Remove group privileges
os.setgroups([])
# Try setting the new uid/gid
os.setgid(running_gid)
os.setuid(running_uid)
# Ensure a very conservative umask
os.umask(0o77)
except Exception:
log.warning("Error dropping privileges to non-root user, continuing")
def setup_script(s: api.Script, env_dir: Path):
"""Setup the script - unpack & install deps"""
# TODO - add local cache check here
if env_dir.exists():
log.debug("Package already exists, not redownloading")
return None
# download and unpack bundle locally into env_dir
sdist = s.download_pkg()
assert tarfile.is_tarfile(sdist), "Invalid sdist file"
shutil.unpack_archive(sdist, extract_dir=env_dir, format="gztar")
sdist.unlink()
comp_r = compileall.compile_dir(env_dir, force=True, workers=1, quiet=1)
if not comp_r:
log.warning("Compiling script bundle failed - errors may occur")
# install deps
if s.requirements:
pip_args = [sys.executable, "-m", "pip", "install"]
if os.getuid() != 0 and not in_venv():
# we're a normal/non-root user outside a venv
pip_args.append("--user")
pip_args.extend(s.requirements)
log.debug(f"Calling pip as '{pip_args}'")
subprocess.run(args=pip_args, check=True)
importlib.invalidate_caches() # ensure new packages are detected
log.info(f"Successfully installed bundle for script {s.id}")
def run_api(run_config: RunnerConfig) -> RunResult:
"""Bootstrap the recursive calls into run"""
script = api.Script.by_id(run_config.script_id)
# is the script compatible with the client runner/api
if not is_version_compatible(
__version__, script.api_version, raise_exception=False):
log.warning(
f"Script developed for an older version of Datapane ({script.api_version}) - "
+ "this run may fail, please update.")
# TODO - we should pull param defaults from script and add in the call
script.call(run_config.env, **run_config.format())
# create the RunResult
script_result = str(api.Result.get()) if api.Result.exists() else None
report_id = None
try:
report = api._report.pop()
log.debug(f"Returning report id {report.id}")
report_id = report.id
except IndexError:
log.debug(
"User script didn't generate report - perhaps result / action only"
)
return RunResult(report_id=report_id, script_result=script_result)
def preview(self, width: int = 600, height: int = 500) -> None:
"""
Preview the report inside your currently running Jupyter notebook
Args:
width: Width of the report preview in Jupyter (default: 600)
height: Height of the report preview in Jupyter (default: 500)
"""
if is_jupyter():
from IPython.display import IFrame
# Remove the previous temp report if it's been generated
if self._tmp_report and self._tmp_report.exists():
self._tmp_report.unlink()
# We need to copy the report HTML to a local temp file,
# as most browsers block iframes to absolute local paths.
tmpfile = DPTmpFile(ext=".html")
if self._last_saved:
# Copy to tmp file if already saved
shutil.copy(self._last_saved, tmpfile.name)
else:
# Else save directly to tmp file
self.save(path=tmpfile.name)
self._tmp_report = tmpfile.file
# NOTE - iframe must be relative path
iframe_src = self._tmp_report.relative_to(Path(".").absolute())
return IFrame(src=str(iframe_src), width=width, height=height)
else:
log.warning("Can't preview - are you running in Jupyter?")
def write_file(self, f: TextIO, dataframe: pd.DataFrame):
n_cells = dataframe.shape[0] * dataframe.shape[1]
if n_cells > self.TABLE_CELLS_LIMIT:
log.warning(
f"Dataframe is has more than {self.TABLE_CELLS_LIMIT} cells. Omitting output."
)
# TODO - this should truncate rather than replace
f.write(
f"<table><tr><td>omitted as over {self.TABLE_CELLS_LIMIT} cells</td></tr></table>"
)
else:
dataframe.to_html(f)
def _setup_template(self) -> Template:
""" Jinja template setup for local rendering """
# check we have the files, download if not
self.assets = ir.files("datapane.resources.local_report")
if not (self.assets / self.asset_js).exists():
log.warning("Can't find report assets, downloading")
do_download_file(f"{self.asset_url}/{self.asset_js}",
self.assets / self.asset_js)
do_download_file(f"{self.asset_url}/{self.asset_css}",
self.assets / self.asset_css)
template_loader = FileSystemLoader(self.assets)
template_env = Environment(loader=template_loader)
template_env.globals["include_raw"] = include_raw
self.template = template_env.get_template("template.html")
def create_initial(cls,
config_file: Path = None,
script: Path = None,
**kw) -> "DatapaneCfg":
raw_config = {}
if config_file:
assert config_file.exists()
else:
config_file = DATAPANE_YAML
if config_file.exists():
# read config from the yaml file
log.debug(f"Reading datapane config file at {config_file}")
with config_file.open("r") as f:
raw_config = yaml.safe_load(f)
elif PYPROJECT_TOML.exists():
# TODO - implement pyproject parsing
log.warning(
"pyproject.toml found but not currently supported - ignoring")
raw_config = {}
elif script:
# we don't have a default config - perhaps in the script file
# TODO - try read config from source-code
abs_script = config_file.parent / script
if script.suffix == ".ipynb":
log.debug("Converting notebook")
mod_code = extract_py_notebook(abs_script)
else:
mod_code = abs_script.read_text()
log.debug("Reading config from python script/notebook")
log.debug(mod_code)
# overwrite config with command-line options
if script:
raw_config.update(script=script)
raw_config.update(kw)
readme = config_file.parent / "README.md"
if readme.exists():
raw_config["description"] = readme.read_text()
elif "description" not in raw_config:
raw_config["description"] = cls.description
dp_cfg = dacite.from_dict(cls,
data=raw_config,
config=dacite.Config(cast=[Path]))
return dp_cfg
def _check_version(name: str, _v: v.Version, ss: SpecifierSet):
if _v not in ss:
log.warning(
f"{name} version {_v} is not supported, your plots may not display correctly, please install version {ss}"
)
