def generate_py_from_ipynb(ipynb_path, output_dir="."):
if not ipynb_path.endswith(".ipynb"):
logger.error("Did not expect file extension. Expected .ipynb, got %s",
os.path.splitext(ipynb_path)[1])
return None
mkdir_p(output_dir)
filename_no_extension = os.path.basename(os.path.splitext(ipynb_path)[0])
output_path = os.path.join(output_dir, filename_no_extension + ".py")
ipynb = jupytext.read(ipynb_path)
jupytext.write(ipynb, output_path)
logger.info("Successfully converted %s -> %s", ipynb_path, output_path)
return output_path
def generate_ipynb_from_py(
template_base_dir: str,
report_name: str,
notebooker_disable_git: bool,
py_template_dir: str,
warn_on_local: Optional[bool] = True,
) -> str:
"""
This method EITHER:
Pulls the latest version of the notebook templates from git, and regenerates templates if there is a new HEAD
OR: finds the local template from the template repository using a relative path
In both cases, this method converts the .py file into an .ipynb file which can be executed by papermill.
:param template_base_dir: The directory in which converted notebook templates reside.
:param report_name: The name of the report which we are running.
:param notebooker_disable_git: Whether or not to pull the latest version from git, if a change is available.
:param py_template_dir: The directory which contains raw python templates. This should be a subdir in a git repo.
:param warn_on_local: Whether to warn when we are searching for notebooks in the notebooker repo itself.
:return: The filepath of the .ipynb which we have just converted.
"""
report_path = convert_report_name_into_path(report_name)
python_template_path = _get_python_template_path(report_path,
warn_on_local,
py_template_dir)
output_template_path = _ipynb_output_path(
template_base_dir, report_path,
_get_output_path_hex(notebooker_disable_git, py_template_dir))
try:
with open(output_template_path, "r") as f:
if f.read():
print("Loading ipynb from cached location: %s",
output_template_path)
return output_template_path
except IOError:
pass
# "touch" the output file
print("Creating ipynb at: %s", output_template_path)
mkdir_p(os.path.dirname(output_template_path))
with open(output_template_path, "w") as f:
os.utime(output_template_path, None)
jupytext_nb = jupytext.read(python_template_path)
jupytext_nb["metadata"]["kernelspec"] = kernel_spec(
) # Override the kernel spec since we want to run it..
jupytext.write(jupytext_nb, output_template_path)
return output_template_path
def test_get_directory_structure():
temp_dir = tempfile.mkdtemp()
try:
paths = [
"hello.py",
"goodbye.py",
"depth/1.py",
"this/is/very/deep.py",
"depth/2.py",
"this/is/deep.py",
"this/report.py",
]
for path in paths:
abspath = os.path.join(temp_dir, path)
if "/" in path:
mkdir_p(os.path.dirname(abspath))
with open(abspath, "w") as f:
f.write("#hello")
expected_structure = {
"hello": None,
"goodbye": None,
"depth": {
"depth/1": None,
"depth/2": None
},
"this": {
"this/report": None,
"is": {
"this/is/deep": None,
"very": {
"this/is/very/deep": None
}
}
},
}
assert get_directory_structure(temp_dir) == expected_structure
finally:
shutil.rmtree(temp_dir)