def fill_notebook(nb_path, args):
"""Code formatting, strip output, file copy, execution and script conversion."""
if not Path(nb_path).exists():
log.info(f"File {nb_path} does not exist.")
return
if args.fmt:
subprocess.run(
[sys.executable, "-m", "gammapy", "jupyter", "--src", nb_path, "black"]
)
subprocess.run(
[sys.executable, "-m", "gammapy", "jupyter", "--src", nb_path, "strip"]
)
log.info(f"Copying notebook {nb_path} to {PATH_NBS}")
shutil.copy(nb_path, PATH_NBS)
# execute notebook
notebook_test(nb_path)
static_nb_path = PATH_NBS / Path(nb_path).absolute().name
subprocess.run(
[
sys.executable,
"-m",
"jupyter",
"nbconvert",
"--to",
"script",
static_nb_path,
]
)
def process_notebook(nb_path):
"""Process notebook before html conversion by nbsphinx."""
if not Path(nb_path).exists():
log.error(f"File {nb_path} does not exist.")
return
# declare paths
folder_path = Path(os.path.dirname(nb_path))
folder_name = os.path.basename(folder_path)
nb_name = nb_path.absolute().name
nb_temp = PATH_TEMP / folder_name / nb_name
env_file = folder_path / "env.yml"
dest_path_temp = PATH_TEMP / folder_name
dest_path_clean = PATH_CLEAN / folder_name
dest_path_filled = PATH_FILLED / folder_name
# clean in recipes
clean_notebook(nb_path)
# copy entire recipes content into temp folder
copy_tree(str(folder_path), str(dest_path_temp))
# fill it in temp folder
if notebook_test(nb_temp):
# copy clean notebook and env.yml in recipes to clean folder
log.info(f"Copying notebook {nb_path} to {dest_path_clean}")
os.makedirs(dest_path_clean, exist_ok=True)
shutil.copy(str(nb_temp), str(dest_path_clean))
shutil.copy(str(env_file), str(dest_path_clean))
# conversion to python scripts in clean folder
static_nb_path = dest_path_clean / nb_name
subprocess.run([
sys.executable,
"-m",
"jupyter",
"nbconvert",
"--to",
"script",
static_nb_path,
])
# add box in temp
add_box(folder_name, nb_temp)
# copy filled notebook and png files in temp to filled folder
log.info(f"Copying notebook {nb_temp} to {dest_path_filled}")
os.makedirs(dest_path_filled, exist_ok=True)
shutil.copy(str(nb_temp), str(dest_path_filled))
images = list(dest_path_temp.glob("*.png"))
for im in images:
shutil.copy(str(im), str(dest_path_filled))
return True
else:
log.error(f"Execution failed for {str(folder_name)}/{nb_name}")
return False
def main():
logging.basicConfig(level=logging.INFO)
if "GAMMAPY_DATA" not in os.environ:
log.info("GAMMAPY_DATA environment variable not set.")
log.info("Running notebook tests requires this environment variable.")
log.info("Exiting now.")
sys.exit()
passed = True
# setup
path_temp = Path("temp")
path_empty_nbs = Path("tutorials")
shutil.rmtree(str(path_temp), ignore_errors=True)
shutil.copytree(str(path_empty_nbs), str(path_temp))
for notebook in get_notebooks():
if requirement_missing(notebook):
log.info(
"Skipping notebook {} because requirement is missing.".format(
notebook["name"]))
continue
filename = notebook["name"] + ".ipynb"
path = path_temp / filename
if not notebook_test(path):
passed = False
# tear down
shutil.rmtree(str(path_temp), ignore_errors=True)
if not passed:
sys.exit("Some tests failed. Existing now.")
def main():
if "GAMMAPY_DATA" not in os.environ:
logging.info("GAMMAPY_DATA environment variable not set.")
logging.info("Running notebook tests requires this environment variable.")
logging.info("Exiting now.")
sys.exit()
passed = True
yamlfile = get_notebooks()
dirnbs = Path("tutorials")
for notebook in yamlfile:
if requirement_missing(notebook):
logging.info(
"Skipping notebook {} because requirement is missing.".format(
notebook["name"]
)
)
continue
filename = notebook["name"] + ".ipynb"
path = dirnbs / filename
if not notebook_test(path):
passed = False
assert passed
def main():
logging.basicConfig(level=logging.INFO)
if "GAMMAPY_DATA" not in os.environ:
log.info("GAMMAPY_DATA environment variable not set.")
log.info("Running notebook tests requires this environment variable.")
log.info("Exiting now.")
sys.exit()
passed = True
# setup
path_temp = Path("temp")
path_temp.mkdir()
try:
for notebook in get_notebooks():
if requirement_missing(notebook):
log.info(
f"Skipping notebook (requirement missing): {notebook['name']}"
)
continue
filename = notebook["name"] + ".ipynb"
path_dest = path_temp / filename
src_path = notebook["url"].replace(URL_GAMMAPY_MASTER, "")
shutil.copyfile(src_path, path_dest)
if not notebook_test(path_dest):
passed = False
finally:
# tear down
shutil.rmtree(path_temp, ignore_errors=True)
if not passed:
sys.exit("Some tests failed. Existing now.")
def build_notebooks(args):
if "GAMMAPY_DATA" not in os.environ:
log.info("GAMMAPY_DATA environment variable not set.")
log.info("Running notebook tests requires this environment variable.")
log.info("Exiting now.")
sys.exit()
# prepare folder structure
pathsrc = Path(args.src)
path_temp = Path("temp")
path_empty_nbs = Path("tutorials")
path_filled_nbs = Path("docs") / "notebooks"
path_static_nbs = Path("docs") / "_static" / "notebooks"
shutil.rmtree(path_temp, ignore_errors=True)
path_temp.mkdir(parents=True, exist_ok=True)
path_filled_nbs.mkdir(parents=True, exist_ok=True)
path_static_nbs.mkdir(parents=True, exist_ok=True)
if pathsrc == path_empty_nbs:
shutil.rmtree(path_temp, ignore_errors=True)
shutil.rmtree(path_static_nbs, ignore_errors=True)
shutil.rmtree(path_filled_nbs, ignore_errors=True)
shutil.copytree(path_empty_nbs, path_temp, ignore=ignorefiles)
elif pathsrc.exists():
notebookname = pathsrc.name
pathdest = path_temp / notebookname
shutil.copyfile(pathsrc, pathdest)
else:
log.info("Notebook file does not exist.")
sys.exit()
subprocess.run(
[sys.executable, "-m", "gammapy", "jupyter", "--src", "temp", "black"])
subprocess.run(
[sys.executable, "-m", "gammapy", "jupyter", "--src", "temp", "strip"])
# test /run
for path in path_temp.glob("*.ipynb"):
notebook_test(path)
# convert into scripts
# copy generated filled notebooks to doc
if pathsrc == path_empty_nbs:
# copytree is needed to copy subfolder images
shutil.copytree(path_empty_nbs, path_static_nbs, ignore=ignorefiles)
for path in path_static_nbs.glob("*.ipynb"):
subprocess.run([
sys.executable,
"-m",
"jupyter",
"nbconvert",
"--to",
"script",
str(path),
])
shutil.copytree(path_temp, path_filled_nbs, ignore=ignorefiles)
else:
pathsrc = path_temp / notebookname
pathdest = path_static_nbs / notebookname
shutil.copyfile(pathsrc, pathdest)
subprocess.run([
sys.executable,
"-m",
"jupyter",
"nbconvert",
"--to",
"script",
str(pathdest),
])
pathdest = path_filled_nbs / notebookname
shutil.copyfile(pathsrc, pathdest)
# tear down
shutil.rmtree(path_temp, ignore_errors=True)
def build_notebooks(args):
if "GAMMAPY_DATA" not in os.environ:
logging.info("GAMMAPY_DATA environment variable not set.")
logging.info(
"Running notebook tests requires this environment variable.")
logging.info("Exiting now.")
sys.exit()
# prepare folder structure
pathsrc = Path(args.src)
path_temp = Path("temp")
path_empty_nbs = Path("tutorials")
path_filled_nbs = Path("docs") / "notebooks"
path_static_nbs = Path("docs") / "_static" / "notebooks"
rmtree(str(path_temp), ignore_errors=True)
path_temp.mkdir(parents=True, exist_ok=True)
path_filled_nbs.mkdir(parents=True, exist_ok=True)
path_static_nbs.mkdir(parents=True, exist_ok=True)
if pathsrc == path_empty_nbs:
rmtree(str(path_temp), ignore_errors=True)
rmtree(str(path_static_nbs), ignore_errors=True)
rmtree(str(path_filled_nbs), ignore_errors=True)
copytree(str(path_empty_nbs), str(path_temp), ignore=ignorefiles)
elif pathsrc.exists():
notebookname = pathsrc.name
pathdest = path_temp / notebookname
copyfile(str(pathsrc), str(pathdest))
else:
logging.info("Notebook file does not exist.")
sys.exit()
# strip and blackformat
subprocess.call("gammapy jupyter --src temp black", shell=True)
subprocess.call("gammapy jupyter --src temp strip", shell=True)
# test /run
passed = True
for path in path_temp.glob("*.ipynb"):
if not notebook_test(path):
passed = False
# convert into scripts
# copy generated filled notebooks to doc
# if passed:
if pathsrc == path_empty_nbs:
# copytree is needed to copy subfolder images
copytree(str(path_empty_nbs), str(path_static_nbs), ignore=ignoreall)
for path in path_static_nbs.glob("*.ipynb"):
subprocess.call("jupyter nbconvert --to script '{}'".format(
str(path)),
shell=True)
copytree(str(path_temp), str(path_filled_nbs), ignore=ignorefiles)
else:
pathsrc = path_temp / notebookname
pathdest = path_static_nbs / notebookname
copyfile(str(pathsrc), str(pathdest))
subprocess.call("jupyter nbconvert --to script '{}'".format(
str(pathdest)),
shell=True)
pathdest = path_filled_nbs / notebookname
copyfile(str(pathsrc), str(pathdest))
# else:
# logging.info("Tests have not passed.")
# logging.info("Tutorials not ready for documentation building process.")
# rmtree(str(path_static_nbs), ignore_errors=True)
# tear down
rmtree(str(path_temp), ignore_errors=True)
