def fill_notebook(nb_path, args): """Code formatting, strip output, file copy, execution and script conversion.""" if not Path(nb_path).exists(): log.info(f"File {nb_path} does not exist.") return if args.fmt: subprocess.run( [sys.executable, "-m", "gammapy", "jupyter", "--src", nb_path, "black"] ) subprocess.run( [sys.executable, "-m", "gammapy", "jupyter", "--src", nb_path, "strip"] ) log.info(f"Copying notebook {nb_path} to {PATH_NBS}") shutil.copy(nb_path, PATH_NBS) # execute notebook notebook_test(nb_path) static_nb_path = PATH_NBS / Path(nb_path).absolute().name subprocess.run( [ sys.executable, "-m", "jupyter", "nbconvert", "--to", "script", static_nb_path, ] )
def process_notebook(nb_path): """Process notebook before html conversion by nbsphinx.""" if not Path(nb_path).exists(): log.error(f"File {nb_path} does not exist.") return # declare paths folder_path = Path(os.path.dirname(nb_path)) folder_name = os.path.basename(folder_path) nb_name = nb_path.absolute().name nb_temp = PATH_TEMP / folder_name / nb_name env_file = folder_path / "env.yml" dest_path_temp = PATH_TEMP / folder_name dest_path_clean = PATH_CLEAN / folder_name dest_path_filled = PATH_FILLED / folder_name # clean in recipes clean_notebook(nb_path) # copy entire recipes content into temp folder copy_tree(str(folder_path), str(dest_path_temp)) # fill it in temp folder if notebook_test(nb_temp): # copy clean notebook and env.yml in recipes to clean folder log.info(f"Copying notebook {nb_path} to {dest_path_clean}") os.makedirs(dest_path_clean, exist_ok=True) shutil.copy(str(nb_temp), str(dest_path_clean)) shutil.copy(str(env_file), str(dest_path_clean)) # conversion to python scripts in clean folder static_nb_path = dest_path_clean / nb_name subprocess.run([ sys.executable, "-m", "jupyter", "nbconvert", "--to", "script", static_nb_path, ]) # add box in temp add_box(folder_name, nb_temp) # copy filled notebook and png files in temp to filled folder log.info(f"Copying notebook {nb_temp} to {dest_path_filled}") os.makedirs(dest_path_filled, exist_ok=True) shutil.copy(str(nb_temp), str(dest_path_filled)) images = list(dest_path_temp.glob("*.png")) for im in images: shutil.copy(str(im), str(dest_path_filled)) return True else: log.error(f"Execution failed for {str(folder_name)}/{nb_name}") return False
def main(): logging.basicConfig(level=logging.INFO) if "GAMMAPY_DATA" not in os.environ: log.info("GAMMAPY_DATA environment variable not set.") log.info("Running notebook tests requires this environment variable.") log.info("Exiting now.") sys.exit() passed = True # setup path_temp = Path("temp") path_empty_nbs = Path("tutorials") shutil.rmtree(str(path_temp), ignore_errors=True) shutil.copytree(str(path_empty_nbs), str(path_temp)) for notebook in get_notebooks(): if requirement_missing(notebook): log.info( "Skipping notebook {} because requirement is missing.".format( notebook["name"])) continue filename = notebook["name"] + ".ipynb" path = path_temp / filename if not notebook_test(path): passed = False # tear down shutil.rmtree(str(path_temp), ignore_errors=True) if not passed: sys.exit("Some tests failed. Existing now.")
def main(): if "GAMMAPY_DATA" not in os.environ: logging.info("GAMMAPY_DATA environment variable not set.") logging.info("Running notebook tests requires this environment variable.") logging.info("Exiting now.") sys.exit() passed = True yamlfile = get_notebooks() dirnbs = Path("tutorials") for notebook in yamlfile: if requirement_missing(notebook): logging.info( "Skipping notebook {} because requirement is missing.".format( notebook["name"] ) ) continue filename = notebook["name"] + ".ipynb" path = dirnbs / filename if not notebook_test(path): passed = False assert passed
def main(): logging.basicConfig(level=logging.INFO) if "GAMMAPY_DATA" not in os.environ: log.info("GAMMAPY_DATA environment variable not set.") log.info("Running notebook tests requires this environment variable.") log.info("Exiting now.") sys.exit() passed = True # setup path_temp = Path("temp") path_temp.mkdir() try: for notebook in get_notebooks(): if requirement_missing(notebook): log.info( f"Skipping notebook (requirement missing): {notebook['name']}" ) continue filename = notebook["name"] + ".ipynb" path_dest = path_temp / filename src_path = notebook["url"].replace(URL_GAMMAPY_MASTER, "") shutil.copyfile(src_path, path_dest) if not notebook_test(path_dest): passed = False finally: # tear down shutil.rmtree(path_temp, ignore_errors=True) if not passed: sys.exit("Some tests failed. Existing now.")
def build_notebooks(args): if "GAMMAPY_DATA" not in os.environ: log.info("GAMMAPY_DATA environment variable not set.") log.info("Running notebook tests requires this environment variable.") log.info("Exiting now.") sys.exit() # prepare folder structure pathsrc = Path(args.src) path_temp = Path("temp") path_empty_nbs = Path("tutorials") path_filled_nbs = Path("docs") / "notebooks" path_static_nbs = Path("docs") / "_static" / "notebooks" shutil.rmtree(path_temp, ignore_errors=True) path_temp.mkdir(parents=True, exist_ok=True) path_filled_nbs.mkdir(parents=True, exist_ok=True) path_static_nbs.mkdir(parents=True, exist_ok=True) if pathsrc == path_empty_nbs: shutil.rmtree(path_temp, ignore_errors=True) shutil.rmtree(path_static_nbs, ignore_errors=True) shutil.rmtree(path_filled_nbs, ignore_errors=True) shutil.copytree(path_empty_nbs, path_temp, ignore=ignorefiles) elif pathsrc.exists(): notebookname = pathsrc.name pathdest = path_temp / notebookname shutil.copyfile(pathsrc, pathdest) else: log.info("Notebook file does not exist.") sys.exit() subprocess.run( [sys.executable, "-m", "gammapy", "jupyter", "--src", "temp", "black"]) subprocess.run( [sys.executable, "-m", "gammapy", "jupyter", "--src", "temp", "strip"]) # test /run for path in path_temp.glob("*.ipynb"): notebook_test(path) # convert into scripts # copy generated filled notebooks to doc if pathsrc == path_empty_nbs: # copytree is needed to copy subfolder images shutil.copytree(path_empty_nbs, path_static_nbs, ignore=ignorefiles) for path in path_static_nbs.glob("*.ipynb"): subprocess.run([ sys.executable, "-m", "jupyter", "nbconvert", "--to", "script", str(path), ]) shutil.copytree(path_temp, path_filled_nbs, ignore=ignorefiles) else: pathsrc = path_temp / notebookname pathdest = path_static_nbs / notebookname shutil.copyfile(pathsrc, pathdest) subprocess.run([ sys.executable, "-m", "jupyter", "nbconvert", "--to", "script", str(pathdest), ]) pathdest = path_filled_nbs / notebookname shutil.copyfile(pathsrc, pathdest) # tear down shutil.rmtree(path_temp, ignore_errors=True)
def build_notebooks(args): if "GAMMAPY_DATA" not in os.environ: logging.info("GAMMAPY_DATA environment variable not set.") logging.info( "Running notebook tests requires this environment variable.") logging.info("Exiting now.") sys.exit() # prepare folder structure pathsrc = Path(args.src) path_temp = Path("temp") path_empty_nbs = Path("tutorials") path_filled_nbs = Path("docs") / "notebooks" path_static_nbs = Path("docs") / "_static" / "notebooks" rmtree(str(path_temp), ignore_errors=True) path_temp.mkdir(parents=True, exist_ok=True) path_filled_nbs.mkdir(parents=True, exist_ok=True) path_static_nbs.mkdir(parents=True, exist_ok=True) if pathsrc == path_empty_nbs: rmtree(str(path_temp), ignore_errors=True) rmtree(str(path_static_nbs), ignore_errors=True) rmtree(str(path_filled_nbs), ignore_errors=True) copytree(str(path_empty_nbs), str(path_temp), ignore=ignorefiles) elif pathsrc.exists(): notebookname = pathsrc.name pathdest = path_temp / notebookname copyfile(str(pathsrc), str(pathdest)) else: logging.info("Notebook file does not exist.") sys.exit() # strip and blackformat subprocess.call("gammapy jupyter --src temp black", shell=True) subprocess.call("gammapy jupyter --src temp strip", shell=True) # test /run passed = True for path in path_temp.glob("*.ipynb"): if not notebook_test(path): passed = False # convert into scripts # copy generated filled notebooks to doc # if passed: if pathsrc == path_empty_nbs: # copytree is needed to copy subfolder images copytree(str(path_empty_nbs), str(path_static_nbs), ignore=ignoreall) for path in path_static_nbs.glob("*.ipynb"): subprocess.call("jupyter nbconvert --to script '{}'".format( str(path)), shell=True) copytree(str(path_temp), str(path_filled_nbs), ignore=ignorefiles) else: pathsrc = path_temp / notebookname pathdest = path_static_nbs / notebookname copyfile(str(pathsrc), str(pathdest)) subprocess.call("jupyter nbconvert --to script '{}'".format( str(pathdest)), shell=True) pathdest = path_filled_nbs / notebookname copyfile(str(pathsrc), str(pathdest)) # else: # logging.info("Tests have not passed.") # logging.info("Tutorials not ready for documentation building process.") # rmtree(str(path_static_nbs), ignore_errors=True) # tear down rmtree(str(path_temp), ignore_errors=True)