def test_notebook_runner(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
notebook = os.path.split(__file__)[-1].replace(".ipynb",
"").replace(".py",
"")[5:]
temp = get_temp_folder(__file__, "temp_" + notebook)
nbfile = os.path.join(temp, "..", "..", "..", "_doc", "notebooks",
"%s.ipynb" % notebook)
if not os.path.exists(nbfile):
raise FileNotFoundError(nbfile)
addpath = [
os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
os.path.normpath(
os.path.join(temp, "..", "..", "..", "..", "pyquickhelper",
"src")),
]
outfile = os.path.join(temp, "out_notebook.ipynb")
assert not os.path.exists(outfile)
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"pyensae")
out = run_notebook(nbfile,
working_dir=temp,
outfilename=outfile,
additional_path=addpath,
kernel_name=kernel_name)
fLOG(out)
assert os.path.exists(outfile)
def test_notebook_runner_flat2db3(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
notebook = os.path.split(
__file__)[-1].replace(".ipynb", "").replace(".py", "")[5:]
temp = get_temp_folder(__file__, "temp_" + notebook)
nbfile = os.path.join(
temp,
"..",
"..",
"..",
"_doc",
"notebooks",
"%s.ipynb" %
notebook)
if not os.path.exists(nbfile):
raise FileNotFoundError(nbfile)
addpath = [os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
os.path.normpath(
os.path.join(
temp,
"..",
"..",
"..",
"..",
"pyquickhelper",
"src")),
os.path.normpath(
os.path.join(
temp,
"..",
"..",
"..",
"..",
"pymyinstall",
"src")),
]
outfile = os.path.join(temp, "out_notebook.ipynb")
assert not os.path.exists(outfile)
def valid(code):
return 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' not in code
if "travis" in sys.executable:
return
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"pyensae")
out = run_notebook(nbfile,
working_dir=temp,
outfilename=outfile,
additional_path=addpath,
valid=valid,
fLOG=fLOG,
kernel_name=kernel_name)
fLOG(out)
assert os.path.exists(outfile)
def execute_notebooks(folder, notebooks, filter,
clean_function=None,
fLOG=noLOG,
deepfLOG=noLOG,
valid_cell=None):
"""
execute a list of notebooks
@param folder folder
@param notebooks list of notebooks
@param filter function which validate the notebooks
@param clean_function cleaning function to apply to the code before running it
@param fLOG logging function
@param deepfLOG logging function used to run the notebook
@param valid_cell to disable the execution of a cell
@return dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })
The signature of function ``filter`` is::
def filter( i, filename) : return True or False
"""
def _valid_cell(cell):
if valid_cell is not None:
if not valid_cell(cell):
return False
if "%system" in cell:
return False
if "df.plot(...)" in cell:
return False
if 'df["difference"] = ...' in cell:
return False
if 'print(next(it))' in cell:
return False
return True
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"ensae_projects")
addpath = get_additional_paths()
results = {}
for i, note in enumerate(notebooks):
if filter(i, note):
fLOG("******", i, os.path.split(note)[-1])
outfile = os.path.join(folder, "out_" + os.path.split(note)[-1])
try:
stat, out = run_notebook(note, working_dir=folder, outfilename=outfile,
additional_path=addpath, valid=valid_cell,
clean_function=clean_function, fLOG=deepfLOG,
kernel_name=kernel_name)
if not os.path.exists(outfile):
raise FileNotFoundError(outfile)
results[note] = (True, stat, out)
except Exception as e:
results[note] = (False, None, e)
return results
def execute_notebooks(folder, notebooks, filter, clean_function=None,
fLOG=noLOG, deepfLOG=noLOG, replacements=None):
"""
execute a list of notebooks
@param folder folder
@param notebooks list of notebooks
@param filter function which validates the notebooks to test (True means will be tested)
@param clean_function cleaning function to apply to the code before running it
@param fLOG logging function
@param deepfLOG logging function used to run the notebook
@param replacements replacements
@return dictionary { notebook_file: (isSuccess, outout) }
The signature of function ``filter`` is::
def filter( i, filename) : return True or False
"""
def valid_cell(cell):
if "%system" in cell:
return False
if "df.plot(...)" in cell:
return False
if 'df["difference"] = ...' in cell:
return False
if 'remote_open' in cell:
return None
if 'blobpassword' in cell:
return None
return True
addpath = get_additional_paths()
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"ensae_teaching_cs")
results = {}
tested = []
for i, note in enumerate(notebooks):
if filter(i, note):
fLOG("******", i, os.path.split(note)[-1])
tested.append(note)
outfile = os.path.join(folder, "out_" + os.path.split(note)[-1])
try:
stat, out = run_notebook(note, working_dir=folder, outfilename=outfile,
additional_path=addpath, valid=valid_cell,
clean_function=clean_function, fLOG=deepfLOG,
kernel_name=kernel_name, replacements=replacements)
if not os.path.exists(outfile):
raise FileNotFoundError(outfile)
results[note] = (True, stat, out)
except Exception as e:
results[note] = (False, None, e)
if len(tested) == 0:
raise Exception("no notebook were tested with '{0}'".format(filter))
return results
def test_notebook_runner_magic_command(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
notebook = os.path.split(
__file__)[-1].replace(".ipynb", "").replace(".py", "")[5:]
temp = get_temp_folder(__file__, "temp_" + notebook)
nbfile = os.path.join(
temp,
"..",
"..",
"..",
"_doc",
"notebooks",
"%s.ipynb" %
notebook)
if not os.path.exists(nbfile):
raise FileNotFoundError(nbfile)
addpath = [os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
os.path.normpath(
os.path.join(
temp,
"..",
"..",
"..",
"..",
"pyquickhelper",
"src")),
]
cps = [os.path.normpath(nbfile),
os.path.join(os.path.dirname(nbfile), "pyensae_sql_magic.ipynb")]
for cp in cps:
if not os.path.exists(cp):
raise FileNotFoundError(cp)
shutil.copy(cp, temp)
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"pyensae")
outfile = os.path.join(temp, "out_notebook.ipynb")
assert not os.path.exists(outfile)
out = run_notebook(
nbfile,
working_dir=temp,
outfilename=outfile,
additional_path=addpath,
kernel_name=kernel_name)
fLOG(out)
assert os.path.exists(outfile)
def execute_notebooks(folder,
notebooks,
filter,
clean_function=None,
fLOG=noLOG,
deepfLOG=noLOG,
replacements=None,
dump=None,
additional_path=None):
"""
Executes a list of notebooks.
@param folder folder
@param notebooks list of notebooks
@param filter function which validates the notebooks to test (True means will be tested)
@param clean_function cleaning function to apply to the code before running it
@param fLOG logging function
@param deepfLOG logging function used to run the notebook
@param replacements replacements
@param dump see function `execute_notebook_list_finalize_ut <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html#pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_
@param additional_path additional path to add
@return dictionary { notebook_file: (isSuccess, outout) }
The signature of function ``filter`` is::
def filter(i, filename):
return True or False
"""
def valid_cell(cell):
if "%system" in cell:
return False
if "df.plot(...)" in cell:
return False
if 'df["difference"] = ...' in cell:
return False
if 'remote_open' in cell:
return None
if 'blobpassword' in cell:
return None
if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell:
return False
if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell:
return False
if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell:
return False
if "%%SPEAK fr-FR" in cell:
return False
if " noeud tri n'est pas encore défini" in cell:
return False
return True
addpaths = get_additional_paths()
if additional_path is not None:
addpaths += additional_path
kernel_name = None if is_travis_or_appveyor(
) else install_python_kernel_for_unittest("ensae_teaching_cs")
if filter:
notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
if len(notebooks) == 0:
raise ValueError("Empty list of notebooks.")
res = execute_notebook_list(folder,
notebooks,
fLOG=fLOG,
clean_function=clean_function,
valid=valid_cell,
additional_path=addpaths,
kernel_name=kernel_name,
replacements=replacements)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=dump)
return res
