def test_notebook_decouverte(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks
from actuariat_python.automation.notebook_test_helper import clean_function_notebook
temp = get_temp_folder(__file__, "temp_decouverte")
keepnote = [_ for _ in ls_notebooks("decouverte")]
self.assertTrue(len(keepnote) > 0)
# copy data
folder = os.path.dirname(keepnote[0])
data = [os.path.join(folder, "UN_Data.csv")]
for dt in data:
shutil.copy(dt, temp)
# run the notebooks
res = execute_notebooks(temp,
keepnote,
lambda i, n: True,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=actuariat_python)
def test_long_long_notebook_population(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fix_tkinter_issues_virtualenv()
from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
if "travis" in sys.executable:
# matplotlib is still failing
warnings.warn(
"travis, unable to test TestNotebookPopulation.test_notebook_sessions"
)
return
temp = get_temp_folder(__file__, "temp_sessions_long_long")
keepnote = [
_ for _ in ls_notebooks("sessions") if
"seance6_graphes_correction" in _ or "seance6_graphes_enonce" in _
]
self.assertTrue(len(keepnote) > 0)
clog = CustomLog(temp)
res = execute_notebooks(temp,
keepnote,
lambda i, n: "deviner" not in n,
fLOG=fLOG,
clean_function=clean_function_notebook,
detailed_log=clog)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=actuariat_python)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
this = os.path.join(temp, "..", "data",
"fr.openfoodfacts.org.products.head100.csv")
this = os.path.normpath(this)
def clean_function(cell):
cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv",
this.replace("\\", "/"))
return cell
import pyquickhelper
import jyquickhelper
import pyensae
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, thismodule])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid,
clean_function=clean_function)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_notebook_example_video(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_exemples_video")
keepnote = ls_notebooks("exemples")
self.assertTrue(len(keepnote) > 0)
source = os.path.join(os.path.dirname(keepnote[0]), "data")
images = os.path.join(temp, 'data')
os.mkdir(images)
for img in os.listdir(source):
shutil.copy(os.path.join(source, img), images)
if is_travis_or_appveyor() == "circleci":
def clean(cell):
cell = clean_function_notebook(cell)
# ValueError: Cannot embed the 'gif' image format (circleci)
cell = cell.replace('Image("video.gif")',
'# Image("video.gif")')
return cell
else:
clean = clean_function_notebook
res = execute_notebooks(temp, keepnote,
lambda i, n: "video" in n,
fLOG=fLOG, clean_function=clean)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=code_beatrix)
def a_test_notebook_runner(self,
name,
folder,
valid=None,
copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
import pyquickhelper
import jyquickhelper
import pyensae
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, thismodule])
res = execute_notebook_list(temp,
keepnote,
additional_path=add_path,
valid=valid)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_skip_run_notebook_javascript(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
if "javascript" in f:
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
def valid(cell):
return True
import jyquickhelper
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))]
try:
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=pyquickhelper)
except Exception as e:
# Issue with permission.
warnings.warn("Unable to test this notebook due to " + str(e))
def test_run_notebook(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=src.python3_module_template)
def test_notebook_example(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_exemples_long")
keepnote = ls_notebooks("exemples")
self.assertTrue(len(keepnote) > 0)
source = os.path.join(os.path.dirname(keepnote[0]), "data")
data = os.path.join(temp, 'data')
os.mkdir(data)
for img in os.listdir(source):
shutil.copy(os.path.join(source, img), data)
try:
res = execute_notebooks(temp, keepnote,
lambda i, n: "poppins" in n,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=code_beatrix)
except Exception as e: # pylint: disable=W0703
if "RegexMatchError" not in str(e):
raise e
warnings.warn("pytube issue: {}".format(e))
return
def test_notebook_example(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_exemples_long_devoxx")
keepnote = ls_notebooks(os.path.join("ateliers", "devoxx2018"))
self.assertTrue(len(keepnote) > 0)
sources = [os.path.join(os.path.dirname(keepnote[0]), "finlaby.jpg")]
for img in sources:
shutil.copy(img, temp)
try:
res = execute_notebooks(temp,
keepnote,
lambda i, n: "devoxx" in n,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=code_beatrix)
except Exception as e: # pylint: disable=W0703
if "RegexMatchError" not in str(e):
raise e
warnings.warn("pytube issue: {}".format(e))
return
def test_skip_run_notebook_git(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
if "git_data" in f:
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
def valid(cell):
if "git log --log-size --abbrev --follow" in cell:
return False
return True
import jyquickhelper
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))]
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=pyquickhelper)
def test_notebook_internet(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fix_tkinter_issues_virtualenv()
if "travis" in sys.executable:
# matplotlib is still failing
warnings.warn(
"travis, unable to test TestNotebookInternet.test_notebook_internet"
)
return
from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
from mlstatpy.data.wikipedia import download_pageviews
self.assertTrue(download_pageviews)
temp = get_temp_folder(__file__, "temp_internet")
keepnote = [
_ for _ in ls_notebooks("internet")
if "wikipedia_stats_correction" not in _
]
self.assertTrue(len(keepnote) > 0)
for k in keepnote:
fLOG(k)
res = execute_notebooks(temp,
keepnote,
lambda i, n: True,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=actuariat_python)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
if not os.path.exists(doc):
raise FileNotFoundError(doc)
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_pyq")
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
if "example_pyquickhelper" in f:
code_init = "form1={'version': 'modified', 'module': 'anything'}"
keepnote.append((os.path.join(fnb, f), code_init))
elif "having_a_form" in f:
code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}"
code_init += "\ncredential={'password': '******', 'login': '******'}"
code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}"
keepnote.append((os.path.join(fnb, f), code_init))
elif "javascript" in f:
# We skip due to connectivity issues.
pass
elif "git_data" in f:
# Too long.
pass
else:
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
def valid(cell):
if "open_html_form" in cell:
return False
if "open_window_params" in cell:
return False
if '<div style="position:absolute' in cell:
return False
if "build_script.bat" in cell:
return False
return True
import jyquickhelper
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(
os.path.abspath(os.path.dirname(jyquickhelper.__file__)),
".."))
]
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# notebooks are not converted into python 2.7, so not tested
return
ci = is_travis_or_appveyor()
if ci is None:
try:
kernel_name = install_python_kernel_for_unittest(
"python3_module_template")
except PermissionError as e:
raise Exception(
"Unable to change the kernel name. ci='{0}'".format(
ci)) from e
else:
kernel_name = None
temp = get_temp_folder(__file__, "temp_run_notebooks")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
# function to tell that a can be run
def valid(cell):
if "open_html_form" in cell:
return False
if "open_window_params" in cell:
return False
if '<div style="position:absolute' in cell:
return False
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths,
kernel_name=kernel_name)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=src.project_name)
def a_test_notebook_runner(filename,
name,
folder,
valid=None,
copy_files=None,
modules=None,
fLOG=noLOG):
"""
Runs and tests a specific list of notebooks.
The function raises an exception if the execution fails.
@param filename test filename
@param name substring to look into notebook filenames
@param folder where to look for notebooks
@param valid skip cells if valid is False, None for all valid
@param copy_files files to copy before running the notebooks.
@param modules list of extra dependencies
@param fLOG logging function
"""
filename = os.path.abspath(filename)
temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name))
doc = os.path.normpath(
os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder))
if not os.path.exists(doc):
raise FileNotFoundError(doc)
keepnote = [
os.path.join(doc, _) for _ in os.listdir(doc)
if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _
]
if len(keepnote) == 0:
raise AssertionError("No found notebook in '{0}'\n{1}".format(
doc, "\n".join(os.listdir(doc))))
if copy_files is not None:
for name_ in copy_files:
dest = os.path.join(temp, name_)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
src_file = os.path.join(doc, name_)
fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format(
src_file, dest_dir))
shutil.copy(src_file, dest_dir)
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]
if modules:
base.extend(modules)
add_path = pyq_get_additional_paths(base)
res = execute_notebook_list(temp,
keepnote,
additional_path=add_path,
valid=valid,
fLOG=fLOG)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook_js(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from sklearn.datasets import load_iris
self.assertTrue(load_iris is not None)
temp = get_temp_folder(__file__, "temp_run_notebooks_js")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \
"lightning" not in f and "pydy" not in f and "bokeh" not in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if 'bar.render(path="render.png")' in cell:
return False
if 'make_a_snapshot' in cell:
return False
return True
# file to copy
for cop in ["pydy.svg"]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
keepnote = [_ for _ in keepnote if 'mpld3' not in _]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook_js_pydy(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from sklearn.datasets import load_iris
self.assertTrue(load_iris is not None)
temp = get_temp_folder(__file__, "temp_run_notebooks_js_pydy")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \
"lightning" not in f and "pydy" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if 'bar.render(path="render.png")' in cell:
return False
return True
# file to copy
for cop in ["pydy.svg"]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
]
keepnote = [_ for _ in keepnote if 'mpld3' not in _]
# run the notebooks
try:
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=jupytalk)
except Exception as e:
if 'can only concatenate list (not "tuple") to list' in str(e):
warnings.warn("Pydy needs to be updated for Python 3.7")
return
else:
raise e
def test_run_notebook_im(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_im")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \
(is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f):
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if "animation.FuncAnimation" in cell:
return False
return True
# file to copy
for cop in [
"green_tripdata_2015-12_sample.csv",
"NYPD_Motor_Vehicle_Collisions_sample.csv",
"NYPD_Motor_Vehicle_Collisions_small.csv"
]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_long_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "nlp"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if "open_html_form" in cell:
return False
if "open_html_form" in cell:
return False
if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell:
return False
if '<div style="position:absolute' in cell:
return False
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
# run the notebooks
clog = CustomLog(temp)
clog("START")
res = execute_notebook_list(temp,
keepnote,
fLOG=clog,
deepfLOG=clog,
valid=valid,
additional_path=addpaths)
clog("END")
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "nlp"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if "open_html_form" in cell:
return False
if "open_window_params" in cell:
return False
if '<div style="position:absolute' in cell:
return False
if "completion.dot" in cell and is_travis_or_appveyor(
) == "travis":
return False
if 'Image("completion.png")' in cell and is_travis_or_appveyor(
) == "travis":
return False
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks")
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks"))
keepnote = []
for nb in os.listdir(fnb):
if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \
"ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb:
keepnote.append(os.path.join(fnb, nb))
def valid(cell):
if "snakeviz" in cell:
return False
if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell:
return False
if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell:
return False
return True
def replace_cell(cell):
return cell.replace(
"%encoding pyensae_sql_magic.ipynb",
"%encoding {0}/pyensae_sql_magic.ipynb".format(fnb))
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pymyinstall", "src"))
]
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths,
clean_function=replace_cell)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyensae)
def test_notebook_algorithm(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_algorithm")
keepnote = ls_notebooks("algorithmes")
self.assertTrue(len(keepnote) > 0)
res = execute_notebooks(temp,
keepnote,
lambda i, n: "deviner" not in n,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix)
def test_run_notebook(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_pyq")
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
if "example_pyquickhelper" in f:
code_init = "form1={'version': 'modified', 'module': 'anything'}"
keepnote.append((os.path.join(fnb, f), code_init))
elif "having_a_form" in f:
code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}"
code_init += "\ncredential={'password': '******', 'login': '******'}"
code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}"
keepnote.append((os.path.join(fnb, f), code_init))
elif "javascript" in f:
# We skip due to connectivity issues.
pass
elif "git_data" in f:
# Too long.
pass
else:
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
def valid(cell):
if "open_html_form" in cell:
return False
if "open_window_params" in cell:
return False
if '<div style="position:absolute' in cell:
return False
if "build_script.bat" in cell:
return False
return True
import jyquickhelper
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))]
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook_ml(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_ml")
try:
from mlinsights.mlmodel import PiecewiseRegressor # pylint: disable=W0611
piecewise = True
except ImportError:
piecewise = False
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "ml"))
keepnote = []
for f in os.listdir(fnb):
if "piecewise_linear_regression" in f and not piecewise:
continue
if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def a_test_notebook_runner(self, name, folder, valid=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
import pyquickhelper
import jyquickhelper
import pyensae
import jupytalk
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, jupytalk])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(filename, name, folder, valid=None, copy_files=None, modules=None, fLOG=noLOG):
"""
Runs and tests a specific list of notebooks.
The function raises an exception if the execution fails.
@param filename test filename
@param name substring to look into notebook filenames
@param folder where to look for notebooks
@param valid skip cells if valid is False, None for all valid
@param copy_files files to copy before running the notebooks.
@param modules list of extra dependencies
@param fLOG logging function
"""
filename = os.path.abspath(filename)
temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name))
doc = os.path.normpath(os.path.join(
temp, "..", "..", "..", "_doc", "notebooks", folder))
if not os.path.exists(doc):
raise FileNotFoundError(doc)
keepnote = [os.path.join(doc, _) for _ in os.listdir(
doc) if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _]
if len(keepnote) == 0:
raise AssertionError("No found notebook in '{0}'\n{1}".format(
doc, "\n".join(os.listdir(doc))))
if copy_files is not None:
for name_ in copy_files:
dest = os.path.join(temp, name_)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
src_file = os.path.join(doc, name_)
fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format(
src_file, dest_dir))
shutil.copy(src_file, dest_dir)
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]
if modules:
base.extend(modules)
add_path = pyq_get_additional_paths(base)
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid, fLOG=fLOG)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
def a_test_notebook_session(self, name):
from actuariat_python.automation.notebook_test_helper import (
ls_notebooks, execute_notebooks, clean_function_notebook)
temp = get_temp_folder(__file__, "temp_sessions_" + name.split('.')[0])
keepnote = [
_ for _ in ls_notebooks("sessions")
if "seance5_approche_fonctionnelle_enonce" not in _
and "seance6_graphes_ml_enonce" not in _
and "election_carte_electorale_correction" not in _
and "seance6_graphes_correction" not in _
and "seance6_graphes_enonce" not in _ and "ways" not in _
]
if is_travis_or_appveyor():
keepnote = [
_ for _ in keepnote if "election_carte_electorale" not in _
]
if name is not None:
keepnote = list(filter(lambda n: name in n, keepnote))
if len(keepnote) == 0:
return
for k in keepnote:
fLOG(k)
# copy data
fold = os.path.dirname(keepnote[0])
files = [os.path.join(fold, "pop-totale-france.txt")]
for name_ in files:
shutil.copy(name_, temp)
data_tem = os.path.join(temp, "data")
if not os.path.exists(data_tem):
os.mkdir(data_tem)
files = [
os.path.join(fold, "data", "housing.data"),
os.path.join(fold, "data", "housing.names"),
os.path.join(fold, "data", "multiTimeline.csv"),
]
for name_ in files:
shutil.copy(name_, data_tem)
res = execute_notebooks(temp,
keepnote,
lambda i, n: "deviner" not in n,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=actuariat_python)
def test_run_notebook_lightning_js(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() is None:
warnings.warn(
"The unit test relies on lightning-python but it shares the same name as another one. We disable it.")
return
temp = get_temp_folder(__file__, "temp_run_notebooks_lightning_js")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "lightning" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# file to copy
for cop in ["pydy.svg"]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=jupytalk)
def test_notebook_examen(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fix_tkinter_issues_virtualenv()
from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
temp = get_temp_folder(__file__, "temp_examen")
keepnote = ls_notebooks("examen")
self.assertTrue(len(keepnote) > 0)
res = execute_notebooks(temp,
keepnote,
lambda i, n: "deviner" not in n,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=actuariat_python)
def test_run_notebook_im(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_im")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \
(is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f):
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if "animation.FuncAnimation" in cell:
return False
return True
# file to copy
for cop in ["green_tripdata_2015-12_sample.csv",
"NYPD_Motor_Vehicle_Collisions_sample.csv",
"NYPD_Motor_Vehicle_Collisions_small.csv"]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(self, name, folder, valid=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
import pyquickhelper # pylint: disable=C0415
import jyquickhelper # pylint: disable=C0415
import pyensae # pylint: disable=C0415
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, thismodule])
res = execute_notebook_list(temp,
keepnote,
additional_path=add_path,
valid=valid)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_run_notebook_pyjs(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(
f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f:
if is_travis_or_appveyor() == "travis":
if "pyjs_brython" in f:
warnings.warn("Travis, skipping " + f)
continue
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_long_run_notebook(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if "open_html_form" in cell:
return False
if "open_html_form" in cell:
return False
if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell:
return False
if '<div style="position:absolute' in cell:
return False
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
]
# run the notebooks
clog = CustomLog(temp)
clog("START")
res = execute_notebook_list(temp, keepnote, fLOG=clog, deepfLOG=clog, valid=valid,
additional_path=addpaths)
clog("END")
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_dsgarden")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "dsgarden"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
# function to tell that a can be run
def valid(cell):
if "open_html_form" in cell:
return False
if "open_window_params" in cell:
return False
if '<div style="position:absolute' in cell:
return False
if 'run_dot' in cell and is_travis_or_appveyor() == "travis":
return False
if "completion.dot" in cell and is_travis_or_appveyor() == "travis":
return False
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks")
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = []
for nb in os.listdir(fnb):
if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \
"ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb:
keepnote.append(os.path.join(fnb, nb))
def valid(cell):
if "snakeviz" in cell:
return False
if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell:
return False
if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell:
return False
return True
def replace_cell(cell):
return cell.replace("%encoding pyensae_sql_magic.ipynb",
"%encoding {0}/pyensae_sql_magic.ipynb".format(fnb))
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pymyinstall", "src"))
]
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths,
clean_function=replace_cell)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=pyensae)
def test_run_notebook_pandas(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_numpy")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "numpy"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
keepnote.append(os.path.join(fnb, f))
# run the notebooks
res = execute_notebook_list(temp, keepnote, fLOG=fLOG)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=teachpyx)
def test_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks"))
keepnote = []
if not os.path.exists(fnb):
return
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
keepnote.append(os.path.join(fnb, f))
self.assertGreater(len(keepnote), 0)
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=lightmlboard)
def test_skip_run_notebook_javascript(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb":
if "javascript" in f:
keepnote.append(os.path.join(fnb, f))
self.assertTrue(len(keepnote) > 0)
def valid(cell):
return True
import jyquickhelper
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(
os.path.abspath(os.path.dirname(jyquickhelper.__file__)),
".."))
]
try:
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res,
fLOG=fLOG,
dump=pyquickhelper)
except Exception as e:
# Issue with permission.
warnings.warn("Unable to test this notebook due to " + str(e))
def test_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_image")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "image"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src"))
]
# copy
shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp)
# run the notebooks
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook_ml(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_ml")
try:
from mlinsights.mlmodel import PiecewiseRegressor # pylint: disable=W0611
piecewise = True
except ImportError:
piecewise = False
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "ml"))
keepnote = []
for f in os.listdir(fnb):
if "piecewise_linear_regression" in f and not piecewise:
continue
if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=mlstatpy)
def test_notebook_ai(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_ai")
keepnote = ls_notebooks("ai")
self.assertTrue(len(keepnote) > 0)
source = os.path.join(os.path.dirname(keepnote[0]), "images")
images = os.path.join(temp, 'images')
os.mkdir(images)
for img in os.listdir(source):
shutil.copy(os.path.join(source, img), images)
res = execute_notebooks(temp,
keepnote,
lambda i, n: True,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix)
def test_cache_urls(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_cache_urls")
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = [os.path.join(fnb, "example_about_files.ipynb")]
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))]
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=None, additional_path=addpaths,
cache_urls=["https://docs.python.org/3.4/library/urllib.request.html"])
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook_pyjs(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f:
if is_travis_or_appveyor() == "travis":
if "pyjs_brython" in f:
warnings.warn("Travis, skipping " + f)
continue
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_image")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "image"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src"))
]
# copy
shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp)
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook_gui(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_gui")
# selection of notebooks
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "gui_" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if ".show" in cell:
return False
return True
# additionnal path to add
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src"))
]
# run the notebooks
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
this = os.path.join(temp, "..", "data",
"fr.openfoodfacts.org.products.head100.csv")
this = os.path.normpath(this)
def clean_function(cell):
cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv",
this.replace("\\", "/"))
return cell
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid,
clean_function=clean_function)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=ensae_teaching_cs)
def execute_notebooks(folder, notebooks, filter, clean_function=None,
fLOG=noLOG, deepfLOG=noLOG, replacements=None, dump=None,
additional_path=None):
"""
Executes a list of notebooks.
@param folder folder
@param notebooks list of notebooks
@param filter function which validates the notebooks to test (True means will be tested)
@param clean_function cleaning function to apply to the code before running it
@param fLOG logging function
@param deepfLOG logging function used to run the notebook
@param replacements replacements
@param dump see function `execute_notebook_list_finalize_ut
<http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html#
pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_
@param additional_path additional path to add
@return dictionary { notebook_file: (isSuccess, outout) }
The signature of function ``filter`` is::
def filter(i, filename):
return True or False
"""
def valid_cell(cell):
if "%system" in cell:
return False
if "df.plot(...)" in cell:
return False
if 'df["difference"] = ...' in cell:
return False
if 'remote_open' in cell:
return None
if 'blobpassword' in cell:
return None
if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell:
return False
if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell:
return False
if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell:
return False
if "%%SPEAK fr-FR" in cell:
return False
if " noeud tri n'est pas encore défini" in cell:
return False
if "nuplet[1] = 5" in cell:
return False
if cell == "dico[0]":
return False
if cell == "dico[ [4,6] ] = 6":
return False
return True
addpaths = get_additional_paths()
if additional_path is not None:
addpaths += additional_path
if filter:
notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
if len(notebooks) == 0:
raise ValueError("Empty list of notebooks.")
res = execute_notebook_list(folder, notebooks, fLOG=fLOG, clean_function=clean_function,
valid=valid_cell, additional_path=addpaths,
replacements=replacements)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=dump)
return res
