def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_skip_run_notebook_javascript(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "javascript" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): return True import jyquickhelper addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))] try: res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper) except Exception as e: # Issue with permission. warnings.warn("Unable to test this notebook due to " + str(e))
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) this = os.path.join(temp, "..", "data", "fr.openfoodfacts.org.products.head100.csv") this = os.path.normpath(this) def clean_function(cell): cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv", this.replace("\\", "/")) return cell import pyquickhelper import jyquickhelper import pyensae add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, thismodule]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid, clean_function=clean_function) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) import pyquickhelper import jyquickhelper import pyensae add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, thismodule]) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_skip_run_notebook_git(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "git_data" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "git log --log-size --abbrev --follow" in cell: return False return True import jyquickhelper addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=src.python3_module_template)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyq") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "example_pyquickhelper" in f: code_init = "form1={'version': 'modified', 'module': 'anything'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "having_a_form" in f: code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}" code_init += "\ncredential={'password': '******', 'login': '******'}" code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "javascript" in f: # We skip due to connectivity issues. pass elif "git_data" in f: # Too long. pass else: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "build_script.bat" in cell: return False return True import jyquickhelper addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join( os.path.abspath(os.path.dirname(jyquickhelper.__file__)), "..")) ] res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # notebooks are not converted into python 2.7, so not tested return ci = is_travis_or_appveyor() if ci is None: try: kernel_name = install_python_kernel_for_unittest( "python3_module_template") except PermissionError as e: raise Exception( "Unable to change the kernel name. ci='{0}'".format( ci)) from e else: kernel_name = None temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=src.project_name)
def a_test_notebook_runner(filename, name, folder, valid=None, copy_files=None, modules=None, fLOG=noLOG): """ Runs and tests a specific list of notebooks. The function raises an exception if the execution fails. @param filename test filename @param name substring to look into notebook filenames @param folder where to look for notebooks @param valid skip cells if valid is False, None for all valid @param copy_files files to copy before running the notebooks. @param modules list of extra dependencies @param fLOG logging function """ filename = os.path.abspath(filename) temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name)) doc = os.path.normpath( os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [ os.path.join(doc, _) for _ in os.listdir(doc) if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _ ] if len(keepnote) == 0: raise AssertionError("No found notebook in '{0}'\n{1}".format( doc, "\n".join(os.listdir(doc)))) if copy_files is not None: for name_ in copy_files: dest = os.path.join(temp, name_) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.mkdir(dest_dir) src_file = os.path.join(doc, name_) fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format( src_file, dest_dir)) shutil.copy(src_file, dest_dir) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs] if modules: base.extend(modules) add_path = pyq_get_additional_paths(base) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid, fLOG=fLOG) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook_js(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from sklearn.datasets import load_iris self.assertTrue(load_iris is not None) temp = get_temp_folder(__file__, "temp_run_notebooks_js") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \ "lightning" not in f and "pydy" not in f and "bokeh" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if 'bar.render(path="render.png")' in cell: return False if 'make_a_snapshot' in cell: return False return True # file to copy for cop in ["pydy.svg"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] keepnote = [_ for _ in keepnote if 'mpld3' not in _] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook_js_pydy(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from sklearn.datasets import load_iris self.assertTrue(load_iris is not None) temp = get_temp_folder(__file__, "temp_run_notebooks_js_pydy") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \ "lightning" not in f and "pydy" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if 'bar.render(path="render.png")' in cell: return False return True # file to copy for cop in ["pydy.svg"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] keepnote = [_ for _ in keepnote if 'mpld3' not in _] # run the notebooks try: res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk) except Exception as e: if 'can only concatenate list (not "tuple") to list' in str(e): warnings.warn("Pydy needs to be updated for Python 3.7") return else: raise e
def test_run_notebook_im(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_im") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \ (is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f): keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "animation.FuncAnimation" in cell: return False return True # file to copy for cop in [ "green_tripdata_2015-12_sample.csv", "NYPD_Motor_Vehicle_Collisions_sample.csv", "NYPD_Motor_Vehicle_Collisions_small.csv" ]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_nlp") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "completion.dot" in cell and is_travis_or_appveyor( ) == "travis": return False if 'Image("completion.png")' in cell and is_travis_or_appveyor( ) == "travis": return False return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for nb in os.listdir(fnb): if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \ "ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb: keepnote.append(os.path.join(fnb, nb)) def valid(cell): if "snakeviz" in cell: return False if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell: return False if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell: return False return True def replace_cell(cell): return cell.replace( "%encoding pyensae_sql_magic.ipynb", "%encoding {0}/pyensae_sql_magic.ipynb".format(fnb)) addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pymyinstall", "src")) ] res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, clean_function=replace_cell) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyensae)
def test_long_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_nlp") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_html_form" in cell: return False if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks clog = CustomLog(temp) clog("START") res = execute_notebook_list(temp, keepnote, fLOG=clog, deepfLOG=clog, valid=valid, additional_path=addpaths) clog("END") execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyq") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "example_pyquickhelper" in f: code_init = "form1={'version': 'modified', 'module': 'anything'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "having_a_form" in f: code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}" code_init += "\ncredential={'password': '******', 'login': '******'}" code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "javascript" in f: # We skip due to connectivity issues. pass elif "git_data" in f: # Too long. pass else: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "build_script.bat" in cell: return False return True import jyquickhelper addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook_ml(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_ml") try: from mlinsights.mlmodel import PiecewiseRegressor # pylint: disable=W0611 piecewise = True except ImportError: piecewise = False # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "ml")) keepnote = [] for f in os.listdir(fnb): if "piecewise_linear_regression" in f and not piecewise: continue if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def a_test_notebook_runner(self, name, folder, valid=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) import pyquickhelper import jyquickhelper import pyensae import jupytalk add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, jupytalk]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(filename, name, folder, valid=None, copy_files=None, modules=None, fLOG=noLOG): """ Runs and tests a specific list of notebooks. The function raises an exception if the execution fails. @param filename test filename @param name substring to look into notebook filenames @param folder where to look for notebooks @param valid skip cells if valid is False, None for all valid @param copy_files files to copy before running the notebooks. @param modules list of extra dependencies @param fLOG logging function """ filename = os.path.abspath(filename) temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name)) doc = os.path.normpath(os.path.join( temp, "..", "..", "..", "_doc", "notebooks", folder)) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [os.path.join(doc, _) for _ in os.listdir( doc) if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _] if len(keepnote) == 0: raise AssertionError("No found notebook in '{0}'\n{1}".format( doc, "\n".join(os.listdir(doc)))) if copy_files is not None: for name_ in copy_files: dest = os.path.join(temp, name_) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.mkdir(dest_dir) src_file = os.path.join(doc, name_) fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format( src_file, dest_dir)) shutil.copy(src_file, dest_dir) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs] if modules: base.extend(modules) add_path = pyq_get_additional_paths(base) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid, fLOG=fLOG) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook_im(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_im") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \ (is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f): keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "animation.FuncAnimation" in cell: return False return True # file to copy for cop in ["green_tripdata_2015-12_sample.csv", "NYPD_Motor_Vehicle_Collisions_sample.csv", "NYPD_Motor_Vehicle_Collisions_small.csv"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(self, name, folder, valid=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) import pyquickhelper # pylint: disable=C0415 import jyquickhelper # pylint: disable=C0415 import pyensae # pylint: disable=C0415 add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, thismodule]) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_run_notebook_lightning_js(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() is None: warnings.warn( "The unit test relies on lightning-python but it shares the same name as another one. We disable it.") return temp = get_temp_folder(__file__, "temp_run_notebooks_lightning_js") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "lightning" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # file to copy for cop in ["pydy.svg"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def test_long_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_nlp") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_html_form" in cell: return False if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks clog = CustomLog(temp) clog("START") res = execute_notebook_list(temp, keepnote, fLOG=clog, deepfLOG=clog, valid=valid, additional_path=addpaths) clog("END") execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook_pyjs(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext( f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f: if is_travis_or_appveyor() == "travis": if "pyjs_brython" in f: warnings.warn("Travis, skipping " + f) continue keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def execute_notebooks(folder, notebooks, filter, clean_function=None, fLOG=noLOG, deepfLOG=noLOG, detailed_log=None): """ Executes a list of notebooks. @param folder folder @param notebooks list of notebooks @param filter function which validate the notebooks @param clean_function cleaning function to apply to the code before running it @param fLOG logging function @param deepfLOG logging function used to run the notebook @param detailed_log log the output while running the notebook (when the notebook execution fails due to timeout @return dictionary tuple (statistics, { notebook_file: (isSuccess, outout) }) The signature of function ``filter`` is:: def filter(i, filename): return True or False """ def valid_cell(cell): if "%system" in cell: return False # pragma: no cover if "df.plot(...)" in cell: return False # pragma: no cover if 'df["difference"] = ...' in cell: return False # pragma: no cover if 'print(next(it))' in cell: return False # pragma: no cover if "est d'indice 8 et non plus 9" in cell: return False return True addpaths = get_additional_paths() if filter: notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] if len(notebooks) == 0: raise ValueError( # pragma: no cover "Empty list of notebooks.") if clean_function is None: clean_function = clean_function_notebook # pragma: no cover return execute_notebook_list( folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths, clean_function=clean_function)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_dsgarden") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "dsgarden")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if 'run_dot' in cell and is_travis_or_appveyor() == "travis": return False if "completion.dot" in cell and is_travis_or_appveyor() == "travis": return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for nb in os.listdir(fnb): if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \ "ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb: keepnote.append(os.path.join(fnb, nb)) def valid(cell): if "snakeviz" in cell: return False if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell: return False if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell: return False return True def replace_cell(cell): return cell.replace("%encoding pyensae_sql_magic.ipynb", "%encoding {0}/pyensae_sql_magic.ipynb".format(fnb)) addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pymyinstall", "src")) ] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, clean_function=replace_cell) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyensae)
def test_skip_run_notebook_javascript(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "javascript" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): return True import jyquickhelper addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join( os.path.abspath(os.path.dirname(jyquickhelper.__file__)), "..")) ] try: res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper) except Exception as e: # Issue with permission. warnings.warn("Unable to test this notebook due to " + str(e))
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_image") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "image")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # copy shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp) # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] if not os.path.exists(fnb): return for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) self.assertGreater(len(keepnote), 0) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=lightmlboard)
def test_run_notebook_pandas(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_numpy") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "numpy")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=teachpyx)
def execute_notebooks(folder, notebooks, filter, clean_function=None, fLOG=noLOG, deepfLOG=noLOG): """ Executes a list of notebooks. @param folder folder @param notebooks list of notebooks @param filter function which validate the notebooks @param clean_function cleaning function to apply to the code before running it @param fLOG logging function @param deepfLOG logging function used to run the notebook @return dictionary { notebook_file: (isSuccess, outout) } The signature of function ``filter`` is:: def filter( i, filename) : return True or False """ def valid_cell(cell): if "%system" in cell: return False if "df.plot(...)" in cell: return False if 'df["difference"] = ...' in cell: return False return True addpaths = get_additional_paths() if filter: notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] if len(notebooks) == 0: raise ValueError("Empty list of notebooks.") return execute_notebook_list(folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths, clean_function=clean_function)
def test_run_notebook_ml(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_ml") try: from mlinsights.mlmodel import PiecewiseRegressor # pylint: disable=W0611 piecewise = True except ImportError: piecewise = False # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "ml")) keepnote = [] for f in os.listdir(fnb): if "piecewise_linear_regression" in f and not piecewise: continue if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_cache_urls(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_cache_urls") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [os.path.join(fnb, "example_about_files.ipynb")] addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=None, additional_path=addpaths, cache_urls=["https://docs.python.org/3.4/library/urllib.request.html"]) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_skip_run_notebook_git(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "git_data" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "git log --log-size --abbrev --follow" in cell: return False return True import jyquickhelper addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join( os.path.abspath(os.path.dirname(jyquickhelper.__file__)), "..")) ] res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook_pyjs(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f: if is_travis_or_appveyor() == "travis": if "pyjs_brython" in f: warnings.warn("Travis, skipping " + f) continue keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_image") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "image")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # copy shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp) # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) this = os.path.join(temp, "..", "data", "fr.openfoodfacts.org.products.head100.csv") this = os.path.normpath(this) def clean_function(cell): cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv", this.replace("\\", "/")) return cell import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid, clean_function=clean_function) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook_gui(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_gui") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "gui_" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if ".show" in cell: return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook_im2(self): """ If the test does not end, it is probably due to PyQt4 needed by ete3. Try ``import PyQt4.QtCore``. Try to remove PyQt5, uninstall PyQt4, reinstall it from http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyqt4 (Windows). """ fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_virtual_environment() and sys.platform.startswith("win"): pp = os.environ.get('PYTHONPATH', '') if "SECONDTRY" in pp: raise Exception( "Not working**EXE\n{0}\n**PP\n{1}\n****".format( sys.executable, pp)) # We need to run this file with the main python. # Otherwise it fails for tables: DLL load failed. import numpy rootn = os.path.normpath( os.path.join(os.path.dirname(numpy.__file__), "..", "..")) exe = os.path.normpath(os.path.join(rootn, "..", "python.exe")) cmd = '"{0}" -u "{1}"'.format(exe, os.path.abspath(__file__)) import jyquickhelper import pyquickhelper add = ["SECONDTRY"] for mod in [pyquickhelper, jyquickhelper]: add.append( os.path.normpath( os.path.join(os.path.dirname(mod.__file__), ".."))) fLOG("set PYTHONPATH={0}".format(";".join(add))) os.environ['PYTHONPATH'] = ";".join(add) out, err = run_cmd(cmd, wait=True, fLOG=fLOG) if len(err) > 0: lines = err.split("\n") lines = [ _.lower().strip() for _ in lines if len(_) > 0 and _[0] not in (" ", "-", ".", "-") ] lines = [ _ for _ in lines if "warning" not in _ and not _.startswith("ran ") and _ != "ok" ] if len(lines) > 0: raise Exception( "--CMD:\n{0}\n--OUT:\n{1}\n--ERR\n{2}\n--ERR2\n{3}\n--PP\n{4}" .format(cmd, out, err, "\n".join(lines), pp)) return temp = get_temp_folder(__file__, "temp_run_notebooks_im2") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext( f)[-1] == ".ipynb" and "im_" in f and "ete" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # file to copy for cop in [ "green_tripdata_2015-12_sample.csv", "NYPD_Motor_Vehicle_Collisions_sample.csv", "NYPD_Motor_Vehicle_Collisions_small.csv" ]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] os.environ["DISPLAY"] = ":0" clog = CustomLog(temp) # run the notebooks import jupytalk res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, detailed_log=clog) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # notebooks are not converted into python 2.7, so not tested return if "travis" in sys.executable: # requires too many dependencies return if compare_module_version(IPython.__version__, "4.0.0") < 0: # IPython is not recent enough return kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest( "pymyinstall") temp = get_temp_folder(__file__, "temp_run_notebooks") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] pyq = os.path.join(os.path.dirname(pyquickhelper.__file__), '..') code_init = "import sys\nsys.path.append('{0}')\n".format( pyq.replace("\\", "\\\\")) for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "install_module" in f: continue else: keepnote.append((os.path.join(fnb, f), code_init)) assert len(keepnote) > 0 def valid(cell): if "snakeviz" in cell: return False return True addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))] if "travis" in sys.executable: keepnote = [_ for _ in keepnote if "javascript_extension" not in _ and (not is_travis_or_appveyor() or "example_xgboost" not in _)] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name) assert len(res) > 0 fails = [(os.path.split(k)[-1], v) for k, v in sorted(res.items()) if not v[0]] for f in fails: fLOG(f) for k, v in sorted(res.items()): name = os.path.split(k)[-1] fLOG(name, v[0], v[1]) if len(fails) > 0: raise fails[0][1][-1]
def execute_notebooks(folder, notebooks, filter, clean_function=None, fLOG=noLOG, deepfLOG=noLOG, replacements=None, dump=None, additional_path=None): """ Executes a list of notebooks. @param folder folder @param notebooks list of notebooks @param filter function which validates the notebooks to test (True means will be tested) @param clean_function cleaning function to apply to the code before running it @param fLOG logging function @param deepfLOG logging function used to run the notebook @param replacements replacements @param dump see function `execute_notebook_list_finalize_ut <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html#pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_ @param additional_path additional path to add @return dictionary { notebook_file: (isSuccess, outout) } The signature of function ``filter`` is:: def filter(i, filename): return True or False """ def valid_cell(cell): if "%system" in cell: return False if "df.plot(...)" in cell: return False if 'df["difference"] = ...' in cell: return False if 'remote_open' in cell: return None if 'blobpassword' in cell: return None if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell: return False if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell: return False if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell: return False if "%%SPEAK fr-FR" in cell: return False if " noeud tri n'est pas encore défini" in cell: return False return True addpaths = get_additional_paths() if additional_path is not None: addpaths += additional_path kernel_name = None if is_travis_or_appveyor( ) else install_python_kernel_for_unittest("ensae_teaching_cs") if filter: notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] if len(notebooks) == 0: raise ValueError("Empty list of notebooks.") res = execute_notebook_list(folder, notebooks, fLOG=fLOG, clean_function=clean_function, valid=valid_cell, additional_path=addpaths, kernel_name=kernel_name, replacements=replacements) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=dump) return res
def execute_notebooks(folder, notebooks, filter, clean_function=None, fLOG=noLOG, deepfLOG=noLOG, replacements=None, dump=None, additional_path=None): """ Executes a list of notebooks. @param folder folder @param notebooks list of notebooks @param filter function which validates the notebooks to test (True means will be tested) @param clean_function cleaning function to apply to the code before running it @param fLOG logging function @param deepfLOG logging function used to run the notebook @param replacements replacements @param dump see function `execute_notebook_list_finalize_ut <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html# pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_ @param additional_path additional path to add @return dictionary { notebook_file: (isSuccess, outout) } The signature of function ``filter`` is:: def filter(i, filename): return True or False """ def valid_cell(cell): if "%system" in cell: return False if "df.plot(...)" in cell: return False if 'df["difference"] = ...' in cell: return False if 'remote_open' in cell: return None if 'blobpassword' in cell: return None if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell: return False if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell: return False if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell: return False if "%%SPEAK fr-FR" in cell: return False if " noeud tri n'est pas encore défini" in cell: return False if "nuplet[1] = 5" in cell: return False if cell == "dico[0]": return False if cell == "dico[ [4,6] ] = 6": return False return True addpaths = get_additional_paths() if additional_path is not None: addpaths += additional_path if filter: notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] if len(notebooks) == 0: raise ValueError("Empty list of notebooks.") res = execute_notebook_list(folder, notebooks, fLOG=fLOG, clean_function=clean_function, valid=valid_cell, additional_path=addpaths, replacements=replacements) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=dump) return res
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # notebooks are not converted into python 2.7, so not tested return if compare_module_version(IPython.__version__, "4.0.0") < 0: # IPython is not recnt enough return kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest( "python3_module_template") temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": keepnote.append(os.path.join(fnb, f)) assert len(keepnote) > 0 # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), ] # creation of a kernel kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest( "python3_module_template") # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name) # final checkings assert len(res) > 0 fails = [(os.path.split(k)[-1], v) for k, v in sorted(res.items()) if not v[0]] for f in fails: fLOG(f) for k, v in sorted(res.items()): name = os.path.split(k)[-1] fLOG(name, v[0], v[1]) if len(fails) > 0: raise fails[0][1][-1]