def test_notebook_decouverte(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks from actuariat_python.automation.notebook_test_helper import clean_function_notebook temp = get_temp_folder(__file__, "temp_decouverte") keepnote = [_ for _ in ls_notebooks("decouverte")] self.assertTrue(len(keepnote) > 0) # copy data folder = os.path.dirname(keepnote[0]) data = [os.path.join(folder, "UN_Data.csv")] for dt in data: shutil.copy(dt, temp) # run the notebooks res = execute_notebooks(temp, keepnote, lambda i, n: True, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=actuariat_python)
def test_long_long_notebook_population(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") fix_tkinter_issues_virtualenv() from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook if "travis" in sys.executable: # matplotlib is still failing warnings.warn( "travis, unable to test TestNotebookPopulation.test_notebook_sessions" ) return temp = get_temp_folder(__file__, "temp_sessions_long_long") keepnote = [ _ for _ in ls_notebooks("sessions") if "seance6_graphes_correction" in _ or "seance6_graphes_enonce" in _ ] self.assertTrue(len(keepnote) > 0) clog = CustomLog(temp) res = execute_notebooks(temp, keepnote, lambda i, n: "deviner" not in n, fLOG=fLOG, clean_function=clean_function_notebook, detailed_log=clog) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=actuariat_python)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) this = os.path.join(temp, "..", "data", "fr.openfoodfacts.org.products.head100.csv") this = os.path.normpath(this) def clean_function(cell): cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv", this.replace("\\", "/")) return cell import pyquickhelper import jyquickhelper import pyensae add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, thismodule]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid, clean_function=clean_function) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_notebook_example_video(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_exemples_video") keepnote = ls_notebooks("exemples") self.assertTrue(len(keepnote) > 0) source = os.path.join(os.path.dirname(keepnote[0]), "data") images = os.path.join(temp, 'data') os.mkdir(images) for img in os.listdir(source): shutil.copy(os.path.join(source, img), images) if is_travis_or_appveyor() == "circleci": def clean(cell): cell = clean_function_notebook(cell) # ValueError: Cannot embed the 'gif' image format (circleci) cell = cell.replace('Image("video.gif")', '# Image("video.gif")') return cell else: clean = clean_function_notebook res = execute_notebooks(temp, keepnote, lambda i, n: "video" in n, fLOG=fLOG, clean_function=clean) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=code_beatrix)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) import pyquickhelper import jyquickhelper import pyensae add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, thismodule]) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_skip_run_notebook_javascript(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "javascript" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): return True import jyquickhelper addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))] try: res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper) except Exception as e: # Issue with permission. warnings.warn("Unable to test this notebook due to " + str(e))
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=src.python3_module_template)
def test_notebook_example(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_exemples_long") keepnote = ls_notebooks("exemples") self.assertTrue(len(keepnote) > 0) source = os.path.join(os.path.dirname(keepnote[0]), "data") data = os.path.join(temp, 'data') os.mkdir(data) for img in os.listdir(source): shutil.copy(os.path.join(source, img), data) try: res = execute_notebooks(temp, keepnote, lambda i, n: "poppins" in n, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=code_beatrix) except Exception as e: # pylint: disable=W0703 if "RegexMatchError" not in str(e): raise e warnings.warn("pytube issue: {}".format(e)) return
def test_notebook_example(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_exemples_long_devoxx") keepnote = ls_notebooks(os.path.join("ateliers", "devoxx2018")) self.assertTrue(len(keepnote) > 0) sources = [os.path.join(os.path.dirname(keepnote[0]), "finlaby.jpg")] for img in sources: shutil.copy(img, temp) try: res = execute_notebooks(temp, keepnote, lambda i, n: "devoxx" in n, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix) except Exception as e: # pylint: disable=W0703 if "RegexMatchError" not in str(e): raise e warnings.warn("pytube issue: {}".format(e)) return
def test_skip_run_notebook_git(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "git_data" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "git log --log-size --abbrev --follow" in cell: return False return True import jyquickhelper addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_notebook_internet(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") fix_tkinter_issues_virtualenv() if "travis" in sys.executable: # matplotlib is still failing warnings.warn( "travis, unable to test TestNotebookInternet.test_notebook_internet" ) return from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook from mlstatpy.data.wikipedia import download_pageviews self.assertTrue(download_pageviews) temp = get_temp_folder(__file__, "temp_internet") keepnote = [ _ for _ in ls_notebooks("internet") if "wikipedia_stats_correction" not in _ ] self.assertTrue(len(keepnote) > 0) for k in keepnote: fLOG(k) res = execute_notebooks(temp, keepnote, lambda i, n: True, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=actuariat_python)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyq") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "example_pyquickhelper" in f: code_init = "form1={'version': 'modified', 'module': 'anything'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "having_a_form" in f: code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}" code_init += "\ncredential={'password': '******', 'login': '******'}" code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "javascript" in f: # We skip due to connectivity issues. pass elif "git_data" in f: # Too long. pass else: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "build_script.bat" in cell: return False return True import jyquickhelper addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join( os.path.abspath(os.path.dirname(jyquickhelper.__file__)), "..")) ] res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # notebooks are not converted into python 2.7, so not tested return ci = is_travis_or_appveyor() if ci is None: try: kernel_name = install_python_kernel_for_unittest( "python3_module_template") except PermissionError as e: raise Exception( "Unable to change the kernel name. ci='{0}'".format( ci)) from e else: kernel_name = None temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=src.project_name)
def a_test_notebook_runner(filename, name, folder, valid=None, copy_files=None, modules=None, fLOG=noLOG): """ Runs and tests a specific list of notebooks. The function raises an exception if the execution fails. @param filename test filename @param name substring to look into notebook filenames @param folder where to look for notebooks @param valid skip cells if valid is False, None for all valid @param copy_files files to copy before running the notebooks. @param modules list of extra dependencies @param fLOG logging function """ filename = os.path.abspath(filename) temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name)) doc = os.path.normpath( os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [ os.path.join(doc, _) for _ in os.listdir(doc) if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _ ] if len(keepnote) == 0: raise AssertionError("No found notebook in '{0}'\n{1}".format( doc, "\n".join(os.listdir(doc)))) if copy_files is not None: for name_ in copy_files: dest = os.path.join(temp, name_) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.mkdir(dest_dir) src_file = os.path.join(doc, name_) fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format( src_file, dest_dir)) shutil.copy(src_file, dest_dir) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs] if modules: base.extend(modules) add_path = pyq_get_additional_paths(base) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid, fLOG=fLOG) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_run_notebook_js(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from sklearn.datasets import load_iris self.assertTrue(load_iris is not None) temp = get_temp_folder(__file__, "temp_run_notebooks_js") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \ "lightning" not in f and "pydy" not in f and "bokeh" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if 'bar.render(path="render.png")' in cell: return False if 'make_a_snapshot' in cell: return False return True # file to copy for cop in ["pydy.svg"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] keepnote = [_ for _ in keepnote if 'mpld3' not in _] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook_js_pydy(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from sklearn.datasets import load_iris self.assertTrue(load_iris is not None) temp = get_temp_folder(__file__, "temp_run_notebooks_js_pydy") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \ "lightning" not in f and "pydy" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if 'bar.render(path="render.png")' in cell: return False return True # file to copy for cop in ["pydy.svg"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] keepnote = [_ for _ in keepnote if 'mpld3' not in _] # run the notebooks try: res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk) except Exception as e: if 'can only concatenate list (not "tuple") to list' in str(e): warnings.warn("Pydy needs to be updated for Python 3.7") return else: raise e
def test_run_notebook_im(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_im") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \ (is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f): keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "animation.FuncAnimation" in cell: return False return True # file to copy for cop in [ "green_tripdata_2015-12_sample.csv", "NYPD_Motor_Vehicle_Collisions_sample.csv", "NYPD_Motor_Vehicle_Collisions_small.csv" ]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_long_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_nlp") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_html_form" in cell: return False if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks clog = CustomLog(temp) clog("START") res = execute_notebook_list(temp, keepnote, fLOG=clog, deepfLOG=clog, valid=valid, additional_path=addpaths) clog("END") execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_nlp") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "completion.dot" in cell and is_travis_or_appveyor( ) == "travis": return False if 'Image("completion.png")' in cell and is_travis_or_appveyor( ) == "travis": return False return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for nb in os.listdir(fnb): if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \ "ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb: keepnote.append(os.path.join(fnb, nb)) def valid(cell): if "snakeviz" in cell: return False if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell: return False if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell: return False return True def replace_cell(cell): return cell.replace( "%encoding pyensae_sql_magic.ipynb", "%encoding {0}/pyensae_sql_magic.ipynb".format(fnb)) addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pymyinstall", "src")) ] res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, clean_function=replace_cell) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyensae)
def test_notebook_algorithm(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_algorithm") keepnote = ls_notebooks("algorithmes") self.assertTrue(len(keepnote) > 0) res = execute_notebooks(temp, keepnote, lambda i, n: "deviner" not in n, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyq") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "example_pyquickhelper" in f: code_init = "form1={'version': 'modified', 'module': 'anything'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "having_a_form" in f: code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}" code_init += "\ncredential={'password': '******', 'login': '******'}" code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}" keepnote.append((os.path.join(fnb, f), code_init)) elif "javascript" in f: # We skip due to connectivity issues. pass elif "git_data" in f: # Too long. pass else: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "build_script.bat" in cell: return False return True import jyquickhelper addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook_ml(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_ml") try: from mlinsights.mlmodel import PiecewiseRegressor # pylint: disable=W0611 piecewise = True except ImportError: piecewise = False # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "ml")) keepnote = [] for f in os.listdir(fnb): if "piecewise_linear_regression" in f and not piecewise: continue if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def a_test_notebook_runner(self, name, folder, valid=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) import pyquickhelper import jyquickhelper import pyensae import jupytalk add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, jupytalk]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(filename, name, folder, valid=None, copy_files=None, modules=None, fLOG=noLOG): """ Runs and tests a specific list of notebooks. The function raises an exception if the execution fails. @param filename test filename @param name substring to look into notebook filenames @param folder where to look for notebooks @param valid skip cells if valid is False, None for all valid @param copy_files files to copy before running the notebooks. @param modules list of extra dependencies @param fLOG logging function """ filename = os.path.abspath(filename) temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name)) doc = os.path.normpath(os.path.join( temp, "..", "..", "..", "_doc", "notebooks", folder)) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [os.path.join(doc, _) for _ in os.listdir( doc) if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _] if len(keepnote) == 0: raise AssertionError("No found notebook in '{0}'\n{1}".format( doc, "\n".join(os.listdir(doc)))) if copy_files is not None: for name_ in copy_files: dest = os.path.join(temp, name_) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.mkdir(dest_dir) src_file = os.path.join(doc, name_) fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format( src_file, dest_dir)) shutil.copy(src_file, dest_dir) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs] if modules: base.extend(modules) add_path = pyq_get_additional_paths(base) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid, fLOG=fLOG) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
def a_test_notebook_session(self, name): from actuariat_python.automation.notebook_test_helper import ( ls_notebooks, execute_notebooks, clean_function_notebook) temp = get_temp_folder(__file__, "temp_sessions_" + name.split('.')[0]) keepnote = [ _ for _ in ls_notebooks("sessions") if "seance5_approche_fonctionnelle_enonce" not in _ and "seance6_graphes_ml_enonce" not in _ and "election_carte_electorale_correction" not in _ and "seance6_graphes_correction" not in _ and "seance6_graphes_enonce" not in _ and "ways" not in _ ] if is_travis_or_appveyor(): keepnote = [ _ for _ in keepnote if "election_carte_electorale" not in _ ] if name is not None: keepnote = list(filter(lambda n: name in n, keepnote)) if len(keepnote) == 0: return for k in keepnote: fLOG(k) # copy data fold = os.path.dirname(keepnote[0]) files = [os.path.join(fold, "pop-totale-france.txt")] for name_ in files: shutil.copy(name_, temp) data_tem = os.path.join(temp, "data") if not os.path.exists(data_tem): os.mkdir(data_tem) files = [ os.path.join(fold, "data", "housing.data"), os.path.join(fold, "data", "housing.names"), os.path.join(fold, "data", "multiTimeline.csv"), ] for name_ in files: shutil.copy(name_, data_tem) res = execute_notebooks(temp, keepnote, lambda i, n: "deviner" not in n, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=actuariat_python)
def test_run_notebook_lightning_js(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() is None: warnings.warn( "The unit test relies on lightning-python but it shares the same name as another one. We disable it.") return temp = get_temp_folder(__file__, "temp_run_notebooks_lightning_js") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "lightning" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # file to copy for cop in ["pydy.svg"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def test_notebook_examen(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") fix_tkinter_issues_virtualenv() from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook temp = get_temp_folder(__file__, "temp_examen") keepnote = ls_notebooks("examen") self.assertTrue(len(keepnote) > 0) res = execute_notebooks(temp, keepnote, lambda i, n: "deviner" not in n, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=actuariat_python)
def test_run_notebook_im(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_im") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \ (is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f): keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "animation.FuncAnimation" in cell: return False return True # file to copy for cop in ["green_tripdata_2015-12_sample.csv", "NYPD_Motor_Vehicle_Collisions_sample.csv", "NYPD_Motor_Vehicle_Collisions_small.csv"]: fsrc = os.path.join(fnb, cop) if os.path.exists(fsrc): dest = temp shutil.copy(fsrc, dest) # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(self, name, folder, valid=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) import pyquickhelper # pylint: disable=C0415 import jyquickhelper # pylint: disable=C0415 import pyensae # pylint: disable=C0415 add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, thismodule]) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_run_notebook_pyjs(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext( f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f: if is_travis_or_appveyor() == "travis": if "pyjs_brython" in f: warnings.warn("Travis, skipping " + f) continue keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_long_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_nlp") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_html_form" in cell: return False if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell: return False if '<div style="position:absolute' in cell: return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks clog = CustomLog(temp) clog("START") res = execute_notebook_list(temp, keepnote, fLOG=clog, deepfLOG=clog, valid=valid, additional_path=addpaths) clog("END") execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_dsgarden") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "dsgarden")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) # function to tell that a can be run def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if 'run_dot' in cell and is_travis_or_appveyor() == "travis": return False if "completion.dot" in cell and is_travis_or_appveyor() == "travis": return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for nb in os.listdir(fnb): if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \ "ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb: keepnote.append(os.path.join(fnb, nb)) def valid(cell): if "snakeviz" in cell: return False if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell: return False if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell: return False return True def replace_cell(cell): return cell.replace("%encoding pyensae_sql_magic.ipynb", "%encoding {0}/pyensae_sql_magic.ipynb".format(fnb)) addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pymyinstall", "src")) ] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, clean_function=replace_cell) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyensae)
def test_run_notebook_pandas(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_numpy") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "numpy")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=teachpyx)
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] if not os.path.exists(fnb): return for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) self.assertGreater(len(keepnote), 0) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=lightmlboard)
def test_skip_run_notebook_javascript(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long") fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb": if "javascript" in f: keepnote.append(os.path.join(fnb, f)) self.assertTrue(len(keepnote) > 0) def valid(cell): return True import jyquickhelper addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join( os.path.abspath(os.path.dirname(jyquickhelper.__file__)), "..")) ] try: res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper) except Exception as e: # Issue with permission. warnings.warn("Unable to test this notebook due to " + str(e))
def test_run_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_image") # selection of notebooks fnb = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "image")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [ os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # copy shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp) # run the notebooks res = execute_notebook_list(temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook_ml(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_ml") try: from mlinsights.mlmodel import PiecewiseRegressor # pylint: disable=W0611 piecewise = True except ImportError: piecewise = False # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "ml")) keepnote = [] for f in os.listdir(fnb): if "piecewise_linear_regression" in f and not piecewise: continue if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_notebook_ai(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_ai") keepnote = ls_notebooks("ai") self.assertTrue(len(keepnote) > 0) source = os.path.join(os.path.dirname(keepnote[0]), "images") images = os.path.join(temp, 'images') os.mkdir(images) for img in os.listdir(source): shutil.copy(os.path.join(source, img), images) res = execute_notebooks(temp, keepnote, lambda i, n: True, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix)
def test_cache_urls(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_cache_urls") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [os.path.join(fnb, "example_about_files.ipynb")] addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=None, additional_path=addpaths, cache_urls=["https://docs.python.org/3.4/library/urllib.request.html"]) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_run_notebook_pyjs(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f: if is_travis_or_appveyor() == "travis": if "pyjs_brython" in f: warnings.warn("Travis, skipping " + f) continue keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def test_run_notebook(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_image") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "image")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # copy shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp) # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=mlstatpy)
def test_run_notebook_gui(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_run_notebooks_gui") # selection of notebooks fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata")) keepnote = [] for f in os.listdir(fnb): if os.path.splitext(f)[-1] == ".ipynb" and "gui_" in f: keepnote.append(os.path.join(fnb, f)) # function to tell that a can be run def valid(cell): if ".show" in cell: return False return True # additionnal path to add addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")), os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")) ] # run the notebooks res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) this = os.path.join(temp, "..", "data", "fr.openfoodfacts.org.products.head100.csv") this = os.path.normpath(this) def clean_function(cell): cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv", this.replace("\\", "/")) return cell import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid, clean_function=clean_function) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=ensae_teaching_cs)
def execute_notebooks(folder, notebooks, filter, clean_function=None, fLOG=noLOG, deepfLOG=noLOG, replacements=None, dump=None, additional_path=None): """ Executes a list of notebooks. @param folder folder @param notebooks list of notebooks @param filter function which validates the notebooks to test (True means will be tested) @param clean_function cleaning function to apply to the code before running it @param fLOG logging function @param deepfLOG logging function used to run the notebook @param replacements replacements @param dump see function `execute_notebook_list_finalize_ut <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html# pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_ @param additional_path additional path to add @return dictionary { notebook_file: (isSuccess, outout) } The signature of function ``filter`` is:: def filter(i, filename): return True or False """ def valid_cell(cell): if "%system" in cell: return False if "df.plot(...)" in cell: return False if 'df["difference"] = ...' in cell: return False if 'remote_open' in cell: return None if 'blobpassword' in cell: return None if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell: return False if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell: return False if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell: return False if "%%SPEAK fr-FR" in cell: return False if " noeud tri n'est pas encore défini" in cell: return False if "nuplet[1] = 5" in cell: return False if cell == "dico[0]": return False if cell == "dico[ [4,6] ] = 6": return False return True addpaths = get_additional_paths() if additional_path is not None: addpaths += additional_path if filter: notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] if len(notebooks) == 0: raise ValueError("Empty list of notebooks.") res = execute_notebook_list(folder, notebooks, fLOG=fLOG, clean_function=clean_function, valid=valid_cell, additional_path=addpaths, replacements=replacements) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=dump) return res