def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
        temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
        doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
        if not os.path.exists(doc):
            raise FileNotFoundError(doc)
        keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
        self.assertTrue(len(keepnote) > 0)

        if copy_folder is not None:
            if not os.path.exists(copy_folder):
                raise FileNotFoundError(copy_folder)
            dest = os.path.split(copy_folder)[-1]
            dest = os.path.join(temp, dest)
            if not os.path.exists(dest):
                os.mkdir(dest)
            synchronize_folder(copy_folder, dest, fLOG=fLOG)

        import pyquickhelper
        import jyquickhelper
        import pyensae
        import ensae_teaching_cs
        add_path = get_additional_paths(
            [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
        res = execute_notebook_list(
            temp, keepnote, additional_path=add_path, valid=valid)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=ensae_teaching_cs)
    def test_skip_run_notebook_javascript(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")

        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "javascript" in f:
                    keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        def valid(cell):
            return True

        import jyquickhelper
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))]

        try:
            res = execute_notebook_list(
                temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
            execute_notebook_list_finalize_ut(
                res, fLOG=fLOG, dump=pyquickhelper)
        except Exception as e:
            # Issue with permission.
            warnings.warn("Unable to test this notebook due to " + str(e))
示例#3
0
    def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
        temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
        doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
        self.assertTrue(os.path.exists(doc))
        keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
        self.assertTrue(len(keepnote) > 0)

        if copy_folder is not None:
            if not os.path.exists(copy_folder):
                raise FileNotFoundError(copy_folder)
            dest = os.path.split(copy_folder)[-1]
            dest = os.path.join(temp, dest)
            if not os.path.exists(dest):
                os.mkdir(dest)
            synchronize_folder(copy_folder, dest, fLOG=fLOG)

        this = os.path.join(temp, "..", "data",
                            "fr.openfoodfacts.org.products.head100.csv")
        this = os.path.normpath(this)

        def clean_function(cell):
            cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv",
                                this.replace("\\", "/"))
            return cell

        import pyquickhelper
        import jyquickhelper
        import pyensae
        add_path = get_additional_paths(
            [jyquickhelper, pyquickhelper, pyensae, thismodule])
        res = execute_notebook_list(
            temp, keepnote, additional_path=add_path, valid=valid,
            clean_function=clean_function)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
    def a_test_notebook_runner(self,
                               name,
                               folder,
                               valid=None,
                               copy_folder=None):
        temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
        doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
        self.assertTrue(os.path.exists(doc))
        keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
        self.assertTrue(len(keepnote) > 0)

        if copy_folder is not None:
            if not os.path.exists(copy_folder):
                raise FileNotFoundError(copy_folder)
            dest = os.path.split(copy_folder)[-1]
            dest = os.path.join(temp, dest)
            if not os.path.exists(dest):
                os.mkdir(dest)
            synchronize_folder(copy_folder, dest, fLOG=fLOG)

        import pyquickhelper
        import jyquickhelper
        import pyensae
        add_path = get_additional_paths(
            [jyquickhelper, pyquickhelper, pyensae, thismodule])
        res = execute_notebook_list(temp,
                                    keepnote,
                                    additional_path=add_path,
                                    valid=valid)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
    def test_skip_run_notebook_git(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")

        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "git_data" in f:
                    keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        def valid(cell):
            if "git log --log-size --abbrev --follow" in cell:
                return False
            return True

        import jyquickhelper
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))]

        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=pyquickhelper)
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        temp = get_temp_folder(__file__, "temp_run_notebooks")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=src.python3_module_template)
    def test_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_pyq")

        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "example_pyquickhelper" in f:
                    code_init = "form1={'version': 'modified', 'module': 'anything'}"
                    keepnote.append((os.path.join(fnb, f), code_init))
                elif "having_a_form" in f:
                    code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}"
                    code_init += "\ncredential={'password': '******', 'login': '******'}"
                    code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}"
                    keepnote.append((os.path.join(fnb, f), code_init))
                elif "javascript" in f:
                    # We skip due to connectivity issues.
                    pass
                elif "git_data" in f:
                    # Too long.
                    pass
                else:
                    keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_window_params" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            if "build_script.bat" in cell:
                return False
            return True

        import jyquickhelper
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(
                    os.path.abspath(os.path.dirname(jyquickhelper.__file__)),
                    ".."))
        ]

        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper)
    def test_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2:
            # notebooks are not converted into python 2.7, so not tested
            return

        ci = is_travis_or_appveyor()
        if ci is None:
            try:
                kernel_name = install_python_kernel_for_unittest(
                    "python3_module_template")
            except PermissionError as e:
                raise Exception(
                    "Unable to change the kernel name. ci='{0}'".format(
                        ci)) from e
        else:
            kernel_name = None
        temp = get_temp_folder(__file__, "temp_run_notebooks")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        # function to tell that a can be run
        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_window_params" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
        ]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths,
                                    kernel_name=kernel_name)
        execute_notebook_list_finalize_ut(res,
                                          fLOG=fLOG,
                                          dump=src.project_name)
def a_test_notebook_runner(filename,
                           name,
                           folder,
                           valid=None,
                           copy_files=None,
                           modules=None,
                           fLOG=noLOG):
    """
    Runs and tests a specific list of notebooks.
    The function raises an exception if the execution fails.

    @param      filename        test filename
    @param      name            substring to look into notebook filenames
    @param      folder          where to look for notebooks
    @param      valid           skip cells if valid is False, None for all valid
    @param      copy_files      files to copy before running the notebooks.
    @param      modules         list of extra dependencies
    @param      fLOG            logging function
    """
    filename = os.path.abspath(filename)
    temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name))
    doc = os.path.normpath(
        os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder))
    if not os.path.exists(doc):
        raise FileNotFoundError(doc)
    keepnote = [
        os.path.join(doc, _) for _ in os.listdir(doc)
        if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _
    ]
    if len(keepnote) == 0:
        raise AssertionError("No found notebook in '{0}'\n{1}".format(
            doc, "\n".join(os.listdir(doc))))

    if copy_files is not None:
        for name_ in copy_files:
            dest = os.path.join(temp, name_)
            dest_dir = os.path.dirname(dest)
            if not os.path.exists(dest_dir):
                os.mkdir(dest_dir)
            src_file = os.path.join(doc, name_)
            fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format(
                src_file, dest_dir))
            shutil.copy(src_file, dest_dir)

    import pyquickhelper
    import jyquickhelper
    import pyensae
    import ensae_teaching_cs
    base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]
    if modules:
        base.extend(modules)
    add_path = pyq_get_additional_paths(base)
    res = execute_notebook_list(temp,
                                keepnote,
                                additional_path=add_path,
                                valid=valid,
                                fLOG=fLOG)
    execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
    def test_run_notebook_js(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        from sklearn.datasets import load_iris
        self.assertTrue(load_iris is not None)
        temp = get_temp_folder(__file__, "temp_run_notebooks_js")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \
                    "lightning" not in f and "pydy" not in f and "bokeh" not in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if 'bar.render(path="render.png")' in cell:
                return False
            if 'make_a_snapshot' in cell:
                return False
            return True

        # file to copy
        for cop in ["pydy.svg"]:
            fsrc = os.path.join(fnb, cop)
            if os.path.exists(fsrc):
                dest = temp
                shutil.copy(fsrc, dest)

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src"))
        ]

        keepnote = [_ for _ in keepnote if 'mpld3' not in _]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
    def test_run_notebook_js_pydy(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        from sklearn.datasets import load_iris
        self.assertTrue(load_iris is not None)
        temp = get_temp_folder(__file__, "temp_run_notebooks_js_pydy")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "pyjs_" not in f and \
                    "lightning" not in f and "pydy" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if 'bar.render(path="render.png")' in cell:
                return False
            return True

        # file to copy
        for cop in ["pydy.svg"]:
            fsrc = os.path.join(fnb, cop)
            if os.path.exists(fsrc):
                dest = temp
                shutil.copy(fsrc, dest)

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        keepnote = [_ for _ in keepnote if 'mpld3' not in _]

        # run the notebooks
        try:
            res = execute_notebook_list(
                temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
            execute_notebook_list_finalize_ut(
                res, fLOG=fLOG, dump=jupytalk)
        except Exception as e:
            if 'can only concatenate list (not "tuple") to list' in str(e):
                warnings.warn("Pydy needs to be updated for Python 3.7")
                return
            else:
                raise e
    def test_run_notebook_im(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_im")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \
                    (is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f):
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if "animation.FuncAnimation" in cell:
                return False
            return True

        # file to copy
        for cop in [
                "green_tripdata_2015-12_sample.csv",
                "NYPD_Motor_Vehicle_Collisions_sample.csv",
                "NYPD_Motor_Vehicle_Collisions_small.csv"
        ]:
            fsrc = os.path.join(fnb, cop)
            if os.path.exists(fsrc):
                dest = temp
                shutil.copy(fsrc, dest)

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
示例#13
0
    def test_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "nlp"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_window_params" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            if "completion.dot" in cell and is_travis_or_appveyor(
            ) == "travis":
                return False
            if 'Image("completion.png")' in cell and is_travis_or_appveyor(
            ) == "travis":
                return False
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
示例#14
0
    def test_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks")

        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks"))
        keepnote = []
        for nb in os.listdir(fnb):
            if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \
                    "ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb:
                keepnote.append(os.path.join(fnb, nb))

        def valid(cell):
            if "snakeviz" in cell:
                return False
            if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell:
                return False
            if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell:
                return False
            return True

        def replace_cell(cell):
            return cell.replace(
                "%encoding pyensae_sql_magic.ipynb",
                "%encoding {0}/pyensae_sql_magic.ipynb".format(fnb))

        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pymyinstall", "src"))
        ]

        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths,
                                    clean_function=replace_cell)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyensae)
    def test_long_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "nlp"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_html_form" in cell:
                return False
            if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        clog = CustomLog(temp)
        clog("START")
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=clog,
                                    deepfLOG=clog,
                                    valid=valid,
                                    additional_path=addpaths)
        clog("END")
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_pyq")

        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "example_pyquickhelper" in f:
                    code_init = "form1={'version': 'modified', 'module': 'anything'}"
                    keepnote.append((os.path.join(fnb, f), code_init))
                elif "having_a_form" in f:
                    code_init = "myvar='my value'\nform1={'version': 'modified', 'module': 'anything'}"
                    code_init += "\ncredential={'password': '******', 'login': '******'}"
                    code_init += "\nmy_address={'last_name': 'dupre', 'combined': 'xavier dupre', 'first_name': 'xavier'}"
                    keepnote.append((os.path.join(fnb, f), code_init))
                elif "javascript" in f:
                    # We skip due to connectivity issues.
                    pass
                elif "git_data" in f:
                    # Too long.
                    pass
                else:
                    keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_window_params" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            if "build_script.bat" in cell:
                return False
            return True

        import jyquickhelper
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(jyquickhelper.__file__)), ".."))]

        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=pyquickhelper)
示例#17
0
    def test_run_notebook_ml(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_ml")

        try:
            from mlinsights.mlmodel import PiecewiseRegressor  # pylint: disable=W0611
            piecewise = True
        except ImportError:
            piecewise = False

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "ml"))
        keepnote = []
        for f in os.listdir(fnb):
            if "piecewise_linear_regression" in f and not piecewise:
                continue
            if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
    def a_test_notebook_runner(self, name, folder, valid=None):
        temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
        doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
        self.assertTrue(os.path.exists(doc))
        keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
        self.assertTrue(len(keepnote) > 0)

        import pyquickhelper
        import jyquickhelper
        import pyensae
        import jupytalk
        add_path = get_additional_paths(
            [jyquickhelper, pyquickhelper, pyensae, jupytalk])
        res = execute_notebook_list(
            temp, keepnote, additional_path=add_path, valid=valid)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def a_test_notebook_runner(filename, name, folder, valid=None, copy_files=None, modules=None, fLOG=noLOG):
    """
    Runs and tests a specific list of notebooks.
    The function raises an exception if the execution fails.

    @param      filename        test filename
    @param      name            substring to look into notebook filenames
    @param      folder          where to look for notebooks
    @param      valid           skip cells if valid is False, None for all valid
    @param      copy_files      files to copy before running the notebooks.
    @param      modules         list of extra dependencies
    @param      fLOG            logging function
    """
    filename = os.path.abspath(filename)
    temp = get_temp_folder(filename, "temp_notebook_123_{0}".format(name))
    doc = os.path.normpath(os.path.join(
        temp, "..", "..", "..", "_doc", "notebooks", folder))
    if not os.path.exists(doc):
        raise FileNotFoundError(doc)
    keepnote = [os.path.join(doc, _) for _ in os.listdir(
        doc) if name in _ and ".ipynb" in _ and ".ipynb_checkpoints" not in _]
    if len(keepnote) == 0:
        raise AssertionError("No found notebook in '{0}'\n{1}".format(
            doc, "\n".join(os.listdir(doc))))

    if copy_files is not None:
        for name_ in copy_files:
            dest = os.path.join(temp, name_)
            dest_dir = os.path.dirname(dest)
            if not os.path.exists(dest_dir):
                os.mkdir(dest_dir)
            src_file = os.path.join(doc, name_)
            fLOG("[a_test_notebook_runner] copy '{0}' to '{1}'.".format(
                src_file, dest_dir))
            shutil.copy(src_file, dest_dir)

    import pyquickhelper
    import jyquickhelper
    import pyensae
    import ensae_teaching_cs
    base = [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]
    if modules:
        base.extend(modules)
    add_path = pyq_get_additional_paths(base)
    res = execute_notebook_list(
        temp, keepnote, additional_path=add_path, valid=valid, fLOG=fLOG)
    execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=ensae_teaching_cs)
    def test_run_notebook_im(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_im")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "im_" in f and "ete" not in f and 'ggplot' not in f and \
                    (is_travis_or_appveyor() != 'appveyor' or 'im_mpl_scatter_density' not in f):
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if "animation.FuncAnimation" in cell:
                return False
            return True

        # file to copy
        for cop in ["green_tripdata_2015-12_sample.csv",
                    "NYPD_Motor_Vehicle_Collisions_sample.csv",
                    "NYPD_Motor_Vehicle_Collisions_small.csv"]:
            fsrc = os.path.join(fnb, cop)
            if os.path.exists(fsrc):
                dest = temp
                shutil.copy(fsrc, dest)

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=jupytalk)
示例#21
0
    def a_test_notebook_runner(self, name, folder, valid=None):
        temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
        doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
        self.assertTrue(os.path.exists(doc))
        keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
        self.assertTrue(len(keepnote) > 0)

        import pyquickhelper  # pylint: disable=C0415
        import jyquickhelper  # pylint: disable=C0415
        import pyensae  # pylint: disable=C0415
        add_path = get_additional_paths(
            [jyquickhelper, pyquickhelper, pyensae, thismodule])
        res = execute_notebook_list(temp,
                                    keepnote,
                                    additional_path=add_path,
                                    valid=valid)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
    def test_run_notebook_lightning_js(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor() is None:
            warnings.warn(
                "The unit test relies on lightning-python but it shares the same name as another one. We disable it.")
            return

        temp = get_temp_folder(__file__, "temp_run_notebooks_lightning_js")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "js_" in f and "lightning" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # file to copy
        for cop in ["pydy.svg"]:
            fsrc = os.path.join(fnb, cop)
            if os.path.exists(fsrc):
                dest = temp
                shutil.copy(fsrc, dest)

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=jupytalk)
    def test_long_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "nlp"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_html_form" in cell:
                return False
            if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        clog = CustomLog(temp)
        clog("START")
        res = execute_notebook_list(temp, keepnote, fLOG=clog, deepfLOG=clog, valid=valid,
                                    additional_path=addpaths)
        clog("END")
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=mlstatpy)
    def test_run_notebook_pyjs(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(
                    f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f:
                if is_travis_or_appveyor() == "travis":
                    if "pyjs_brython" in f:
                        warnings.warn("Travis, skipping " + f)
                        continue
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
示例#25
0
def execute_notebooks(folder, notebooks, filter, clean_function=None,
                      fLOG=noLOG, deepfLOG=noLOG, detailed_log=None):
    """
    Executes a list of notebooks.

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validate the notebooks
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @param      detailed_log    log the output while running the notebook (when the notebook execution fails due to timeout
    @return                     dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })

    The signature of function ``filter`` is::

        def filter(i, filename):
            return True or False

    """

    def valid_cell(cell):
        if "%system" in cell:
            return False  # pragma: no cover
        if "df.plot(...)" in cell:
            return False  # pragma: no cover
        if 'df["difference"] = ...' in cell:
            return False  # pragma: no cover
        if 'print(next(it))' in cell:
            return False  # pragma: no cover
        if "est d'indice 8 et non plus 9" in cell:
            return False
        return True

    addpaths = get_additional_paths()
    if filter:
        notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
    if len(notebooks) == 0:
        raise ValueError(  # pragma: no cover
            "Empty list of notebooks.")
    if clean_function is None:
        clean_function = clean_function_notebook  # pragma: no cover
    return execute_notebook_list(
        folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths,
        clean_function=clean_function)
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_dsgarden")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "dsgarden"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
                keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        # function to tell that a can be run
        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_window_params" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            if 'run_dot' in cell and is_travis_or_appveyor() == "travis":
                return False
            if "completion.dot" in cell and is_travis_or_appveyor() == "travis":
                return False
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=mlstatpy)
示例#27
0
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks")

        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        for nb in os.listdir(fnb):
            if ".ipynb" in nb and "velib" not in nb and "azure" not in nb and \
                    "ssh" not in nb and "checkpoints" not in nb and "hadoop" not in nb:
                keepnote.append(os.path.join(fnb, nb))

        def valid(cell):
            if "snakeviz" in cell:
                return False
            if 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' in cell:
                return False
            if 'pandas.read_csv("flatfile_tab_pos2.txt"' in cell:
                return False
            return True

        def replace_cell(cell):
            return cell.replace("%encoding pyensae_sql_magic.ipynb",
                                "%encoding {0}/pyensae_sql_magic.ipynb".format(fnb))

        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pymyinstall", "src"))
        ]

        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths,
            clean_function=replace_cell)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=pyensae)
    def test_skip_run_notebook_javascript(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")

        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "javascript" in f:
                    keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        def valid(cell):
            return True

        import jyquickhelper
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(
                    os.path.abspath(os.path.dirname(jyquickhelper.__file__)),
                    ".."))
        ]

        try:
            res = execute_notebook_list(temp,
                                        keepnote,
                                        fLOG=fLOG,
                                        valid=valid,
                                        additional_path=addpaths)
            execute_notebook_list_finalize_ut(res,
                                              fLOG=fLOG,
                                              dump=pyquickhelper)
        except Exception as e:
            # Issue with permission.
            warnings.warn("Unable to test this notebook due to " + str(e))
    def test_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_image")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "image"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src"))
        ]

        # copy
        shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp)

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
示例#30
0
    def test_run_notebook(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks"))
        keepnote = []
        if not os.path.exists(fnb):
            return
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
                keepnote.append(os.path.join(fnb, f))
        self.assertGreater(len(keepnote), 0)

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=lightmlboard)
示例#31
0
    def test_run_notebook_pandas(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_numpy")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "numpy"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
                keepnote.append(os.path.join(fnb, f))

        # run the notebooks
        res = execute_notebook_list(temp, keepnote, fLOG=fLOG)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=teachpyx)
def execute_notebooks(folder,
                      notebooks,
                      filter,
                      clean_function=None,
                      fLOG=noLOG,
                      deepfLOG=noLOG):
    """
    Executes a list of notebooks.

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validate the notebooks
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @return                     dictionary { notebook_file: (isSuccess, outout) }

    The signature of function ``filter`` is::

        def filter( i, filename) : return True or False

    """
    def valid_cell(cell):
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        return True

    addpaths = get_additional_paths()
    if filter:
        notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
    if len(notebooks) == 0:
        raise ValueError("Empty list of notebooks.")
    return execute_notebook_list(folder,
                                 notebooks,
                                 fLOG=fLOG,
                                 valid=valid_cell,
                                 additional_path=addpaths,
                                 clean_function=clean_function)
示例#33
0
    def test_run_notebook_ml(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_ml")

        try:
            from mlinsights.mlmodel import PiecewiseRegressor  # pylint: disable=W0611
            piecewise = True
        except ImportError:
            piecewise = False

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "ml"))
        keepnote = []
        for f in os.listdir(fnb):
            if "piecewise_linear_regression" in f and not piecewise:
                continue
            if os.path.splitext(f)[-1] == ".ipynb" and "_long" not in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=mlstatpy)
示例#34
0
    def test_cache_urls(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_cache_urls")

        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = [os.path.join(fnb, "example_about_files.ipynb")]

        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))]

        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=None, additional_path=addpaths,
            cache_urls=["https://docs.python.org/3.4/library/urllib.request.html"])
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=pyquickhelper)
    def test_skip_run_notebook_git(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_skip_run_notebooks_pyq_long")

        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "git_data" in f:
                    keepnote.append(os.path.join(fnb, f))
        self.assertTrue(len(keepnote) > 0)

        def valid(cell):
            if "git log --log-size --abbrev --follow" in cell:
                return False
            return True

        import jyquickhelper
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(
                    os.path.abspath(os.path.dirname(jyquickhelper.__file__)),
                    ".."))
        ]

        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=pyquickhelper)
    def test_run_notebook_pyjs(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_pyjs")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "pyjs_" in f and "mpld3" not in f:
                if is_travis_or_appveyor() == "travis":
                    if "pyjs_brython" in f:
                        warnings.warn("Travis, skipping " + f)
                        continue
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=jupytalk)
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_image")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "image"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "long" not in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src"))
        ]

        # copy
        shutil.copy(os.path.join(fnb, "eglise_zoom2.jpg"), temp)

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=mlstatpy)
    def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
        temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
        doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
        self.assertTrue(os.path.exists(doc))
        keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
        self.assertTrue(len(keepnote) > 0)

        if copy_folder is not None:
            if not os.path.exists(copy_folder):
                raise FileNotFoundError(copy_folder)
            dest = os.path.split(copy_folder)[-1]
            dest = os.path.join(temp, dest)
            if not os.path.exists(dest):
                os.mkdir(dest)
            synchronize_folder(copy_folder, dest, fLOG=fLOG)

        this = os.path.join(temp, "..", "data",
                            "fr.openfoodfacts.org.products.head100.csv")
        this = os.path.normpath(this)

        def clean_function(cell):
            cell = cell.replace("c:/temp/fr.openfoodfacts.org.products.csv",
                                this.replace("\\", "/"))
            return cell

        import pyquickhelper
        import jyquickhelper
        import pyensae
        import ensae_teaching_cs
        add_path = get_additional_paths(
            [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
        res = execute_notebook_list(
            temp, keepnote, additional_path=add_path, valid=valid,
            clean_function=clean_function)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=ensae_teaching_cs)
    def test_run_notebook_gui(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_run_notebooks_gui")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb" and "gui_" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            if ".show" in cell:
                return False
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "jyquickhelper", "src")),
            os.path.normpath(os.path.join(
                os.path.abspath(os.path.dirname(__file__)), "..", "..", "..", "pyquickhelper", "src"))
        ]

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths)
        execute_notebook_list_finalize_ut(
            res, fLOG=fLOG, dump=jupytalk)
    def test_run_notebook_im2(self):
        """
        If the test does not end, it is probably due to PyQt4 needed by ete3.
        Try ``import PyQt4.QtCore``.
        Try to remove PyQt5, uninstall PyQt4, reinstall it from
        http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyqt4 (Windows).
        """
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        if is_virtual_environment() and sys.platform.startswith("win"):
            pp = os.environ.get('PYTHONPATH', '')
            if "SECONDTRY" in pp:
                raise Exception(
                    "Not working**EXE\n{0}\n**PP\n{1}\n****".format(
                        sys.executable, pp))
            # We need to run this file with the main python.
            # Otherwise it fails for tables: DLL load failed.
            import numpy
            rootn = os.path.normpath(
                os.path.join(os.path.dirname(numpy.__file__), "..", ".."))
            exe = os.path.normpath(os.path.join(rootn, "..", "python.exe"))
            cmd = '"{0}" -u "{1}"'.format(exe, os.path.abspath(__file__))
            import jyquickhelper
            import pyquickhelper
            add = ["SECONDTRY"]
            for mod in [pyquickhelper, jyquickhelper]:
                add.append(
                    os.path.normpath(
                        os.path.join(os.path.dirname(mod.__file__), "..")))
            fLOG("set PYTHONPATH={0}".format(";".join(add)))
            os.environ['PYTHONPATH'] = ";".join(add)
            out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
            if len(err) > 0:
                lines = err.split("\n")
                lines = [
                    _.lower().strip() for _ in lines
                    if len(_) > 0 and _[0] not in (" ", "-", ".", "-")
                ]
                lines = [
                    _ for _ in lines if "warning" not in _
                    and not _.startswith("ran ") and _ != "ok"
                ]
                if len(lines) > 0:
                    raise Exception(
                        "--CMD:\n{0}\n--OUT:\n{1}\n--ERR\n{2}\n--ERR2\n{3}\n--PP\n{4}"
                        .format(cmd, out, err, "\n".join(lines), pp))
            return

        temp = get_temp_folder(__file__, "temp_run_notebooks_im2")

        # selection of notebooks
        fnb = os.path.normpath(
            os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                         "..", "_doc", "notebooks", "2016", "pydata"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(
                    f)[-1] == ".ipynb" and "im_" in f and "ete" in f:
                keepnote.append(os.path.join(fnb, f))

        # function to tell that a can be run
        def valid(cell):
            return True

        # file to copy
        for cop in [
                "green_tripdata_2015-12_sample.csv",
                "NYPD_Motor_Vehicle_Collisions_sample.csv",
                "NYPD_Motor_Vehicle_Collisions_small.csv"
        ]:
            fsrc = os.path.join(fnb, cop)
            if os.path.exists(fsrc):
                dest = temp
                shutil.copy(fsrc, dest)

        # additionnal path to add
        addpaths = [
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "pyquickhelper", "src")),
            os.path.normpath(
                os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
                             "..", "..", "jyquickhelper", "src"))
        ]

        os.environ["DISPLAY"] = ":0"

        clog = CustomLog(temp)

        # run the notebooks
        import jupytalk
        res = execute_notebook_list(temp,
                                    keepnote,
                                    fLOG=fLOG,
                                    valid=valid,
                                    additional_path=addpaths,
                                    detailed_log=clog)
        execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
示例#41
0
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2:
            # notebooks are not converted into python 2.7, so not tested
            return

        if "travis" in sys.executable:
            # requires too many dependencies
            return

        if compare_module_version(IPython.__version__, "4.0.0") < 0:
            # IPython is not recent enough
            return

        kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
            "pymyinstall")

        temp = get_temp_folder(__file__, "temp_run_notebooks")

        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        pyq = os.path.join(os.path.dirname(pyquickhelper.__file__), '..')
        code_init = "import sys\nsys.path.append('{0}')\n".format(
            pyq.replace("\\", "\\\\"))
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                if "install_module" in f:
                    continue
                else:
                    keepnote.append((os.path.join(fnb, f), code_init))
        assert len(keepnote) > 0

        def valid(cell):
            if "snakeviz" in cell:
                return False
            return True

        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))]

        if "travis" in sys.executable:
            keepnote = [_ for _ in keepnote if "javascript_extension" not in _ and
                        (not is_travis_or_appveyor() or "example_xgboost" not in _)]

        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths,
            kernel_name=kernel_name)
        assert len(res) > 0
        fails = [(os.path.split(k)[-1], v)
                 for k, v in sorted(res.items()) if not v[0]]
        for f in fails:
            fLOG(f)
        for k, v in sorted(res.items()):
            name = os.path.split(k)[-1]
            fLOG(name, v[0], v[1])
        if len(fails) > 0:
            raise fails[0][1][-1]
示例#42
0
def execute_notebooks(folder,
                      notebooks,
                      filter,
                      clean_function=None,
                      fLOG=noLOG,
                      deepfLOG=noLOG,
                      replacements=None,
                      dump=None,
                      additional_path=None):
    """
    Executes a list of notebooks.

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validates the notebooks to test (True means will be tested)
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @param      replacements    replacements
    @param      dump            see function `execute_notebook_list_finalize_ut <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html#pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_
    @param      additional_path additional path to add
    @return                     dictionary { notebook_file: (isSuccess, outout) }

    The signature of function ``filter`` is::

        def filter(i, filename):
            return True or False
    """
    def valid_cell(cell):
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        if 'remote_open' in cell:
            return None
        if 'blobpassword' in cell:
            return None
        if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell:
            return False
        if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell:
            return False
        if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell:
            return False
        if "%%SPEAK fr-FR" in cell:
            return False
        if " noeud tri n'est pas encore défini" in cell:
            return False
        return True

    addpaths = get_additional_paths()
    if additional_path is not None:
        addpaths += additional_path
    kernel_name = None if is_travis_or_appveyor(
    ) else install_python_kernel_for_unittest("ensae_teaching_cs")
    if filter:
        notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
    if len(notebooks) == 0:
        raise ValueError("Empty list of notebooks.")
    res = execute_notebook_list(folder,
                                notebooks,
                                fLOG=fLOG,
                                clean_function=clean_function,
                                valid=valid_cell,
                                additional_path=addpaths,
                                kernel_name=kernel_name,
                                replacements=replacements)
    execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=dump)
    return res
def execute_notebooks(folder, notebooks, filter, clean_function=None,
                      fLOG=noLOG, deepfLOG=noLOG, replacements=None, dump=None,
                      additional_path=None):
    """
    Executes a list of notebooks.

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validates the notebooks to test (True means will be tested)
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @param      replacements    replacements
    @param      dump            see function `execute_notebook_list_finalize_ut
                                <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/ipythonhelper/run_notebook.html#
                                pyquickhelper.ipythonhelper.run_notebook.execute_notebook_list_finalize_ut>`_
    @param      additional_path additional path to add
    @return                     dictionary { notebook_file: (isSuccess, outout) }

    The signature of function ``filter`` is::

        def filter(i, filename):
            return True or False
    """

    def valid_cell(cell):
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        if 'remote_open' in cell:
            return None
        if 'blobpassword' in cell:
            return None
        if 'String.Join(",", a.Select(c=>c.ToString()).ToArray())' in cell:
            return False
        if 'Speech.VocalSynthesis("ENSAE", "fr-FR","","")' in cell:
            return False
        if 'Speech.VocalSynthesis(text, lang, voice, filename)' in cell:
            return False
        if "%%SPEAK fr-FR" in cell:
            return False
        if " noeud tri n'est pas encore défini" in cell:
            return False
        if "nuplet[1] = 5" in cell:
            return False
        if cell == "dico[0]":
            return False
        if cell == "dico[ [4,6] ] = 6":
            return False
        return True

    addpaths = get_additional_paths()
    if additional_path is not None:
        addpaths += additional_path
    if filter:
        notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
    if len(notebooks) == 0:
        raise ValueError("Empty list of notebooks.")
    res = execute_notebook_list(folder, notebooks, fLOG=fLOG, clean_function=clean_function,
                                valid=valid_cell, additional_path=addpaths,
                                replacements=replacements)
    execute_notebook_list_finalize_ut(
        res, fLOG=fLOG, dump=dump)
    return res
示例#44
0
    def test_run_notebook(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2:
            # notebooks are not converted into python 2.7, so not tested
            return

        if compare_module_version(IPython.__version__, "4.0.0") < 0:
            # IPython is not recnt enough
            return

        kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
            "python3_module_template")

        temp = get_temp_folder(__file__, "temp_run_notebooks")

        # selection of notebooks
        fnb = os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
        keepnote = []
        for f in os.listdir(fnb):
            if os.path.splitext(f)[-1] == ".ipynb":
                keepnote.append(os.path.join(fnb, f))
        assert len(keepnote) > 0

        # function to tell that a can be run
        def valid(cell):
            if "open_html_form" in cell:
                return False
            if "open_window_params" in cell:
                return False
            if '<div style="position:absolute' in cell:
                return False
            return True

        # additionnal path to add
        addpaths = [os.path.normpath(os.path.join(
            os.path.abspath(os.path.dirname(__file__)), "..", "..", "src")),
        ]

        # creation of a kernel
        kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
            "python3_module_template")

        # run the notebooks
        res = execute_notebook_list(
            temp, keepnote, fLOG=fLOG, valid=valid, additional_path=addpaths, kernel_name=kernel_name)

        # final checkings
        assert len(res) > 0
        fails = [(os.path.split(k)[-1], v)
                 for k, v in sorted(res.items()) if not v[0]]
        for f in fails:
            fLOG(f)
        for k, v in sorted(res.items()):
            name = os.path.split(k)[-1]
            fLOG(name, v[0], v[1])
        if len(fails) > 0:
            raise fails[0][1][-1]