예제 #1
0
    def test_notebook_runner_flat2db3(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        notebook = os.path.split(
            __file__)[-1].replace(".ipynb", "").replace(".py", "")[5:]
        temp = get_temp_folder(__file__, "temp_" + notebook)
        nbfile = os.path.join(
            temp,
            "..",
            "..",
            "..",
            "_doc",
            "notebooks",
            "%s.ipynb" %
            notebook)
        if not os.path.exists(nbfile):
            raise FileNotFoundError(nbfile)
        addpath = [os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
                   os.path.normpath(
            os.path.join(
                temp,
                "..",
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")),
            os.path.normpath(
            os.path.join(
                temp,
                "..",
                "..",
                "..",
                "..",
                "pymyinstall",
                "src")),
        ]
        outfile = os.path.join(temp, "out_notebook.ipynb")
        assert not os.path.exists(outfile)

        def valid(code):
            return 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' not in code

        if "travis" in sys.executable:
            return

        kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
            "pyensae")

        out = run_notebook(nbfile,
                           working_dir=temp,
                           outfilename=outfile,
                           additional_path=addpath,
                           valid=valid,
                           fLOG=fLOG,
                           kernel_name=kernel_name)
        fLOG(out)
        assert os.path.exists(outfile)
예제 #2
0
    def test_notebook_runner(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")
        notebook = os.path.split(__file__)[-1].replace(".ipynb",
                                                       "").replace(".py",
                                                                   "")[5:]
        temp = get_temp_folder(__file__, "temp_" + notebook)
        nbfile = os.path.join(temp, "..", "..", "..", "_doc", "notebooks",
                              "%s.ipynb" % notebook)
        if not os.path.exists(nbfile):
            raise FileNotFoundError(nbfile)
        addpath = [
            os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
            os.path.normpath(
                os.path.join(temp, "..", "..", "..", "..", "pyquickhelper",
                             "src")),
        ]
        outfile = os.path.join(temp, "out_notebook.ipynb")
        assert not os.path.exists(outfile)

        kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
            "pyensae")

        out = run_notebook(nbfile,
                           working_dir=temp,
                           outfilename=outfile,
                           additional_path=addpath,
                           kernel_name=kernel_name)
        fLOG(out)
        assert os.path.exists(outfile)
    def test_notebook_runner_custom_log(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")
        temp = get_temp_folder(self.test_notebook_runner_custom_log)
        nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb")
        assert os.path.exists(nbfile)
        addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
        assert os.path.exists(addpath)

        outfile = os.path.join(temp, "out_notebook.ipynb")
        assert not os.path.exists(outfile)

        custom = CustomLog(temp)
        stat, out = run_notebook(nbfile,
                                 working_dir=temp,
                                 outfilename=outfile,
                                 additional_path=[addpath],
                                 detailed_log=custom)
        fLOG(stat)
        fLOG(out)
        assert os.path.exists(outfile)
        assert "No module named 'pyquickhelper'" not in out
        assert "datetime.datetime(2015, 3, 2" in out
        out = os.path.join(temp, "log_custom_000.txt")
        if not os.path.exists(out):
            raise FileNotFoundError(out)
    def test_notebook_runner_custom_log(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        temp = get_temp_folder(self.test_notebook_runner_custom_log)
        nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb")
        assert os.path.exists(nbfile)
        addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
        assert os.path.exists(addpath)

        outfile = os.path.join(temp, "out_notebook.ipynb")
        assert not os.path.exists(outfile)

        custom = CustomLog(temp)
        stat, out = run_notebook(nbfile, working_dir=temp, outfilename=outfile,
                                 additional_path=[addpath],
                                 detailed_log=custom)
        fLOG(stat)
        fLOG(out)
        assert os.path.exists(outfile)
        assert "No module named 'pyquickhelper'" not in out
        assert "datetime.datetime(2015, 3, 2" in out
        out = os.path.join(temp, "log_custom_000.txt")
        if not os.path.exists(out):
            raise FileNotFoundError(out)
예제 #5
0
    def test_notebook_runner(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")
        from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks
        assert ls_notebooks
        temp = get_temp_folder(__file__, "temp_notebook1")
        nbfile = os.path.join(temp, "..", "..", "..", "_doc", "notebooks",
                              "td2a", "td2a_correction_session_1.ipynb")
        nbfile = os.path.normpath(nbfile)
        assert os.path.exists(nbfile)
        addpath = [os.path.dirname(location), os.path.dirname(location2)]
        addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath]
        addpath.append(os.path.abspath(os.path.dirname(init_file)))

        outfile = os.path.join(temp, "out_" + os.path.split(nbfile)[-1])
        assert not os.path.exists(outfile)
        stat, out = run_notebook(nbfile,
                                 working_dir=temp,
                                 outfilename=outfile,
                                 additional_path=addpath,
                                 valid=lambda code: '%system' not in code,
                                 fLOG=fLOG)
        fLOG(out)
        assert os.path.exists(outfile)
예제 #6
0
    def test_notebook_runner_exc(self):
        temp = get_temp_folder(__file__, "temp_notebook")
        nbfile = os.path.join(temp, "..", "data", "simple_example_exc.ipynb")
        self.assertExists(nbfile)
        addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
        self.assertExists(addpath)

        outfile = os.path.join(temp, "out_notebook.ipynb")
        self.assertNotExists(outfile)

        try:
            run_notebook(nbfile,
                         working_dir=temp,
                         outfilename=outfile,
                         additional_path=[addpath])
        except NotebookError as e:
            self.assertIn("name 'str2datetimes' is not defined", str(e))
def execute_notebooks(folder, notebooks, filter,
                      clean_function=None,
                      fLOG=noLOG,
                      deepfLOG=noLOG,
                      valid_cell=None):
    """
    execute a list of notebooks

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validate the notebooks
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @param      valid_cell      to disable the execution of a cell
    @return                     dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })

    The signature of function ``filter`` is::

        def filter( i, filename) : return True or False

    """

    def _valid_cell(cell):
        if valid_cell is not None:
            if not valid_cell(cell):
                return False
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        if 'print(next(it))' in cell:
            return False
        return True

    kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
        "ensae_projects")
    addpath = get_additional_paths()
    results = {}
    for i, note in enumerate(notebooks):
        if filter(i, note):
            fLOG("******", i, os.path.split(note)[-1])
            outfile = os.path.join(folder, "out_" + os.path.split(note)[-1])
            try:
                stat, out = run_notebook(note, working_dir=folder, outfilename=outfile,
                                         additional_path=addpath, valid=valid_cell,
                                         clean_function=clean_function, fLOG=deepfLOG,
                                         kernel_name=kernel_name)
                if not os.path.exists(outfile):
                    raise FileNotFoundError(outfile)
                results[note] = (True, stat, out)
            except Exception as e:
                results[note] = (False, None, e)
    return results
def execute_notebooks(folder, notebooks, filter, clean_function=None,
                      fLOG=noLOG, deepfLOG=noLOG, replacements=None):
    """
    execute a list of notebooks

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validates the notebooks to test (True means will be tested)
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @param      replacements    replacements
    @return                     dictionary { notebook_file: (isSuccess, outout) }

    The signature of function ``filter`` is::

        def filter( i, filename) : return True or False

    """

    def valid_cell(cell):
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        if 'remote_open' in cell:
            return None
        if 'blobpassword' in cell:
            return None
        return True

    addpath = get_additional_paths()
    kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
        "ensae_teaching_cs")
    results = {}
    tested = []
    for i, note in enumerate(notebooks):
        if filter(i, note):
            fLOG("******", i, os.path.split(note)[-1])
            tested.append(note)
            outfile = os.path.join(folder, "out_" + os.path.split(note)[-1])
            try:
                stat, out = run_notebook(note, working_dir=folder, outfilename=outfile,
                                         additional_path=addpath, valid=valid_cell,
                                         clean_function=clean_function, fLOG=deepfLOG,
                                         kernel_name=kernel_name, replacements=replacements)
                if not os.path.exists(outfile):
                    raise FileNotFoundError(outfile)
                results[note] = (True, stat, out)
            except Exception as e:
                results[note] = (False, None, e)
    if len(tested) == 0:
        raise Exception("no notebook were tested with '{0}'".format(filter))
    return results
예제 #9
0
    def test_notebook_runner_magic_command(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        notebook = os.path.split(
            __file__)[-1].replace(".ipynb", "").replace(".py", "")[5:]
        temp = get_temp_folder(__file__, "temp_" + notebook)
        nbfile = os.path.join(
            temp,
            "..",
            "..",
            "..",
            "_doc",
            "notebooks",
            "%s.ipynb" %
            notebook)
        if not os.path.exists(nbfile):
            raise FileNotFoundError(nbfile)
        addpath = [os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
                   os.path.normpath(
            os.path.join(
                temp,
                "..",
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")),
        ]

        cps = [os.path.normpath(nbfile),
               os.path.join(os.path.dirname(nbfile), "pyensae_sql_magic.ipynb")]
        for cp in cps:
            if not os.path.exists(cp):
                raise FileNotFoundError(cp)
            shutil.copy(cp, temp)

        kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
            "pyensae")

        outfile = os.path.join(temp, "out_notebook.ipynb")
        assert not os.path.exists(outfile)
        out = run_notebook(
            nbfile,
            working_dir=temp,
            outfilename=outfile,
            additional_path=addpath,
            kernel_name=kernel_name)
        fLOG(out)
        assert os.path.exists(outfile)
예제 #10
0
    def test_notebook_runner(self):
        temp = get_temp_folder(__file__, "temp_notebook")
        nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb")
        self.assertExists(nbfile)
        addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
        self.assertExists(addpath)

        outfile = os.path.join(temp, "out_notebook.ipynb")
        self.assertNotExists(outfile)

        stat, out = run_notebook(nbfile,
                                 working_dir=temp,
                                 outfilename=outfile,
                                 additional_path=[addpath])
        self.assertExists(outfile)
        self.assertNotIn("No module named 'pyquickhelper'", out)
        self.assertIn("datetime.datetime(2015, 3, 2", out)
        self.assertIsInstance(stat, dict)
예제 #11
0
    def test_notebook_runner_empty(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        temp = get_temp_folder(__file__, "temp_notebook_empty")
        nbfile = os.path.join(
            temp,
            "..",
            "data",
            "td2a_cenonce_session_4B.ipynb")
        assert os.path.exists(nbfile)
        addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
        assert os.path.exists(addpath)

        outfile = os.path.join(temp, "out_notebook.ipynb")
        assert not os.path.exists(outfile)
        out = run_notebook(nbfile, working_dir=temp, outfilename=outfile,
                           additional_path=[addpath])
        fLOG(out)
        assert os.path.exists(outfile)
        assert "No module named 'pyquickhelper'" not in out