def test_notebook_runner_ensae_sklearn(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.automation.notebook_test_helper import copy_data_file from src.ensae_teaching_cs.helpers.size_helper import total_size self.assertTrue(total_size) temp = get_temp_folder(__file__, "temp_notebook_ensae_sklearn") keepnote = ls_notebooks("sklearn_ensae_course") self.assertTrue(len(keepnote) > 0) copy_data_file("sklearn_ensae_course", "iris_setosa.jpg", temp, fLOG=fLOG) copy_data_file("sklearn_ensae_course", "iris_versicolor.jpg", temp, fLOG=fLOG) copy_data_file("sklearn_ensae_course", "iris_virginica.jpg", temp, fLOG=fLOG) execute_notebooks(temp, keepnote, lambda i, n: True, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs, additional_path=[os.path.dirname(keepnote[0])])
def test_notebook_runner_2a_eco(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # Requires authentification. return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False if "Scraping" in n: return False if "2.ipynb" in n: return False if "flask" in n.lower(): # flask from a notebook does not work return False return True execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_2_enonce") keepnote = ls_notebooks("td2a") if is_travis_or_appveyor() in ('travis', 'appveyor'): warnings.warn( "travis or appveyor, unable to test TestNotebookRunner2a_2_enonce.test_notebook_runner" ) return fold = os.path.dirname(keepnote[0]) for png in os.listdir(fold): if ".png" not in png: continue fLOG("copy", png) shutil.copy(os.path.join(fold, png), temp) self.assertTrue(len(keepnote) > 0) execute_notebooks(temp, keepnote, lambda i, n: "_2" in n and "enonce" in n and "_2D" not in n and "_2B" not in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_ml(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_ml4") keepnote = ls_notebooks("td2a_ml") keepnote = [_ for _ in keepnote if "overfitting" in _] shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False if "Scraping" in n: return False if "deep_python" in n: return False if "h2o" in n: # h2o is not working from a virtual environment return False if "td2a" in os.path.split(n)[-1]: # already tested by others tests return False if "libraries" in n: return False return True execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_correction_1_7(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook1a_correction_1_7") keepnote = ls_notebooks("td1a") fold = os.path.dirname(keepnote[0]) cp = os.path.join(temp, "..", "data", "seance4_excel.txt") shutil.copy(cp, temp) cp = os.path.join(temp, "..", "data", "seance4_excel.xlsx") shutil.copy(cp, temp) cp = os.path.join(fold, "td2_1.png") shutil.copy(cp, temp) execute_notebooks(temp, keepnote, lambda i, n: "_12" not in n and "session6." not in n and "session8." not in n and "session9." not in n and "session_10." not in n and "session_11." not in n and "session5." not in n and "deviner" not in n and "correction" in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_bayesian(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_") keepnote = ls_notebooks("2a") def filter(i, n): if "bayesian_with_python" in n: return True return False if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_long(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if "travis" in sys.executable: # skip travis and R warnings.warn( "travis, unable to test TestNotebookRunner2a_long.test_notebook_runner_2a_long") return if "R_HOME" not in os.environ or not os.path.exists(os.environ["R_HOME"]): paths = [r"C:\Program Files\R\R-3.2.4revised", r"C:\Program Files\R\R-3.2.4", r"C:\Program Files\R\R-3.2.3", r"C:\Program Files\R\R-3.2.2"] for path in paths: if os.path.exists(path): os.environ["R_HOME"] = path break if "R_HOME" not in os.environ: warnings.warn("No installed R") return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks from src.ensae_teaching_cs.automation.notebook_test_helper import clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_long_") keepnote = ls_notebooks("2a") execute_notebooks(temp, keepnote, (lambda i, n: "python_r" in n), fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") # this notebook describes how to distribute the work with multiple processors # it requires to start multiple clusters first (a command line) # and to stop them afterwards # it still needs to be implemented # we skip ! warnings.warn( "TODO: implement a unit test testing the distribution on multiple processors" ) do_test = False if do_test: temp = get_temp_folder(__file__, "temp_notebook2a_2_enonce_2D") keepnote = ls_notebooks("td2a") execute_notebooks( temp, keepnote, lambda i, n: "_2" in n and "enonce" in n and "_2D" in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_eco_sncf(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) folder_note = os.path.split(keepnote[0])[0] jsfile = os.path.join(folder_note, "stop_areas.json") shutil.copy(jsfile, temp) def filter(i, n): if "SNCF" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_enonce_algo(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_algo") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a keepnote = ls_notebooks("td1a_algo") def filter(i, n): if is_travis_or_appveyor() == "travis": if "graph1exo_parcours" in n or "graph4exos" in n: # Graphviz is installed but cannot be found. return False if "graph_spectral_clustering" in n: # Graphviz is installed but cannot be found. return False return "BJKST" in n or ("enonce" not in n and "correction" not in n) execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_") keepnote = ls_notebooks("2a") def filter(i, n): if "seance_5" in n: return False if not sys.platform.startswith("win") and "_convert" in n: return False if is_travis_or_appveyor() and "notebook_convert.ipynb" in n: # this one requires pandoc return False if "git_" not in n and "python_r" not in n and "csharp" not in n: return True else: return False if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_long_notebook_runner_2a_ml(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_ml_deep") keepnote = ls_notebooks("td2a_deep") shutil.copy(simple_database(), temp) def filter(i, n): if "deep_python" in n: return True return False if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return clog = CustomLog(temp) execute_notebooks(temp, keepnote, filter, fLOG=clog, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def a_test_notebook_runner(self, name, folder): from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.helpers.size_helper import total_size self.assertTrue(total_size) temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) keepnote = ls_notebooks(folder) self.assertTrue(len(keepnote) > 0) def custom_clean_function_1a(cell): if "..." in cell: return "" if "b[0,0] = 44444444" in cell: return "" return clean_function_1a(cell) replacements = { 'input("Entrez un nombre")': 'random.randint(0, 100)', 'input(message)': 'random.randint(0, 100)' } execute_notebooks(temp, keepnote, lambda i, n: name in n, fLOG=fLOG, replacements=replacements, clean_function=custom_clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks temp = get_temp_folder(__file__, "temp_notebook2a_5") keepnote = ls_notebooks("2a") execute_notebooks(temp, keepnote, (lambda i, n: "_5" in n), fLOG=fLOG, dump=src.ensae_teaching_cs)
def test_notebook_runner_correction(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_3A_correction") keepnote = ls_notebooks("td2a_ml") execute_notebooks(temp, keepnote, (lambda i, n: "_3A" in n and "correction" in n), clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_enonce_8(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook1a_enonce_8") keepnote = ls_notebooks("td1a_algo") execute_notebooks(temp, keepnote, lambda i, n: "cenonce_session8." in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_1") keepnote = ls_notebooks("competitions/2016") execute_notebooks(temp, keepnote, (lambda i, n: "solution" not in n), clean_function=clean_function_1a, fLOG=fLOG, dump=src.ensae_teaching_cs)
def test_notebook_runner_exposelong2(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebookexposelong2_") keepnote = ls_notebooks("expose") execute_notebooks(temp, keepnote, (lambda i, n: "paris_parcours" in n), fLOG=fLOG, deepfLOG=fLOG if __name__ == "__main__" else noLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_correction(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks temp = get_temp_folder(__file__, "temp_notebook2a_4_correction") keepnote = ls_notebooks("td2a_ml") execute_notebooks(temp, keepnote, lambda i, n: "_4" in n and "correction" in n, fLOG=fLOG, replacements=self.get_replacements(), dump=src.ensae_teaching_cs)
def test_notebook_runner_enonce_12(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks temp = get_temp_folder(__file__, "temp_notebook1a_enonce_12") keepnote = ls_notebooks("td1a_dfnp") execute_notebooks(temp, keepnote, (lambda i, n: "cenonce_session_12" in n), fLOG=fLOG, deepfLOG=fLOG if __name__ == "__main__" else noLOG, clean_function=TestNotebookRunner1a_enonce_12.clean_function, dump=src.ensae_teaching_cs)
def a_test_notebook_runner_1a(self, name, folder): from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.helpers.size_helper import total_size self.assertTrue(total_size) temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) keepnote = ls_notebooks(folder) self.assertTrue(len(keepnote) > 0) execute_notebooks(temp, keepnote, lambda i, n: name in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_exams(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebookexams_") keepnote = ls_notebooks("exams") execute_notebooks(temp, keepnote, lambda i, n: "git_" not in n and "interro_rapide_20_minutes_2014_11" not in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_soft_cpp(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_soft") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a keepnote = ls_notebooks("td1a_soft") execute_notebooks(temp, keepnote, lambda i, n: "edit_correction" in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_eleves(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks temp = get_temp_folder(__file__, "temp_notebook_eleves") keepnote = ls_notebooks("notebook_eleves/2016-2017") self.assertTrue(len(keepnote) > 0) execute_notebooks(temp, keepnote, (lambda i, n: True), fLOG=fLOG, replacements=self.get_replacements(), dump=src.ensae_teaching_cs)
def test_notebook_runner_enonce(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sql") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) execute_notebooks(temp, keepnote, (lambda i, n: "sql" in n), fLOG=fLOG, replacements=self.get_replacements(), dump=src.ensae_teaching_cs)
def test_notebook_runner_correction_9(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, copy_data_file temp = get_temp_folder(__file__, "temp_notebook1a_correction_9") keepnote = ls_notebooks("td1a_algo") copy_data_file("td1a", "seance4_excel.txt", temp, fLOG=fLOG) copy_data_file("td1a", "seance4_excel.xlsx", temp, fLOG=fLOG) execute_notebooks(temp, keepnote, (lambda i, n: "_12" not in n and "session9." in n and "correction" in n), fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_expose1(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebookexpose1_") keepnote = ls_notebooks("expose") execute_notebooks( temp, keepnote, lambda i, n: "velib" not in n and "paris_parcours" not in n and "ml_table_mortalite" not in n and "huge" not in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_2_corection") keepnote = ls_notebooks("td2a") shutil.copy(simple_database(), temp) execute_notebooks( temp, keepnote, (lambda i, n: "n_2" in n and "correction" in n and "_2B" not in n), fLOG=fLOG, dump=src.ensae_teaching_cs)
def common_notebook_runner_2a_eco_nlp_enonce(self, sub): from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_eco_nlp_" + sub) keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 if sub == "correction": folder = os.path.join(temp, "ressources_googleplus") os.mkdir(folder) folder_note = os.path.split(keepnote[0])[0] jsfile = os.path.join( folder_note, "ressources_googleplus", "107033731246200681024.json") shutil.copy(jsfile, folder) def filter(i, n): if "Traitement_automatique" not in n: return False if sub == "enonce": if "correction" in n: return False elif sub not in n: return False return True res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook2a_1") keepnote = ls_notebooks("td2a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "_1" in n, fLOG=fLOG) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_3a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a if is_travis_or_appveyor(): # too long for appveyor # not available on linux return temp = get_temp_folder(__file__, "temp_notebook3a") keepnote = ls_notebooks("td3a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: True, fLOG=fLOG, clean_function=clean_function_1a) assert len(res) > 0 fails = [(os.path.split(k)[-1], v) for k, v in sorted(res.items()) if not v[0]] for f in fails: fLOG(f) if len(fails) > 0: e = str(fails[0][1][-1]) fLOG(str(e).replace("\n", " EOL ")) raise fails[0][1][-1] else: fLOG("success")
def test_notebook_runner_enonce_9(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": # requires MKL and this is not yet a good story return temp = get_temp_folder(__file__, "temp_notebook1a_enonce_9") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a keepnote = ls_notebooks("td1a_algo") execute_notebooks(temp, keepnote, lambda i, n: "cenonce_session9." in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_long(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if "travis" in sys.executable: # skip travis and R warnings.warn( "travis, unable to test TestNotebookRunner2a_long.test_notebook_runner_2a_long" ) return if "R_HOME" not in os.environ or not os.path.exists( os.environ["R_HOME"]): paths = [ r"C:\Program Files\R\R-3.2.2", r"C:\Program Files\R\R-3.1.2" ] for path in paths: if os.path.exists(path): os.environ["R_HOME"] = path break temp = get_temp_folder(__file__, "temp_notebook2a_long_") keepnote = ls_notebooks("2a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "python_r" in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_correction_1_7(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook1a_correction_1_7") keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 cp = os.path.join(temp, "..", "data", "seance4_excel.txt") shutil.copy(cp, temp) cp = os.path.join(temp, "..", "data", "seance4_excel.xlsx") shutil.copy(cp, temp) res = execute_notebooks(temp, keepnote, lambda i, n: "_12" not in n and "session6." not in n and "session8." not in n and "session9." not in n and "session_10." not in n and "session_11." not in n and "correction" in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a_long(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if "travis" in sys.executable: # skip travis and R warnings.warn( "travis, unable to test TestNotebookRunner2a_long.test_notebook_runner_2a_long") return if "R_HOME" not in os.environ or not os.path.exists(os.environ["R_HOME"]): paths = [r"C:\Program Files\R\R-3.2.2", r"C:\Program Files\R\R-3.1.2"] for path in paths: if os.path.exists(path): os.environ["R_HOME"] = path break temp = get_temp_folder(__file__, "temp_notebook2a_long_") keepnote = ls_notebooks("2a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "python_r" in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_") keepnote = ls_notebooks("2a") assert len(keepnote) > 0 def filter(i, n): if not sys.platform.startswith("win") and "_convert" in n: return False if "git_" not in n and "python_r" not in n and "csharp" not in n: return True if is_travis_or_appveyor() and "notebook_convert.ipynb" in n: # this one requires pandoc return False return False if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a_eco_scraping(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_scraping") keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 shutil.copy(simple_database(), temp) def filter(i, n): if "Scraping" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a_eco_sncf(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf") keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 shutil.copy(simple_database(), temp) folder_note = os.path.split(keepnote[0])[0] jsfile = os.path.join(folder_note, "stop_areas.json") shutil.copy(jsfile, temp) def filter(i, n): if "SNCF" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_2_corection_2B") keepnote = ls_notebooks("td2a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "_2" in n and "correction" in n and "_2B" in n, fLOG=fLOG) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_1a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_") keepnote = ls_notebooks("1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "deviner" not in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_enonce(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, unittest_raise_exception_notebook, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_3A_enonce") keepnote = ls_notebooks("td2a_ml") assert len(keepnote) > 0 res = execute_notebooks( temp, keepnote, lambda i, n: "_3A" in n and "enonce" in n, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_exams(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebookexams_") keepnote = ls_notebooks("exams") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "git" not in n and "interro_rapide_20_minutes_2014_11" not in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_correction_11(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_correction_11") keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "correction_session_11" in n, fLOG=fLOG, clean_function=TestNotebookRunner1a_correction_11.clean_function) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_enonce_10(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_enonce_10") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "cenonce_session_10." in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_exposelong3(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebookexposelong3_") keepnote = ls_notebooks("expose") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "mortalite" in n, fLOG=fLOG, deepfLOG=fLOG if __name__ == "__main__" else noLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_expose1(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebookexpose1_") keepnote = ls_notebooks("expose") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "velib" not in n and "paris_parcours" not in n and "ml_table_mortalite" not in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if "travis" in sys.executable: # skip C# on linux warnings.warn( "travis, unable to test TestNotebookRunner2a_csharp.test_notebook_runner_2a") return if not sys.platform.startswith("win"): return temp = get_temp_folder(__file__, "temp_notebook2a_sharp") keepnote = ls_notebooks("2a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "csharp" in n, fLOG=fLOG, clean_function=clean_function_1a) assert len(res) > 0 fails = [(os.path.split(k)[-1], v) for k, v in sorted(res.items()) if not v[0]] for f in fails: fLOG(f) if len(fails) > 0: e = str(fails[0][1][-1]) if "Audio device error encountered" in str(e): # maybe the script is running on a virtual machine (no Audia # device) if os.environ["USERNAME"] == "ensaestudent" or \ os.environ["USERNAME"] == "vsxavierdupre" or \ "paris" in os.environ["COMPUTERNAME"].lower() or \ os.environ["USERNAME"].endswith("$"): # anonymous Jenkins configuration # I would prefer to catch a proper exception # it just exclude one user only used on remotre machines fLOG("no audio") return elif "<class 'int'>-" in str(e): # issue with conversion from 3 to double return fLOG(str(e).replace("\n", " EOL ")) raise fails[0][1][-1] else: fLOG("success")
def test_notebook_runner_correction(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": # requires MKL return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_4_correction") keepnote = ls_notebooks("td2a_ml") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "_4" in n and "correction" in n, fLOG=fLOG, replacements=self.get_replacements()) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_exposelong1(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebookexposelong1_") keepnote = ls_notebooks("expose") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "velib" in n, fLOG=fLOG, deepfLOG=fLOG if __name__ == "__main__" else noLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks from src.ensae_teaching_cs.automation.notebook_test_helper import ( unittest_raise_exception_notebook, clean_function_1a, ) temp = get_temp_folder(__file__, "temp_notebook2a_1") keepnote = ls_notebooks("competitions/2016") assert len(keepnote) > 0 res = execute_notebooks( temp, keepnote, lambda i, n: "solution" not in n, clean_function=clean_function_1a, fLOG=fLOG ) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_1a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.helpers.size_helper import total_size assert total_size temp = get_temp_folder(__file__, "temp_notebook1a_") keepnote = ls_notebooks("1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "deviner" not in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_enonce_9(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": # requires MKL and this is not yet a good story return temp = get_temp_folder(__file__, "temp_notebook1a_enonce_9") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "cenonce_session9." in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_enonce_12(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": # issue with MKL on travis return temp = get_temp_folder(__file__, "temp_notebook1a_enonce_12") keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "cenonce_session_12" in n, fLOG=fLOG, deepfLOG=fLOG if __name__ == "__main__" else noLOG, clean_function=TestNotebookRunner1a_enonce_12.clean_function) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_2_enonce_2B") # The folder must remain, I did not find a way to install # a package without a message box. os.environ["R_USER"] = "******" keepnote = ls_notebooks("td2a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "_2" in n and "enonce" in n and "_2B" in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_enonce(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sql") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) assert len(keepnote) > 0 res = execute_notebooks( temp, keepnote, lambda i, n: "sql" in n, fLOG=fLOG, replacements=self.get_replacements()) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_correction_10(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook1a_correction_10") keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 if is_travis_or_appveyor() == "travis": warnings.warn("too long") return res = execute_notebooks(temp, keepnote, lambda i, n: "_12" not in n and "session_10." in n and "correction" in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_2_enonce") keepnote = ls_notebooks("td2a") assert len(keepnote) > 0 if is_travis_or_appveyor(): warnings.warn( "travis or appveyor, unable to test TestNotebookRunner2a_2_enonce.test_notebook_runner") return res = execute_notebooks(temp, keepnote, lambda i, n: "_2" in n and "enonce" in n and "_2D" not in n and "_2B" not in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks from src.ensae_teaching_cs.automation.notebook_test_helper import unittest_raise_exception_notebook, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_1_tiny") keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 for k in keepnote: if "_1" in k: fLOG("*********", k) res = execute_notebooks( temp, keepnote, lambda i, n: "_1" in n and "td2_eco_rappels_1a" in n, clean_function=clean_function_1a, fLOG=fLOG) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_enonce_1_7(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_enonce_1_7") keepnote = ls_notebooks("td1a") assert len(keepnote) > 0 res = execute_notebooks(temp, keepnote, lambda i, n: "_12" not in n and "cenonce_session1." not in n and "cenonce_session6." not in n and "cenonce_session8." not in n and "cenonce_session9." not in n and "cenonce_session_10." not in n and "cenonce_session_11." not in n and "enonce" in n, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a_ml(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco") keepnote = ls_notebooks("td2a_ml") assert len(keepnote) > 0 shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False if "Scraping" in n: return False if "deep_python" in n: return False if "h2o" in n: # h2o is not working from a virtual environment return False if "td2a" in os.path.split(n)[-1]: # already tested by others tests return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)