def test_encrypt_decrypt(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") password = "******" * 2 temp = get_temp_folder(__file__, "temp_encrypt") temp2 = get_temp_folder(__file__, "temp_encrypt2") tempmm = get_temp_folder(__file__, "temp_encrypt_status") cstatus = os.path.join(tempmm, "crypt_status.txt") cmap = os.path.join(tempmm, "crypt_map.txt") srcf = os.path.abspath(os.path.join(temp, "..")) sys.argv = ["", srcf, temp, password, "--status", cstatus, "--map", cmap] encrypt(fLOG=fLOG) this = __file__ sys.argv = ["", temp, temp2, password] decrypt(fLOG=fLOG) with open(__file__, "rb") as f: c1 = f.read() with open(os.path.join(temp2, os.path.split(this)[-1]), "rb") as f: c2 = f.read() self.assertEqual(c1, c2) fLOG("end")
def test_auc_file(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") ans = [1, 1, 1, 1, 1, 0, 0, 0, 0] score = [i * 0.8 + 0.1 for i in ans] score[0] = 0.4 score[-1] = 0.6 fLOG(score) t1 = get_temp_folder(__file__, "temp_answers") t2 = get_temp_folder(__file__, "temp_scores") f1 = "answer.txt" f2 = "answer.txt" fu1 = os.path.join(t1, f1) fu2 = os.path.join(t2, f2) out = os.path.join(t2, "scores.txt") with open(fu1, "w") as f: f.write("\n".join(str(_) for _ in ans)) with open(fu2, "w") as f: f.write("\n".join(str(_) for _ in score)) private_codalab_wrapper_binary_classification( AUC, "AUC", t1, t2, output=out) assert os.path.exists(out) with open(out, "r") as f: code = f.read() fLOG("**", code) self.assertEqual(code, "AUC:0.95")
def test_install_revealjs(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_install_revealjs") dest = get_temp_folder(__file__, "temp_install_revealjs_dest") fs = download_revealjs(temp, dest, fLOG=fLOG) fLOG(fs) assert len(fs) > 0 for a in fs: assert os.path.exists(a)
def test_rst_only(self): from docutils import nodes as skip_ content = """ test a directive ================ .. only:: html only for html .. only:: rst only for rst """.replace(" ", "") if sys.version_info[0] >= 3: content = content.replace('u"', '"') tives = [("cmdref", CmdRef, cmdref_node, visit_cmdref_node, depart_cmdref_node)] text = rst2html(content, # fLOG=fLOG, writer="rst", keep_warnings=True, directives=tives, extlinks={'issue': ('http://%s', '_issue_')}) temp = get_temp_folder(__file__, "temp_only") with open(os.path.join(temp, "out_cmdref.rst"), "w", encoding="utf8") as f: f.write(text) t1 = "only for rst" if t1 not in text: raise Exception(text) t1 = "only for html" if t1 in text: raise Exception(text) text = rst2html(content, # fLOG=fLOG, writer="html", keep_warnings=True, directives=tives, extlinks={'issue': ('http://%s', '_issue_')}) temp = get_temp_folder(__file__, "temp_only") with open(os.path.join(temp, "out_cmdref.rst"), "w", encoding="utf8") as f: f.write(text) t1 = "only for rst" if t1 in text: raise Exception(text) t1 = "only for html" if t1 not in text: raise Exception(text)
def test_auc_multi_file(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") t1 = get_temp_folder(__file__, "temp_multi_answers") t2 = get_temp_folder(__file__, "temp_multi_scores") t3 = get_temp_folder(__file__, "temp_multi_scores2") truth = os.path.join( t1, "..", "data", "tbl_test_dossier.Y.dummy.truth.txt") ans = os.path.join(t1, "..", "data", "tbl_test_dossier.Y.dummy.txt") f1 = "answer.txt" f2 = "answer.txt" shutil.copy(truth, os.path.join(t1, f1)) shutil.copy(ans, os.path.join(t2, f2)) with open(truth, "r") as f: lines = [_.strip("\r\n ").split("\t") for _ in f.readlines()] lines = [[l[0], l[1], "1.0", "1.0"] for l in lines] f3 = "answer.txt" with open(os.path.join(t3, f3), "w") as f: f.write("\n".join("\t".join(_) for _ in lines)) temp = get_temp_folder(__file__, "temp_multi_out") # = out = os.path.join(temp, "outpute.txt") private_codalab_wrapper_multi_classification(AUC_multi_multi, ["orientation", "nature"], t1, t3, output=out) assert os.path.exists(out) with open(out, "r") as f: code = f.read() self.assertEqual( code, "orientation_ERR:0.0\norientation_AUC:1.0\nnature_ERR:0.0\nnature_AUC:1.0") # dummy fLOG("------------ dummy") out = os.path.join(temp, "outputd.txt") private_codalab_wrapper_multi_classification(AUC_multi_multi, ["orientation", "nature"], t1, t2, output=out, ignored=["nul"]) assert os.path.exists(out) with open(out, "r") as f: code = f.read() fLOG("**", code) self.assertEqual( code, "orientation_ERR:0.7183754333828628\norientation_AUC:0.5862428771453444\nnature_ERR:0.8236750866765725\nnature_AUC:0.5160556240766593")
def test_sphinx_ext_video_latex(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_sphinx_ext_video_latex") fLOG('custom app init') src_ = self.setup_format(temp) app = CustomSphinxApp(src_, temp, buildername="latex") fLOG('custom app build') app.build() fLOG('custom app done') index = os.path.join(temp, "pyq-video.tex") self.assertExists(index) with open(index, "r", encoding="utf-8") as f: content = f.read() self.assertNotIn("unable to find", content) self.assertIn('mur.mp4}', content) index = os.path.join(temp, "mur.mp4") self.assertExists(index) index = os.path.join(temp, "jol", "mur2.mp4") self.assertExists(index) index = os.path.join(temp, "jol", 'im', "mur3.mp4") self.assertExists(index) if is_travis_or_appveyor() not in ('travis', 'appveyor'): latex = find_latex_path() fLOG("latex-compile", latex) compile_latex_output_final(temp, latex, doall=True) fLOG("compilatione done") index = os.path.join(temp, "pyq-video.pdf") self.assertExists(index)
def test_join_multiple2(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") filename = os.path.join(os.path.split( __file__)[0], "data", "database_linked.zip") temp = get_temp_folder(__file__, "temp_join_multiple2") filename = unzip(filename, temp) assert os.path.exists(filename) db = Database(filename, LOG=fLOG) db.connect() where = {"bucket": ("==", "bu###1")} n1 = db.JoinTreeNode("profile_QSSH", where=where, parent_key="query", key="query") n2 = db.JoinTreeNode("url_QSSH", where=where, parent_key=('url', 'pos'), key=('url', 'pos')) n1.append(n2) sql, fields = db.inner_joins(n1, execute=False, create_index=False) view = db.execute_view(sql) assert view == [('facebbooklogin', 1, 0, 'bu###1', 86, 0, 'digg.com/security/Hackers_Put_Social_Networks_In_Crosshairs', 'digg.com/security/Hackers_Put_Social_Networks_In_Crosshairs', 1, 0, 1, 1, 0, 0, 0, 0)] assert "WHERE" in sql db.close()
def test_image_video_puzzle_girafe(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_image_video_girafe") if is_travis_or_appveyor() in ("travis",): # pygame.error: No available video device return import pygame if is_travis_or_appveyor() == "circleci": # os.environ["SDL_VIDEODRIVER"] = "x11" flags = pygame.NOFRAME else: flags = 0 pygame_simulation(pygame, fLOG=fLOG, folder=temp, delay=200 if __name__ == "__main__" else 2, flags=flags) files = os.listdir(temp) assert len(files) > 9 png = [os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png"] assert len(png) > 0 out = os.path.join(temp, "puzzle_girafe.avi") v = make_video(png, out, size=(500, 500), format="XVID", fps=4) assert v is not None
def test_image_video_epidemic(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_image_video_epidemic") if is_travis_or_appveyor() in ("travis",): # pygame.error: No available video device return import pygame if is_travis_or_appveyor() == "circleci": # os.environ["SDL_VIDEODRIVER"] = "x11" flags = pygame.NOFRAME else: flags = 0 pygame_simulation(pygame, fLOG=fLOG, iter=10, folder=temp, flags=flags) files = os.listdir(temp) self.assertTrue(len(files) > 9) png = [os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png"] self.assertTrue(len(png) > 0) out = os.path.join(temp, "epidemic.avi") v = make_video(png, out, size=(300, 300), format="XVID") self.assertTrue(v is not None)
def test_notebook_latex_problems(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join( path, "..", "..", "_doc", "notebooks", "td2a_ml")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if _.endswith(".ipynb") and "problems" in _] nbs.sort() print(nbs) formats = ["pdf", "ipynb", "html", "python", "rst", "docx"] if is_travis_or_appveyor() is not None: warnings.warn( "travis, appveyor, unable to test %s" % self._testMethodName) return temp = get_temp_folder(__file__, "temp_nb_td2A_ml_bug_latex") res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) assert os.path.exists(_[0])
def test_example_pydy(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_example_pydy") fix_tkinter_issues_virtualenv() import matplotlib.pyplot as plt fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 8)) try: example_pydy(ax=ax) except Exception as e: if 'can only concatenate list (not "tuple") to list' in str(e): warnings.warn("Pydy needs to be updated for Python 3.7") return else: raise e self.assertNotEmpty(ax) img = os.path.join(temp, "img.png") fig.savefig(img) self.assertExists(img) if __name__ == "__main__": fig.show() plt.close('all') fLOG("end")
def test_benchmark(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_grid_benchmark") params = [dict(value=random.randint(10, 20), name="name%d" % i, shortname="m%d" % i) for i in range(0, 2)] datasets = [dict(X=pandas.DataFrame([[0, 1], [0, 1]]), name="set1", shortname="s1"), dict(X=pandas.DataFrame([[1, 1], [1, 1]]), name="set2", shortname="s2"), ] bench = ATestOverGridBenchMark("TestName", datasets, fLOG=fLOG, clog=temp, cache_file=os.path.join(temp, "cache.pickle")) bench.run(params) df = bench.to_df() ht = df.to_html(float_format="%1.3f", index=False) self.assertTrue(len(df) > 0) self.assertTrue(ht is not None) self.assertEqual(df.shape[0], 4) report = os.path.join(temp, "report.html") csv = os.path.join(temp, "report.csv") rst = os.path.join(temp, "report.rst") bench.report(filehtml=report, filecsv=csv, filerst=rst, title="A Title", description="description") self.assertTrue(os.path.exists(report)) self.assertTrue(os.path.exists(csv)) self.assertTrue(os.path.exists(rst))
def test_folium(self): temp = get_temp_folder(__file__, "temp_folium") outfile = os.path.join(temp, "osm.map") map_osm = folium.Map(location=[48.85, 2.34]) map_osm.save(outfile=outfile) self.assertTrue(os.path.exists(outfile))
def test_notebook_rst_svg(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_nb_rst_svg") nbs = [os.path.normpath(os.path.join( temp, '..', "data", "rst_notebooks", "notebook_with_svg.ipynb"))] formats = ["rst"] res = process_notebooks(nbs, temp, temp, formats=formats, fLOG=fLOG) name = res[0][0] with open(name, 'r', encoding='utf-8') as f: content = f.read() self.assertIn('SVG in a notebook.', content) self.assertIn('.. image::', content) nb = 0 for line in content.split('\n'): if '.. image::' in line: name = line.replace('.. image::', '').strip(' \r\t') dest = os.path.join(temp, name) self.assertExists(dest) nb += 1 self.assertGreater(nb, 0)
def test_enumerate_feedback(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") exp = [ "<p>ok</p>", ] temp = get_temp_folder(__file__, "temp_enumerate_feedback") data = os.path.abspath(os.path.dirname(__file__)) data = os.path.join(data, "data") xls = os.path.join(data, "groupes_eleves_pitch.xlsx") df = pandas.read_excel(xls, sheetname=0, index=False) mails = list(enumerate_feedback(df, exc=False, fLOG=fLOG, begin="BEGIN", end="END", subject="SUBJECT", col_name="Nom", cols=["Pitch", "Code"])) for i, m in enumerate(mails): fLOG("------------", i) name = os.path.join(temp, "m%d.html" % i) with open(name, "w", encoding="utf-8") as f: f.write(m[1]) if i < len(exp): if exp[i] not in m[1]: raise Exception("EXP\n{0}\nRES\n{1}".format(exp[i], m[1]))
def test_notebook_raw(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "data")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if "TD_2A" in _] self.assertGreater(len(nbs), 0) formats = ["latex", "present", "ipynb", "html", "python", "rst", "pdf"] if sys.platform.startswith("win"): formats.append("docx") temp = get_temp_folder(__file__, "temp_nb_bug_raw") if is_travis_or_appveyor() in ('travis', 'appveyor'): return res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) self.assertExists(_[0]) check = os.path.join(temp, "TD_2A_Eco_Web_Scraping.tex") with open(check, "r", encoding="utf8") as f: content = f.read() if "\\begin{verbatim" not in content: raise Exception(content)
def test_module_c(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) file = os.path.join(path, "data", "pyd", "stdchelper.cp37-win_amd64.pyd") self.assertExists(file) mo = import_module( None, file, fLOG, additional_sys_path=None, first_try=True) self.assertIsInstance(mo, tuple) self.assertEqual(len(mo), 2) self.assertTrue(hasattr(mo[0], '__doc__')) if 'stdchelper' in sys.modules: del sys.modules['stdchelper'] temp = get_temp_folder(__file__, "temp_module_c") store_obj = {} actions = copy_source_files(os.path.dirname(file), temp, fLOG=fLOG) store_obj = {} indexes = {} add_file_rst(temp, store_obj, actions, fLOG=fLOG, rootrep=("module_c.", ""), indexes=indexes) self.assertNotEmpty(store_obj) self.assertEqual(len(store_obj), 4) if len(actions) != 2: raise Exception("{0}\n{1}".format( len(actions), "\n".join(str(_) for _ in actions))) self.assertEqual(len(indexes), 1)
def test_status(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": warnings.warn("run_cmd no end on travis") return temp = get_temp_folder(__file__, "temp_status") outfile = os.path.join(temp, "modules.xlsx") this = os.path.abspath(os.path.dirname(__file__)) script = os.path.normpath(os.path.join( this, "..", "..", "src", "pymyinstall", "cli", "pymy_status.py")) cmd = "{0} -u {1} {2}".format( sys.executable, script, "numpy --out={0}".format(outfile)) fLOG(cmd) out, err = run_cmd(cmd, wait=True) if len(out) == 0: if is_travis_or_appveyor() == "appveyor": warnings.warn( "CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) return else: raise Exception( "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) if len(err) > 0: raise Exception( "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) if not os.path.exists(outfile): raise Exception(outfile) fLOG(out)
def test_post_list(self): # the test will fail if you add a file in data/blog others # with rst files which is not a blog post directives.register_directive("blogpost", BlogPostDirective) path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.join(path, "data", "blog") out = get_temp_folder(__file__, "temp_post_list") p = BlogPostList(fold) cats = p.get_categories() months = p.get_months() self.assertEqual(cats, ['documentation', 'example']) self.assertEqual(months, ['2015-04']) res = p.write_aggregated(out) self.assertTrue(len(res) >= 4) for r in res: if r in (None, ''): raise ValueError("An empty value in {0}".format(res)) if not os.path.exists(r): raise FileNotFoundError("Unable to find '{0}'".format(r)) if 'main_0000.rst' in r: with open(r, 'r', encoding='utf-8') as f: content = f.read() self.assertIn('...', content)
def test_profiling(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_profiling") data = os.path.join(temp, "..", "data", "sample1000.txt") with open(data, "r", encoding="utf-8") as f: lines = [_.strip(" \n\r\t") for _ in f.readlines()] def profile_exe(): res = self.gain_dynamique_moyen_par_mot(lines, [1.0] * len(lines)) return res def prof(n, show): pr = cProfile.Profile() pr.enable() profile_exe() pr.disable() s = io.StringIO() ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() rem = os.path.normpath(os.path.join(temp, "..", "..", "..")) res = s.getvalue().replace(rem, "") if show: fLOG(res) with open(os.path.join(temp, "profiling%d.txt" % n), "w") as f: f.write(res) prof(1, show=False) prof(2, show=True)
def test_american_cities(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[:2] <= (3, 4): warnings.warn( "Issue with Python 3.4, bug probably related to wrong pointers") return fix_tkinter_issues_virtualenv() import matplotlib.pyplot as plt import mpld3 filter = {"NewYork": "NY", "Chicago": "CH", "SanFrancisco": "SF", "Seattle": "Sea"} temp = get_temp_folder(__file__, "temp_american_cities") data = os.path.join(temp, "..", "data", "american_cities.txt") df = pandas.read_csv(data) df["Longitude"] = -df["Longitude"] df["City"] = df["City"].apply(lambda v: filter.get(v, "")) fig, ax = plt.subplots(figsize=(32, 32)) df = df[df.Latitude < 52] df = df[df.Longitude > -130].copy() ax = graph_cities(df, ax=ax, markersize=3, fontcolor=(0, 1.0, 0), fontsize='40', fontname="Courrier", fontweight="bold") assert ax is not None img = os.path.join(temp, "img.png") fig.savefig(img) assert os.path.exists(img) name2 = os.path.join(temp, "picture.html") mpld3.save_html(fig, name2) assert os.path.exists(name2) if __name__ == "__main__": fig.show() plt.close('all') fLOG("end")
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) if not os.path.exists(doc): raise FileNotFoundError(doc) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) import pyquickhelper import jyquickhelper import pyensae import ensae_teaching_cs add_path = get_additional_paths( [jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs]) res = execute_notebook_list( temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_graph_graphviz(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") code = """ namespace hello { public static class world { public static double function(double x, doubly y) { return x+y ; } } } """ clparser, cllexer = get_parser_lexer("C#") parser = parse_code(code, clparser, cllexer) tree = parser.parse() st = get_tree_graph(tree, parser) dot = st.to_dot() # fLOG(dot) assert len(dot) > 0 if "travis" not in sys.executable: temp = get_temp_folder(__file__, "temp_dot_grammar") name = os.path.join(temp, "graph.dot") with open(name, "w") as f: f.write(dot) img = os.path.join(temp, "graph.png") run_dot(name, img) assert os.path.exists(img)
def test_notebook_runner_2a_eco_sncf(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf") keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 shutil.copy(simple_database(), temp) folder_note = os.path.split(keepnote[0])[0] jsfile = os.path.join(folder_note, "stop_areas.json") shutil.copy(jsfile, temp) def filter(i, n): if "SNCF" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_flat2db3(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") notebook = os.path.split( __file__)[-1].replace(".ipynb", "").replace(".py", "")[5:] temp = get_temp_folder(__file__, "temp_" + notebook) nbfile = os.path.join( temp, "..", "..", "..", "_doc", "notebooks", "%s.ipynb" % notebook) if not os.path.exists(nbfile): raise FileNotFoundError(nbfile) addpath = [os.path.normpath(os.path.join(temp, "..", "..", "..", "src")), os.path.normpath( os.path.join( temp, "..", "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join( temp, "..", "..", "..", "..", "pymyinstall", "src")), ] outfile = os.path.join(temp, "out_notebook.ipynb") assert not os.path.exists(outfile) def valid(code): return 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' not in code if "travis" in sys.executable: return kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest( "pyensae") out = run_notebook(nbfile, working_dir=temp, outfilename=outfile, additional_path=addpath, valid=valid, fLOG=fLOG, kernel_name=kernel_name) fLOG(out) assert os.path.exists(outfile)
def test_notebook_js(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "notebooks_js")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _] formats = ["slides", "present", "ipynb", "html", "python", "rst", "pdf"] if sys.platform.startswith("win"): formats.append("docx") temp = get_temp_folder(__file__, "temp_nb_bug_js") res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: if not os.path.exists(_[0]): raise Exception(_[0]) check = os.path.join(temp, "using_qgrid_with_jsdf.tex") with open(check, "r", encoding="utf8") as f: content = f.read() if "\\section{" not in content: raise Exception(content) checks = [os.path.join(temp, "reveal.js"), os.path.join(temp, "require.js")] for check in checks: if not os.path.exists(check): raise Exception(check)
def test_notebook_runner_2a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_") keepnote = ls_notebooks("2a") assert len(keepnote) > 0 def filter(i, n): if not sys.platform.startswith("win") and "_convert" in n: return False if "git_" not in n and "python_r" not in n and "csharp" not in n: return True if is_travis_or_appveyor() and "notebook_convert.ipynb" in n: # this one requires pandoc return False return False if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_ensae_2016_answers(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no stored password return password = keyring.get_password( "cpt", os.environ["COMPUTERNAME"] + "pwd") r = data_cpt_ENSAE_2016_11_blind_set(password) self.assertEqual(len(r), 7500) truth = r r = data_cpt_ENSAE_2016_11_blind_set("dummy") self.assertEqual(len(r), 7500) s = sum(r) assert 0 <= s <= len(r) auc = AUC(truth, r) fLOG(auc) assert 0 <= auc <= 1 temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers") out = os.path.join(temp, "answer.txt") with open(out, "w") as f: f.write("\n".join(str(_) for _ in r)) zip_files(os.path.join("submission.zip"), [out])
def test_download_zip(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") fold = get_temp_folder(__file__, "temp_download") url = "https://docs.python.org/3/library/ftplib.html" f = download(url, fold) fLOG(f) self.assertTrue(os.path.exists(f)) if not f.endswith("ftplib.html"): raise Exception(f) out1 = os.path.join(fold, "try.html.gz") gzip_files(out1, [f], fLOG=fLOG) self.assertTrue(os.path.exists(out1)) out2 = os.path.join(fold, "try.zip") zip_files(out2, [f], fLOG=fLOG) self.assertTrue(os.path.exists(out2)) if is_travis_or_appveyor() in ("circleci", None): out7 = os.path.join(fold, "try.7z") zip7_files(out7, [out1, out2], fLOG=fLOG, temp_folder=fold) if not os.path.exists(out7): raise FileNotFoundError(out7) else: fLOG("skip 7z")
def test_notebook_runner_2a_ml(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_ml4") keepnote = ls_notebooks("td2a_ml") keepnote = [_ for _ in keepnote if "overfitting" in _] shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False if "Scraping" in n: return False if "deep_python" in n: return False if "h2o" in n: # h2o is not working from a virtual environment return False if "td2a" in os.path.split(n)[-1]: # already tested by others tests return False if "libraries" in n: return False return True execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=ensae_teaching_cs)
def test_notebook_runner_enonce_10(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook1a_enonce_10") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a keepnote = ls_notebooks("td1a_dfnp") fold = os.path.dirname(keepnote[0]) for png in os.listdir(fold): if ".png" not in png: continue fLOG("copy", png) shutil.copy(os.path.join(fold, png), temp) execute_notebooks(temp, keepnote, lambda i, n: "cenonce_session_10." in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_added(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from pyensae.datasource import DownloadDataException temp = get_temp_folder(__file__, "temp_added") text = besancon_df(local=True, fLOG=fLOG) assert text is not None assert isinstance(text, str) try: text2 = added(local=False, cache_folder=temp, fLOG=fLOG) assert text2 assert isinstance(text2, str) except (ConnectionResetError, DownloadDataException) as e: warnings.warn("Cannot check remote added.txt.\n" + str(e)) return
def test_notebook_runner_2a_algo(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_algo") keepnote = ls_notebooks("td2a_algo") shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False return True def clean_function_1a_upgraded(code): code = clean_function_1a(code) rep = "[1000, 2000, 5000, 10000, 12000, 15000, 17000, 20000]" by = "[1000, 2000]" code = code.replace(rep, by) rep = "[10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000]" by = "[10, 20, 50, 100, 200]" code = code.replace(rep, by) return code if is_travis_or_appveyor() == "travis": # execution does not stop return fold = os.path.dirname(keepnote[0]) for png in os.listdir(fold): if ".png" not in png: continue fLOG("copy", png) shutil.copy(os.path.join(fold, png), temp) execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a_upgraded, dump=src.ensae_teaching_cs)
def test_validate_sklearn_operators_dump_all(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") logger = getLogger('skl2onnx') logger.disabled = True verbose = 1 if __name__ == "__main__" else 0 temp = get_temp_folder(__file__, "temp_validate_sklearn_operators_dump_all") self.assertRaise( lambda: list( enumerate_validated_operator_opsets( verbose, models={"DecisionTreeClassifier"}, filter_exp=lambda m, p: '64' not in p, fLOG=fLOG, dump_all=True)), ValueError) rows = list( enumerate_validated_operator_opsets( verbose, models={"DecisionTreeClassifier"}, filter_exp=lambda m, p: '64' not in p, fLOG=fLOG, dump_all=True, dump_folder=temp)) self.assertGreater(len(rows), 1) df = DataFrame(rows) self.assertGreater(df.shape[1], 1) fLOG("output results") df.to_csv(os.path.join(temp, "sklearn_opsets_report.csv"), index=False) df.to_excel(os.path.join(temp, "sklearn_opsets_report.xlsx"), index=False) stored = os.path.join( temp, ("dump-i-python-DecisionTreeClassifier-default-b-cl-tree._classes." "DecisionTreeClassifierzipmapFalse-op%d-nf4.pkl" % TARGET_OPSET)) with open(stored, "rb") as f: obj = pickle.load(f) self.assertIn('onnx_bytes', obj) self.assertIn('skl_model', obj) self.assertIn('X_test', obj) self.assertIn('Xort_test', obj)
def test_american_cities(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[:2] <= (3, 4): warnings.warn( "Issue with Python 3.4, bug probably related to wrong pointers" ) return fix_tkinter_issues_virtualenv() import matplotlib.pyplot as plt filter = { "NewYork": "NY", "Chicago": "CH", "SanFrancisco": "SF", "Seattle": "Sea" } temp = get_temp_folder(__file__, "temp_american_cities") data = os.path.join(temp, "..", "data", "american_cities.txt") df = pandas.read_csv(data) df["Longitude"] = -df["Longitude"] df["City"] = df["City"].apply(lambda v: filter.get(v, "")) fig, ax = plt.subplots(figsize=(32, 32)) df = df[df.Latitude < 52] df = df[df.Longitude > -130].copy() ax = graph_cities(df, ax=ax, markersize=3, fontcolor=(0, 1.0, 0), fontsize='40', fontweight="bold") assert ax is not None img = os.path.join(temp, "img.png") fig.savefig(img) assert os.path.exists(img) # mpld3 is deprecated. # name2 = os.path.join(temp, "picture.html") # mpld3.save_html(fig, name2) # assert os.path.exists(name2) if __name__ == "__main__": fig.show() plt.close('all') fLOG("end")
def test_notebook_number(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook_number") nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb") self.assertTrue(os.path.exists(nbfile)) outfile = os.path.join(temp, "out_nb.ipynb") res = remove_execution_number(nbfile, outfile) self.assertTrue(res is not None) self.assertTrue(os.path.exists(outfile)) self.assertTrue('"execution_count": null' in res) if is_travis_or_appveyor() == "appveyor": change_file_status(outfile) change_file_status(temp, strict=True) else: warnings.warn( "linux, unable to test change_file_status: TestNotebookNumber.test_notebook_number")
def test_venv_docutils08_readme(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") fold = os.path.dirname(os.path.abspath(__file__)) readme = os.path.join(fold, "..", "..", "README.rst") self.assertTrue(os.path.exists(readme)) with open(readme, "r", encoding="utf8") as f: content = f.read() self.assertTrue(len(content) > 0) temp = get_temp_folder(__file__, "temp_readme") if __name__ != "__main__": # does not work from a virtual environment return from pyquickhelper.pycode import check_readme_syntax check_readme_syntax(readme, folder=temp, fLOG=fLOG)
def test_notebook_runner_2a_cffi(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_") keepnote = ls_notebooks("2a") def filter(i, n): if "cffi" not in n: return False return True execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=ensae_teaching_cs)
def test_cli_benchmark_replay(self): temp = get_temp_folder(__file__, "temp_benchmark_replay") out1 = os.path.join(temp, "raw.csv") st = BufferedPrint() out1 = os.path.join(temp, "raw.csv") st = BufferedPrint() main(args=["validate_runtime", "--n_features", "4", "-nu", "3", "-re", "3", "-o", "11", "-op", "11", "-v", "2", "--out_raw", out1, "-b", "0", "--runtime", "python_compiled", "--models", "KMeans", "--dtype", "32", "--dump_all", '1', '--dump_folder', temp], fLOG=st.fprint) out = os.path.join(temp, "res.xlsx") main(args=["benchmark_replay", "--folder", temp, "--out", out, '--verbose', '2'], fLOG=st.fprint) res = str(st) self.assertExists(out) self.assertIn("'folder'", res)
def test_notebook_runner_flat2db3(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") notebook = os.path.split(__file__)[-1].replace(".ipynb", "").replace(".py", "")[5:] temp = get_temp_folder(__file__, "temp_" + notebook) nbfile = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", "%s.ipynb" % notebook) if not os.path.exists(nbfile): raise FileNotFoundError(nbfile) addpath = [ os.path.normpath(os.path.join(temp, "..", "..", "..", "src")), os.path.normpath( os.path.join(temp, "..", "..", "..", "..", "pyquickhelper", "src")), os.path.normpath( os.path.join(temp, "..", "..", "..", "..", "pymyinstall", "src")), ] outfile = os.path.join(temp, "out_notebook.ipynb") assert not os.path.exists(outfile) def valid(code): return 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' not in code if "travis" in sys.executable: return kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest( "pyensae") out = run_notebook(nbfile, working_dir=temp, outfilename=outfile, additional_path=addpath, valid=valid, fLOG=fLOG, kernel_name=kernel_name) fLOG(out) assert os.path.exists(outfile)
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None): temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder) self.assertTrue(os.path.exists(doc)) keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _] self.assertTrue(len(keepnote) > 0) if copy_folder is not None: if not os.path.exists(copy_folder): raise FileNotFoundError(copy_folder) dest = os.path.split(copy_folder)[-1] dest = os.path.join(temp, dest) if not os.path.exists(dest): os.mkdir(dest) synchronize_folder(copy_folder, dest, fLOG=fLOG) def new_valid(cell): if "print(next(it))\nprint(next(it))" in cell: return False return True if valid is None: valid = new_valid import pyquickhelper import jyquickhelper import pymyinstall import pyensae import pyrsslocal add_path = get_additional_paths([ jyquickhelper, pyquickhelper, pyensae, pymyinstall, pyrsslocal, thismodule ]) res = execute_notebook_list(temp, keepnote, additional_path=add_path, valid=valid) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_search_vectors_zip(self): temp = get_temp_folder(__file__, "temp_search_vectors_zip") res = [] for i in range(20): h = i * 0.05 h2 = 1 - i * 0.05 res.append( dict(ind=i * 5, meta1="m%d" % i, meta2="m%d" % (i + 1), f1=h, f2=h2)) df = pandas.DataFrame(res) se = SearchEngineVectors(n_neighbors=5) r = repr(se) self.assertEqual( r.replace("\n", "").replace(" ", ""), 'SearchEngineVectors(n_neighbors=5)') se.fit(data=None, features=df[["f1", "f2"]].values, metadata=df[["ind", "meta1", "meta2"]]) score, ind, meta = se.kneighbors([0.5, 0.5]) self.assertIsInstance(ind, (list, numpy.ndarray)) self.assertEqual(len(ind), 5) self.assertEqual(ind[0], 10) self.assertIsInstance(score, numpy.ndarray) self.assertEqual(score.shape, (5, )) self.assertEqual(score[0], 0) dest = os.path.join(temp, "se.zip") se.to_zip(dest, encoding='utf-8') se2 = SearchEngineVectors.read_zip(dest, encoding='utf-8') score2, ind2, meta2 = se2.kneighbors([0.5, 0.5]) self.assertEqualArray(score, score2) self.assertEqualArray(ind, ind2) self.assertEqualDataFrame(meta, meta2) self.assertEqual(se.pknn, se2.pknn)
def test_svg2img(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_svg2png") svg = """<svg width="200" height="180"> <rect x="50" y="20" width="150" height="150" style="fill:blue;stroke:pink;fill-opacity:0.1;" /> <ellipse cx="40" cy="50" rx="20" ry="30" style="fill:yellow" /> <ellipse cx="20" cy="50" rx="90" ry="20" style="fill:white" /> </svg> """ img = svg2img(svg) name = os.path.join(temp, "image.png") img.save(name) self.assertExists(name) svg = """<svg> <rect x="50" y="20" width="150" height="150" style="fill:blue;stroke:pink;fill-opacity:0.1;" /> <ellipse cx="40" cy="50" rx="20" ry="30" style="fill:yellow" /> <ellipse cx="20" cy="50" rx="90" ry="20" style="fill:white" /> </svg> """ size = guess_svg_size(svg) self.assertEqual(size, (180, 40)) img = svg2img(svg) name = os.path.join(temp, "image2.png") img.save(name) self.assertExists(name) img = svg2img(svg, dpi=200) name = os.path.join(temp, "image3.png") img.save(name) self.assertExists(name) img = svg2img(svg, scale=2.) name = os.path.join(temp, "image4.png") img.save(name) self.assertExists(name)
def test_import_exam_vigenere(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return from ensae_teaching_cs.td_1a.vigenere import code_vigenere temp = get_temp_folder(__file__, "temp_import_exam_vigenere") pwd = get_password("exam", "ensae_teaching_cs,key") pwd = pwd.encode("ascii") this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere") assert os.path.exists(dst) with open(dst, "rb") as f: content = f.read() content = code_vigenere(content, pwd, binary=True, decode=True) decr = os.path.join(temp, "onemod_vigenere.py") with open(decr, "wb") as f: f.write(content) assert os.path.exists(decr) fLOG("importing") sys.path.append(temp) import onemod as temp_module del sys.path[-1] answer = temp_module.build(120) exp = """*** H H *** H *** H *** H H *** H *** H *** H H *** H *** H *** H H *** H *** H""".replace(" ", "") self.assertEqual(answer, exp)
def test_notebook_ai(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_ai") keepnote = ls_notebooks("ai") self.assertTrue(len(keepnote) > 0) source = os.path.join(os.path.dirname(keepnote[0]), "images") images = os.path.join(temp, 'images') os.mkdir(images) for img in os.listdir(source): shutil.copy(os.path.join(source, img), images) res = execute_notebooks(temp, keepnote, lambda i, n: True, fLOG=fLOG, clean_function=clean_function_notebook) execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix)
def test_video_download(self): temp = get_temp_folder(__file__, "temp_video_download") try: download_youtube_video('vHcfbOqYztU', output_path=temp) except RegexMatchError as e: if "zero match" not in str(e): raise e import pytube warnings.warn( "RegexMatchError: pytube version {} - pytube issue: {}".format( pytube.__version__, e)) return except KeyError as e: import pytube warnings.warn( "KeyError: pytube version {} - pytube issue: {}".format( pytube.__version__, e)) return exp = os.path.join(temp, "vidéo tres courte.mp4") self.assertExists(exp)
def test_gradient(self): temp = get_temp_folder(__file__, "temp_segment_gradient") img = os.path.join(temp, "..", "data", "eglise_zoom2.jpg") grad = _calcule_gradient(img, color=0) self.assertEqual(grad.shape, (308, 408, 2)) for d in [-2, -1, 0, 1, 2]: imgrad = plot_gradient(img, grad, direction=d) grfile = os.path.join(temp, "gradient-%d.png" % d) imgrad.save(grfile) self.assertExists(grfile) with open(os.path.join(temp, "..", "data", "gradient--2.png"), 'rb') as f: c1 = f.read() with open(os.path.join(temp, "..", "data", "gradient--2b.png"), 'rb') as f: c1b = f.read() with open(os.path.join(temp, "gradient--2.png"), 'rb') as f: c2 = f.read() self.assertIn(c2, (c1, c1b))
def test_zip_numpy(self): df = numpy.zeros((3, 4)) df[2, 3] = 1 temp = get_temp_folder(__file__, "temp_zip") name = os.path.join(temp, "df.zip") to_zip(df, name, "arr.npy") df2 = read_zip(name, "arr.npy") self.assertEqualArray(df, df2) st = io.BytesIO() zp = zipfile.ZipFile(st, 'w') to_zip(df, zp, "arr.npy") zp.close() st = io.BytesIO(st.getvalue()) zp = zipfile.ZipFile(st, 'r') df3 = read_zip(zp, "arr.npy") zp.close() self.assertEqualArray(df, df3)
def test_cache_urls(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_cache_urls") fnb = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks")) keepnote = [os.path.join(fnb, "example_about_files.ipynb")] addpaths = [os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))] res = execute_notebook_list( temp, keepnote, fLOG=fLOG, valid=None, additional_path=addpaths, cache_urls=["https://docs.python.org/3.4/library/urllib.request.html"]) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=pyquickhelper)
def test_convert_music_file(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_convert_music_file") data = os.path.join(temp, "..", "data") ffmpeg = os.path.join(data, "ffmpeg.zip") assert os.path.exists(ffmpeg) ff = unzip(ffmpeg, temp) mp3 = os.path.join(data, "podcasts_example.mp3") wav = os.path.join(temp, "podcasts_example.wav") if is_travis_or_appveyor() == "travis": warnings.warn("pydub is not available") return convert_music_file(mp3, wav, ffmpeg=ff) assert os.path.exists(wav) assert os.stat(wav).st_size > 90000
def test_downloadlink_rst(self): name = self.get_name() content = """ :downloadlink:`rst::http://f.html` :downloadlink:`rst::{0}` :downloadlink:`{0} <rst::{0}>` """.replace(" ", "").format(name) out = rst2html(content, writer="rst", keep_warnings=True, directives=None) out = out.replace("\n", " ") self.assertNotIn('Unknown interpreted text role', out) self.assertIn(':downloadlink:`test_rst_builder.py', out) self.assertNotIn("test_rst_builder.py>`test_rst_builder.py", out) temp = get_temp_folder(__file__, "temp_downloadlink_rst") with open(os.path.join(temp, "out.rst"), "w", encoding="utf8") as f: f.write(out)
def test_maml_mlnet(self): temp = get_temp_folder(__file__, "temp_maml_mlnet") iris = datasets.load_iris() X = iris.data y = iris.target df = pandas.DataFrame( X, columns=['Slength', 'Swidth', 'Plength', 'Pwidth']) df["Label"] = y df = df[["Label"] + ['Slength', 'Swidth', 'Plength', 'Pwidth']] dest = os.path.join(temp, "iris_data_id.txt") df.to_csv(dest, sep=',', index=False) fct = mlnet("ReturnMLClass", TestMamlMlnet._script, redirect=True) cl, _, __ = fct(dest) cl.Train() res = cl.Predict(3.4, 5.4, 3.2, 5.6) label = res.PredictedLabel score = list(res.Score) self.assertEqual(label, 3) self.assertEqual(len(score), 3)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks from src.ensae_teaching_cs.automation.notebook_test_helper import clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_1") keepnote = ls_notebooks("td2a") fold = os.path.dirname(keepnote[0]) for png in os.listdir(fold): if ".png" not in png: continue fLOG("copy", png) shutil.copy(os.path.join(fold, png), temp) self.assertTrue(len(keepnote) > 0) execute_notebooks(temp, keepnote, (lambda i, n: "_1" in n), clean_function=clean_function_1a, fLOG=fLOG, dump=src.ensae_teaching_cs)
def test_perf_onnxruntime_logreg(self): res = onnxruntime_perf_binary_classifiers()[0] bp = BenchPerf(res['pbefore'], res['pafter'], res['fct']) results = list(bp.enumerate_run_benchs(repeat=10, verbose=True)) results_df = pandas.DataFrame(results) temp = get_temp_folder(__file__, "temp_perf_onnxruntime_logreg") out = os.path.join(temp, "onnxruntime_logreg.perf.csv") results_df.to_csv(out, index=False) self.assertExists(out) subset = { 'sklearn', 'numpy', 'pandas', 'onnxruntime', 'skl2onnx', 'mlprodict' } df = pandas.DataFrame(machine_information(subset)) out = os.path.join(temp, "onnxruntime_logreg.time.csv") df.to_csv(out, index=False) self.assertExists(out)
def test_notebook_add_slides_bug(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook_add_slides_bug") nbfile = os.path.join(temp, "..", "data", "pyensae_text2table.ipynb") assert os.path.exists(nbfile) nb = read_nb(nbfile, kernel=False) for i, cell in enumerate(nb.iter_cells()): ls = nb.cell_height(cell) #fLOG(i,"-",nb.cell_type(cell), l) #if l > 400: fLOG(cell) assert ls > 0 new_tags = nb.add_tag_slide() for k, v in sorted(new_tags.items()): a, b, c = v fLOG(k, a, b) assert len(new_tags) > 0
def test_notebook_runner_2a_eco_nlp_long1(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_eco_nlp_long1") keepnote = ls_notebooks("td2a_eco2") def filter(i, n): if "td2a_Seance_7_Analyse_de_textes" in n: return True return False execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=ensae_teaching_cs)
def test_video_frame(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") import gizeh temp = get_temp_folder(__file__, "temp_video_frame") def make_frame(t): # See example: https://zulko.github.io/moviepy/getting_started/videoclips.html#videoclip surface = gizeh.Surface(128, 128) radius = 100 * (1 + (t * (2 - t))**2) / 6 circle = gizeh.circle(radius, xy=(64, 64), fill=(1, 0, 0)) circle.draw(surface) return surface.get_npimage() vid = video_frame(make_frame) exp = os.path.join(temp, "courte.gif") video_save(vid, exp, fps=20, duration=2) self.assertExists(exp) clean_video(vid)
def test_notebook_latex_problems(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath( os.path.join(path, "..", "..", "_doc", "notebooks", "td2a_ml")) nbs = [ os.path.join(fold, _) for _ in os.listdir(fold) if _.endswith(".ipynb") and "problems" in _ ] nbs.sort() formats = ["pdf", "ipynb", "html", "python", "rst"] temp = get_temp_folder(__file__, "temp_nb_td2A_ml_bug_latex") res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) self.assertTrue(os.path.exists(_[0]))
def a_test_notebook_runner(self, name, folder): from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.helpers.size_helper import total_size self.assertTrue(total_size) temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name)) keepnote = ls_notebooks(folder) self.assertTrue(len(keepnote) > 0) replacements = { 'input("Entrez un nombre")': 'random.randint(0, 100)', 'input(message)': 'random.randint(0, 100)' } execute_notebooks(temp, keepnote, lambda i, n: name in n, fLOG=fLOG, replacements=replacements, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_crypt_helper(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_crypt") input = os.path.abspath(__file__.replace(".pyc", ".py")) output = os.path.join(temp, "crypted.bin") output2 = os.path.join(temp, "decrypted.txt") key = "01" * 8 encrypt_data(key, input, output) decrypt_data(key, output, output2) with open(input, "r") as f: t1 = f.read() with open(output2, "r") as f: t2 = f.read() self.assertEqual(t1, t2) assert len(t1) > 0