def test_image_video_epidemic(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_image_video_epidemic") if is_travis_or_appveyor() in ("travis",): # pygame.error: No available video device return import pygame if is_travis_or_appveyor() == "circleci": # os.environ["SDL_VIDEODRIVER"] = "x11" flags = pygame.NOFRAME else: flags = 0 pygame_simulation(pygame, fLOG=fLOG, iter=10, folder=temp, flags=flags) files = os.listdir(temp) self.assertTrue(len(files) > 9) png = [os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png"] self.assertTrue(len(png) > 0) out = os.path.join(temp, "epidemic.avi") v = make_video(png, out, size=(300, 300), format="XVID") self.assertTrue(v is not None)
def test_image_video_puzzle_girafe(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_image_video_girafe") if is_travis_or_appveyor() in ("travis",): # pygame.error: No available video device return import pygame if is_travis_or_appveyor() == "circleci": # os.environ["SDL_VIDEODRIVER"] = "x11" flags = pygame.NOFRAME else: flags = 0 pygame_simulation(pygame, fLOG=fLOG, folder=temp, delay=200 if __name__ == "__main__" else 2, flags=flags) files = os.listdir(temp) assert len(files) > 9 png = [os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png"] assert len(png) > 0 out = os.path.join(temp, "puzzle_girafe.avi") v = make_video(png, out, size=(500, 500), format="XVID", fps=4) assert v is not None
def test_notebook_runner_2a_eco_scraping(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_scraping") keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 shutil.copy(simple_database(), temp) def filter(i, n): if "Scraping" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": # too long for appveyor return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook temp = get_temp_folder(__file__, "temp_notebook2a_") keepnote = ls_notebooks("2a") assert len(keepnote) > 0 def filter(i, n): if not sys.platform.startswith("win") and "_convert" in n: return False if "git_" not in n and "python_r" not in n and "csharp" not in n: return True if is_travis_or_appveyor() and "notebook_convert.ipynb" in n: # this one requires pandoc return False return False if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_kruskal_pygame_simulation(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_kruskal_pygame_simulation") if is_travis_or_appveyor() in ("travis",): # pygame.error: No available video device return import pygame if is_travis_or_appveyor() == "circleci": # os.environ["SDL_VIDEODRIVER"] = "x11" flags = pygame.NOFRAME else: flags = 0 pygame.init() pygame_simulation(fLOG=fLOG, max_iter=10 if __name__ != "__main__" else 10000, pygame=pygame, folder=temp, flags=flags) files = os.listdir(temp) self.assertGreater(len(files), 9) png = [os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png"] self.assertGreater(len(png), 0) out = os.path.join(temp, "tsp_kruskal.avi") v = make_video(png, out, size=(800, 500), format="XVID", fps=20) self.assertNotEmpty(v)
def test_status(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": warnings.warn("run_cmd no end on travis") return temp = get_temp_folder(__file__, "temp_status") outfile = os.path.join(temp, "modules.xlsx") this = os.path.abspath(os.path.dirname(__file__)) script = os.path.normpath(os.path.join( this, "..", "..", "src", "pymyinstall", "cli", "pymy_status.py")) cmd = "{0} -u {1} {2}".format( sys.executable, script, "numpy --out={0}".format(outfile)) fLOG(cmd) out, err = run_cmd(cmd, wait=True) if len(out) == 0: if is_travis_or_appveyor() == "appveyor": warnings.warn( "CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) return else: raise Exception( "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) if len(err) > 0: raise Exception( "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) if not os.path.exists(outfile): raise Exception(outfile) fLOG(out)
def test_script_help(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "travis": warnings.warn("run_cmd no end on travis") return script = os.path.join(os.path.dirname(os.path.abspath( __file__)), "..", "..", "src", "pymyinstall", "cli", "pymy_install.py") if not os.path.exists(script): raise Exception(script) scriptu = os.path.join(os.path.dirname(os.path.abspath( __file__)), "..", "..", "src", "pymyinstall", "cli", "pymy_update.py") if not os.path.exists(script): raise Exception(script) exe = sys.executable cmd = exe + " " + script + " --help" out, err = run_cmd(cmd, wait=True, fLOG=fLOG) if "usage: pymy_install.py" not in out: raise Exception(out) cmd = exe + " " + scriptu + " --help" out, err = run_cmd(cmd, wait=True, fLOG=fLOG) if "usage: pymy_update.py" not in out: if is_travis_or_appveyor() == "appveyor": warnings.warn( "CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) else: raise Exception( "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
def test_diff_full(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") seq1 = "zab zab2 zabc3 zabcd zabc4".split() seq2 = "ab ab2 abc3 abc4 abc adb".split() diff = SequenceMatcher(a=seq1, b=seq2) h = 20 size = 500, 500 white = 255, 255, 255 if is_travis_or_appveyor() in ("travis",): # pygame.error: No available video device return import pygame if is_travis_or_appveyor() == "circleci": # os.environ["SDL_VIDEODRIVER"] = "x11" flags = pygame.NOFRAME else: flags = 0 pygame, screen, fonts = get_pygame_screen_font(h, size, flags=flags) from ensae_teaching_cs.helpers.pygame_helper import wait_event bars = [random.randint(10, 500) / 500.0 for s in seq2] screen.fill(white) build_diff_image(pygame, screen, h=h, maxw=size[1], seq1=seq1, seq2=seq2, diff=diff, fonts=fonts, bars=bars) pygame.display.flip() temp = get_temp_folder(__file__, "temp_video_diff_full") for i in range(0, 21): screen.fill(white) build_diff_image(pygame, screen, h=h, maxw=size[0], seq1=seq1, seq2=seq2, diff=diff, fonts=fonts, bars=bars, progress=i / 20.0, prev_bars=None) pygame.time.wait(60) pygame.display.flip() pygame.image.save(screen, os.path.join(temp, "diff%d.png" % i)) if __name__ == "__main__": from ensae_teaching_cs.helpers.video_helper import make_video png = [os.path.join(temp, _) for _ in os.listdir(temp) if ".png" in _] out = os.path.join(temp, "diff.avi") make_video(png, out, size=(350, 250), format="XVID", fps=5) wait_event(pygame) for font in fonts.values(): del font pygame.quit()
def valid(cell): if "open_html_form" in cell: return False if "open_window_params" in cell: return False if '<div style="position:absolute' in cell: return False if "completion.dot" in cell and is_travis_or_appveyor() == "travis": return False if 'Image("completion.png")' in cell and is_travis_or_appveyor() == "travis": return False return True
def test_selenium_image(self): if is_travis_or_appveyor(): return temp = get_temp_folder(__file__, "temp_selenium_image") img = os.path.join(temp, "image_selenium.png") url = "http://www.xavierdupre.fr/" navigator = "chrome" # download_chromedriver(dest=temp) # os.environ["PATH"] += ";" + temp res = webshot(img, url, navigator=navigator) self.assertExists(img) self.assertEqual(len(res), 1) self.assertEqual(len(res[0]), 2) if navigator != "opera": # not available on splinter img = os.path.join(temp, "image_splinter.png") res = webshot(img, url, module='splinter', navigator=navigator) img = res[0][1] self.assertExists(img) self.assertEqual(len(res), 1) self.assertEqual(len(res[0]), 2) else: warnings.warn("opera not available on splinter")
def test_graph2(self): """ This test is failing with Python 3.4 if many pictures are drawn. """ fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": return fix_tkinter_issues_virtualenv() from matplotlib import pyplot as plt cache = os.path.abspath(os.path.split(__file__)[0]) cache = os.path.join(cache, "temp_cache2") stock = StockPrices("BNP.PA", folder=cache) stock2 = StockPrices("CA.PA", folder=cache) fig, ax = plt.subplots(figsize=(16, 8)) ax = stock.plot(ax=ax) ax = stock2.plot(ax=ax) img = os.path.abspath( os.path.join( os.path.split(__file__)[0], "temp_image2.png")) if os.path.exists(img): os.remove(img) fig.savefig(img) plt.close('all') assert os.path.exists(img)
def test_theano_logreg(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # it requires latex return from theano import config fLOG(config) from src.ensae_teaching_cs.examples.theano_logreg import theano_sgd_optimization_mnist, theano_predict temp = get_temp_folder(__file__, "temp__theano_logreg") dataset = "mnist.pkl.gz" if not os.path.exists(dataset): download_data( dataset, website="http://deeplearning.net/data/mnist/") model = os.path.join(temp, "log_reg_theano.bin") theano_sgd_optimization_mnist( dataset=dataset, saved_model=model, n_epochs=2, fLOG=fLOG) pred = theano_predict(model, dataset, 10) fLOG(pred) fLOG(type(pred)) self.assertEqual(len(pred), 10)
def test_image_video_kohonen(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_image_video_tsp_kohonen") if is_travis_or_appveyor() == "travis": warnings.warn("pygame is not available") return import pygame pygame_simulation(pygame, fLOG=fLOG, folder=temp, nb=200 if __name__ == "__main__" else 20, size=(400, 250)) files = os.listdir(temp) assert len(files) > 9 png = [ os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png" ] assert len(png) > 0 out = os.path.join(temp, "tsp_kohonen.avi") v = make_video(png, out, size=(200, 125), format="XVID", fps=20) assert v is not None
def test_joined_schemas(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": warnings.warn("disabled on appveyor") return temp = get_temp_folder(__file__, "temp_joined_schemas") try: df = merge_schema() except PasswordException: # the password is not available on this machine return assert len(df) > 0 outexc = os.path.join(temp, "schemas.xlsx") df.to_excel(outexc, index=False) fLOG(df.columns) txt = df2rsthtml(df, format="rst") assert "nan\t" not in txt fLOG("\n" + txt) outfile = os.path.join(temp, "schemas.rst") with open(outfile, "w", encoding="utf-8") as f: f.write(txt)
def test_read_all_elections(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) temp = os.path.join(path, "temp_read_all_elections") if not os.path.exists(temp): os.mkdir(temp) if is_travis_or_appveyor() == "travis": warnings.warn( "The test cannot read an excel file using xlrd. Disabling it.") return for year in [2012, 2002, 2007]: for level in ["Départements", "Cantons", ]: fLOG("******reading ", year, level) file = os.path.join(path, "data", "election_%d.xlsx" % year) if not os.path.exists(file): file = os.path.join(temp, "election_%d.xlsx" % year) if not os.path.exists(file): zip = os.path.join(path, "data", "elections.zip") unzip(zip, temp) file = os.path.join(temp, "election_%d.xlsx" % year) assert os.path.exists(file) el = ElectionResults(year=year, file=file, level=level) fLOG("saving") xls = os.path.join(temp, "out_%d_%s_0.xlsx" % (year, level)) el.T0.to_excel(xls) xls = os.path.join(temp, "out_%d_%s_1.xlsx" % (year, level)) el.T1.to_excel(xls) res = el.vote_transfer() fLOG(res)
def test_sphinx_ext_video_latex(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_sphinx_ext_video_latex") fLOG('custom app init') src_ = self.setup_format(temp) app = CustomSphinxApp(src_, temp, buildername="latex") fLOG('custom app build') app.build() fLOG('custom app done') index = os.path.join(temp, "pyq-video.tex") self.assertExists(index) with open(index, "r", encoding="utf-8") as f: content = f.read() self.assertNotIn("unable to find", content) self.assertIn('mur.mp4}', content) index = os.path.join(temp, "mur.mp4") self.assertExists(index) index = os.path.join(temp, "jol", "mur2.mp4") self.assertExists(index) index = os.path.join(temp, "jol", 'im', "mur3.mp4") self.assertExists(index) if is_travis_or_appveyor() not in ('travis', 'appveyor'): latex = find_latex_path() fLOG("latex-compile", latex) compile_latex_output_final(temp, latex, doall=True) fLOG("compilatione done") index = os.path.join(temp, "pyq-video.pdf") self.assertExists(index)
def test_notebook_runner(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a temp = get_temp_folder(__file__, "temp_notebook2a_2_enonce") keepnote = ls_notebooks("td2a") if is_travis_or_appveyor() in ('travis', 'appveyor'): warnings.warn( "travis or appveyor, unable to test TestNotebookRunner2a_2_enonce.test_notebook_runner" ) return fold = os.path.dirname(keepnote[0]) for png in os.listdir(fold): if ".png" not in png: continue fLOG("copy", png) shutil.copy(os.path.join(fold, png), temp) self.assertTrue(len(keepnote) > 0) execute_notebooks(temp, keepnote, lambda i, n: "_2" in n and "enonce" in n and "_2D" not in n and "_2B" not in n, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_rst2html_png_bug(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() in ('travis', 'appveyor'): # It requires latex. return if sys.version_info[:2] <= (2, 7): # i don't want to fix it for Python 2.7 return temp = get_temp_folder(__file__, "temp_rst2html_png_latex") rst = os.path.join(os.path.abspath( os.path.dirname(__file__)), "data", "puzzle_girafe.rst") with open(rst, "r", encoding="utf-8") as f: content = f.read() text = rst2html(content, fLOG=fLOG, outdir=temp, warnings_log=True, imgmath_latex_preamble=""" \\newcommand{\\acc}[1]{\\left\\{#1\\right\\}} \\newcommand{\\cro}[1]{\\left[#1\\right]} \\newcommand{\\pa}[1]{\\left(#1\\right)} \\newcommand{\\girafedec}[3]{ \\begin{array}{ccccc} #1 &=& #2 &+& #3 \\\\ a' &=& a &-& o \\end{array}} \\newcommand{\\vecteur}[2]{\\pa{#1,\\dots,#2}} \\newcommand{\\R}[0]{\\mathbb{R}} \\newcommand{\\N}[0]{\\mathbb{N}} """) # fLOG(text) ji = os.path.join(temp, "out.html") with open(ji, "w", encoding="utf-8") as f: f.write(text)
def test_exe(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") command = "dir" if sys.platform.startswith("win32") else "ls" yml = """ language: python python: - {{Python35}} before_script: - %s after_script: - %s {{PLATFORM}} script: - %s """.replace(" ", "") % (command, command, command) temp = get_temp_folder(__file__, "temp_yaml_exe") context = dict(Python34="fake", Python35=os.path.dirname(sys.executable), Python27=None, Anaconda3=None, Anaconda2=None, WinPython35=None, project_name="pyquickhelper", root_path="ROOT", PLATFORM="win32") obj, name = load_yaml(yml, context=context) self.assertTrue(name is not None) res = list(enumerate_convert_yaml_into_instructions( obj, variables=context)) for r, var in res: conv = convert_sequence_into_batch_file(r, variables=var) if ("%s " % command) not in conv: raise Exception("{0}\n--\n{1}".format(command, conv)) fLOG("####", conv) ext = "bat" if command == "dir" else "sh" name = os.path.join(temp, "yml.%s" % ext) with open(name, "w") as f: f.write(conv) if is_travis_or_appveyor() == "__travis": # linux, unable to test TestYamlExe.test_exe. pass else: if sys.platform.startswith("win"): cmd = name else: cmd = "bash " + name try: out, err = run_cmd(cmd, wait=True) except PermissionError as e: raise Exception( "Unable to execute '{0}' which contains\n{1}".format(name, conv)) from e fLOG("###") fLOG(out) if "BEFORE_SCRIPT" not in out: raise Exception( "{0}\nERR\n{2}\n#########\n{1}".format(out, conv, err)) if "AFTER_SCRIPT" not in out: raise Exception( "{0}\nERR\n{2}\n#########\n{1}".format(out, conv, err)) if "SCRIPT" not in out: raise Exception( "{0}\nERR\n{2}\n#########\n{1}".format(out, conv, err))
def test_notebook_latex(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "notebooks_latex")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _] formats = ["ipynb", "html", "python", "rst", "pdf"] if sys.platform.startswith("win"): formats.append("docx") temp = os.path.join(path, "temp_nb_bug_latex") if not os.path.exists(temp): os.mkdir(temp) for file in os.listdir(temp): os.remove(os.path.join(temp, file)) if is_travis_or_appveyor() in ('travis', 'appveyor'): return res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) self.assertExists(_[0])
def test_clone_repo(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") fold = os.path.abspath(os.path.split(__file__)[0]) temp = os.path.join(fold, "temp_clone_repo") if os.path.exists(temp): removedirs(temp, use_command_line=True) if not os.path.exists(temp): os.mkdir(temp) if is_travis_or_appveyor() in ("travis", "appveyor"): return to = os.path.join(temp, "pq") out, err = clone(to, "github.com", "sdpython", "pyquickhelper") fLOG("OUT:", out) fLOG("ERR:", err) self.assertTrue("Cloning into" in err) self.assertTrue( os.path.exists( os.path.join(to, "src", "pyquickhelper", "__init__.py"))) out, err = rebase(to, "github.com", "sdpython", "pyquickhelper") fLOG("OUT:", out) fLOG("ERR:", err) r = change_file_status(temp) self.assertTrue(len(r) > 0)
def test_style_test(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") run_lint = is_travis_or_appveyor(env=['NAME_JENKINS']) is None _private_test_style_test(fLOG, run_lint, verbose='-v' in sys.argv)
def test_notebook_latex1(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) nbfold = os.path.normpath( os.path.join(path, "..", "..", "_doc", "notebooks")) nbs = [ os.path.join(nbfold, "notebook_eleves", "2014_2015", "2015_kmeans.ipynb"), # os.path.join(nbfold, "notebook_eleves", "2014_2015", "2015_page_rank.ipynb"), # os.path.join(nbfold, "notebook_eleves", "2014_2015", "2015_factorisation_matrice.ipynb"), ] formats = [ "pdf", ] temp = get_temp_folder(__file__, "temp_nb_bug_latex1") if is_travis_or_appveyor(): warnings.warn( "travis or appveyor, unable to test TestNoteBooksBugLatex_ecs.test_notebook_latex1" ) return res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) assert os.path.exists(_[0])
def test_notebook_latex_problems(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join( path, "..", "..", "_doc", "notebooks", "td2a_eco")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if _.endswith(".ipynb") and "TD2A_eco_API_SNCF_corrige" in _] nbs.sort() formats = ["pdf", "ipynb", "html", "python", "rst", "docx"] if is_travis_or_appveyor() in ('appveyor', 'travis'): warnings.warn( "travis, appveyor, unable to test %s" % self._testMethodName) return temp = get_temp_folder(__file__, "temp_nb_td2A_eco_bug_latex2") res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) self.assertTrue(os.path.exists(_[0]))
def test_install_deps(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") this = os.path.abspath(os.path.dirname(__file__)) script = os.path.normpath(os.path.join(this, "..", "..", "src", "pymyinstall", "cli", "pymy_deps.py")) cmd = "{0} -u {1} {2}".format(sys.executable, script, "pandas") try: out, err = run_cmd(cmd, wait=True, fLOG=fLOG, communicate=True, timeout=120) except subprocess.CalledProcessError as e: mes = "CMD\n{0}\nOUT\n{1}\nERR\n{2}".format(e.cmd, e.output, e.stderr.read() if e.stderr else "") raise Exception(mes) from e out = out.strip() if len(out) == 0: if is_travis_or_appveyor() == "appveyor": warnings.warn("CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) else: raise Exception("cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err)) else: fLOG(out) if sys.version_info[0] == 2: # failing on Python 2.7 return if "pandas" not in out: raise Exception(out) if "['pandas']" not in out: raise Exception(out)
def test_sphinx_ext_video_latex(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_sphinx_ext_video_latex") fLOG('custom app init') src_ = self.setup_format(temp) app = CustomSphinxApp(src_, temp, buildername="latex") fLOG('custom app build') app.build() fLOG('custom app done') index = os.path.join(temp, "pyq-video.tex") self.assertExists(index) with open(index, "r", encoding="utf-8") as f: content = f.read() self.assertNotIn("unable to find", content) self.assertIn('mur.mp4}', content) index = os.path.join(temp, "mur.mp4") self.assertExists(index) index = os.path.join(temp, "jol", "mur2.mp4") self.assertExists(index) index = os.path.join(temp, "jol", 'im', "mur3.mp4") self.assertExists(index) if is_travis_or_appveyor() not in ('travis', 'appveyor'): latex = find_latex_path() fLOG("latex-compile", latex) compile_latex_output_final(temp, latex, doall=True) fLOG("compilatione done") index = os.path.join(temp, "pyq-video.pdf") self.assertExists(index)
def test_graph2(self): """ This test is failing with Python 3.4 if many pictures are drawn. """ fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() == "appveyor": return fix_tkinter_issues_virtualenv() from matplotlib import pyplot as plt cache = os.path.abspath(os.path.split(__file__)[0]) cache = os.path.join(cache, "temp_cache2") stock = StockPrices("BNP.PA", folder=cache) stock2 = StockPrices("CA.PA", folder=cache) fig, ax = plt.subplots(figsize=(16, 8)) ax = stock.plot(ax=ax) ax = stock2.plot(ax=ax) img = os.path.abspath( os.path.join(os.path.split(__file__)[0], "temp_image2.png")) if os.path.exists(img): os.remove(img) fig.savefig(img) plt.close('all') assert os.path.exists(img)
def test_optimisation(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") sol2 = exercice_particulier2() x2 = sol2['x'] d = abs(x2[0] - 0.428571428055853) + abs(x2[1] - 0.2857142848749249) assert d < 1e-5 if is_travis_or_appveyor() == "travis": # skip mkl return sol1 = exercice_particulier1() sol2 = exercice_particulier2() fLOG("cvxopt") fLOG(sol1) fLOG("solution:", sol1['x'].T) fLOG("Arrow_Hurwicz") fLOG(sol2) fLOG("solution:", sol2['x']) x1 = sol1['x'] x2 = sol2['x'] d1 = x1[0] - x2[0] d2 = x1[1] - x2[1] fLOG(d1, d2) assert abs(d1) < 1e-5 assert abs(d2) < 1e-5
def test_rst2html_png(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() in ('travis', 'appveyor'): # It requires latex. return temp = get_temp_folder(__file__, "temp_rst2html_png") rst = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "hermionne.rst") with open(rst, "r", encoding="utf-8") as f: content = f.read() text = rst2html(content) ji = os.path.join(temp, "out.html") with open(ji, "w", encoding="utf-8") as f: f.write(text) text2 = rst2html(content, layout="sphinx") ji = os.path.join(temp, "out_sphinx.html") with open(ji, "w", encoding="utf-8") as f: f.write(text) self.assertTrue(len(text2) > len(text))
def test_scene(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_scene") s = Scene(Repere(), math.pi / 1.5, 400, 300) s.ajoute_source(Source(Vecteur(0, 10, 10), Couleur(1, 1, 1))) s.ajoute_source(Source(Vecteur(10, 10, 5), Couleur(0.5, 0.5, 0.5))) s.ajoute_objet(Sphere(Vecteur(0, 0, 12), 3, Couleur(1, 0, 0))) s.ajoute_objet( Sphere(Vecteur(0, -400, 12), 396, Couleur(0.5, 0.5, 0.5))) fLOG(s) if is_travis_or_appveyor() == "travis": warnings.warn("pygame is not available") return import pygame from src.ensae_teaching_cs.helpers.pygame_helper import wait_event screen = pygame.display.set_mode(s.dim) screen.fill((255, 255, 255)) s.construit_image(screen, pygame=pygame, fLOG=fLOG) pygame.image.save(screen, os.path.join(temp, "scene.png")) if __name__ == "__main__": wait_event(pygame)
def test_install_set_schedule(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") this = os.path.abspath(os.path.dirname(__file__)) script = os.path.normpath( os.path.join(this, "..", "..", "src", "pymyinstall", "cli", "pymy_install.py")) cmd = "{0} -u {1} {2}".format(sys.executable, script, "--set=pyquickhelper --schedule") try: out, err = run_cmd(cmd, wait=True, fLOG=fLOG, communicate=True, timeout=60) except subprocess.CalledProcessError as e: mes = "CMD\n{0}\nOUT\n{1}\nERR\n{2}".format( e.cmd, e.output, e.stderr.read() if e.stderr else "") raise Exception(mes) from e if len(out) == 0: if is_travis_or_appveyor() == "appveyor": warnings.warn( "CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format( cmd, out, err)) else: raise Exception("cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format( cmd, out, err))
def test_reap_children(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.platform.startswith("win"): cmd = "pause" else: cmd = 'ls -la | less' temp = get_temp_folder(__file__, "temp_reap_children") clog = CustomLog(temp) proc, _ = run_cmd(cmd, wait=False, fLOG=clog) self.assertTrue(_ is None) clog('proc={} pid={}'.format(proc, proc.pid)) ki = reap_children(fLOG=clog, subset={proc.pid}) clog('ki={0}'.format(ki)) if ki is None and not is_travis_or_appveyor( ) and __name__ != '__main__': warnings.warn( "reap_children could not be fully tested ki is None.") return self.assertTrue(ki is not None) self.assertEqual(len(ki), 1) # fLOG(ki) # To avoid a warning. proc.returncode = 0
def test_notebook(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "notebooks")) nbs = [ os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _ ] formats = ["ipynb", "python", "rst", "pdf"] temp = get_temp_folder(__file__, "temp_nb_bug") if is_travis_or_appveyor() in ('travis', 'appveyor'): return res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: if not os.path.exists(_[0]): raise Exception(_[0]) check = os.path.join(temp, "td1a_correction_session4.tex") with open(check, "r", encoding="utf8") as f: content = f.read() if "\\section{" not in content: raise Exception(content)
def test_status_helper(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") res = get_installed_modules( fLOG=fLOG, stop=10, pypi=True, short_list=["pep8"]) for f in res: fLOG(f) if sys.version_info[0] > 2: if is_travis_or_appveyor(): self.assertEqual(len(res), 1) else: warnings.warn( "This test does not work well on a virtual environment [TestStatusHelper].") res = get_installed_modules( fLOG=fLOG, stop=10, pypi=True, short_list=["dataspyre"]) fLOG(res) res = get_installed_modules(fLOG=fLOG, stop=10, pypi=True) for f in res: fLOG(f) if sys.version_info[0] == 2: # we disable the test for Python 2.7 warnings.warn( "TestStatusHelper.test_status_helper disabled on Python 2.7") return self.assertEqual(len(res), 10)
def test_notebook_slides(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "notebooks")) nbs = [ os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _ ] formats = ["slides"] temp = get_temp_folder(__file__, "temp_nb_bug_slides") if is_travis_or_appveyor() in ('travis', 'appveyor'): return res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: if not os.path.exists(_[0]): raise Exception(_[0]) checks = [ os.path.join(temp, "reveal.js"), os.path.join(temp, "require.js") ] for check in checks: if not os.path.exists(check): raise Exception(check)
def test_convert_slides_api_rst(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() in ('travis', 'appveyor'): # no latex, no pandoc return path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath( os.path.join( path, "..", "..", "_doc", "notebooks")) nb = os.path.join(fold, "example_pyquickhelper.ipynb") self.assertExists(nb) nbr = read_nb(nb, kernel=False) temp = get_temp_folder(__file__, "temp_nb_api_rst") outfile = os.path.join(temp, "out_nb_slides.rst") res = nb2rst(nbr, outfile) self.assertEqual(len(res), 1) for r in res: self.assertExists(r)
def test_notebook_runner_2a_eco(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # Requires authentification. return from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False if "Scraping" in n: return False if "2.ipynb" in n: return False if "flask" in n.lower(): # flask from a notebook does not work return False if "td2a_TD5_Traitement_automatique_des_langues_en_Python" in n: return False return True execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=ensae_teaching_cs)
def test_voice_listen(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no keys return if sys.platform.startswith("win"): from src.ensae_teaching_cs.pythonnet import vocal_recognition_listening try: for i, text in enumerate(vocal_recognition_listening()): fLOG("listented", text) if i >= 2: break except Exception as e: if "Audio device error encountered" in str(e): # maybe the script is running on a virtual machine (no # Audia device) if os.environ["USERNAME"] == "ensaestudent" or \ os.environ["USERNAME"] == "vsxavierdupre" or \ os.environ["USERNAME"] == "vsxavierdupre" or \ "DOUZE2016" in os.environ["COMPUTERNAME"] or \ os.environ["USERNAME"] == "appveyor" or \ "paris" in os.environ["COMPUTERNAME"].lower() or \ os.environ["USERNAME"].endswith("$"): # anonymous Jenkins configuration # I would prefer to catch a proper exception # it just exclude one user only used on remotre # machines return raise Exception("USERNAME: "******"USERNAME", "-"))
def test_notebook_runner_2a_eco_nlp_long(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # Requires authentification. return from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_nlp_long") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) folder = os.path.join(temp, "ressources_googleplus") if not os.path.exists(folder): os.mkdir(folder) jsfile = os.path.join(os.path.dirname(keepnote[0]), "ressources_googleplus", "107033731246200681024.json") shutil.copy(jsfile, folder) def filter(i, n): if "td2a_TD5_Traitement_automatique_des_langues_en_Python" in n: return True return False execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=ensae_teaching_cs)
def test_cpu_gpu(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # it requires latex return from theano import function, config, shared import theano.tensor as T import numpy import time vlen = 10 * 30 * 768 # 10 x #cores x # threads per core iters = 1000 rng = numpy.random.RandomState(22) x = shared(numpy.asarray(rng.rand(vlen), config.floatX)) f = function([], T.exp(x)) fLOG(f.maker.fgraph.toposort()) t0 = time.time() for i in range(iters): r = f() t1 = time.time() fLOG("Looping %d times took %f seconds" % (iters, t1 - t0)) fLOG("Result is %s" % (r,)) if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]): fLOG('Used the cpu') else: fLOG('Used the gpu')
def test_rst2html_autoclass(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor() in ('travis', 'appveyor'): # It requires latex. return if sys.version_info[:2] <= (2, 7): # i don't want to fix it for Python 2.7 return content = """ ====== title1 ====== .. autoclass:: pyquickhelper.sphinxext.sphinx_runpython_extension.RunPythonDirective :members: """.replace(" ", "") temp = get_temp_folder(__file__, "temp_rst2html_autoclass") text = rst2html(content, outdir=temp, layout="sphinx", writer="rst") ji = os.path.join(temp, "out.rst") with open(ji, "w", encoding="utf-8") as f: f.write(text) self.assertIn("* ``:indent:<int>`` to indent the output", text)
def test_selenium_image(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): return temp = get_temp_folder(__file__, "temp_selenium_image") img = os.path.join(temp, "image_selenium.png") url = "http://www.xavierdupre.fr/" navigator = "chrome" # download_chromedriver(dest=temp) # os.environ["PATH"] += ";" + temp res = webshot(img, url, navigator=navigator, fLOG=fLOG) assert os.path.exists(img) fLOG(res) self.assertEqual(len(res), 1) self.assertEqual(len(res[0]), 2) if navigator != "opera": # not available on splinter img = os.path.join(temp, "image_splinter.png") res = webshot(img, url, module='splinter', navigator=navigator, fLOG=fLOG) img = res[0][1] assert os.path.exists(img) fLOG(res) self.assertEqual(len(res), 1) self.assertEqual(len(res[0]), 2) else: warnings.warn("opera not available on splinter")
def test_read_all_elections(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) temp = os.path.join(path, "temp_read_all_elections") if not os.path.exists(temp): os.mkdir(temp) if is_travis_or_appveyor() == "travis": warnings.warn( "The test cannot read an excel file using xlrd. Disabling it.") return for year in [2012, 2002, 2007]: for level in ["Départements", "Cantons", ]: fLOG("******reading ", year, level) file = os.path.join(path, "data", "election_%d.xlsx" % year) if not os.path.exists(file): file = os.path.join(temp, "election_%d.xlsx" % year) if not os.path.exists(file): zip_ = os.path.join(path, "data", "elections.zip") unzip(zip_, temp) file = os.path.join(temp, "election_%d.xlsx" % year) assert os.path.exists(file) el = ElectionResults(year=year, file=file, level=level) fLOG("saving") xls = os.path.join(temp, "out_%d_%s_0.xlsx" % (year, level)) el.T0.to_excel(xls) xls = os.path.join(temp, "out_%d_%s_1.xlsx" % (year, level)) el.T1.to_excel(xls) res = el.vote_transfer() fLOG(res)
def test_notebook_read_write(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_notebook_read_write") nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb") self.assertTrue(os.path.exists(nbfile)) # For some reason, if this instruction is included in the build, # the travis build completes but never ends. # This is removed from the whole list on tavis. if is_travis_or_appveyor() == "travis": # This test prevents travis from ending. The process never stops. return nb = read_nb(nbfile, kernel=False) outfile = os.path.join(temp, "out_notebook.ipynb") nb.to_json(outfile) assert os.path.exists(outfile) with open(nbfile, "r", encoding="utf8") as f: c1 = f.read().replace("\r", "") with open(outfile, "r", encoding="utf8") as f: c2 = f.read().replace("\r", "") if c1 != c2: l1 = c1.strip("\n ").split("\n") l2 = c2.strip("\n ").split("\n") for i, cc in enumerate(zip(l1, l2)): a, b = cc if a.strip(" \n") != b.strip(" \n"): raise Exception( "difference at line {0}\n1: [{1}]-[{3}]\n2: [{2}]-[{4}]".format(i, a, b, type(a), type(b))) if len(l1) != len(l2): raise Exception("different length")
def test_notebook_raw(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "data")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if "TD_2A" in _] self.assertGreater(len(nbs), 0) formats = ["latex", "present", "ipynb", "html", "python", "rst", "pdf"] if sys.platform.startswith("win"): formats.append("docx") temp = get_temp_folder(__file__, "temp_nb_bug_raw") if is_travis_or_appveyor() in ('travis', 'appveyor'): return res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) self.assertExists(_[0]) check = os.path.join(temp, "TD_2A_Eco_Web_Scraping.tex") with open(check, "r", encoding="utf8") as f: content = f.read() if "\\begin{verbatim" not in content: raise Exception(content)
def test_crypt_file(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) import keyring this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted") if not os.path.exists(dst) or os.stat(dst).st_size < 10: fLOG("crypt") pwd = keyring.get_password("exam", os.environ["COMPUTERNAME"] + "key") pwd += "*" * (16 - len(pwd)) pwd = pwd.encode("ascii") fLOG(type(pwd)) this = os.path.join(this, "cryptcode.py") assert os.path.exists(this) encrypt_stream(pwd, this, dst) fLOG(os.stat(dst).st_size)
def test_notebook_runner_2a_eco(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # Requires authentification. return from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) def filter(i, n): if "SNCF" in n: return False if "Scraping" in n: return False if "2.ipynb" in n: return False if "flask" in n.lower(): # flask from a notebook does not work return False if "td2a_TD5_Traitement_automatique_des_langues_en_Python" in n: return False return True execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_eco_sncf(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf") keepnote = ls_notebooks("td2a_eco") assert len(keepnote) > 0 shutil.copy(simple_database(), temp) folder_note = os.path.split(keepnote[0])[0] jsfile = os.path.join(folder_note, "stop_areas.json") shutil.copy(jsfile, temp) def filter(i, n): if "SNCF" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return res = execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a) unittest_raise_exception_notebook(res, fLOG)
def test_image_video_epidemic(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_image_video_voisinage") if is_travis_or_appveyor() == "travis": warnings.warn("pygame is not available") return import pygame pygame_simulation(pygame, fLOG=fLOG, folder=temp, max_iter=150 if __name__ == "__main__" else 10) files = os.listdir(temp) assert len(files) > 9 png = [ os.path.join(temp, _) for _ in files if os.path.splitext(_)[-1] == ".png" ] assert len(png) > 0 out = os.path.join(temp, "voisinage.avi") v = make_video(png, out, size=(500, 500), format="XVID") assert v is not None
def test_ensae_2016_answers(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no stored password return password = keyring.get_password( "cpt", os.environ["COMPUTERNAME"] + "pwd") r = data_cpt_ENSAE_2016_11_blind_set(password) self.assertEqual(len(r), 7500) truth = r r = data_cpt_ENSAE_2016_11_blind_set("dummy") self.assertEqual(len(r), 7500) s = sum(r) assert 0 <= s <= len(r) auc = AUC(truth, r) fLOG(auc) assert 0 <= auc <= 1 temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers") out = os.path.join(temp, "answer.txt") with open(out, "w") as f: f.write("\n".join(str(_) for _ in r)) zip_files(os.path.join("submission.zip"), [out])
def test_notebook_runner_2a_eco_sncf(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a from src.ensae_teaching_cs.data import simple_database temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf") keepnote = ls_notebooks("td2a_eco") shutil.copy(simple_database(), temp) folder_note = os.path.split(keepnote[0])[0] jsfile = os.path.join(folder_note, "stop_areas.json") shutil.copy(jsfile, temp) def filter(i, n): if "SNCF" not in n: return False return True if is_travis_or_appveyor() == "travis": warnings.warn("execution does not stop") return execute_notebooks(temp, keepnote, filter, fLOG=fLOG, clean_function=clean_function_1a, dump=src.ensae_teaching_cs)
def test_download_zip(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") fold = get_temp_folder(__file__, "temp_download") url = "https://docs.python.org/3/library/ftplib.html" f = download(url, fold) fLOG(f) self.assertTrue(os.path.exists(f)) if not f.endswith("ftplib.html"): raise Exception(f) out1 = os.path.join(fold, "try.html.gz") gzip_files(out1, [f], fLOG=fLOG) self.assertTrue(os.path.exists(out1)) out2 = os.path.join(fold, "try.zip") zip_files(out2, [f], fLOG=fLOG) self.assertTrue(os.path.exists(out2)) if is_travis_or_appveyor() in ("circleci", None): out7 = os.path.join(fold, "try.7z") zip7_files(out7, [out1, out2], fLOG=fLOG, temp_folder=fold) if not os.path.exists(out7): raise FileNotFoundError(out7) else: fLOG("skip 7z")
def test_notebook_example(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_exemples") keepnote = ls_notebooks("exemples") self.assertTrue(len(keepnote) > 0) source = os.path.join(os.path.dirname(keepnote[0]), "data") images = os.path.join(temp, 'data') os.mkdir(images) for img in os.listdir(source): shutil.copy(os.path.join(source, img), images) if is_travis_or_appveyor() == "circleci": def clean(cell): cell = clean_function_notebook(cell) # ValueError: Cannot embed the 'gif' image format (circleci) cell = cell.replace('Image("video.gif")', '# Image("video.gif")') return cell else: clean = clean_function_notebook res = execute_notebooks(temp, keepnote, lambda i, n: "poppins" not in n and "video" not in n, fLOG=fLOG, clean_function=clean) execute_notebook_list_finalize_ut( res, fLOG=fLOG, dump=code_beatrix)
def test_notebook_latex_problems(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join( path, "..", "..", "_doc", "notebooks", "td2a_ml")) nbs = [os.path.join(fold, _) for _ in os.listdir(fold) if _.endswith(".ipynb") and "problems" in _] nbs.sort() print(nbs) formats = ["pdf", "ipynb", "html", "python", "rst", "docx"] if is_travis_or_appveyor() is not None: warnings.warn( "travis, appveyor, unable to test %s" % self._testMethodName) return temp = get_temp_folder(__file__, "temp_nb_td2A_ml_bug_latex") res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) assert os.path.exists(_[0])
def test_cpu_gpu(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # it requires latex return from theano import function, config, shared import theano.tensor as T import numpy import time vlen = 10 * 30 * 768 # 10 x #cores x # threads per core iters = 1000 rng = numpy.random.RandomState(22) x = shared(numpy.asarray(rng.rand(vlen), config.floatX)) f = function([], T.exp(x)) fLOG(f.maker.fgraph.toposort()) t0 = time.time() for i in range(iters): r = f() t1 = time.time() fLOG("Looping %d times took %f seconds" % (iters, t1 - t0)) fLOG("Result is %s" % (r, )) if numpy.any( [isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]): fLOG('Used the cpu') else: fLOG('Used the gpu')
def test_crypt_file_vigenere(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return from src.ensae_teaching_cs.td_1a.vigenere import code_vigenere import keyring this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere") if not os.path.exists(dst) or os.stat(dst).st_size < 10: fLOG("crypt") pwd = keyring.get_password("exam", os.environ["COMPUTERNAME"] + "key") pwd = pwd.encode("ascii") this = os.path.join(this, "cryptcode.py") assert os.path.exists(this) with open(this, "rb") as f: content = f.read() code = code_vigenere(content, pwd, binary=True) with open(dst, "wb") as f: f.write(code)
def test_theano_logreg(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # it requires latex return from theano import config fLOG(config) from src.ensae_teaching_cs.examples.theano_logreg import theano_sgd_optimization_mnist, theano_predict temp = get_temp_folder(__file__, "temp__theano_logreg") dataset = "mnist.pkl.gz" if not os.path.exists(dataset): download_data(dataset, website="http://deeplearning.net/data/mnist/") model = os.path.join(temp, "log_reg_theano.bin") theano_sgd_optimization_mnist(dataset=dataset, saved_model=model, n_epochs=2, fLOG=fLOG) pred = theano_predict(model, dataset, 10) fLOG(pred) fLOG(type(pred)) self.assertEqual(len(pred), 10)
def test_ensae_2016_answers(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no stored password return password = keyring.get_password("cpt", os.environ["COMPUTERNAME"] + "pwd") r = data_cpt_ENSAE_2016_11_blind_set(password) self.assertEqual(len(r), 7500) truth = r r = data_cpt_ENSAE_2016_11_blind_set("dummy") self.assertEqual(len(r), 7500) s = sum(r) assert 0 <= s <= len(r) auc = AUC(truth, r) fLOG(auc) assert 0 <= auc <= 1 temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers") out = os.path.join(temp, "answer.txt") with open(out, "w") as f: f.write("\n".join(str(_) for _ in r)) zip_files(os.path.join("submission.zip"), [out])
def test_notebook_utf8(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath(os.path.join(path, "notebooks_utf8")) nbs = [ os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _ ] formats = ["latex", "pdf", "rst", "html"] temp = os.path.join(path, "temp_nb_bug_utf8") if not os.path.exists(temp): os.mkdir(temp) for file in os.listdir(temp): os.remove(os.path.join(temp, file)) if is_travis_or_appveyor() in ('travis', 'appveyor'): return setup_environment_for_help() res = process_notebooks(nbs, temp, temp, formats=formats) fLOG("*****", len(res)) for _ in res: fLOG(_) self.assertExists(_[0]) with open(os.path.join(temp, "simple_example.tex"), "r", encoding="utf8") as f: content = f.read() exp = "textquestiondown" if exp not in content: raise Exception(content)