def test_image_video_epidemic(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_image_video_epidemic")
if is_travis_or_appveyor() in ("travis",):
# pygame.error: No available video device
return
import pygame
if is_travis_or_appveyor() == "circleci":
# os.environ["SDL_VIDEODRIVER"] = "x11"
flags = pygame.NOFRAME
else:
flags = 0
pygame_simulation(pygame, fLOG=fLOG, iter=10, folder=temp, flags=flags)
files = os.listdir(temp)
self.assertTrue(len(files) > 9)
png = [os.path.join(temp, _)
for _ in files if os.path.splitext(_)[-1] == ".png"]
self.assertTrue(len(png) > 0)
out = os.path.join(temp, "epidemic.avi")
v = make_video(png, out, size=(300, 300), format="XVID")
self.assertTrue(v is not None)
def test_image_video_puzzle_girafe(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_image_video_girafe")
if is_travis_or_appveyor() in ("travis",):
# pygame.error: No available video device
return
import pygame
if is_travis_or_appveyor() == "circleci":
# os.environ["SDL_VIDEODRIVER"] = "x11"
flags = pygame.NOFRAME
else:
flags = 0
pygame_simulation(pygame, fLOG=fLOG, folder=temp,
delay=200 if __name__ == "__main__" else 2,
flags=flags)
files = os.listdir(temp)
assert len(files) > 9
png = [os.path.join(temp, _)
for _ in files if os.path.splitext(_)[-1] == ".png"]
assert len(png) > 0
out = os.path.join(temp, "puzzle_girafe.avi")
v = make_video(png, out, size=(500, 500), format="XVID", fps=4)
assert v is not None
def test_notebook_runner_2a_eco_scraping(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
# too long for appveyor
return
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco_scraping")
keepnote = ls_notebooks("td2a_eco")
assert len(keepnote) > 0
shutil.copy(simple_database(), temp)
def filter(i, n):
if "Scraping" not in n:
return False
return True
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
res = execute_notebooks(temp, keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a)
unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_2a(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
# too long for appveyor
return
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook
temp = get_temp_folder(__file__, "temp_notebook2a_")
keepnote = ls_notebooks("2a")
assert len(keepnote) > 0
def filter(i, n):
if not sys.platform.startswith("win") and "_convert" in n:
return False
if "git_" not in n and "python_r" not in n and "csharp" not in n:
return True
if is_travis_or_appveyor() and "notebook_convert.ipynb" in n:
# this one requires pandoc
return False
return False
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
res = execute_notebooks(temp, keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a)
unittest_raise_exception_notebook(res, fLOG)
def test_kruskal_pygame_simulation(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_kruskal_pygame_simulation")
if is_travis_or_appveyor() in ("travis",):
# pygame.error: No available video device
return
import pygame
if is_travis_or_appveyor() == "circleci":
# os.environ["SDL_VIDEODRIVER"] = "x11"
flags = pygame.NOFRAME
else:
flags = 0
pygame.init()
pygame_simulation(fLOG=fLOG, max_iter=10 if __name__ != "__main__" else 10000,
pygame=pygame, folder=temp, flags=flags)
files = os.listdir(temp)
self.assertGreater(len(files), 9)
png = [os.path.join(temp, _)
for _ in files if os.path.splitext(_)[-1] == ".png"]
self.assertGreater(len(png), 0)
out = os.path.join(temp, "tsp_kruskal.avi")
v = make_video(png, out, size=(800, 500), format="XVID", fps=20)
self.assertNotEmpty(v)
def test_status(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "travis":
warnings.warn("run_cmd no end on travis")
return
temp = get_temp_folder(__file__, "temp_status")
outfile = os.path.join(temp, "modules.xlsx")
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(
this, "..", "..", "src", "pymyinstall", "cli", "pymy_status.py"))
cmd = "{0} -u {1} {2}".format(
sys.executable, script, "numpy --out={0}".format(outfile))
fLOG(cmd)
out, err = run_cmd(cmd, wait=True)
if len(out) == 0:
if is_travis_or_appveyor() == "appveyor":
warnings.warn(
"CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
return
else:
raise Exception(
"cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
if len(err) > 0:
raise Exception(
"cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
if not os.path.exists(outfile):
raise Exception(outfile)
fLOG(out)
def test_script_help(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "travis":
warnings.warn("run_cmd no end on travis")
return
script = os.path.join(os.path.dirname(os.path.abspath(
__file__)), "..", "..", "src", "pymyinstall", "cli", "pymy_install.py")
if not os.path.exists(script):
raise Exception(script)
scriptu = os.path.join(os.path.dirname(os.path.abspath(
__file__)), "..", "..", "src", "pymyinstall", "cli", "pymy_update.py")
if not os.path.exists(script):
raise Exception(script)
exe = sys.executable
cmd = exe + " " + script + " --help"
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
if "usage: pymy_install.py" not in out:
raise Exception(out)
cmd = exe + " " + scriptu + " --help"
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
if "usage: pymy_update.py" not in out:
if is_travis_or_appveyor() == "appveyor":
warnings.warn(
"CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
else:
raise Exception(
"cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
def test_diff_full(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
seq1 = "zab zab2 zabc3 zabcd zabc4".split()
seq2 = "ab ab2 abc3 abc4 abc adb".split()
diff = SequenceMatcher(a=seq1, b=seq2)
h = 20
size = 500, 500
white = 255, 255, 255
if is_travis_or_appveyor() in ("travis",):
# pygame.error: No available video device
return
import pygame
if is_travis_or_appveyor() == "circleci":
# os.environ["SDL_VIDEODRIVER"] = "x11"
flags = pygame.NOFRAME
else:
flags = 0
pygame, screen, fonts = get_pygame_screen_font(h, size, flags=flags)
from ensae_teaching_cs.helpers.pygame_helper import wait_event
bars = [random.randint(10, 500) / 500.0 for s in seq2]
screen.fill(white)
build_diff_image(pygame, screen, h=h, maxw=size[1], seq1=seq1, seq2=seq2, diff=diff,
fonts=fonts, bars=bars)
pygame.display.flip()
temp = get_temp_folder(__file__, "temp_video_diff_full")
for i in range(0, 21):
screen.fill(white)
build_diff_image(pygame, screen, h=h, maxw=size[0], seq1=seq1, seq2=seq2, diff=diff,
fonts=fonts, bars=bars, progress=i / 20.0, prev_bars=None)
pygame.time.wait(60)
pygame.display.flip()
pygame.image.save(screen, os.path.join(temp, "diff%d.png" % i))
if __name__ == "__main__":
from ensae_teaching_cs.helpers.video_helper import make_video
png = [os.path.join(temp, _)
for _ in os.listdir(temp) if ".png" in _]
out = os.path.join(temp, "diff.avi")
make_video(png, out, size=(350, 250), format="XVID", fps=5)
wait_event(pygame)
for font in fonts.values():
del font
pygame.quit()
def valid(cell):
if "open_html_form" in cell:
return False
if "open_window_params" in cell:
return False
if '<div style="position:absolute' in cell:
return False
if "completion.dot" in cell and is_travis_or_appveyor() == "travis":
return False
if 'Image("completion.png")' in cell and is_travis_or_appveyor() == "travis":
return False
return True
def test_selenium_image(self):
if is_travis_or_appveyor():
return
temp = get_temp_folder(__file__, "temp_selenium_image")
img = os.path.join(temp, "image_selenium.png")
url = "http://www.xavierdupre.fr/"
navigator = "chrome"
# download_chromedriver(dest=temp)
# os.environ["PATH"] += ";" + temp
res = webshot(img, url, navigator=navigator)
self.assertExists(img)
self.assertEqual(len(res), 1)
self.assertEqual(len(res[0]), 2)
if navigator != "opera":
# not available on splinter
img = os.path.join(temp, "image_splinter.png")
res = webshot(img, url, module='splinter', navigator=navigator)
img = res[0][1]
self.assertExists(img)
self.assertEqual(len(res), 1)
self.assertEqual(len(res[0]), 2)
else:
warnings.warn("opera not available on splinter")
def test_graph2(self):
"""
This test is failing with Python 3.4 if many pictures are drawn.
"""
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
return
fix_tkinter_issues_virtualenv()
from matplotlib import pyplot as plt
cache = os.path.abspath(os.path.split(__file__)[0])
cache = os.path.join(cache, "temp_cache2")
stock = StockPrices("BNP.PA", folder=cache)
stock2 = StockPrices("CA.PA", folder=cache)
fig, ax = plt.subplots(figsize=(16, 8))
ax = stock.plot(ax=ax)
ax = stock2.plot(ax=ax)
img = os.path.abspath(
os.path.join(
os.path.split(__file__)[0],
"temp_image2.png"))
if os.path.exists(img):
os.remove(img)
fig.savefig(img)
plt.close('all')
assert os.path.exists(img)
def test_theano_logreg(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# it requires latex
return
from theano import config
fLOG(config)
from src.ensae_teaching_cs.examples.theano_logreg import theano_sgd_optimization_mnist, theano_predict
temp = get_temp_folder(__file__, "temp__theano_logreg")
dataset = "mnist.pkl.gz"
if not os.path.exists(dataset):
download_data(
dataset, website="http://deeplearning.net/data/mnist/")
model = os.path.join(temp, "log_reg_theano.bin")
theano_sgd_optimization_mnist(
dataset=dataset, saved_model=model, n_epochs=2, fLOG=fLOG)
pred = theano_predict(model, dataset, 10)
fLOG(pred)
fLOG(type(pred))
self.assertEqual(len(pred), 10)
def test_image_video_kohonen(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_image_video_tsp_kohonen")
if is_travis_or_appveyor() == "travis":
warnings.warn("pygame is not available")
return
import pygame
pygame_simulation(pygame,
fLOG=fLOG,
folder=temp,
nb=200 if __name__ == "__main__" else 20,
size=(400, 250))
files = os.listdir(temp)
assert len(files) > 9
png = [
os.path.join(temp, _) for _ in files
if os.path.splitext(_)[-1] == ".png"
]
assert len(png) > 0
out = os.path.join(temp, "tsp_kohonen.avi")
v = make_video(png, out, size=(200, 125), format="XVID", fps=20)
assert v is not None
def test_joined_schemas(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
warnings.warn("disabled on appveyor")
return
temp = get_temp_folder(__file__, "temp_joined_schemas")
try:
df = merge_schema()
except PasswordException:
# the password is not available on this machine
return
assert len(df) > 0
outexc = os.path.join(temp, "schemas.xlsx")
df.to_excel(outexc, index=False)
fLOG(df.columns)
txt = df2rsthtml(df, format="rst")
assert "nan\t" not in txt
fLOG("\n" + txt)
outfile = os.path.join(temp, "schemas.rst")
with open(outfile, "w", encoding="utf-8") as f:
f.write(txt)
def test_read_all_elections(self):
fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
temp = os.path.join(path, "temp_read_all_elections")
if not os.path.exists(temp):
os.mkdir(temp)
if is_travis_or_appveyor() == "travis":
warnings.warn(
"The test cannot read an excel file using xlrd. Disabling it.")
return
for year in [2012, 2002, 2007]:
for level in ["Départements", "Cantons", ]:
fLOG("******reading ", year, level)
file = os.path.join(path, "data", "election_%d.xlsx" % year)
if not os.path.exists(file):
file = os.path.join(temp, "election_%d.xlsx" % year)
if not os.path.exists(file):
zip = os.path.join(path, "data", "elections.zip")
unzip(zip, temp)
file = os.path.join(temp, "election_%d.xlsx" % year)
assert os.path.exists(file)
el = ElectionResults(year=year, file=file, level=level)
fLOG("saving")
xls = os.path.join(temp, "out_%d_%s_0.xlsx" % (year, level))
el.T0.to_excel(xls)
xls = os.path.join(temp, "out_%d_%s_1.xlsx" % (year, level))
el.T1.to_excel(xls)
res = el.vote_transfer()
fLOG(res)
def test_sphinx_ext_video_latex(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_sphinx_ext_video_latex")
fLOG('custom app init')
src_ = self.setup_format(temp)
app = CustomSphinxApp(src_, temp, buildername="latex")
fLOG('custom app build')
app.build()
fLOG('custom app done')
index = os.path.join(temp, "pyq-video.tex")
self.assertExists(index)
with open(index, "r", encoding="utf-8") as f:
content = f.read()
self.assertNotIn("unable to find", content)
self.assertIn('mur.mp4}', content)
index = os.path.join(temp, "mur.mp4")
self.assertExists(index)
index = os.path.join(temp, "jol", "mur2.mp4")
self.assertExists(index)
index = os.path.join(temp, "jol", 'im', "mur3.mp4")
self.assertExists(index)
if is_travis_or_appveyor() not in ('travis', 'appveyor'):
latex = find_latex_path()
fLOG("latex-compile", latex)
compile_latex_output_final(temp, latex, doall=True)
fLOG("compilatione done")
index = os.path.join(temp, "pyq-video.pdf")
self.assertExists(index)
def test_notebook_runner(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
temp = get_temp_folder(__file__, "temp_notebook2a_2_enonce")
keepnote = ls_notebooks("td2a")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
warnings.warn(
"travis or appveyor, unable to test TestNotebookRunner2a_2_enonce.test_notebook_runner"
)
return
fold = os.path.dirname(keepnote[0])
for png in os.listdir(fold):
if ".png" not in png:
continue
fLOG("copy", png)
shutil.copy(os.path.join(fold, png), temp)
self.assertTrue(len(keepnote) > 0)
execute_notebooks(temp,
keepnote,
lambda i, n: "_2" in n and "enonce" in n and "_2D"
not in n and "_2B" not in n,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_rst2html_png_bug(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
# It requires latex.
return
if sys.version_info[:2] <= (2, 7):
# i don't want to fix it for Python 2.7
return
temp = get_temp_folder(__file__, "temp_rst2html_png_latex")
rst = os.path.join(os.path.abspath(
os.path.dirname(__file__)), "data", "puzzle_girafe.rst")
with open(rst, "r", encoding="utf-8") as f:
content = f.read()
text = rst2html(content, fLOG=fLOG, outdir=temp, warnings_log=True,
imgmath_latex_preamble="""
\\newcommand{\\acc}[1]{\\left\\{#1\\right\\}}
\\newcommand{\\cro}[1]{\\left[#1\\right]}
\\newcommand{\\pa}[1]{\\left(#1\\right)}
\\newcommand{\\girafedec}[3]{ \\begin{array}{ccccc} #1 &=& #2 &+& #3 \\\\ a' &=& a &-& o \\end{array}}
\\newcommand{\\vecteur}[2]{\\pa{#1,\\dots,#2}}
\\newcommand{\\R}[0]{\\mathbb{R}}
\\newcommand{\\N}[0]{\\mathbb{N}}
""")
# fLOG(text)
ji = os.path.join(temp, "out.html")
with open(ji, "w", encoding="utf-8") as f:
f.write(text)
def test_exe(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
command = "dir" if sys.platform.startswith("win32") else "ls"
yml = """
language: python
python:
- {{Python35}}
before_script:
- %s
after_script:
- %s {{PLATFORM}}
script:
- %s
""".replace(" ", "") % (command, command, command)
temp = get_temp_folder(__file__, "temp_yaml_exe")
context = dict(Python34="fake", Python35=os.path.dirname(sys.executable),
Python27=None, Anaconda3=None, Anaconda2=None,
WinPython35=None, project_name="pyquickhelper",
root_path="ROOT", PLATFORM="win32")
obj, name = load_yaml(yml, context=context)
self.assertTrue(name is not None)
res = list(enumerate_convert_yaml_into_instructions(
obj, variables=context))
for r, var in res:
conv = convert_sequence_into_batch_file(r, variables=var)
if ("%s " % command) not in conv:
raise Exception("{0}\n--\n{1}".format(command, conv))
fLOG("####", conv)
ext = "bat" if command == "dir" else "sh"
name = os.path.join(temp, "yml.%s" % ext)
with open(name, "w") as f:
f.write(conv)
if is_travis_or_appveyor() == "__travis":
# linux, unable to test TestYamlExe.test_exe.
pass
else:
if sys.platform.startswith("win"):
cmd = name
else:
cmd = "bash " + name
try:
out, err = run_cmd(cmd, wait=True)
except PermissionError as e:
raise Exception(
"Unable to execute '{0}' which contains\n{1}".format(name, conv)) from e
fLOG("###")
fLOG(out)
if "BEFORE_SCRIPT" not in out:
raise Exception(
"{0}\nERR\n{2}\n#########\n{1}".format(out, conv, err))
if "AFTER_SCRIPT" not in out:
raise Exception(
"{0}\nERR\n{2}\n#########\n{1}".format(out, conv, err))
if "SCRIPT" not in out:
raise Exception(
"{0}\nERR\n{2}\n#########\n{1}".format(out, conv, err))
def test_notebook_latex(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "notebooks_latex"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if ".ipynb" in _]
formats = ["ipynb", "html", "python", "rst", "pdf"]
if sys.platform.startswith("win"):
formats.append("docx")
temp = os.path.join(path, "temp_nb_bug_latex")
if not os.path.exists(temp):
os.mkdir(temp)
for file in os.listdir(temp):
os.remove(os.path.join(temp, file))
if is_travis_or_appveyor() in ('travis', 'appveyor'):
return
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertExists(_[0])
def test_clone_repo(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = os.path.abspath(os.path.split(__file__)[0])
temp = os.path.join(fold, "temp_clone_repo")
if os.path.exists(temp):
removedirs(temp, use_command_line=True)
if not os.path.exists(temp):
os.mkdir(temp)
if is_travis_or_appveyor() in ("travis", "appveyor"):
return
to = os.path.join(temp, "pq")
out, err = clone(to, "github.com", "sdpython", "pyquickhelper")
fLOG("OUT:", out)
fLOG("ERR:", err)
self.assertTrue("Cloning into" in err)
self.assertTrue(
os.path.exists(
os.path.join(to, "src", "pyquickhelper", "__init__.py")))
out, err = rebase(to, "github.com", "sdpython", "pyquickhelper")
fLOG("OUT:", out)
fLOG("ERR:", err)
r = change_file_status(temp)
self.assertTrue(len(r) > 0)
def test_style_test(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
run_lint = is_travis_or_appveyor(env=['NAME_JENKINS']) is None
_private_test_style_test(fLOG, run_lint, verbose='-v' in sys.argv)
def test_notebook_latex1(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
nbfold = os.path.normpath(
os.path.join(path, "..", "..", "_doc", "notebooks"))
nbs = [
os.path.join(nbfold, "notebook_eleves", "2014_2015",
"2015_kmeans.ipynb"),
# os.path.join(nbfold, "notebook_eleves", "2014_2015", "2015_page_rank.ipynb"),
# os.path.join(nbfold, "notebook_eleves", "2014_2015", "2015_factorisation_matrice.ipynb"),
]
formats = [
"pdf",
]
temp = get_temp_folder(__file__, "temp_nb_bug_latex1")
if is_travis_or_appveyor():
warnings.warn(
"travis or appveyor, unable to test TestNoteBooksBugLatex_ecs.test_notebook_latex1"
)
return
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
assert os.path.exists(_[0])
def test_notebook_latex_problems(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(
path, "..", "..", "_doc", "notebooks", "td2a_eco"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if _.endswith(".ipynb") and "TD2A_eco_API_SNCF_corrige" in _]
nbs.sort()
formats = ["pdf", "ipynb", "html", "python", "rst", "docx"]
if is_travis_or_appveyor() in ('appveyor', 'travis'):
warnings.warn(
"travis, appveyor, unable to test %s" % self._testMethodName)
return
temp = get_temp_folder(__file__, "temp_nb_td2A_eco_bug_latex2")
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertTrue(os.path.exists(_[0]))
def test_install_deps(self):
fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(this, "..", "..", "src", "pymyinstall", "cli", "pymy_deps.py"))
cmd = "{0} -u {1} {2}".format(sys.executable, script, "pandas")
try:
out, err = run_cmd(cmd, wait=True, fLOG=fLOG, communicate=True, timeout=120)
except subprocess.CalledProcessError as e:
mes = "CMD\n{0}\nOUT\n{1}\nERR\n{2}".format(e.cmd, e.output, e.stderr.read() if e.stderr else "")
raise Exception(mes) from e
out = out.strip()
if len(out) == 0:
if is_travis_or_appveyor() == "appveyor":
warnings.warn("CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
else:
raise Exception("cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
else:
fLOG(out)
if sys.version_info[0] == 2:
# failing on Python 2.7
return
if "pandas" not in out:
raise Exception(out)
if "['pandas']" not in out:
raise Exception(out)
def test_sphinx_ext_video_latex(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_sphinx_ext_video_latex")
fLOG('custom app init')
src_ = self.setup_format(temp)
app = CustomSphinxApp(src_, temp, buildername="latex")
fLOG('custom app build')
app.build()
fLOG('custom app done')
index = os.path.join(temp, "pyq-video.tex")
self.assertExists(index)
with open(index, "r", encoding="utf-8") as f:
content = f.read()
self.assertNotIn("unable to find", content)
self.assertIn('mur.mp4}', content)
index = os.path.join(temp, "mur.mp4")
self.assertExists(index)
index = os.path.join(temp, "jol", "mur2.mp4")
self.assertExists(index)
index = os.path.join(temp, "jol", 'im', "mur3.mp4")
self.assertExists(index)
if is_travis_or_appveyor() not in ('travis', 'appveyor'):
latex = find_latex_path()
fLOG("latex-compile", latex)
compile_latex_output_final(temp, latex, doall=True)
fLOG("compilatione done")
index = os.path.join(temp, "pyq-video.pdf")
self.assertExists(index)
def test_graph2(self):
"""
This test is failing with Python 3.4 if many pictures are drawn.
"""
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
return
fix_tkinter_issues_virtualenv()
from matplotlib import pyplot as plt
cache = os.path.abspath(os.path.split(__file__)[0])
cache = os.path.join(cache, "temp_cache2")
stock = StockPrices("BNP.PA", folder=cache)
stock2 = StockPrices("CA.PA", folder=cache)
fig, ax = plt.subplots(figsize=(16, 8))
ax = stock.plot(ax=ax)
ax = stock2.plot(ax=ax)
img = os.path.abspath(
os.path.join(os.path.split(__file__)[0], "temp_image2.png"))
if os.path.exists(img):
os.remove(img)
fig.savefig(img)
plt.close('all')
assert os.path.exists(img)
def test_optimisation(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
sol2 = exercice_particulier2()
x2 = sol2['x']
d = abs(x2[0] - 0.428571428055853) + abs(x2[1] - 0.2857142848749249)
assert d < 1e-5
if is_travis_or_appveyor() == "travis":
# skip mkl
return
sol1 = exercice_particulier1()
sol2 = exercice_particulier2()
fLOG("cvxopt")
fLOG(sol1)
fLOG("solution:", sol1['x'].T)
fLOG("Arrow_Hurwicz")
fLOG(sol2)
fLOG("solution:", sol2['x'])
x1 = sol1['x']
x2 = sol2['x']
d1 = x1[0] - x2[0]
d2 = x1[1] - x2[1]
fLOG(d1, d2)
assert abs(d1) < 1e-5
assert abs(d2) < 1e-5
def test_rst2html_png(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
# It requires latex.
return
temp = get_temp_folder(__file__, "temp_rst2html_png")
rst = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data",
"hermionne.rst")
with open(rst, "r", encoding="utf-8") as f:
content = f.read()
text = rst2html(content)
ji = os.path.join(temp, "out.html")
with open(ji, "w", encoding="utf-8") as f:
f.write(text)
text2 = rst2html(content, layout="sphinx")
ji = os.path.join(temp, "out_sphinx.html")
with open(ji, "w", encoding="utf-8") as f:
f.write(text)
self.assertTrue(len(text2) > len(text))
def test_scene(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_scene")
s = Scene(Repere(), math.pi / 1.5, 400, 300)
s.ajoute_source(Source(Vecteur(0, 10, 10), Couleur(1, 1, 1)))
s.ajoute_source(Source(Vecteur(10, 10, 5), Couleur(0.5, 0.5, 0.5)))
s.ajoute_objet(Sphere(Vecteur(0, 0, 12), 3, Couleur(1, 0, 0)))
s.ajoute_objet(
Sphere(Vecteur(0, -400, 12), 396, Couleur(0.5, 0.5, 0.5)))
fLOG(s)
if is_travis_or_appveyor() == "travis":
warnings.warn("pygame is not available")
return
import pygame
from src.ensae_teaching_cs.helpers.pygame_helper import wait_event
screen = pygame.display.set_mode(s.dim)
screen.fill((255, 255, 255))
s.construit_image(screen, pygame=pygame, fLOG=fLOG)
pygame.image.save(screen, os.path.join(temp, "scene.png"))
if __name__ == "__main__":
wait_event(pygame)
def test_install_set_schedule(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(
os.path.join(this, "..", "..", "src", "pymyinstall", "cli",
"pymy_install.py"))
cmd = "{0} -u {1} {2}".format(sys.executable, script,
"--set=pyquickhelper --schedule")
try:
out, err = run_cmd(cmd,
wait=True,
fLOG=fLOG,
communicate=True,
timeout=60)
except subprocess.CalledProcessError as e:
mes = "CMD\n{0}\nOUT\n{1}\nERR\n{2}".format(
e.cmd, e.output,
e.stderr.read() if e.stderr else "")
raise Exception(mes) from e
if len(out) == 0:
if is_travis_or_appveyor() == "appveyor":
warnings.warn(
"CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(
cmd, out, err))
else:
raise Exception("cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(
cmd, out, err))
def test_reap_children(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.platform.startswith("win"):
cmd = "pause"
else:
cmd = 'ls -la | less'
temp = get_temp_folder(__file__, "temp_reap_children")
clog = CustomLog(temp)
proc, _ = run_cmd(cmd, wait=False, fLOG=clog)
self.assertTrue(_ is None)
clog('proc={} pid={}'.format(proc, proc.pid))
ki = reap_children(fLOG=clog, subset={proc.pid})
clog('ki={0}'.format(ki))
if ki is None and not is_travis_or_appveyor(
) and __name__ != '__main__':
warnings.warn(
"reap_children could not be fully tested ki is None.")
return
self.assertTrue(ki is not None)
self.assertEqual(len(ki), 1)
# fLOG(ki)
# To avoid a warning.
proc.returncode = 0
def test_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "notebooks"))
nbs = [
os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _
]
formats = ["ipynb", "python", "rst", "pdf"]
temp = get_temp_folder(__file__, "temp_nb_bug")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
return
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
if not os.path.exists(_[0]):
raise Exception(_[0])
check = os.path.join(temp, "td1a_correction_session4.tex")
with open(check, "r", encoding="utf8") as f:
content = f.read()
if "\\section{" not in content:
raise Exception(content)
def test_status_helper(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
res = get_installed_modules(
fLOG=fLOG, stop=10, pypi=True, short_list=["pep8"])
for f in res:
fLOG(f)
if sys.version_info[0] > 2:
if is_travis_or_appveyor():
self.assertEqual(len(res), 1)
else:
warnings.warn(
"This test does not work well on a virtual environment [TestStatusHelper].")
res = get_installed_modules(
fLOG=fLOG, stop=10, pypi=True, short_list=["dataspyre"])
fLOG(res)
res = get_installed_modules(fLOG=fLOG, stop=10, pypi=True)
for f in res:
fLOG(f)
if sys.version_info[0] == 2:
# we disable the test for Python 2.7
warnings.warn(
"TestStatusHelper.test_status_helper disabled on Python 2.7")
return
self.assertEqual(len(res), 10)
def test_notebook_slides(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "notebooks"))
nbs = [
os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _
]
formats = ["slides"]
temp = get_temp_folder(__file__, "temp_nb_bug_slides")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
return
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
if not os.path.exists(_[0]):
raise Exception(_[0])
checks = [
os.path.join(temp, "reveal.js"),
os.path.join(temp, "require.js")
]
for check in checks:
if not os.path.exists(check):
raise Exception(check)
def test_convert_slides_api_rst(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
# no latex, no pandoc
return
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(
os.path.join(
path,
"..",
"..",
"_doc",
"notebooks"))
nb = os.path.join(fold, "example_pyquickhelper.ipynb")
self.assertExists(nb)
nbr = read_nb(nb, kernel=False)
temp = get_temp_folder(__file__, "temp_nb_api_rst")
outfile = os.path.join(temp, "out_nb_slides.rst")
res = nb2rst(nbr, outfile)
self.assertEqual(len(res), 1)
for r in res:
self.assertExists(r)
def test_notebook_runner_2a_eco(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# Requires authentification.
return
from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco")
keepnote = ls_notebooks("td2a_eco")
shutil.copy(simple_database(), temp)
def filter(i, n):
if "SNCF" in n:
return False
if "Scraping" in n:
return False
if "2.ipynb" in n:
return False
if "flask" in n.lower():
# flask from a notebook does not work
return False
if "td2a_TD5_Traitement_automatique_des_langues_en_Python" in n:
return False
return True
execute_notebooks(temp, keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=ensae_teaching_cs)
def test_voice_listen(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no keys
return
if sys.platform.startswith("win"):
from src.ensae_teaching_cs.pythonnet import vocal_recognition_listening
try:
for i, text in enumerate(vocal_recognition_listening()):
fLOG("listented", text)
if i >= 2:
break
except Exception as e:
if "Audio device error encountered" in str(e):
# maybe the script is running on a virtual machine (no
# Audia device)
if os.environ["USERNAME"] == "ensaestudent" or \
os.environ["USERNAME"] == "vsxavierdupre" or \
os.environ["USERNAME"] == "vsxavierdupre" or \
"DOUZE2016" in os.environ["COMPUTERNAME"] or \
os.environ["USERNAME"] == "appveyor" or \
"paris" in os.environ["COMPUTERNAME"].lower() or \
os.environ["USERNAME"].endswith("$"): # anonymous Jenkins configuration
# I would prefer to catch a proper exception
# it just exclude one user only used on remotre
# machines
return
raise Exception("USERNAME: "******"USERNAME", "-"))
def test_notebook_runner_2a_eco_nlp_long(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# Requires authentification.
return
from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco_nlp_long")
keepnote = ls_notebooks("td2a_eco")
shutil.copy(simple_database(), temp)
folder = os.path.join(temp, "ressources_googleplus")
if not os.path.exists(folder):
os.mkdir(folder)
jsfile = os.path.join(os.path.dirname(keepnote[0]),
"ressources_googleplus", "107033731246200681024.json")
shutil.copy(jsfile, folder)
def filter(i, n):
if "td2a_TD5_Traitement_automatique_des_langues_en_Python" in n:
return True
return False
execute_notebooks(temp, keepnote, filter, fLOG=fLOG,
clean_function=clean_function_1a,
dump=ensae_teaching_cs)
def test_cpu_gpu(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# it requires latex
return
from theano import function, config, shared
import theano.tensor as T
import numpy
import time
vlen = 10 * 30 * 768 # 10 x #cores x # threads per core
iters = 1000
rng = numpy.random.RandomState(22)
x = shared(numpy.asarray(rng.rand(vlen), config.floatX))
f = function([], T.exp(x))
fLOG(f.maker.fgraph.toposort())
t0 = time.time()
for i in range(iters):
r = f()
t1 = time.time()
fLOG("Looping %d times took %f seconds" % (iters, t1 - t0))
fLOG("Result is %s" % (r,))
if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]):
fLOG('Used the cpu')
else:
fLOG('Used the gpu')
def test_rst2html_autoclass(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
# It requires latex.
return
if sys.version_info[:2] <= (2, 7):
# i don't want to fix it for Python 2.7
return
content = """
======
title1
======
.. autoclass:: pyquickhelper.sphinxext.sphinx_runpython_extension.RunPythonDirective
:members:
""".replace(" ", "")
temp = get_temp_folder(__file__, "temp_rst2html_autoclass")
text = rst2html(content, outdir=temp, layout="sphinx", writer="rst")
ji = os.path.join(temp, "out.rst")
with open(ji, "w", encoding="utf-8") as f:
f.write(text)
self.assertIn("* ``:indent:<int>`` to indent the output", text)
def test_selenium_image(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
return
temp = get_temp_folder(__file__, "temp_selenium_image")
img = os.path.join(temp, "image_selenium.png")
url = "http://www.xavierdupre.fr/"
navigator = "chrome"
# download_chromedriver(dest=temp)
# os.environ["PATH"] += ";" + temp
res = webshot(img, url, navigator=navigator, fLOG=fLOG)
assert os.path.exists(img)
fLOG(res)
self.assertEqual(len(res), 1)
self.assertEqual(len(res[0]), 2)
if navigator != "opera":
# not available on splinter
img = os.path.join(temp, "image_splinter.png")
res = webshot(img, url, module='splinter',
navigator=navigator, fLOG=fLOG)
img = res[0][1]
assert os.path.exists(img)
fLOG(res)
self.assertEqual(len(res), 1)
self.assertEqual(len(res[0]), 2)
else:
warnings.warn("opera not available on splinter")
def test_read_all_elections(self):
fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
temp = os.path.join(path, "temp_read_all_elections")
if not os.path.exists(temp):
os.mkdir(temp)
if is_travis_or_appveyor() == "travis":
warnings.warn(
"The test cannot read an excel file using xlrd. Disabling it.")
return
for year in [2012, 2002, 2007]:
for level in ["Départements", "Cantons", ]:
fLOG("******reading ", year, level)
file = os.path.join(path, "data", "election_%d.xlsx" % year)
if not os.path.exists(file):
file = os.path.join(temp, "election_%d.xlsx" % year)
if not os.path.exists(file):
zip_ = os.path.join(path, "data", "elections.zip")
unzip(zip_, temp)
file = os.path.join(temp, "election_%d.xlsx" % year)
assert os.path.exists(file)
el = ElectionResults(year=year, file=file, level=level)
fLOG("saving")
xls = os.path.join(temp, "out_%d_%s_0.xlsx" % (year, level))
el.T0.to_excel(xls)
xls = os.path.join(temp, "out_%d_%s_1.xlsx" % (year, level))
el.T1.to_excel(xls)
res = el.vote_transfer()
fLOG(res)
def test_notebook_read_write(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_notebook_read_write")
nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb")
self.assertTrue(os.path.exists(nbfile))
# For some reason, if this instruction is included in the build,
# the travis build completes but never ends.
# This is removed from the whole list on tavis.
if is_travis_or_appveyor() == "travis":
# This test prevents travis from ending. The process never stops.
return
nb = read_nb(nbfile, kernel=False)
outfile = os.path.join(temp, "out_notebook.ipynb")
nb.to_json(outfile)
assert os.path.exists(outfile)
with open(nbfile, "r", encoding="utf8") as f:
c1 = f.read().replace("\r", "")
with open(outfile, "r", encoding="utf8") as f:
c2 = f.read().replace("\r", "")
if c1 != c2:
l1 = c1.strip("\n ").split("\n")
l2 = c2.strip("\n ").split("\n")
for i, cc in enumerate(zip(l1, l2)):
a, b = cc
if a.strip(" \n") != b.strip(" \n"):
raise Exception(
"difference at line {0}\n1: [{1}]-[{3}]\n2: [{2}]-[{4}]".format(i, a, b, type(a), type(b)))
if len(l1) != len(l2):
raise Exception("different length")
def test_notebook_raw(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "data"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if "TD_2A" in _]
self.assertGreater(len(nbs), 0)
formats = ["latex", "present", "ipynb", "html",
"python", "rst", "pdf"]
if sys.platform.startswith("win"):
formats.append("docx")
temp = get_temp_folder(__file__, "temp_nb_bug_raw")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
return
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertExists(_[0])
check = os.path.join(temp, "TD_2A_Eco_Web_Scraping.tex")
with open(check, "r", encoding="utf8") as f:
content = f.read()
if "\\begin{verbatim" not in content:
raise Exception(content)
def test_crypt_file(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = keyring.get_password("exam",
os.environ["COMPUTERNAME"] + "key")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def test_notebook_runner_2a_eco(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# Requires authentification.
return
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco")
keepnote = ls_notebooks("td2a_eco")
shutil.copy(simple_database(), temp)
def filter(i, n):
if "SNCF" in n:
return False
if "Scraping" in n:
return False
if "2.ipynb" in n:
return False
if "flask" in n.lower():
# flask from a notebook does not work
return False
if "td2a_TD5_Traitement_automatique_des_langues_en_Python" in n:
return False
return True
execute_notebooks(temp,
keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_notebook_runner_2a_eco_sncf(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf")
keepnote = ls_notebooks("td2a_eco")
assert len(keepnote) > 0
shutil.copy(simple_database(), temp)
folder_note = os.path.split(keepnote[0])[0]
jsfile = os.path.join(folder_note, "stop_areas.json")
shutil.copy(jsfile, temp)
def filter(i, n):
if "SNCF" not in n:
return False
return True
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
res = execute_notebooks(temp, keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a)
unittest_raise_exception_notebook(res, fLOG)
def test_image_video_epidemic(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_image_video_voisinage")
if is_travis_or_appveyor() == "travis":
warnings.warn("pygame is not available")
return
import pygame
pygame_simulation(pygame,
fLOG=fLOG,
folder=temp,
max_iter=150 if __name__ == "__main__" else 10)
files = os.listdir(temp)
assert len(files) > 9
png = [
os.path.join(temp, _) for _ in files
if os.path.splitext(_)[-1] == ".png"
]
assert len(png) > 0
out = os.path.join(temp, "voisinage.avi")
v = make_video(png, out, size=(500, 500), format="XVID")
assert v is not None
def test_ensae_2016_answers(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no stored password
return
password = keyring.get_password(
"cpt", os.environ["COMPUTERNAME"] + "pwd")
r = data_cpt_ENSAE_2016_11_blind_set(password)
self.assertEqual(len(r), 7500)
truth = r
r = data_cpt_ENSAE_2016_11_blind_set("dummy")
self.assertEqual(len(r), 7500)
s = sum(r)
assert 0 <= s <= len(r)
auc = AUC(truth, r)
fLOG(auc)
assert 0 <= auc <= 1
temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers")
out = os.path.join(temp, "answer.txt")
with open(out, "w") as f:
f.write("\n".join(str(_) for _ in r))
zip_files(os.path.join("submission.zip"), [out])
def test_notebook_runner_2a_eco_sncf(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf")
keepnote = ls_notebooks("td2a_eco")
shutil.copy(simple_database(), temp)
folder_note = os.path.split(keepnote[0])[0]
jsfile = os.path.join(folder_note, "stop_areas.json")
shutil.copy(jsfile, temp)
def filter(i, n):
if "SNCF" not in n:
return False
return True
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
execute_notebooks(temp,
keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_download_zip(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = get_temp_folder(__file__, "temp_download")
url = "https://docs.python.org/3/library/ftplib.html"
f = download(url, fold)
fLOG(f)
self.assertTrue(os.path.exists(f))
if not f.endswith("ftplib.html"):
raise Exception(f)
out1 = os.path.join(fold, "try.html.gz")
gzip_files(out1, [f], fLOG=fLOG)
self.assertTrue(os.path.exists(out1))
out2 = os.path.join(fold, "try.zip")
zip_files(out2, [f], fLOG=fLOG)
self.assertTrue(os.path.exists(out2))
if is_travis_or_appveyor() in ("circleci", None):
out7 = os.path.join(fold, "try.7z")
zip7_files(out7, [out1, out2], fLOG=fLOG, temp_folder=fold)
if not os.path.exists(out7):
raise FileNotFoundError(out7)
else:
fLOG("skip 7z")
def test_notebook_example(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_exemples")
keepnote = ls_notebooks("exemples")
self.assertTrue(len(keepnote) > 0)
source = os.path.join(os.path.dirname(keepnote[0]), "data")
images = os.path.join(temp, 'data')
os.mkdir(images)
for img in os.listdir(source):
shutil.copy(os.path.join(source, img), images)
if is_travis_or_appveyor() == "circleci":
def clean(cell):
cell = clean_function_notebook(cell)
# ValueError: Cannot embed the 'gif' image format (circleci)
cell = cell.replace('Image("video.gif")',
'# Image("video.gif")')
return cell
else:
clean = clean_function_notebook
res = execute_notebooks(temp, keepnote,
lambda i, n: "poppins" not in n and "video" not in n,
fLOG=fLOG, clean_function=clean)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=code_beatrix)
def test_notebook_latex_problems(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(
path, "..", "..", "_doc", "notebooks", "td2a_ml"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if _.endswith(".ipynb") and "problems" in _]
nbs.sort()
print(nbs)
formats = ["pdf", "ipynb", "html", "python", "rst", "docx"]
if is_travis_or_appveyor() is not None:
warnings.warn(
"travis, appveyor, unable to test %s" % self._testMethodName)
return
temp = get_temp_folder(__file__, "temp_nb_td2A_ml_bug_latex")
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
assert os.path.exists(_[0])
def test_cpu_gpu(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# it requires latex
return
from theano import function, config, shared
import theano.tensor as T
import numpy
import time
vlen = 10 * 30 * 768 # 10 x #cores x # threads per core
iters = 1000
rng = numpy.random.RandomState(22)
x = shared(numpy.asarray(rng.rand(vlen), config.floatX))
f = function([], T.exp(x))
fLOG(f.maker.fgraph.toposort())
t0 = time.time()
for i in range(iters):
r = f()
t1 = time.time()
fLOG("Looping %d times took %f seconds" % (iters, t1 - t0))
fLOG("Result is %s" % (r, ))
if numpy.any(
[isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]):
fLOG('Used the cpu')
else:
fLOG('Used the gpu')
def test_crypt_file_vigenere(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
from src.ensae_teaching_cs.td_1a.vigenere import code_vigenere
import keyring
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = keyring.get_password("exam",
os.environ["COMPUTERNAME"] + "key")
pwd = pwd.encode("ascii")
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
with open(this, "rb") as f:
content = f.read()
code = code_vigenere(content, pwd, binary=True)
with open(dst, "wb") as f:
f.write(code)
def test_theano_logreg(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# it requires latex
return
from theano import config
fLOG(config)
from src.ensae_teaching_cs.examples.theano_logreg import theano_sgd_optimization_mnist, theano_predict
temp = get_temp_folder(__file__, "temp__theano_logreg")
dataset = "mnist.pkl.gz"
if not os.path.exists(dataset):
download_data(dataset,
website="http://deeplearning.net/data/mnist/")
model = os.path.join(temp, "log_reg_theano.bin")
theano_sgd_optimization_mnist(dataset=dataset,
saved_model=model,
n_epochs=2,
fLOG=fLOG)
pred = theano_predict(model, dataset, 10)
fLOG(pred)
fLOG(type(pred))
self.assertEqual(len(pred), 10)
def test_ensae_2016_answers(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no stored password
return
password = keyring.get_password("cpt",
os.environ["COMPUTERNAME"] + "pwd")
r = data_cpt_ENSAE_2016_11_blind_set(password)
self.assertEqual(len(r), 7500)
truth = r
r = data_cpt_ENSAE_2016_11_blind_set("dummy")
self.assertEqual(len(r), 7500)
s = sum(r)
assert 0 <= s <= len(r)
auc = AUC(truth, r)
fLOG(auc)
assert 0 <= auc <= 1
temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers")
out = os.path.join(temp, "answer.txt")
with open(out, "w") as f:
f.write("\n".join(str(_) for _ in r))
zip_files(os.path.join("submission.zip"), [out])
def test_notebook_utf8(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "notebooks_utf8"))
nbs = [
os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _
]
formats = ["latex", "pdf", "rst", "html"]
temp = os.path.join(path, "temp_nb_bug_utf8")
if not os.path.exists(temp):
os.mkdir(temp)
for file in os.listdir(temp):
os.remove(os.path.join(temp, file))
if is_travis_or_appveyor() in ('travis', 'appveyor'):
return
setup_environment_for_help()
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertExists(_[0])
with open(os.path.join(temp, "simple_example.tex"),
"r",
encoding="utf8") as f:
content = f.read()
exp = "textquestiondown"
if exp not in content:
raise Exception(content)
