def test_encrypt_decrypt(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
password = "******" * 2
temp = get_temp_folder(__file__, "temp_encrypt")
temp2 = get_temp_folder(__file__, "temp_encrypt2")
tempmm = get_temp_folder(__file__, "temp_encrypt_status")
cstatus = os.path.join(tempmm, "crypt_status.txt")
cmap = os.path.join(tempmm, "crypt_map.txt")
srcf = os.path.abspath(os.path.join(temp, ".."))
sys.argv = ["", srcf, temp, password,
"--status", cstatus,
"--map", cmap]
encrypt(fLOG=fLOG)
this = __file__
sys.argv = ["", temp, temp2, password]
decrypt(fLOG=fLOG)
with open(__file__, "rb") as f:
c1 = f.read()
with open(os.path.join(temp2, os.path.split(this)[-1]), "rb") as f:
c2 = f.read()
self.assertEqual(c1, c2)
fLOG("end")
def test_auc_file(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
ans = [1, 1, 1, 1, 1, 0, 0, 0, 0]
score = [i * 0.8 + 0.1 for i in ans]
score[0] = 0.4
score[-1] = 0.6
fLOG(score)
t1 = get_temp_folder(__file__, "temp_answers")
t2 = get_temp_folder(__file__, "temp_scores")
f1 = "answer.txt"
f2 = "answer.txt"
fu1 = os.path.join(t1, f1)
fu2 = os.path.join(t2, f2)
out = os.path.join(t2, "scores.txt")
with open(fu1, "w") as f:
f.write("\n".join(str(_) for _ in ans))
with open(fu2, "w") as f:
f.write("\n".join(str(_) for _ in score))
private_codalab_wrapper_binary_classification(
AUC, "AUC", t1, t2, output=out)
assert os.path.exists(out)
with open(out, "r") as f:
code = f.read()
fLOG("**", code)
self.assertEqual(code, "AUC:0.95")
def test_install_revealjs(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_install_revealjs")
dest = get_temp_folder(__file__, "temp_install_revealjs_dest")
fs = download_revealjs(temp, dest, fLOG=fLOG)
fLOG(fs)
assert len(fs) > 0
for a in fs:
assert os.path.exists(a)
def test_rst_only(self):
from docutils import nodes as skip_
content = """
test a directive
================
.. only:: html
only for html
.. only:: rst
only for rst
""".replace(" ", "")
if sys.version_info[0] >= 3:
content = content.replace('u"', '"')
tives = [("cmdref", CmdRef, cmdref_node,
visit_cmdref_node, depart_cmdref_node)]
text = rst2html(content, # fLOG=fLOG,
writer="rst", keep_warnings=True,
directives=tives, extlinks={'issue': ('http://%s', '_issue_')})
temp = get_temp_folder(__file__, "temp_only")
with open(os.path.join(temp, "out_cmdref.rst"), "w", encoding="utf8") as f:
f.write(text)
t1 = "only for rst"
if t1 not in text:
raise Exception(text)
t1 = "only for html"
if t1 in text:
raise Exception(text)
text = rst2html(content, # fLOG=fLOG,
writer="html", keep_warnings=True,
directives=tives, extlinks={'issue': ('http://%s', '_issue_')})
temp = get_temp_folder(__file__, "temp_only")
with open(os.path.join(temp, "out_cmdref.rst"), "w", encoding="utf8") as f:
f.write(text)
t1 = "only for rst"
if t1 in text:
raise Exception(text)
t1 = "only for html"
if t1 not in text:
raise Exception(text)
def test_auc_multi_file(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
t1 = get_temp_folder(__file__, "temp_multi_answers")
t2 = get_temp_folder(__file__, "temp_multi_scores")
t3 = get_temp_folder(__file__, "temp_multi_scores2")
truth = os.path.join(
t1, "..", "data", "tbl_test_dossier.Y.dummy.truth.txt")
ans = os.path.join(t1, "..", "data", "tbl_test_dossier.Y.dummy.txt")
f1 = "answer.txt"
f2 = "answer.txt"
shutil.copy(truth, os.path.join(t1, f1))
shutil.copy(ans, os.path.join(t2, f2))
with open(truth, "r") as f:
lines = [_.strip("\r\n ").split("\t") for _ in f.readlines()]
lines = [[l[0], l[1], "1.0", "1.0"] for l in lines]
f3 = "answer.txt"
with open(os.path.join(t3, f3), "w") as f:
f.write("\n".join("\t".join(_) for _ in lines))
temp = get_temp_folder(__file__, "temp_multi_out")
# =
out = os.path.join(temp, "outpute.txt")
private_codalab_wrapper_multi_classification(AUC_multi_multi,
["orientation", "nature"],
t1, t3, output=out)
assert os.path.exists(out)
with open(out, "r") as f:
code = f.read()
self.assertEqual(
code, "orientation_ERR:0.0\norientation_AUC:1.0\nnature_ERR:0.0\nnature_AUC:1.0")
# dummy
fLOG("------------ dummy")
out = os.path.join(temp, "outputd.txt")
private_codalab_wrapper_multi_classification(AUC_multi_multi,
["orientation", "nature"],
t1, t2, output=out, ignored=["nul"])
assert os.path.exists(out)
with open(out, "r") as f:
code = f.read()
fLOG("**", code)
self.assertEqual(
code, "orientation_ERR:0.7183754333828628\norientation_AUC:0.5862428771453444\nnature_ERR:0.8236750866765725\nnature_AUC:0.5160556240766593")
def test_sphinx_ext_video_latex(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_sphinx_ext_video_latex")
fLOG('custom app init')
src_ = self.setup_format(temp)
app = CustomSphinxApp(src_, temp, buildername="latex")
fLOG('custom app build')
app.build()
fLOG('custom app done')
index = os.path.join(temp, "pyq-video.tex")
self.assertExists(index)
with open(index, "r", encoding="utf-8") as f:
content = f.read()
self.assertNotIn("unable to find", content)
self.assertIn('mur.mp4}', content)
index = os.path.join(temp, "mur.mp4")
self.assertExists(index)
index = os.path.join(temp, "jol", "mur2.mp4")
self.assertExists(index)
index = os.path.join(temp, "jol", 'im', "mur3.mp4")
self.assertExists(index)
if is_travis_or_appveyor() not in ('travis', 'appveyor'):
latex = find_latex_path()
fLOG("latex-compile", latex)
compile_latex_output_final(temp, latex, doall=True)
fLOG("compilatione done")
index = os.path.join(temp, "pyq-video.pdf")
self.assertExists(index)
def test_join_multiple2(self):
fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")
filename = os.path.join(os.path.split(
__file__)[0], "data", "database_linked.zip")
temp = get_temp_folder(__file__, "temp_join_multiple2")
filename = unzip(filename, temp)
assert os.path.exists(filename)
db = Database(filename, LOG=fLOG)
db.connect()
where = {"bucket": ("==", "bu###1")}
n1 = db.JoinTreeNode("profile_QSSH", where=where,
parent_key="query", key="query")
n2 = db.JoinTreeNode("url_QSSH", where=where,
parent_key=('url', 'pos'), key=('url', 'pos'))
n1.append(n2)
sql, fields = db.inner_joins(n1, execute=False, create_index=False)
view = db.execute_view(sql)
assert view == [('facebbooklogin', 1, 0, 'bu###1', 86, 0,
'digg.com/security/Hackers_Put_Social_Networks_In_Crosshairs',
'digg.com/security/Hackers_Put_Social_Networks_In_Crosshairs',
1, 0, 1, 1, 0, 0, 0, 0)]
assert "WHERE" in sql
db.close()
def test_image_video_puzzle_girafe(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_image_video_girafe")
if is_travis_or_appveyor() in ("travis",):
# pygame.error: No available video device
return
import pygame
if is_travis_or_appveyor() == "circleci":
# os.environ["SDL_VIDEODRIVER"] = "x11"
flags = pygame.NOFRAME
else:
flags = 0
pygame_simulation(pygame, fLOG=fLOG, folder=temp,
delay=200 if __name__ == "__main__" else 2,
flags=flags)
files = os.listdir(temp)
assert len(files) > 9
png = [os.path.join(temp, _)
for _ in files if os.path.splitext(_)[-1] == ".png"]
assert len(png) > 0
out = os.path.join(temp, "puzzle_girafe.avi")
v = make_video(png, out, size=(500, 500), format="XVID", fps=4)
assert v is not None
def test_image_video_epidemic(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_image_video_epidemic")
if is_travis_or_appveyor() in ("travis",):
# pygame.error: No available video device
return
import pygame
if is_travis_or_appveyor() == "circleci":
# os.environ["SDL_VIDEODRIVER"] = "x11"
flags = pygame.NOFRAME
else:
flags = 0
pygame_simulation(pygame, fLOG=fLOG, iter=10, folder=temp, flags=flags)
files = os.listdir(temp)
self.assertTrue(len(files) > 9)
png = [os.path.join(temp, _)
for _ in files if os.path.splitext(_)[-1] == ".png"]
self.assertTrue(len(png) > 0)
out = os.path.join(temp, "epidemic.avi")
v = make_video(png, out, size=(300, 300), format="XVID")
self.assertTrue(v is not None)
def test_notebook_latex_problems(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(
path, "..", "..", "_doc", "notebooks", "td2a_ml"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if _.endswith(".ipynb") and "problems" in _]
nbs.sort()
print(nbs)
formats = ["pdf", "ipynb", "html", "python", "rst", "docx"]
if is_travis_or_appveyor() is not None:
warnings.warn(
"travis, appveyor, unable to test %s" % self._testMethodName)
return
temp = get_temp_folder(__file__, "temp_nb_td2A_ml_bug_latex")
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
assert os.path.exists(_[0])
def test_example_pydy(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_example_pydy")
fix_tkinter_issues_virtualenv()
import matplotlib.pyplot as plt
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 8))
try:
example_pydy(ax=ax)
except Exception as e:
if 'can only concatenate list (not "tuple") to list' in str(e):
warnings.warn("Pydy needs to be updated for Python 3.7")
return
else:
raise e
self.assertNotEmpty(ax)
img = os.path.join(temp, "img.png")
fig.savefig(img)
self.assertExists(img)
if __name__ == "__main__":
fig.show()
plt.close('all')
fLOG("end")
def test_benchmark(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_grid_benchmark")
params = [dict(value=random.randint(10, 20), name="name%d" %
i, shortname="m%d" % i) for i in range(0, 2)]
datasets = [dict(X=pandas.DataFrame([[0, 1], [0, 1]]), name="set1", shortname="s1"),
dict(X=pandas.DataFrame([[1, 1], [1, 1]]), name="set2", shortname="s2"), ]
bench = ATestOverGridBenchMark("TestName", datasets, fLOG=fLOG, clog=temp,
cache_file=os.path.join(temp, "cache.pickle"))
bench.run(params)
df = bench.to_df()
ht = df.to_html(float_format="%1.3f", index=False)
self.assertTrue(len(df) > 0)
self.assertTrue(ht is not None)
self.assertEqual(df.shape[0], 4)
report = os.path.join(temp, "report.html")
csv = os.path.join(temp, "report.csv")
rst = os.path.join(temp, "report.rst")
bench.report(filehtml=report, filecsv=csv, filerst=rst,
title="A Title", description="description")
self.assertTrue(os.path.exists(report))
self.assertTrue(os.path.exists(csv))
self.assertTrue(os.path.exists(rst))
def test_folium(self):
temp = get_temp_folder(__file__, "temp_folium")
outfile = os.path.join(temp, "osm.map")
map_osm = folium.Map(location=[48.85, 2.34])
map_osm.save(outfile=outfile)
self.assertTrue(os.path.exists(outfile))
def test_notebook_rst_svg(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_nb_rst_svg")
nbs = [os.path.normpath(os.path.join(
temp, '..', "data", "rst_notebooks", "notebook_with_svg.ipynb"))]
formats = ["rst"]
res = process_notebooks(nbs, temp, temp, formats=formats, fLOG=fLOG)
name = res[0][0]
with open(name, 'r', encoding='utf-8') as f:
content = f.read()
self.assertIn('SVG in a notebook.', content)
self.assertIn('.. image::', content)
nb = 0
for line in content.split('\n'):
if '.. image::' in line:
name = line.replace('.. image::', '').strip(' \r\t')
dest = os.path.join(temp, name)
self.assertExists(dest)
nb += 1
self.assertGreater(nb, 0)
def test_enumerate_feedback(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
exp = [
"<p>ok</p>",
]
temp = get_temp_folder(__file__, "temp_enumerate_feedback")
data = os.path.abspath(os.path.dirname(__file__))
data = os.path.join(data, "data")
xls = os.path.join(data, "groupes_eleves_pitch.xlsx")
df = pandas.read_excel(xls, sheetname=0, index=False)
mails = list(enumerate_feedback(df, exc=False, fLOG=fLOG,
begin="BEGIN", end="END", subject="SUBJECT",
col_name="Nom", cols=["Pitch", "Code"]))
for i, m in enumerate(mails):
fLOG("------------", i)
name = os.path.join(temp, "m%d.html" % i)
with open(name, "w", encoding="utf-8") as f:
f.write(m[1])
if i < len(exp):
if exp[i] not in m[1]:
raise Exception("EXP\n{0}\nRES\n{1}".format(exp[i], m[1]))
def test_notebook_raw(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "data"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if "TD_2A" in _]
self.assertGreater(len(nbs), 0)
formats = ["latex", "present", "ipynb", "html",
"python", "rst", "pdf"]
if sys.platform.startswith("win"):
formats.append("docx")
temp = get_temp_folder(__file__, "temp_nb_bug_raw")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
return
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertExists(_[0])
check = os.path.join(temp, "TD_2A_Eco_Web_Scraping.tex")
with open(check, "r", encoding="utf8") as f:
content = f.read()
if "\\begin{verbatim" not in content:
raise Exception(content)
def test_module_c(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
file = os.path.join(path, "data", "pyd",
"stdchelper.cp37-win_amd64.pyd")
self.assertExists(file)
mo = import_module(
None, file, fLOG, additional_sys_path=None, first_try=True)
self.assertIsInstance(mo, tuple)
self.assertEqual(len(mo), 2)
self.assertTrue(hasattr(mo[0], '__doc__'))
if 'stdchelper' in sys.modules:
del sys.modules['stdchelper']
temp = get_temp_folder(__file__, "temp_module_c")
store_obj = {}
actions = copy_source_files(os.path.dirname(file), temp, fLOG=fLOG)
store_obj = {}
indexes = {}
add_file_rst(temp, store_obj, actions, fLOG=fLOG,
rootrep=("module_c.", ""), indexes=indexes)
self.assertNotEmpty(store_obj)
self.assertEqual(len(store_obj), 4)
if len(actions) != 2:
raise Exception("{0}\n{1}".format(
len(actions), "\n".join(str(_) for _ in actions)))
self.assertEqual(len(indexes), 1)
def test_status(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "travis":
warnings.warn("run_cmd no end on travis")
return
temp = get_temp_folder(__file__, "temp_status")
outfile = os.path.join(temp, "modules.xlsx")
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(
this, "..", "..", "src", "pymyinstall", "cli", "pymy_status.py"))
cmd = "{0} -u {1} {2}".format(
sys.executable, script, "numpy --out={0}".format(outfile))
fLOG(cmd)
out, err = run_cmd(cmd, wait=True)
if len(out) == 0:
if is_travis_or_appveyor() == "appveyor":
warnings.warn(
"CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
return
else:
raise Exception(
"cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
if len(err) > 0:
raise Exception(
"cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
if not os.path.exists(outfile):
raise Exception(outfile)
fLOG(out)
def test_post_list(self):
# the test will fail if you add a file in data/blog others
# with rst files which is not a blog post
directives.register_directive("blogpost", BlogPostDirective)
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.join(path, "data", "blog")
out = get_temp_folder(__file__, "temp_post_list")
p = BlogPostList(fold)
cats = p.get_categories()
months = p.get_months()
self.assertEqual(cats, ['documentation', 'example'])
self.assertEqual(months, ['2015-04'])
res = p.write_aggregated(out)
self.assertTrue(len(res) >= 4)
for r in res:
if r in (None, ''):
raise ValueError("An empty value in {0}".format(res))
if not os.path.exists(r):
raise FileNotFoundError("Unable to find '{0}'".format(r))
if 'main_0000.rst' in r:
with open(r, 'r', encoding='utf-8') as f:
content = f.read()
self.assertIn('...', content)
def test_profiling(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_profiling")
data = os.path.join(temp, "..", "data", "sample1000.txt")
with open(data, "r", encoding="utf-8") as f:
lines = [_.strip(" \n\r\t") for _ in f.readlines()]
def profile_exe():
res = self.gain_dynamique_moyen_par_mot(lines, [1.0] * len(lines))
return res
def prof(n, show):
pr = cProfile.Profile()
pr.enable()
profile_exe()
pr.disable()
s = io.StringIO()
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
ps.print_stats()
rem = os.path.normpath(os.path.join(temp, "..", "..", ".."))
res = s.getvalue().replace(rem, "")
if show:
fLOG(res)
with open(os.path.join(temp, "profiling%d.txt" % n), "w") as f:
f.write(res)
prof(1, show=False)
prof(2, show=True)
def test_american_cities(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[:2] <= (3, 4):
warnings.warn(
"Issue with Python 3.4, bug probably related to wrong pointers")
return
fix_tkinter_issues_virtualenv()
import matplotlib.pyplot as plt
import mpld3
filter = {"NewYork": "NY", "Chicago": "CH",
"SanFrancisco": "SF", "Seattle": "Sea"}
temp = get_temp_folder(__file__, "temp_american_cities")
data = os.path.join(temp, "..", "data", "american_cities.txt")
df = pandas.read_csv(data)
df["Longitude"] = -df["Longitude"]
df["City"] = df["City"].apply(lambda v: filter.get(v, ""))
fig, ax = plt.subplots(figsize=(32, 32))
df = df[df.Latitude < 52]
df = df[df.Longitude > -130].copy()
ax = graph_cities(df, ax=ax, markersize=3, fontcolor=(0, 1.0, 0), fontsize='40',
fontname="Courrier", fontweight="bold")
assert ax is not None
img = os.path.join(temp, "img.png")
fig.savefig(img)
assert os.path.exists(img)
name2 = os.path.join(temp, "picture.html")
mpld3.save_html(fig, name2)
assert os.path.exists(name2)
if __name__ == "__main__":
fig.show()
plt.close('all')
fLOG("end")
def a_test_notebook_runner(self, name, folder, valid=None, copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
if not os.path.exists(doc):
raise FileNotFoundError(doc)
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
import pyquickhelper
import jyquickhelper
import pyensae
import ensae_teaching_cs
add_path = get_additional_paths(
[jyquickhelper, pyquickhelper, pyensae, ensae_teaching_cs])
res = execute_notebook_list(
temp, keepnote, additional_path=add_path, valid=valid)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=ensae_teaching_cs)
def test_graph_graphviz(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
code = """
namespace hello
{
public static class world
{
public static double function(double x, doubly y)
{
return x+y ;
}
}
}
"""
clparser, cllexer = get_parser_lexer("C#")
parser = parse_code(code, clparser, cllexer)
tree = parser.parse()
st = get_tree_graph(tree, parser)
dot = st.to_dot()
# fLOG(dot)
assert len(dot) > 0
if "travis" not in sys.executable:
temp = get_temp_folder(__file__, "temp_dot_grammar")
name = os.path.join(temp, "graph.dot")
with open(name, "w") as f:
f.write(dot)
img = os.path.join(temp, "graph.png")
run_dot(name, img)
assert os.path.exists(img)
def test_notebook_runner_2a_eco_sncf(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_eco_sncf")
keepnote = ls_notebooks("td2a_eco")
assert len(keepnote) > 0
shutil.copy(simple_database(), temp)
folder_note = os.path.split(keepnote[0])[0]
jsfile = os.path.join(folder_note, "stop_areas.json")
shutil.copy(jsfile, temp)
def filter(i, n):
if "SNCF" not in n:
return False
return True
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
res = execute_notebooks(temp, keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a)
unittest_raise_exception_notebook(res, fLOG)
def test_notebook_runner_flat2db3(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
notebook = os.path.split(
__file__)[-1].replace(".ipynb", "").replace(".py", "")[5:]
temp = get_temp_folder(__file__, "temp_" + notebook)
nbfile = os.path.join(
temp,
"..",
"..",
"..",
"_doc",
"notebooks",
"%s.ipynb" %
notebook)
if not os.path.exists(nbfile):
raise FileNotFoundError(nbfile)
addpath = [os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
os.path.normpath(
os.path.join(
temp,
"..",
"..",
"..",
"..",
"pyquickhelper",
"src")),
os.path.normpath(
os.path.join(
temp,
"..",
"..",
"..",
"..",
"pymyinstall",
"src")),
]
outfile = os.path.join(temp, "out_notebook.ipynb")
assert not os.path.exists(outfile)
def valid(code):
return 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' not in code
if "travis" in sys.executable:
return
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"pyensae")
out = run_notebook(nbfile,
working_dir=temp,
outfilename=outfile,
additional_path=addpath,
valid=valid,
fLOG=fLOG,
kernel_name=kernel_name)
fLOG(out)
assert os.path.exists(outfile)
def test_notebook_js(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "notebooks_js"))
nbs = [os.path.join(fold, _)
for _ in os.listdir(fold) if ".ipynb" in _]
formats = ["slides", "present", "ipynb", "html",
"python", "rst", "pdf"]
if sys.platform.startswith("win"):
formats.append("docx")
temp = get_temp_folder(__file__, "temp_nb_bug_js")
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
if not os.path.exists(_[0]):
raise Exception(_[0])
check = os.path.join(temp, "using_qgrid_with_jsdf.tex")
with open(check, "r", encoding="utf8") as f:
content = f.read()
if "\\section{" not in content:
raise Exception(content)
checks = [os.path.join(temp, "reveal.js"),
os.path.join(temp, "require.js")]
for check in checks:
if not os.path.exists(check):
raise Exception(check)
def test_notebook_runner_2a(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
# too long for appveyor
return
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a, unittest_raise_exception_notebook
temp = get_temp_folder(__file__, "temp_notebook2a_")
keepnote = ls_notebooks("2a")
assert len(keepnote) > 0
def filter(i, n):
if not sys.platform.startswith("win") and "_convert" in n:
return False
if "git_" not in n and "python_r" not in n and "csharp" not in n:
return True
if is_travis_or_appveyor() and "notebook_convert.ipynb" in n:
# this one requires pandoc
return False
return False
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
res = execute_notebooks(temp, keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a)
unittest_raise_exception_notebook(res, fLOG)
def test_ensae_2016_answers(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no stored password
return
password = keyring.get_password(
"cpt", os.environ["COMPUTERNAME"] + "pwd")
r = data_cpt_ENSAE_2016_11_blind_set(password)
self.assertEqual(len(r), 7500)
truth = r
r = data_cpt_ENSAE_2016_11_blind_set("dummy")
self.assertEqual(len(r), 7500)
s = sum(r)
assert 0 <= s <= len(r)
auc = AUC(truth, r)
fLOG(auc)
assert 0 <= auc <= 1
temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers")
out = os.path.join(temp, "answer.txt")
with open(out, "w") as f:
f.write("\n".join(str(_) for _ in r))
zip_files(os.path.join("submission.zip"), [out])
def test_download_zip(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = get_temp_folder(__file__, "temp_download")
url = "https://docs.python.org/3/library/ftplib.html"
f = download(url, fold)
fLOG(f)
self.assertTrue(os.path.exists(f))
if not f.endswith("ftplib.html"):
raise Exception(f)
out1 = os.path.join(fold, "try.html.gz")
gzip_files(out1, [f], fLOG=fLOG)
self.assertTrue(os.path.exists(out1))
out2 = os.path.join(fold, "try.zip")
zip_files(out2, [f], fLOG=fLOG)
self.assertTrue(os.path.exists(out2))
if is_travis_or_appveyor() in ("circleci", None):
out7 = os.path.join(fold, "try.7z")
zip7_files(out7, [out1, out2], fLOG=fLOG, temp_folder=fold)
if not os.path.exists(out7):
raise FileNotFoundError(out7)
else:
fLOG("skip 7z")
def test_notebook_runner_2a_ml(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_ml4")
keepnote = ls_notebooks("td2a_ml")
keepnote = [_ for _ in keepnote if "overfitting" in _]
shutil.copy(simple_database(), temp)
def filter(i, n):
if "SNCF" in n:
return False
if "Scraping" in n:
return False
if "deep_python" in n:
return False
if "h2o" in n:
# h2o is not working from a virtual environment
return False
if "td2a" in os.path.split(n)[-1]:
# already tested by others tests
return False
if "libraries" in n:
return False
return True
execute_notebooks(temp, keepnote, filter, fLOG=fLOG,
clean_function=clean_function_1a,
dump=ensae_teaching_cs)
def test_notebook_runner_enonce_10(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_notebook1a_enonce_10")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
keepnote = ls_notebooks("td1a_dfnp")
fold = os.path.dirname(keepnote[0])
for png in os.listdir(fold):
if ".png" not in png:
continue
fLOG("copy", png)
shutil.copy(os.path.join(fold, png), temp)
execute_notebooks(temp,
keepnote,
lambda i, n: "cenonce_session_10." in n,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_added(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from pyensae.datasource import DownloadDataException
temp = get_temp_folder(__file__, "temp_added")
text = besancon_df(local=True, fLOG=fLOG)
assert text is not None
assert isinstance(text, str)
try:
text2 = added(local=False, cache_folder=temp, fLOG=fLOG)
assert text2
assert isinstance(text2, str)
except (ConnectionResetError, DownloadDataException) as e:
warnings.warn("Cannot check remote added.txt.\n" + str(e))
return
def test_notebook_runner_2a_algo(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
# too long for appveyor
return
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_algo")
keepnote = ls_notebooks("td2a_algo")
shutil.copy(simple_database(), temp)
def filter(i, n):
if "SNCF" in n:
return False
return True
def clean_function_1a_upgraded(code):
code = clean_function_1a(code)
rep = "[1000, 2000, 5000, 10000, 12000, 15000, 17000, 20000]"
by = "[1000, 2000]"
code = code.replace(rep, by)
rep = "[10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000]"
by = "[10, 20, 50, 100, 200]"
code = code.replace(rep, by)
return code
if is_travis_or_appveyor() == "travis":
# execution does not stop
return
fold = os.path.dirname(keepnote[0])
for png in os.listdir(fold):
if ".png" not in png:
continue
fLOG("copy", png)
shutil.copy(os.path.join(fold, png), temp)
execute_notebooks(temp, keepnote, filter, fLOG=fLOG,
clean_function=clean_function_1a_upgraded,
dump=src.ensae_teaching_cs)
def test_validate_sklearn_operators_dump_all(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
logger = getLogger('skl2onnx')
logger.disabled = True
verbose = 1 if __name__ == "__main__" else 0
temp = get_temp_folder(__file__,
"temp_validate_sklearn_operators_dump_all")
self.assertRaise(
lambda: list(
enumerate_validated_operator_opsets(
verbose,
models={"DecisionTreeClassifier"},
filter_exp=lambda m, p: '64' not in p,
fLOG=fLOG,
dump_all=True)), ValueError)
rows = list(
enumerate_validated_operator_opsets(
verbose,
models={"DecisionTreeClassifier"},
filter_exp=lambda m, p: '64' not in p,
fLOG=fLOG,
dump_all=True,
dump_folder=temp))
self.assertGreater(len(rows), 1)
df = DataFrame(rows)
self.assertGreater(df.shape[1], 1)
fLOG("output results")
df.to_csv(os.path.join(temp, "sklearn_opsets_report.csv"), index=False)
df.to_excel(os.path.join(temp, "sklearn_opsets_report.xlsx"),
index=False)
stored = os.path.join(
temp,
("dump-i-python-DecisionTreeClassifier-default-b-cl-tree._classes."
"DecisionTreeClassifierzipmapFalse-op%d-nf4.pkl" % TARGET_OPSET))
with open(stored, "rb") as f:
obj = pickle.load(f)
self.assertIn('onnx_bytes', obj)
self.assertIn('skl_model', obj)
self.assertIn('X_test', obj)
self.assertIn('Xort_test', obj)
def test_american_cities(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[:2] <= (3, 4):
warnings.warn(
"Issue with Python 3.4, bug probably related to wrong pointers"
)
return
fix_tkinter_issues_virtualenv()
import matplotlib.pyplot as plt
filter = {
"NewYork": "NY",
"Chicago": "CH",
"SanFrancisco": "SF",
"Seattle": "Sea"
}
temp = get_temp_folder(__file__, "temp_american_cities")
data = os.path.join(temp, "..", "data", "american_cities.txt")
df = pandas.read_csv(data)
df["Longitude"] = -df["Longitude"]
df["City"] = df["City"].apply(lambda v: filter.get(v, ""))
fig, ax = plt.subplots(figsize=(32, 32))
df = df[df.Latitude < 52]
df = df[df.Longitude > -130].copy()
ax = graph_cities(df,
ax=ax,
markersize=3,
fontcolor=(0, 1.0, 0),
fontsize='40',
fontweight="bold")
assert ax is not None
img = os.path.join(temp, "img.png")
fig.savefig(img)
assert os.path.exists(img)
# mpld3 is deprecated.
# name2 = os.path.join(temp, "picture.html")
# mpld3.save_html(fig, name2)
# assert os.path.exists(name2)
if __name__ == "__main__":
fig.show()
plt.close('all')
fLOG("end")
def test_notebook_number(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_notebook_number")
nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb")
self.assertTrue(os.path.exists(nbfile))
outfile = os.path.join(temp, "out_nb.ipynb")
res = remove_execution_number(nbfile, outfile)
self.assertTrue(res is not None)
self.assertTrue(os.path.exists(outfile))
self.assertTrue('"execution_count": null' in res)
if is_travis_or_appveyor() == "appveyor":
change_file_status(outfile)
change_file_status(temp, strict=True)
else:
warnings.warn(
"linux, unable to test change_file_status: TestNotebookNumber.test_notebook_number")
def test_venv_docutils08_readme(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = os.path.dirname(os.path.abspath(__file__))
readme = os.path.join(fold, "..", "..", "README.rst")
self.assertTrue(os.path.exists(readme))
with open(readme, "r", encoding="utf8") as f:
content = f.read()
self.assertTrue(len(content) > 0)
temp = get_temp_folder(__file__, "temp_readme")
if __name__ != "__main__":
# does not work from a virtual environment
return
from pyquickhelper.pycode import check_readme_syntax
check_readme_syntax(readme, folder=temp, fLOG=fLOG)
def test_notebook_runner_2a_cffi(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
temp = get_temp_folder(__file__, "temp_notebook2a_")
keepnote = ls_notebooks("2a")
def filter(i, n):
if "cffi" not in n:
return False
return True
execute_notebooks(temp,
keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=ensae_teaching_cs)
def test_cli_benchmark_replay(self):
temp = get_temp_folder(__file__, "temp_benchmark_replay")
out1 = os.path.join(temp, "raw.csv")
st = BufferedPrint()
out1 = os.path.join(temp, "raw.csv")
st = BufferedPrint()
main(args=["validate_runtime", "--n_features", "4", "-nu", "3",
"-re", "3", "-o", "11", "-op", "11", "-v", "2", "--out_raw",
out1, "-b", "0",
"--runtime", "python_compiled",
"--models", "KMeans", "--dtype", "32",
"--dump_all", '1', '--dump_folder', temp],
fLOG=st.fprint)
out = os.path.join(temp, "res.xlsx")
main(args=["benchmark_replay", "--folder", temp, "--out", out, '--verbose', '2'],
fLOG=st.fprint)
res = str(st)
self.assertExists(out)
self.assertIn("'folder'", res)
def test_notebook_runner_flat2db3(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
notebook = os.path.split(__file__)[-1].replace(".ipynb",
"").replace(".py",
"")[5:]
temp = get_temp_folder(__file__, "temp_" + notebook)
nbfile = os.path.join(temp, "..", "..", "..", "_doc", "notebooks",
"%s.ipynb" % notebook)
if not os.path.exists(nbfile):
raise FileNotFoundError(nbfile)
addpath = [
os.path.normpath(os.path.join(temp, "..", "..", "..", "src")),
os.path.normpath(
os.path.join(temp, "..", "..", "..", "..", "pyquickhelper",
"src")),
os.path.normpath(
os.path.join(temp, "..", "..", "..", "..", "pymyinstall",
"src")),
]
outfile = os.path.join(temp, "out_notebook.ipynb")
assert not os.path.exists(outfile)
def valid(code):
return 'run_cmd("SQLiteSpy.exe velib_vanves.db3")' not in code
if "travis" in sys.executable:
return
kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
"pyensae")
out = run_notebook(nbfile,
working_dir=temp,
outfilename=outfile,
additional_path=addpath,
valid=valid,
fLOG=fLOG,
kernel_name=kernel_name)
fLOG(out)
assert os.path.exists(outfile)
def a_test_notebook_runner(self,
name,
folder,
valid=None,
copy_folder=None):
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
doc = os.path.join(temp, "..", "..", "..", "_doc", "notebooks", folder)
self.assertTrue(os.path.exists(doc))
keepnote = [os.path.join(doc, _) for _ in os.listdir(doc) if name in _]
self.assertTrue(len(keepnote) > 0)
if copy_folder is not None:
if not os.path.exists(copy_folder):
raise FileNotFoundError(copy_folder)
dest = os.path.split(copy_folder)[-1]
dest = os.path.join(temp, dest)
if not os.path.exists(dest):
os.mkdir(dest)
synchronize_folder(copy_folder, dest, fLOG=fLOG)
def new_valid(cell):
if "print(next(it))\nprint(next(it))" in cell:
return False
return True
if valid is None:
valid = new_valid
import pyquickhelper
import jyquickhelper
import pymyinstall
import pyensae
import pyrsslocal
add_path = get_additional_paths([
jyquickhelper, pyquickhelper, pyensae, pymyinstall, pyrsslocal,
thismodule
])
res = execute_notebook_list(temp,
keepnote,
additional_path=add_path,
valid=valid)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=thismodule)
def test_search_vectors_zip(self):
temp = get_temp_folder(__file__, "temp_search_vectors_zip")
res = []
for i in range(20):
h = i * 0.05
h2 = 1 - i * 0.05
res.append(
dict(ind=i * 5,
meta1="m%d" % i,
meta2="m%d" % (i + 1),
f1=h,
f2=h2))
df = pandas.DataFrame(res)
se = SearchEngineVectors(n_neighbors=5)
r = repr(se)
self.assertEqual(
r.replace("\n", "").replace(" ", ""),
'SearchEngineVectors(n_neighbors=5)')
se.fit(data=None,
features=df[["f1", "f2"]].values,
metadata=df[["ind", "meta1", "meta2"]])
score, ind, meta = se.kneighbors([0.5, 0.5])
self.assertIsInstance(ind, (list, numpy.ndarray))
self.assertEqual(len(ind), 5)
self.assertEqual(ind[0], 10)
self.assertIsInstance(score, numpy.ndarray)
self.assertEqual(score.shape, (5, ))
self.assertEqual(score[0], 0)
dest = os.path.join(temp, "se.zip")
se.to_zip(dest, encoding='utf-8')
se2 = SearchEngineVectors.read_zip(dest, encoding='utf-8')
score2, ind2, meta2 = se2.kneighbors([0.5, 0.5])
self.assertEqualArray(score, score2)
self.assertEqualArray(ind, ind2)
self.assertEqualDataFrame(meta, meta2)
self.assertEqual(se.pknn, se2.pknn)
def test_svg2img(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_svg2png")
svg = """<svg width="200" height="180">
<rect x="50" y="20" width="150" height="150" style="fill:blue;stroke:pink;fill-opacity:0.1;" />
<ellipse cx="40" cy="50" rx="20" ry="30" style="fill:yellow" />
<ellipse cx="20" cy="50" rx="90" ry="20" style="fill:white" />
</svg>
"""
img = svg2img(svg)
name = os.path.join(temp, "image.png")
img.save(name)
self.assertExists(name)
svg = """<svg>
<rect x="50" y="20" width="150" height="150" style="fill:blue;stroke:pink;fill-opacity:0.1;" />
<ellipse cx="40" cy="50" rx="20" ry="30" style="fill:yellow" />
<ellipse cx="20" cy="50" rx="90" ry="20" style="fill:white" />
</svg>
"""
size = guess_svg_size(svg)
self.assertEqual(size, (180, 40))
img = svg2img(svg)
name = os.path.join(temp, "image2.png")
img.save(name)
self.assertExists(name)
img = svg2img(svg, dpi=200)
name = os.path.join(temp, "image3.png")
img.save(name)
self.assertExists(name)
img = svg2img(svg, scale=2.)
name = os.path.join(temp, "image4.png")
img.save(name)
self.assertExists(name)
def test_import_exam_vigenere(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
from ensae_teaching_cs.td_1a.vigenere import code_vigenere
temp = get_temp_folder(__file__, "temp_import_exam_vigenere")
pwd = get_password("exam", "ensae_teaching_cs,key")
pwd = pwd.encode("ascii")
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere")
assert os.path.exists(dst)
with open(dst, "rb") as f:
content = f.read()
content = code_vigenere(content, pwd, binary=True, decode=True)
decr = os.path.join(temp, "onemod_vigenere.py")
with open(decr, "wb") as f:
f.write(content)
assert os.path.exists(decr)
fLOG("importing")
sys.path.append(temp)
import onemod as temp_module
del sys.path[-1]
answer = temp_module.build(120)
exp = """*** H
H *** H
*** H
*** H
H *** H
*** H
*** H
H *** H
*** H
*** H
H *** H
*** H""".replace(" ", "")
self.assertEqual(answer, exp)
def test_notebook_ai(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_ai")
keepnote = ls_notebooks("ai")
self.assertTrue(len(keepnote) > 0)
source = os.path.join(os.path.dirname(keepnote[0]), "images")
images = os.path.join(temp, 'images')
os.mkdir(images)
for img in os.listdir(source):
shutil.copy(os.path.join(source, img), images)
res = execute_notebooks(temp,
keepnote,
lambda i, n: True,
fLOG=fLOG,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=code_beatrix)
def test_video_download(self):
temp = get_temp_folder(__file__, "temp_video_download")
try:
download_youtube_video('vHcfbOqYztU', output_path=temp)
except RegexMatchError as e:
if "zero match" not in str(e):
raise e
import pytube
warnings.warn(
"RegexMatchError: pytube version {} - pytube issue: {}".format(
pytube.__version__, e))
return
except KeyError as e:
import pytube
warnings.warn(
"KeyError: pytube version {} - pytube issue: {}".format(
pytube.__version__, e))
return
exp = os.path.join(temp, "vidéo tres courte.mp4")
self.assertExists(exp)
def test_gradient(self):
temp = get_temp_folder(__file__, "temp_segment_gradient")
img = os.path.join(temp, "..", "data", "eglise_zoom2.jpg")
grad = _calcule_gradient(img, color=0)
self.assertEqual(grad.shape, (308, 408, 2))
for d in [-2, -1, 0, 1, 2]:
imgrad = plot_gradient(img, grad, direction=d)
grfile = os.path.join(temp, "gradient-%d.png" % d)
imgrad.save(grfile)
self.assertExists(grfile)
with open(os.path.join(temp, "..", "data", "gradient--2.png"),
'rb') as f:
c1 = f.read()
with open(os.path.join(temp, "..", "data", "gradient--2b.png"),
'rb') as f:
c1b = f.read()
with open(os.path.join(temp, "gradient--2.png"), 'rb') as f:
c2 = f.read()
self.assertIn(c2, (c1, c1b))
def test_zip_numpy(self):
df = numpy.zeros((3, 4))
df[2, 3] = 1
temp = get_temp_folder(__file__, "temp_zip")
name = os.path.join(temp, "df.zip")
to_zip(df, name, "arr.npy")
df2 = read_zip(name, "arr.npy")
self.assertEqualArray(df, df2)
st = io.BytesIO()
zp = zipfile.ZipFile(st, 'w')
to_zip(df, zp, "arr.npy")
zp.close()
st = io.BytesIO(st.getvalue())
zp = zipfile.ZipFile(st, 'r')
df3 = read_zip(zp, "arr.npy")
zp.close()
self.assertEqualArray(df, df3)
def test_cache_urls(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_cache_urls")
fnb = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "_doc", "notebooks"))
keepnote = [os.path.join(fnb, "example_about_files.ipynb")]
addpaths = [os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", "..", "src"))]
res = execute_notebook_list(
temp, keepnote, fLOG=fLOG, valid=None, additional_path=addpaths,
cache_urls=["https://docs.python.org/3.4/library/urllib.request.html"])
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=pyquickhelper)
def test_convert_music_file(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_convert_music_file")
data = os.path.join(temp, "..", "data")
ffmpeg = os.path.join(data, "ffmpeg.zip")
assert os.path.exists(ffmpeg)
ff = unzip(ffmpeg, temp)
mp3 = os.path.join(data, "podcasts_example.mp3")
wav = os.path.join(temp, "podcasts_example.wav")
if is_travis_or_appveyor() == "travis":
warnings.warn("pydub is not available")
return
convert_music_file(mp3, wav, ffmpeg=ff)
assert os.path.exists(wav)
assert os.stat(wav).st_size > 90000
def test_downloadlink_rst(self):
name = self.get_name()
content = """
:downloadlink:`rst::http://f.html`
:downloadlink:`rst::{0}`
:downloadlink:`{0} <rst::{0}>`
""".replace(" ", "").format(name)
out = rst2html(content,
writer="rst",
keep_warnings=True,
directives=None)
out = out.replace("\n", " ")
self.assertNotIn('Unknown interpreted text role', out)
self.assertIn(':downloadlink:`test_rst_builder.py', out)
self.assertNotIn("test_rst_builder.py>`test_rst_builder.py", out)
temp = get_temp_folder(__file__, "temp_downloadlink_rst")
with open(os.path.join(temp, "out.rst"), "w", encoding="utf8") as f:
f.write(out)
def test_maml_mlnet(self):
temp = get_temp_folder(__file__, "temp_maml_mlnet")
iris = datasets.load_iris()
X = iris.data
y = iris.target
df = pandas.DataFrame(
X, columns=['Slength', 'Swidth', 'Plength', 'Pwidth'])
df["Label"] = y
df = df[["Label"] + ['Slength', 'Swidth', 'Plength', 'Pwidth']]
dest = os.path.join(temp, "iris_data_id.txt")
df.to_csv(dest, sep=',', index=False)
fct = mlnet("ReturnMLClass", TestMamlMlnet._script, redirect=True)
cl, _, __ = fct(dest)
cl.Train()
res = cl.Predict(3.4, 5.4, 3.2, 5.6)
label = res.PredictedLabel
score = list(res.Score)
self.assertEqual(label, 3)
self.assertEqual(len(score), 3)
def test_notebook_runner(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks
from src.ensae_teaching_cs.automation.notebook_test_helper import clean_function_1a
temp = get_temp_folder(__file__, "temp_notebook2a_1")
keepnote = ls_notebooks("td2a")
fold = os.path.dirname(keepnote[0])
for png in os.listdir(fold):
if ".png" not in png:
continue
fLOG("copy", png)
shutil.copy(os.path.join(fold, png), temp)
self.assertTrue(len(keepnote) > 0)
execute_notebooks(temp,
keepnote, (lambda i, n: "_1" in n),
clean_function=clean_function_1a,
fLOG=fLOG,
dump=src.ensae_teaching_cs)
def test_perf_onnxruntime_logreg(self):
res = onnxruntime_perf_binary_classifiers()[0]
bp = BenchPerf(res['pbefore'], res['pafter'], res['fct'])
results = list(bp.enumerate_run_benchs(repeat=10, verbose=True))
results_df = pandas.DataFrame(results)
temp = get_temp_folder(__file__, "temp_perf_onnxruntime_logreg")
out = os.path.join(temp, "onnxruntime_logreg.perf.csv")
results_df.to_csv(out, index=False)
self.assertExists(out)
subset = {
'sklearn', 'numpy', 'pandas', 'onnxruntime', 'skl2onnx',
'mlprodict'
}
df = pandas.DataFrame(machine_information(subset))
out = os.path.join(temp, "onnxruntime_logreg.time.csv")
df.to_csv(out, index=False)
self.assertExists(out)
def test_notebook_add_slides_bug(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_notebook_add_slides_bug")
nbfile = os.path.join(temp, "..", "data", "pyensae_text2table.ipynb")
assert os.path.exists(nbfile)
nb = read_nb(nbfile, kernel=False)
for i, cell in enumerate(nb.iter_cells()):
ls = nb.cell_height(cell)
#fLOG(i,"-",nb.cell_type(cell), l)
#if l > 400: fLOG(cell)
assert ls > 0
new_tags = nb.add_tag_slide()
for k, v in sorted(new_tags.items()):
a, b, c = v
fLOG(k, a, b)
assert len(new_tags) > 0
def test_notebook_runner_2a_eco_nlp_long1(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
temp = get_temp_folder(__file__, "temp_notebook2a_eco_nlp_long1")
keepnote = ls_notebooks("td2a_eco2")
def filter(i, n):
if "td2a_Seance_7_Analyse_de_textes" in n:
return True
return False
execute_notebooks(temp,
keepnote,
filter,
fLOG=fLOG,
clean_function=clean_function_1a,
dump=ensae_teaching_cs)
def test_video_frame(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
import gizeh
temp = get_temp_folder(__file__, "temp_video_frame")
def make_frame(t):
# See example: https://zulko.github.io/moviepy/getting_started/videoclips.html#videoclip
surface = gizeh.Surface(128, 128)
radius = 100 * (1 + (t * (2 - t))**2) / 6
circle = gizeh.circle(radius, xy=(64, 64), fill=(1, 0, 0))
circle.draw(surface)
return surface.get_npimage()
vid = video_frame(make_frame)
exp = os.path.join(temp, "courte.gif")
video_save(vid, exp, fps=20, duration=2)
self.assertExists(exp)
clean_video(vid)
def test_notebook_latex_problems(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(
os.path.join(path, "..", "..", "_doc", "notebooks", "td2a_ml"))
nbs = [
os.path.join(fold, _) for _ in os.listdir(fold)
if _.endswith(".ipynb") and "problems" in _
]
nbs.sort()
formats = ["pdf", "ipynb", "html", "python", "rst"]
temp = get_temp_folder(__file__, "temp_nb_td2A_ml_bug_latex")
res = process_notebooks(nbs, temp, temp, formats=formats)
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertTrue(os.path.exists(_[0]))
def a_test_notebook_runner(self, name, folder):
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from src.ensae_teaching_cs.helpers.size_helper import total_size
self.assertTrue(total_size)
temp = get_temp_folder(__file__, "temp_notebook_123_{0}".format(name))
keepnote = ls_notebooks(folder)
self.assertTrue(len(keepnote) > 0)
replacements = {
'input("Entrez un nombre")': 'random.randint(0, 100)',
'input(message)': 'random.randint(0, 100)'
}
execute_notebooks(temp,
keepnote,
lambda i, n: name in n,
fLOG=fLOG,
replacements=replacements,
clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_crypt_helper(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_crypt")
input = os.path.abspath(__file__.replace(".pyc", ".py"))
output = os.path.join(temp, "crypted.bin")
output2 = os.path.join(temp, "decrypted.txt")
key = "01" * 8
encrypt_data(key, input, output)
decrypt_data(key, output, output2)
with open(input, "r") as f:
t1 = f.read()
with open(output2, "r") as f:
t2 = f.read()
self.assertEqual(t1, t2)
assert len(t1) > 0
