def test_long_long_notebook_population_s6_correction(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fix_tkinter_issues_virtualenv()
from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
if "travis" in sys.executable:
# matplotlib is still failing
warnings.warn(
"travis, unable to test TestNotebookPopulation.test_notebook_sessions")
return
temp = get_temp_folder(__file__, "temp_sessions_long_long")
keepnote = [_ for _ in ls_notebooks("sessions")
if "seance6_graphes_correction" in _]
self.assertTrue(len(keepnote) > 0)
clog = CustomLog(temp)
res = execute_notebooks(temp, keepnote,
lambda i, n: "deviner" not in n,
fLOG=fLOG, clean_function=clean_function_notebook,
detailed_log=clog)
execute_notebook_list_finalize_ut(
res, fLOG=fLOG, dump=actuariat_python)
def test_install_python_minimal(self):
fLOG(__file__, self._testMethodName, OutputPrint=True)
vers = "%d%d" % sys.version_info[:2]
temp = get_temp_folder(__file__,
"temp_python%s_minimal" % vers,
clean=True)
if sys.platform.startswith("win"):
fLOG("BEGIN")
clog = CustomLog(temp)
install_python(install=True,
temp_folder=temp,
fLOG=clog,
modules="minimal",
custom=True,
latest=True,
download_folder=temp + "_download")
fLOG("END")
pyt = os.path.join(temp, "python.exe")
pip = os.path.join(temp, "Scripts", "pip.exe")
if not os.path.exists(pyt):
raise FileNotFoundError(pyt)
if not os.path.exists(pip):
raise FileNotFoundError(pip)
post = os.path.join(temp, "Scripts", "pywin32_postinstall.py")
with open(post, "r") as f:
content = f.read()
if "os.path.exists(dest)" not in content:
raise Exception(
"'os.path.exists(dest)' not found in '{0}'".format(post))
def test_long_notebook_runner_2a_ml(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() == "appveyor":
# too long for appveyor
return
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from src.ensae_teaching_cs.data import simple_database
temp = get_temp_folder(__file__, "temp_notebook2a_ml_deep")
keepnote = ls_notebooks("td2a_ml")
shutil.copy(simple_database(), temp)
def filter(i, n):
if "deep_python" in n:
return True
return False
if is_travis_or_appveyor() == "travis":
warnings.warn("execution does not stop")
return
clog = CustomLog(temp)
execute_notebooks(temp,
keepnote,
filter,
fLOG=clog,
clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_compare_versions(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if not sys.platform.startswith("win"):
# useful only on Windows
return
if sys.version_info[0] == 2:
# not maintaining for Python 2.7
return
mods = [mod for mod in all_set() if mod.kind in ("wheel",)]
fLOG(len(mods))
assert len(mods) > 0
clog = CustomLog(os.path.dirname(__file__),
"windows_modules_versions.txt")
diff = []
for i, mod in enumerate(mods[0:]):
if mod.name in ["PySide", "cgal_bindings", "ad3"]:
# skipping modules not available
continue
if i % 10 == 0:
fLOG(i, mod)
try:
url1 = mod.get_exewheel_url_link(wheel=True)
except MissingWheelException:
url1 = None
try:
url2 = mod.get_exewheel_url_link2(wheel=True, source="2")
except MissingWheelException:
url2 = None
if url1 is None and url2 is None:
fLOG("missing package on both sides", mod)
clog("missing package on both sides", mod)
continue
# fLOG(i, url1, url2)
n1 = url1[-1] if url1 is not None else None
n2 = url2[-1] if url2 is not None else None
v1 = get_wheel_version(n1) if n1 is not None else None
v2 = get_wheel_version(n2) if n2 is not None else None
cmp = compare_version(
v1, v2) if v1 is not None and v2 is not None else -2
if cmp != 0:
fLOG("---", i, v1, v2, mod.name)
clog("---", i, v1, v2, mod.name)
if v1 is not None and v2 is not None and not ("rc1" in v1 or "rc1" in v2):
diff.append(mod)
assert len(diff) <= 10
def test_notebook_runner_correction_8(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
temp = get_temp_folder(__file__, "temp_notebook1a_correction_8")
keepnote = ls_notebooks("td1a_algo")
clog = CustomLog(temp)
execute_notebooks(temp, keepnote, (lambda i, n: "correction_session8." in n),
fLOG=clog, clean_function=clean_function_1a,
dump=src.ensae_teaching_cs)
def test_long_run_notebook(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_run_notebooks_nlp")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "nlp"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(f)[-1] == ".ipynb" and "_long" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
if "open_html_form" in cell:
return False
if "open_html_form" in cell:
return False
if "[50000, 100000, 200000, 500000, 500000, 1000000, 2000000, None]" in cell:
return False
if '<div style="position:absolute' in cell:
return False
return True
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
# run the notebooks
clog = CustomLog(temp)
clog("START")
res = execute_notebook_list(temp,
keepnote,
fLOG=clog,
deepfLOG=clog,
valid=valid,
additional_path=addpaths)
clog("END")
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=mlstatpy)
def test_notebook_runner_correction_8(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
from src.ensae_teaching_cs.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_1a
from src.ensae_teaching_cs.automation.notebook_test_helper import unittest_raise_exception_notebook
temp = get_temp_folder(__file__, "temp_notebook1a_correction_8")
keepnote = ls_notebooks("td1a")
assert len(keepnote) > 0
clog = CustomLog(temp)
res = execute_notebooks(temp,
keepnote,
lambda i, n: "correction_session8." in n,
fLOG=clog,
clean_function=clean_function_1a)
unittest_raise_exception_notebook(res, fLOG)
def test_notebook_rst(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(os.path.join(path, "notebooks_rst"))
nbs = [
os.path.join(fold, _) for _ in os.listdir(fold) if ".ipynb" in _
]
formats = [
"rst",
]
temp = get_temp_folder(__file__, "temp_nb_bug_rst")
clog = CustomLog(temp)
clog("test_notebook_rst")
fold = 'static'
if not os.path.exists(fold):
os.mkdir(fold)
for tpl in ['rst', 'display_priority', 'null']:
sty = os.path.join(fold, '%s.tpl' % tpl)
sr = os.path.join(temp, '..', 'data', '%s.tpl' % tpl)
if not os.path.exists(sr):
raise FileNotFoundError(sr)
if not os.path.exists(sty):
shutil.copy(sr, fold)
if not os.path.exists('%s.tpl' % tpl):
shutil.copy(sr, '.')
clog("process_notebooks: begin")
res = process_notebooks(nbs, temp, temp, formats=formats, fLOG=clog)
clog("process_notebooks: end")
fLOG("*****", len(res))
for _ in res:
fLOG(_)
self.assertExists(_[0])
name = os.path.join(temp, "having_a_form_in_a_notebook.rst")
clog("final checking", name)
with open(name, "r", encoding="utf8") as f:
content = f.read()
clog("final read", name)
exp = "<#Animated-output>`"
if exp in content or exp.lower() not in content:
raise Exception(content)
clog("done")
def test_notebook_video(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
formats = ["pdf", "rst"]
temp = get_temp_folder(__file__, "temp_nb_bug_video")
nbs = [os.path.join(temp, '..', 'data', 'video_notebook.ipynb')]
clog = CustomLog(temp)
clog("test_notebook_rst_audio")
fold = 'static'
if not os.path.exists(fold):
os.mkdir(fold)
for tpl in ['article']:
sty = os.path.join(fold, '%s.tplx' % tpl)
sr = os.path.join(temp, '..', 'data', '%s.tplx' % tpl)
if not os.path.exists(sr):
raise FileNotFoundError(sr)
if not os.path.exists(sty):
shutil.copy(sr, fold)
if not os.path.exists('%s.tplx' % tpl):
shutil.copy(sr, '.')
clog("process_notebooks: begin")
process_notebooks(nbs, temp, temp, formats=formats, fLOG=clog)
clog("process_notebooks: end")
name = os.path.join(temp, "video_notebook.rst")
clog("final checking", name)
with open(name, "r", encoding="utf8") as f:
content = f.read()
clog("final read", name)
exp = ".. raw:: html"
if exp not in content:
raise Exception(content)
clog("done")
name = os.path.join(temp, "video_notebook.tex")
clog("final checking", name)
with open(name, "r", encoding="utf8") as f:
content = f.read()
clog("final read", name)
exp = "<moviepy.video.io.html\\_tools.HTML2 object>"
if exp not in content:
raise Exception(content)
def test_graph_corrplot(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if "travis" in sys.executable:
# it requires scipy which is not included in the requirements.txt
return
temp = get_temp_folder(__file__, "temp_corrplot")
clog = CustomLog(temp)
clog("fix")
fix_tkinter_issues_virtualenv(fLOG=fLOG)
clog("import")
from matplotlib import pyplot as plt
letters = "ABCDEFGHIJKLMNOP"[0:10]
df = pandas.DataFrame(
dict(((k, numpy.random.random(10) + ord(k) - 65) for k in letters)))
df = df.corr()
clog("figure")
subplot_kw = dict(aspect='equal', facecolor='white')
fig, ax = plt.subplots(1, 1, subplot_kw=subplot_kw)
clog("corrplot")
c = Corrplot(df)
clog("plot")
ax = c.plot(fig=fig, ax=ax)
clog("save")
fLOG("save")
img = os.path.join(temp, "corrplot.png")
fig.savefig(img)
fLOG("close")
clog("close")
if __name__ == "__main__":
plt.show()
plt.close('all')
clog("end")
fLOG("end")
assert os.path.exists(img)
def test_install_python(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = os.path.abspath(os.path.split(__file__)[0])
vers = "%d%d" % sys.version_info[:2]
temp = os.path.join(fold, "temp_python%s" % vers)
temp = get_temp_folder(__file__, "temp_python_base%s" % vers)
if sys.platform.startswith("win"):
clog = CustomLog(temp)
install_python(install=True,
temp_folder=temp,
fLOG=clog,
download_folder=temp + "_download")
pyt = os.path.join(temp, "python.exe")
pip = os.path.join(temp, "Scripts", "pip.exe")
if not os.path.exists(pyt):
raise FileNotFoundError(pyt)
if not os.path.exists(pip):
raise FileNotFoundError(pip)
def test_graph_corrplot(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_corrplot")
clog = CustomLog(temp)
clog("fix")
fix_tkinter_issues_virtualenv(fLOG=fLOG)
clog("import")
from matplotlib import pyplot as plt
letters = "ABCDEFGHIJKLMNOP"[0:10]
df = pandas.DataFrame(
dict(
((k, numpy.random.random(10) + ord(k) - 65) for k in letters)))
df = df.corr()
clog("figure")
subplot_kw = dict(aspect='equal', facecolor='white')
fig, ax = plt.subplots(1, 1, subplot_kw=subplot_kw)
clog("corrplot")
c = Corrplot(df)
clog("plot")
for up in ['lower', 'upper', 'method', 'both']:
ax = c.plot(fig=fig, ax=ax, colorbar=up == 'lower')
clog("save")
fLOG("save")
img = os.path.join(temp, "corrplot.png")
fig.savefig(img)
fLOG("close")
clog("close")
if __name__ == "__main__":
plt.show()
plt.close('all')
clog("end")
fLOG("end")
self.assertExists(img)
def test_install_python_pyensae(self):
fLOG(__file__, self._testMethodName, OutputPrint=True)
vers = "%d%d" % sys.version_info[:2]
temp = get_temp_folder(__file__,
"temp_python%s_pyensae" % vers,
clean=False)
if sys.platform.startswith("win"):
clog = CustomLog(temp)
install_python(install=True,
temp_folder=temp,
fLOG=clog,
modules="pyensae",
custom=True,
latest=True,
download_folder=temp + "_download")
pyt = os.path.join(temp, "python.exe")
pip = os.path.join(temp, "Scripts", "pip.exe")
if not os.path.exists(pyt):
raise FileNotFoundError(pyt)
if not os.path.exists(pip):
raise FileNotFoundError(pip)
def test_random_position(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_random_position")
clog = CustomLog(temp)
clog("random_positions")
fLOG("random_positions")
positions = random_positions(24)
self.assertEqual(len(positions), 24)
clog("beginning")
fLOG("beginning")
fix_tkinter_issues_virtualenv()
clog("fix")
fLOG("fix")
import matplotlib.pyplot as plt
clog("plotting")
fLOG("plotting")
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 8))
clog("plotting positions")
fLOG("plotting positions")
plot_positions(positions, ax=ax)
self.assertTrue(ax is not None)
clog("savefig")
img = os.path.join(temp, "img.png")
fig.savefig(img)
self.assertTrue(os.path.exists(img))
clog("noshow")
if __name__ == "__main__":
fig.show()
clog("close")
plt.close('all')
clog("end")
fLOG("end")
def test_notebook_population_gerry_mandering(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fix_tkinter_issues_virtualenv()
from actuariat_python.automation.notebook_test_helper import ls_notebooks, execute_notebooks, clean_function_notebook
temp = get_temp_folder(__file__, "temp_gerry_mandering")
clog = CustomLog(temp)
keepnote = [
_ for _ in ls_notebooks("sessions")
if "election_carte_electorale_correction" in _
]
self.assertTrue(len(keepnote) > 0)
res = execute_notebooks(temp,
keepnote,
lambda i, n: "deviner" not in n,
fLOG=clog,
deepfLOG=clog,
detailed_log=clog,
clean_function=clean_function_notebook)
execute_notebook_list_finalize_ut(res,
fLOG=clog,
dump=actuariat_python)
def test_full_documentation_module_template_rst(self):
"""
This test might fail in sphinx-gallery due to a very long filename.
Please look into the following commit:
https://github.com/sdpython/sphinx-gallery/commit/
3ae9f13250cf25c75e1b17b2fade98b7a9940b0d.
"""
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
# travis fails due to the following:
# sitep = [_ for _ in site.getsitepackages() if "packages" in _]
# AttributeError: 'module' object has no attribute
# 'getsitepackages'
# It fails for python 2.7 (encoding issue).
return
temp = get_temp_folder(__file__,
"temp_full_documentation_module_template_rst",
clean=__name__ != "__main__")
clog = CustomLog(temp)
this_pyq = os.path.normpath(
os.path.abspath(
os.path.join(os.path.dirname(pyquickhelper.__file__), "..")))
class MyStream:
def __init__(self):
self.rows = []
def write(self, text):
clog("[warning*] {0} - '{1}'".format(len(self),
text.strip("\n\r ")))
self.rows.append(text)
def getvalue(self):
return "\n".join(self.rows)
def __len__(self):
return len(self.rows)
rem = os.path.join(temp, "python3_module_template-master", "_doc",
"sphinxdoc", "build")
if os.path.exists(rem):
remove_folder(rem)
url = "https://github.com/sdpython/python3_module_template/archive/master.zip"
fLOG("[ut] download", url)
download(url, temp, fLOG=fLOG, flatten=False)
self.assertTrue(not os.path.exists(os.path.join(temp, "src")))
root = os.path.join(temp, "python3_module_template-master")
with sys_path_append(os.path.join(root, "src")):
# Checks that the unit test might fails.
coucou = os.path.join(
temp, "python3_module_template-master", "_doc", "sphinxdoc",
"source", "gallery",
"python3_module_template.subproject2.exclude_from_code_coverage.NotImplementedClass.__init__.examples"
)
if not os.path.exists(coucou):
fLOG("[ut] creating file '{0}'".format(coucou))
clog("[ut] creating file '{0}'".format(coucou))
dirname = os.path.dirname(coucou)
os.makedirs(dirname)
try:
# replicating what sphinx_gallery does
open(coucou, "w").close()
except Exception as e:
warnings.warn("Unable to create '{0}' due to '{1}'".format(
coucou, e))
else:
fLOG("[ut] file exists '{0}'".format(coucou))
clog("[ut] file exists '{0}'".format(coucou))
# documentation
fLOG("generate documentation", root)
var = "python3_module_template"
# we modify conf.py to let it find pyquickhelper
pyq = os.path.abspath(os.path.dirname(pyquickhelper.__file__))
confpy = os.path.join(root, "_doc", "sphinxdoc", "source",
"conf.py")
if not os.path.exists(confpy):
raise FileNotFoundError("Unable to find '{0}' and\n{1}".format(
confpy, os.listdir(temp)))
with open(confpy, "r", encoding="utf8") as f:
lines = f.read().split("\n")
fi = len(lines) - 1
for i, line in enumerate(lines):
if line.startswith("sys."):
fi = i
break
addition = "sys.path.append(r'{0}')".format(pyq)
lines[fi] = "{0}\n{1}".format(addition, lines[fi])
with open(confpy, "w", encoding="utf8") as f:
f.write("\n".join(lines))
# test
for i in range(0, 3):
fLOG("\n")
fLOG("\n")
fLOG("\n")
fLOG("#################################################", i)
fLOG("#################################################", i)
fLOG("#################################################", i)
# we add access to pyquickhelper
p = os.path.abspath(os.path.dirname(pyquickhelper.__file__))
p = os.path.join(p, 'src')
fLOG("PYTHONPATH=", p)
os.environ["PYTHONPATH"] = p
if p not in sys.path:
pos = len(sys.path)
sys.path.append(p)
else:
pos = -1
if "conf" in sys.modules:
del sys.modules["conf"]
fLOG(
"[test_full_documentation] **********************************"
)
fLOG("[test_full_documentation] begin",
list(roles._roles.keys()))
fLOG(
"[test_full_documentation] **********************************"
)
direct_call = i % 2 == 0
layout = ['rst']
logger1 = getLogger("docassert")
logger2 = getLogger("tocdelay")
log_capture_string = MyStream() # StringIO()
ch = logging.StreamHandler(log_capture_string)
ch.setLevel(logging.DEBUG)
logger1.logger.addHandler(ch)
logger2.logger.addHandler(ch)
with warnings.catch_warnings(record=True) as ww:
warnings.simplefilter("always")
# Change clog for print if it fails on circleli
generate_help_sphinx(
var,
module_name=var,
root=root,
layout=layout,
extra_ext=["tohelp"],
from_repo=False,
direct_call=direct_call,
parallel=1,
fLOG=clog,
extra_paths=[this_pyq],
nbformats=['html', 'ipynb', 'rst', 'slides'])
for w in ww:
if isinstance(w, dict):
rows = ["----"] + [
"{0}={1}".format(k, v)
for k, v in sorted(w.items())
]
sw = "\n".join(rows)
elif isinstance(w, warnings.WarningMessage):
rows = [
"-----",
str(type(w)), w.filename,
str(w.lineno),
str(w.message)
]
sw = "\n".join(rows)
else:
sw = str(w)
if "WARNING:" in sw and "ERROR/" in sw:
raise Exception(
"A warning is not expected:\n{0}".format(sw))
fLOG(
"[test_full_documentation] **********************************"
)
fLOG("[test_full_documentation] END")
fLOG(
"[test_full_documentation] **********************************"
)
lines = log_capture_string.getvalue().split("\n")
for line in lines:
if not line.strip():
continue
if "[docassert]" in line:
raise Exception(line)
if "[tocdelay]" in line:
fLOG(" ", line)
if '[tocdelay] ERROR' in line:
raise Exception(line)
# we clean
if "pyquickhelper" in sys.modules:
del sys.modules["pyquickhelper"]
os.environ["PYTHONPATH"] = ""
if pos >= 0:
del sys.path[pos]
# blog index
blog = os.path.join(root, "_doc", "sphinxdoc", "build", "rst",
"blog", "blogindex.rst")
with open(blog, "r", encoding="utf-8") as f:
content = f.read()
self.assertIn("2015", content)
self.assertIn('<2016/2016-06-11_blogpost_with_label>', content)
spl = content.split("2016-06")
if len(spl) <= 2:
raise Exception("Two expected:\n" + content)
# checkings
files = [
os.path.join(root, "_doc", "sphinxdoc", "build", "rst",
"index.rst"),
os.path.join(root, "_doc", "sphinxdoc", "build", "rst",
"all_notebooks.rst"),
]
for f in files:
if not os.path.exists(f):
raise FileNotFoundError(
"Not found '{0}'\n---\n{1}".format(
f, "\n".join(lines)))
self.assertTrue(not os.path.exists(os.path.join(temp, "_doc")))
rss = os.path.join(root, "_doc", "sphinxdoc", "source", "blog",
"rss.xml")
with open(rss, "r", encoding="utf8") as f:
content_rss = f.read()
self.assertTrue("__BLOG_ROOT__" not in content_rss)
# this should be replaced when uploading the stream onto the website
# the website is unknown when producing the documentation
# it should be resolved when uploading (the documentation could be
# uploaded at different places)
# checks some links were processed
fhtml = os.path.join(temp, "python3_module_template-master",
"_doc", "sphinxdoc", "source",
"all_notebooks.rst")
with open(fhtml, "r", encoding="utf8") as f:
content = f.read()
self.assertTrue('notebooks/custom_notebooks' in content)
# checks slideshow was added
fhtml = os.path.join(temp, "python3_module_template-master",
"build", "notebooks", "bslides",
"custom_notebooks.ipynb")
with open(fhtml, "r", encoding="utf8") as f:
content = f.read()
self.assertTrue('"slide"' in content)
# reveal.js + images
rev = [
os.path.join(root, "_doc", "sphinxdoc", "source",
"phdoc_static", "reveal.js")
]
for r in rev:
if not os.path.exists(r):
raise FileNotFoundError(r)
def test_run_notebook_im2(self):
"""
If the test does not end, it is probably due to PyQt4 needed by ete3.
Try ``import PyQt4.QtCore``.
Try to remove PyQt5, uninstall PyQt4, reinstall it from
http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyqt4 (Windows).
"""
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_virtual_environment() and sys.platform.startswith("win"):
pp = os.environ.get('PYTHONPATH', '')
if "SECONDTRY" in pp:
raise Exception(
"Not working**EXE\n{0}\n**PP\n{1}\n****".format(
sys.executable, pp))
# We need to run this file with the main python.
# Otherwise it fails for tables: DLL load failed.
import numpy
rootn = os.path.normpath(
os.path.join(os.path.dirname(numpy.__file__), "..", ".."))
exe = os.path.normpath(os.path.join(rootn, "..", "python.exe"))
cmd = '"{0}" -u "{1}"'.format(exe, os.path.abspath(__file__))
import jyquickhelper
import pyquickhelper
add = ["SECONDTRY"]
for mod in [pyquickhelper, jyquickhelper]:
add.append(
os.path.normpath(
os.path.join(os.path.dirname(mod.__file__), "..")))
fLOG("set PYTHONPATH={0}".format(";".join(add)))
os.environ['PYTHONPATH'] = ";".join(add)
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
if len(err) > 0:
lines = err.split("\n")
lines = [
_.lower().strip() for _ in lines
if len(_) > 0 and _[0] not in (" ", "-", ".", "-")
]
lines = [
_ for _ in lines if "warning" not in _
and not _.startswith("ran ") and _ != "ok"
]
if len(lines) > 0:
raise Exception(
"--CMD:\n{0}\n--OUT:\n{1}\n--ERR\n{2}\n--ERR2\n{3}\n--PP\n{4}"
.format(cmd, out, err, "\n".join(lines), pp))
return
temp = get_temp_folder(__file__, "temp_run_notebooks_im2")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(
f)[-1] == ".ipynb" and "im_" in f and "ete" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# file to copy
for cop in [
"green_tripdata_2015-12_sample.csv",
"NYPD_Motor_Vehicle_Collisions_sample.csv",
"NYPD_Motor_Vehicle_Collisions_small.csv"
]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
os.environ["DISPLAY"] = ":0"
clog = CustomLog(temp)
# run the notebooks
import jupytalk
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths,
detailed_log=clog)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)
def test_build_dynamic_trie_mks_min(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
data = os.path.join(os.path.abspath(
os.path.dirname(__file__)), "data", "sample20000.txt")
with open(data, "r", encoding="utf-8") as f:
lines = [_.strip("\n\r\t ") for _ in f.readlines()]
queries = [(None, _) for _ in lines]
temp = get_temp_folder(__file__, "temp_build_dynamic_trie_mks_min")
clog = CustomLog(temp)
clog("build trie")
trie = CompletionTrieNode.build(queries)
fLOG(len(queries), len(set(_[1] for _ in queries)),
len(list(trie.leaves())), len(set(trie.leaves())))
self.assertTrue("Cannes 2005" in set(_[1] for _ in queries))
self.assertTrue("Cannes 2005" in set(_.value for _ in trie.leaves()))
clog("precompute")
trie.precompute_stat()
clog("update")
trie.update_stat_dynamic()
clog("loop")
fLOG("loop")
for i, q in enumerate(queries):
if i % 1000 == 0:
clog(i)
fLOG(i)
leave = trie.find(q[1])
if leave.stat is None:
raise Exception("None for {0}".format(leave))
self.assertTrue(hasattr(leave, "stat"))
self.assertTrue(hasattr(leave.stat, "mks0"))
self.assertTrue(hasattr(leave.stat, "mks1"))
sug = leave.all_mks_completions()
nb_ = [(a.value, len([s.value for _, s in b if s.value == q[1]]))
for a, b in sug]
nbf_ = [(a.value, len(b)) for a, b in sug]
nb = sum(_[1] for _ in nb_)
mnb = max(_[1] for _ in nbf_)
if nb == 0 and len(q[1]) > 10:
info = "nb={0} mnb={2} q='{1}'".format(nb, q[1], mnb)
st = leave.stat.str_mks()
text = leave.str_all_completions()
text2 = leave.str_all_completions(use_precompute=False)
raise Exception(
"{4}\n---\nleave='{0}'\n{1}\n---\n{2}\n---\n{3}".format(leave.value, st, text, text2, info))
mk1 = trie.min_keystroke0(leave.value)
try:
mk = trie.min_dynamic_keystroke(leave.value)
mk2 = trie.min_dynamic_keystroke2(leave.value)
except Exception as e:
raise Exception(
"{0}-{1}-{2}-{3}".format(id(trie), id(leave), str(leave), leave.leave)) from e
if mk[0] > mk1[0]:
st = leave.stat.str_mks()
text = leave.str_all_completions()
text2 = leave.str_all_completions(use_precompute=False)
raise Exception("weird {0} > {1} -- leave='{2}'\n{3}\n---\n{4}\n---\n{5}".format(
mk, mk1, leave.value, st, text, text2))
if mk2[0] < mk[0]:
st = leave.stat.str_mks()
text = leave.str_all_completions()
text2 = leave.str_all_completions(use_precompute=False)
raise Exception("weird {0} > {1} -- leave='{2}'\n{3}\n---\n{4}\n---\n{5}".format(
mk, mk2, leave.value, st, text, text2))
clog("end")
fLOG("end")