def test_full_documentation_module_template_rst(self):
"""
This test might fail in sphinx-gallery due to a very long filename.
Please look into the following commit:
https://github.com/sdpython/sphinx-gallery/commit/
3ae9f13250cf25c75e1b17b2fade98b7a9940b0d.
"""
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor() in ('travis', 'appveyor'):
# travis fails due to the following:
# sitep = [_ for _ in site.getsitepackages() if "packages" in _]
# AttributeError: 'module' object has no attribute
# 'getsitepackages'
# It fails for python 2.7 (encoding issue).
return
temp = get_temp_folder(__file__,
"temp_full_documentation_module_template_rst",
clean=__name__ != "__main__")
clog = CustomLog(temp)
this_pyq = os.path.normpath(
os.path.abspath(
os.path.join(os.path.dirname(pyquickhelper.__file__), "..")))
class MyStream:
def __init__(self):
self.rows = []
def write(self, text):
clog("[warning*] {0} - '{1}'".format(len(self),
text.strip("\n\r ")))
self.rows.append(text)
def getvalue(self):
return "\n".join(self.rows)
def __len__(self):
return len(self.rows)
rem = os.path.join(temp, "python3_module_template-master", "_doc",
"sphinxdoc", "build")
if os.path.exists(rem):
remove_folder(rem)
url = "https://github.com/sdpython/python3_module_template/archive/master.zip"
fLOG("[ut] download", url)
download(url, temp, fLOG=fLOG, flatten=False)
self.assertTrue(not os.path.exists(os.path.join(temp, "src")))
root = os.path.join(temp, "python3_module_template-master")
with sys_path_append(os.path.join(root, "src")):
# Checks that the unit test might fails.
coucou = os.path.join(
temp, "python3_module_template-master", "_doc", "sphinxdoc",
"source", "gallery",
"python3_module_template.subproject2.exclude_from_code_coverage.NotImplementedClass.__init__.examples"
)
if not os.path.exists(coucou):
fLOG("[ut] creating file '{0}'".format(coucou))
clog("[ut] creating file '{0}'".format(coucou))
dirname = os.path.dirname(coucou)
os.makedirs(dirname)
try:
# replicating what sphinx_gallery does
open(coucou, "w").close()
except Exception as e:
warnings.warn("Unable to create '{0}' due to '{1}'".format(
coucou, e))
else:
fLOG("[ut] file exists '{0}'".format(coucou))
clog("[ut] file exists '{0}'".format(coucou))
# documentation
fLOG("generate documentation", root)
var = "python3_module_template"
# we modify conf.py to let it find pyquickhelper
pyq = os.path.abspath(os.path.dirname(pyquickhelper.__file__))
confpy = os.path.join(root, "_doc", "sphinxdoc", "source",
"conf.py")
if not os.path.exists(confpy):
raise FileNotFoundError("Unable to find '{0}' and\n{1}".format(
confpy, os.listdir(temp)))
with open(confpy, "r", encoding="utf8") as f:
lines = f.read().split("\n")
fi = len(lines) - 1
for i, line in enumerate(lines):
if line.startswith("sys."):
fi = i
break
addition = "sys.path.append(r'{0}')".format(pyq)
lines[fi] = "{0}\n{1}".format(addition, lines[fi])
with open(confpy, "w", encoding="utf8") as f:
f.write("\n".join(lines))
# test
for i in range(0, 3):
fLOG("\n")
fLOG("\n")
fLOG("\n")
fLOG("#################################################", i)
fLOG("#################################################", i)
fLOG("#################################################", i)
# we add access to pyquickhelper
p = os.path.abspath(os.path.dirname(pyquickhelper.__file__))
p = os.path.join(p, 'src')
fLOG("PYTHONPATH=", p)
os.environ["PYTHONPATH"] = p
if p not in sys.path:
pos = len(sys.path)
sys.path.append(p)
else:
pos = -1
if "conf" in sys.modules:
del sys.modules["conf"]
fLOG(
"[test_full_documentation] **********************************"
)
fLOG("[test_full_documentation] begin",
list(roles._roles.keys()))
fLOG(
"[test_full_documentation] **********************************"
)
direct_call = i % 2 == 0
layout = ['rst']
logger1 = getLogger("docassert")
logger2 = getLogger("tocdelay")
log_capture_string = MyStream() # StringIO()
ch = logging.StreamHandler(log_capture_string)
ch.setLevel(logging.DEBUG)
logger1.logger.addHandler(ch)
logger2.logger.addHandler(ch)
with warnings.catch_warnings(record=True) as ww:
warnings.simplefilter("always")
# Change clog for print if it fails on circleli
generate_help_sphinx(
var,
module_name=var,
root=root,
layout=layout,
extra_ext=["tohelp"],
from_repo=False,
direct_call=direct_call,
parallel=1,
fLOG=clog,
extra_paths=[this_pyq],
nbformats=['html', 'ipynb', 'rst', 'slides'])
for w in ww:
if isinstance(w, dict):
rows = ["----"] + [
"{0}={1}".format(k, v)
for k, v in sorted(w.items())
]
sw = "\n".join(rows)
elif isinstance(w, warnings.WarningMessage):
rows = [
"-----",
str(type(w)), w.filename,
str(w.lineno),
str(w.message)
]
sw = "\n".join(rows)
else:
sw = str(w)
if "WARNING:" in sw and "ERROR/" in sw:
raise Exception(
"A warning is not expected:\n{0}".format(sw))
fLOG(
"[test_full_documentation] **********************************"
)
fLOG("[test_full_documentation] END")
fLOG(
"[test_full_documentation] **********************************"
)
lines = log_capture_string.getvalue().split("\n")
for line in lines:
if not line.strip():
continue
if "[docassert]" in line:
raise Exception(line)
if "[tocdelay]" in line:
fLOG(" ", line)
if '[tocdelay] ERROR' in line:
raise Exception(line)
# we clean
if "pyquickhelper" in sys.modules:
del sys.modules["pyquickhelper"]
os.environ["PYTHONPATH"] = ""
if pos >= 0:
del sys.path[pos]
# blog index
blog = os.path.join(root, "_doc", "sphinxdoc", "build", "rst",
"blog", "blogindex.rst")
with open(blog, "r", encoding="utf-8") as f:
content = f.read()
self.assertIn("2015", content)
self.assertIn('<2016/2016-06-11_blogpost_with_label>', content)
spl = content.split("2016-06")
if len(spl) <= 2:
raise Exception("Two expected:\n" + content)
# checkings
files = [
os.path.join(root, "_doc", "sphinxdoc", "build", "rst",
"index.rst"),
os.path.join(root, "_doc", "sphinxdoc", "build", "rst",
"all_notebooks.rst"),
]
for f in files:
if not os.path.exists(f):
raise FileNotFoundError(
"Not found '{0}'\n---\n{1}".format(
f, "\n".join(lines)))
self.assertTrue(not os.path.exists(os.path.join(temp, "_doc")))
rss = os.path.join(root, "_doc", "sphinxdoc", "source", "blog",
"rss.xml")
with open(rss, "r", encoding="utf8") as f:
content_rss = f.read()
self.assertTrue("__BLOG_ROOT__" not in content_rss)
# this should be replaced when uploading the stream onto the website
# the website is unknown when producing the documentation
# it should be resolved when uploading (the documentation could be
# uploaded at different places)
# checks some links were processed
fhtml = os.path.join(temp, "python3_module_template-master",
"_doc", "sphinxdoc", "source",
"all_notebooks.rst")
with open(fhtml, "r", encoding="utf8") as f:
content = f.read()
self.assertTrue('notebooks/custom_notebooks' in content)
# checks slideshow was added
fhtml = os.path.join(temp, "python3_module_template-master",
"build", "notebooks", "bslides",
"custom_notebooks.ipynb")
with open(fhtml, "r", encoding="utf8") as f:
content = f.read()
self.assertTrue('"slide"' in content)
# reveal.js + images
rev = [
os.path.join(root, "_doc", "sphinxdoc", "source",
"phdoc_static", "reveal.js")
]
for r in rev:
if not os.path.exists(r):
raise FileNotFoundError(r)
def test_build_dynamic_trie_mks_min(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
data = os.path.join(os.path.abspath(
os.path.dirname(__file__)), "data", "sample20000.txt")
with open(data, "r", encoding="utf-8") as f:
lines = [_.strip("\n\r\t ") for _ in f.readlines()]
queries = [(None, _) for _ in lines]
temp = get_temp_folder(__file__, "temp_build_dynamic_trie_mks_min")
clog = CustomLog(temp)
clog("build trie")
trie = CompletionTrieNode.build(queries)
fLOG(len(queries), len(set(_[1] for _ in queries)),
len(list(trie.leaves())), len(set(trie.leaves())))
self.assertTrue("Cannes 2005" in set(_[1] for _ in queries))
self.assertTrue("Cannes 2005" in set(_.value for _ in trie.leaves()))
clog("precompute")
trie.precompute_stat()
clog("update")
trie.update_stat_dynamic()
clog("loop")
fLOG("loop")
for i, q in enumerate(queries):
if i % 1000 == 0:
clog(i)
fLOG(i)
leave = trie.find(q[1])
if leave.stat is None:
raise Exception("None for {0}".format(leave))
self.assertTrue(hasattr(leave, "stat"))
self.assertTrue(hasattr(leave.stat, "mks0"))
self.assertTrue(hasattr(leave.stat, "mks1"))
sug = leave.all_mks_completions()
nb_ = [(a.value, len([s.value for _, s in b if s.value == q[1]]))
for a, b in sug]
nbf_ = [(a.value, len(b)) for a, b in sug]
nb = sum(_[1] for _ in nb_)
mnb = max(_[1] for _ in nbf_)
if nb == 0 and len(q[1]) > 10:
info = "nb={0} mnb={2} q='{1}'".format(nb, q[1], mnb)
st = leave.stat.str_mks()
text = leave.str_all_completions()
text2 = leave.str_all_completions(use_precompute=False)
raise Exception(
"{4}\n---\nleave='{0}'\n{1}\n---\n{2}\n---\n{3}".format(leave.value, st, text, text2, info))
mk1 = trie.min_keystroke0(leave.value)
try:
mk = trie.min_dynamic_keystroke(leave.value)
mk2 = trie.min_dynamic_keystroke2(leave.value)
except Exception as e:
raise Exception(
"{0}-{1}-{2}-{3}".format(id(trie), id(leave), str(leave), leave.leave)) from e
if mk[0] > mk1[0]:
st = leave.stat.str_mks()
text = leave.str_all_completions()
text2 = leave.str_all_completions(use_precompute=False)
raise Exception("weird {0} > {1} -- leave='{2}'\n{3}\n---\n{4}\n---\n{5}".format(
mk, mk1, leave.value, st, text, text2))
if mk2[0] < mk[0]:
st = leave.stat.str_mks()
text = leave.str_all_completions()
text2 = leave.str_all_completions(use_precompute=False)
raise Exception("weird {0} > {1} -- leave='{2}'\n{3}\n---\n{4}\n---\n{5}".format(
mk, mk2, leave.value, st, text, text2))
clog("end")
fLOG("end")
def test_run_notebook_im2(self):
"""
If the test does not end, it is probably due to PyQt4 needed by ete3.
Try ``import PyQt4.QtCore``.
Try to remove PyQt5, uninstall PyQt4, reinstall it from
http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyqt4 (Windows).
"""
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_virtual_environment() and sys.platform.startswith("win"):
pp = os.environ.get('PYTHONPATH', '')
if "SECONDTRY" in pp:
raise Exception(
"Not working**EXE\n{0}\n**PP\n{1}\n****".format(
sys.executable, pp))
# We need to run this file with the main python.
# Otherwise it fails for tables: DLL load failed.
import numpy
rootn = os.path.normpath(
os.path.join(os.path.dirname(numpy.__file__), "..", ".."))
exe = os.path.normpath(os.path.join(rootn, "..", "python.exe"))
cmd = '"{0}" -u "{1}"'.format(exe, os.path.abspath(__file__))
import jyquickhelper
import pyquickhelper
add = ["SECONDTRY"]
for mod in [pyquickhelper, jyquickhelper]:
add.append(
os.path.normpath(
os.path.join(os.path.dirname(mod.__file__), "..")))
fLOG("set PYTHONPATH={0}".format(";".join(add)))
os.environ['PYTHONPATH'] = ";".join(add)
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
if len(err) > 0:
lines = err.split("\n")
lines = [
_.lower().strip() for _ in lines
if len(_) > 0 and _[0] not in (" ", "-", ".", "-")
]
lines = [
_ for _ in lines if "warning" not in _
and not _.startswith("ran ") and _ != "ok"
]
if len(lines) > 0:
raise Exception(
"--CMD:\n{0}\n--OUT:\n{1}\n--ERR\n{2}\n--ERR2\n{3}\n--PP\n{4}"
.format(cmd, out, err, "\n".join(lines), pp))
return
temp = get_temp_folder(__file__, "temp_run_notebooks_im2")
# selection of notebooks
fnb = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "_doc", "notebooks", "2016", "pydata"))
keepnote = []
for f in os.listdir(fnb):
if os.path.splitext(
f)[-1] == ".ipynb" and "im_" in f and "ete" in f:
keepnote.append(os.path.join(fnb, f))
# function to tell that a can be run
def valid(cell):
return True
# file to copy
for cop in [
"green_tripdata_2015-12_sample.csv",
"NYPD_Motor_Vehicle_Collisions_sample.csv",
"NYPD_Motor_Vehicle_Collisions_small.csv"
]:
fsrc = os.path.join(fnb, cop)
if os.path.exists(fsrc):
dest = temp
shutil.copy(fsrc, dest)
# additionnal path to add
addpaths = [
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "pyquickhelper", "src")),
os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
"..", "..", "jyquickhelper", "src"))
]
os.environ["DISPLAY"] = ":0"
clog = CustomLog(temp)
# run the notebooks
import jupytalk
res = execute_notebook_list(temp,
keepnote,
fLOG=fLOG,
valid=valid,
additional_path=addpaths,
detailed_log=clog)
execute_notebook_list_finalize_ut(res, fLOG=fLOG, dump=jupytalk)