def test_notebook_runner_operation(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# written in Python 3
return
temp = get_temp_folder(__file__, "temp_notebook_operation")
nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb")
nbfile2 = os.path.join(
temp, "..", "data", "td2a_cenonce_session_4B.ipynb")
nb1 = read_nb(nbfile, kernel=False)
n1 = len(nb1)
nb2 = read_nb(nbfile2, kernel=False)
n2 = len(nb2)
add = nb1 + nb2
nb1.merge_notebook([nb2])
n3a = len(add)
n3 = len(nb1)
if n1 + n2 != n3:
raise Exception("{0} + {1} != {2}".format(n1, n2, n3))
if n3a != n3:
raise Exception("{0} != {1}".format(n3a, n3))
fLOG(n1, n2, n3, n3a)
outfile = os.path.join(temp, "merge_nb.ipynb")
nb1.to_json(outfile)
assert os.path.exists(outfile)
outfile = os.path.join(temp, "merge_nb_add.ipynb")
add.to_json(outfile)
assert os.path.exists(outfile)
def test_convert_slides_api(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
return
path = os.path.abspath(os.path.split(__file__)[0])
fold = os.path.normpath(
os.path.join(
path,
"..",
"..",
"_doc",
"notebooks"))
nb = os.path.join(fold, "example_pyquickhelper.ipynb")
assert os.path.exists(nb)
nbr = read_nb(nb, kernel=False)
temp = get_temp_folder(__file__, "temp_nb_api")
outfile = os.path.join(temp, "out_nb_slides.slides.html")
res = nb2slides(nbr, outfile)
assert len(res) > 1
for r in res:
assert os.path.exists(r)
outfile = os.path.join(temp, "out_nb_slides.html")
res = nb2html(nbr, outfile)
assert len(res) == 1
for r in res:
assert os.path.exists(r)
def test_regex(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
exp = re.compile(r"(.{3}[\\]\$)")
s = ": [ ['$',"
fLOG(s)
r = exp.finditer(s)
nb = 0
for _ in r:
fLOG("1", _.groups())
nb += 1
nb1 = nb
s = r"\def\PYZdl{\char`\$}"
fLOG(s)
r = exp.finditer(s)
nb = 0
for _ in r:
fLOG("2", _.groups())
nb += 1
assert nb1 == 0
assert nb > 0
def test_pandas_rst_right(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
df = pandas.DataFrame([{"A": "x", "AA": "xx", "AAA": "xxx"},
{"AA": "xxxxxxx", "AAA": "xxx"}])
rst = df2rst(df, align="r")
exp = """+-----+---------+-----+
| A | AA | AAA |
+=====+=========+=====+
| x | xx | xxx |
+-----+---------+-----+
| | xxxxxxx | xxx |
+-----+---------+-----+
""".replace(" ", "")
self.assertEqual(rst, exp)
rst = df2rst(df, align="c")
exp = """+-----+---------+-----+
| A | AA | AAA |
+=====+=========+=====+
| x | xx | xxx |
+-----+---------+-----+
| | xxxxxxx | xxx |
+-----+---------+-----+
""".replace(" ", "")
self.assertEqual(rst, exp)
def test_encrypt_decrypt_file(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# the module returns the following error
# ENCODING ERROR WITH Python 2.7, will not fix it
return
else:
password = "******" * 2
temp = get_temp_folder(__file__, "temp_encrypt_file")
dest = os.path.join(temp, "__file__.enc")
sys.argv = ["", __file__, dest, password]
encrypt_file(fLOG=fLOG)
dest2 = os.path.join(temp, "__file__.py")
sys.argv = ["", dest, dest2, password]
decrypt_file(fLOG=fLOG)
with open(__file__, "rb") as f:
c1 = f.read()
with open(dest2, "rb") as f:
c2 = f.read()
self.assertEqual(c1, c2)
fLOG("end")
def test_jinja2_exception_not_here(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
tmpl = """
{% for i in range(0, len(lll)) %}
print({{ll[i]}})
{% endfor %}
"""
exp = """
print(0)
print(2)
"""
try:
res = apply_template(tmpl, dict(
lll=[0, 2], len=len), engine="jinja2")
assert False
except CustomTemplateException as e:
if "Some parameters are missing or mispelled" not in str(e):
raise Exception(str(e))
return
assert False
fLOG(res)
self.assertEqual(res.replace(" ", "").replace("\n", ""),
exp.replace(" ", "").replace("\n", ""))
def test_notebook_iter(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# written in Python 3
return
temp = get_temp_folder(__file__, "temp_notebook_add_slides_metadata")
nbfile = os.path.join(
temp, "..", "data", "having_a_form_in_a_notebook.ipynb")
assert os.path.exists(nbfile)
nb = read_nb(nbfile, kernel=False)
for i, cell in enumerate(nb.iter_cells()):
ls = nb.cell_height(cell)
#fLOG(i,"-",nb.cell_type(cell), l)
#if l > 400: fLOG(cell)
assert ls > 0
return
new_tags = nb.add_tag_slide()
for k, v in sorted(new_tags.items()):
a, b, c = v
fLOG(k, a, b)
assert len(new_tags) > 0
def test_encryption_stream_fernet_chunck_size(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
return
infile = StreamIO(bytes(list(i % 255 for i in range(0, 10000))))
outst = StreamIO()
r = encrypt_stream("key0" * 8, infile, outst,
algo="fernet", chunksize=256)
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream("key0" * 8, enc, outdec,
algo="fernet", chunksize=256)
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
try:
r3 = decrypt_stream("key1" * 8, enc2, outdec2,
algo="fernet", chunksize=256)
except:
return
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_walk(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
this = os.path.abspath(os.path.dirname(__file__))
ut = os.path.join(this, "..")
fLOG(this)
nb = 0
for root, dirs, files in walk(ut):
nb += 1
if sys.version_info[0] >= 3:
nb2 = 0
for root, dirs, files in walk(ut, neg_filter="*ut_loghelper*"):
nb2 += 1
assert nb2 > 0
assert nb2 < nb
else:
nb2 = 1e6
def filter(d):
return "loghelper" in d or "helpgen" in d
nb3 = 0
for root, dirs, files in walk(ut, neg_filter=filter):
nb3 += 1
assert nb3 > 0
assert nb3 < nb2
def test_encryption_stream(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
return
try:
import Cryptodome as skip___
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
infile = StreamIO(bytes([0, 1, 2, 3, 4]))
outst = StreamIO()
r = encrypt_stream(b"key0" * 4, infile, outst)
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream(b"key0" * 4, enc, outdec)
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
r3 = decrypt_stream(b"key1" * 4, enc2, outdec2)
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_stream_fernet(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
return
infile = StreamIO(bytes([0, 1, 2, 3, 4]))
outst = StreamIO()
r = encrypt_stream("key0" * 8, infile, outst, algo="fernet")
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream("key0" * 8, enc, outdec, algo="fernet")
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
try:
r3 = decrypt_stream("key1" * 8, enc2, outdec2, algo="fernet")
except:
return
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_bytes(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
return
try:
import Cryptodome as skip__
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
infile = bytes([0, 1, 2, 3, 4])
r = encrypt_stream(b"key0" * 4, infile)
assert r is not None
r2 = decrypt_stream(b"key0" * 4, r)
assert r2 is not None
self.assertEqual(infile, r2)
r3 = decrypt_stream(b"key1" * 4, r)
assert r3 is not None
self.assertNotEqual(infile, r3)
def zz_st_jenkins_job_verif(self, platform):
engines_default = dict(anaconda2="c:\\Anaconda",
anaconda3="c:\\Anaconda3",
py35="c:\\Python35_x64",
default="c:\\Python34_x64",
winpython="c:\\APythonENSAE\\python")
srv = JenkinsExt(
"http://localhost:8080/", "user", "password",
mock=True, fLOG=fLOG, engines=engines_default, platform=platform)
if platform == "win":
job = "standalone [conda_update] [anaconda3]"
cmd = srv.get_cmd_standalone(job)
assert "Anaconda3" in cmd
if platform == "win":
job = "pyrsslocal [py35] <-- pyquickhelper, pyensae"
cmd = "\n".join(srv.get_jenkins_script(job))
assert "Python34" not in cmd
else:
try:
job = "pyrsslocal [py35] <-- pyquickhelper, pyensae"
except NotImplementedError:
fLOG("OK FOR", platform)
def test_notebook_runner_empty(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# written in Python 3
return
temp = get_temp_folder(__file__, "temp_notebook_empty")
nbfile = os.path.join(
temp,
"..",
"data",
"td2a_cenonce_session_4B.ipynb")
assert os.path.exists(nbfile)
addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
assert os.path.exists(addpath)
kernel_name = None if is_travis_or_appveyor() is not None else install_python_kernel_for_unittest(
"pyquickhelper")
outfile = os.path.join(temp, "out_notebook.ipynb")
assert not os.path.exists(outfile)
out = run_notebook(nbfile, working_dir=temp, outfilename=outfile,
additional_path=[addpath], fLOG=fLOG,
kernel_name=kernel_name)
fLOG(out)
assert os.path.exists(outfile)
assert "No module named 'pyquickhelper'" not in out
def test_bug_exe(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
command = "dir" if sys.platform.startswith("win32") else "ls"
yml = """
language: python
python:
- {{Python35}}
before:
- %s
after_script:
- %s {{PLATFORM}}
script:
- %s
""" % (command, command, command)
context = dict(Python34="fake", Python35=os.path.dirname(sys.executable),
Python27=None, Anaconda3=None, Anaconda2=None,
WinPython35=None, project_name="pyquickhelper",
root_path="ROOT", PLATFORM="win")
obj, name = load_yaml(yml, context=context)
assert name is not None
try:
res = list(enumerate_convert_yaml_into_instructions(
obj, variables=context))
assert False
assert res
except ValueError as e:
assert "'before'" in str(e)
def test_read_content_ufs(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
url = "https://raw.githubusercontent.com/sdpython/pyquickhelper/master/src/pyquickhelper/ipythonhelper/magic_parser.py"
content = read_content_ufs(url, encoding="utf8")
assert "MagicCommandParser" in content
assert isinstance(content, str # unicode#
)
typstr = str # unicode#
file = typstr(__file__)
file_, ext = os.path.splitext(file)
if ext != ".py":
file = file_ + ".py"
content = read_content_ufs(file, encoding="utf8")
assert "TestDownloadContent" in content
content2 = read_content_ufs(content, encoding="utf8")
self.assertEqual(content2, content)
st = StringIO(content)
content2 = read_content_ufs(st, encoding="utf8")
self.assertEqual(content2, content)
if sys.version_info[0] > 2:
by = BytesIO(content.encode("utf8"))
content2 = read_content_ufs(by, encoding="utf8")
self.assertEqual(content2, content)
def test_notebook_gallery(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
return
temp = get_temp_folder(__file__, "temp_gallery")
fold = os.path.normpath(os.path.join(
temp, "..", "data_gallery", "notebooks"))
assert os.path.exists(fold)
file = os.path.join(temp, "all_notebooks.rst")
build_notebooks_gallery(fold, file, fLOG=fLOG)
if not os.path.exists(file):
raise FileNotFoundError(file)
with open(file, "r", encoding="utf8") as f:
text = f.read()
if "GitHub/pyquickhelper" in text.replace("\\", "/"):
raise Exception(text)
spl = text.split("notebooks/td2a_eco_competition_modeles_logistiques")
if len(spl) != 3:
raise Exception(text)
def test_url_more_recent(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
this = os.path.abspath(os.path.dirname(__file__))
name = "test_syncho.py"
temp = get_temp_folder(__file__, "temp_url_more_recent")
shutil.copy(os.path.join(this, name), temp)
dest = os.path.join(temp, name)
url = "https://github.com/sdpython/pyquickhelper/tree/master/_unittests/ut_loghelper/" + name
f1 = URL.urlopen(url)
r = _first_more_recent(f1, dest)
f1.close()
assert isinstance(r, bool)
url = "http://www.lemonde.fr/"
f1 = URL.urlopen(url)
r = _first_more_recent(f1, dest)
f1.close()
assert isinstance(r, bool)
fLOG(r)
def test_mako_exceptions_not_here(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
tmpl = """
% for i in range(0, len(lll)):
print(${ll[i]})
% endfor
"""
exp = """
print(0)
print(2)
"""
try:
res = apply_template(tmpl, dict(lll=[0, 2]))
assert False
except CustomTemplateException as e:
# fLOG(str(e))
assert "Undefined" in str(e)
return
assert False
fLOG(res)
self.assertEqual(res.replace(" ", "").replace("\n", ""),
exp.replace(" ", "").replace("\n", ""))
def test_clean_pep8(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_pep8_clean")
name = os.path.join(temp, "python_try.py")
with open(name, "w") as f:
f.write("""
import sys
import os
def f1 () :
#g
return [2,3]
""".replace(" ", ""))
r = remove_extra_spaces_and_pep8(name)
assert r > 0
with open(name, "r") as f:
content = f.read()
self.assertEqual(content.strip(), """
import sys
import os
def f1():
# g
return [2, 3]
""".replace(" ", "").strip())
r = remove_extra_spaces_and_pep8(name)
self.assertEqual(r, 0)
def test_build_script(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.platform.startswith("win") and sys.version_info[0] != 2:
sc = get_build_script("pyquickhelper")
lines = sc.split("\n")
for line in lines:
if "__" in line and _default_nofolder not in line:
raise Exception("issue with __ in:\n" +
line + "\nFULL\n" + sc)
scc = get_script_command(
"unittest", "pyquickhelper", requirements=[])
assert "setup.py" in scc
assert "__" not in scc
sccc = get_extra_script_command(
"local_pypi", "pyquickhelper", port=8067, requirements=[])
assert "python" in sccc
if "__" in sccc:
raise Exception(sccc)
else:
# not yet implemented for this platform
return
def test_version(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
url = "http://www.xavierdupre.fr/enseignement/complements/marathon.txt"
repeat = 0
while True:
try:
df = read_url(
url,
sep="\t",
names=[
"ville",
"annee",
"temps",
"secondes"])
break
except ConnectionResetError as e:
if repeat >= 2:
raise e
repeat += 1
assert len(df) > 0
assert len(df.columns) == 4
fLOG(df.head())
def test_jenkins_job_verif(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
this = os.path.abspath(os.path.dirname(__file__))
yml = os.path.abspath(os.path.join(
this, "..", "..", ".local.jenkins.win.yml"))
if not os.path.exists(yml):
yml = os.path.abspath(os.path.join(
this, "..", "..", "..", ".local.jenkins.win.yml"))
if not os.path.exists(yml):
raise FileNotFoundError(yml)
context = dict(Python34=None, Python35=os.path.dirname(sys.executable),
Python36=os.path.dirname(sys.executable),
Python27=None, Anaconda3=None, Anaconda2=None,
WinPython35=None, project_name="pyquickhelper",
root_path="ROOT")
obj, name = load_yaml(yml, context=context)
for k, v in obj.items():
fLOG(k, type(v), v)
assert "python" in obj
assert isinstance(obj["python"], list)
assert name is not None
def test_missing_funcwin_file_split(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
assert _clean_name_variable("s-6") == "s_6"
assert _get_format_zero_nb_integer(5006) == "%04d"
ioout = [io.StringIO(), io.StringIO()]
if sys.version_info[0] == 2:
warnings.warn("skip testing Python 2.7")
return
file_split(os.path.abspath(__file__), out=ioout, header=True)
ioout = [io.StringIO(), io.StringIO()]
s1 = ioout[0].getvalue()
s2 = ioout[0].getvalue()
self.assertEqual(s1[:5], s2[:5])
nb = file_split(os.path.abspath(__file__), out=ioout, header=False)
size = os.stat(os.path.abspath(__file__)).st_size
sa = 0
for l in ioout:
s = l.getvalue()
assert len(s) > 0
sa += len(s)
assert 0 < sa <= size
if sa not in (size, size - nb - 1, size - nb, size - 1):
raise Exception("{0} != {1}, nb={2}".format(sa, size, nb))
def test_encrypt_decrypt(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] == 2:
# the module returns the following error
# ENCODING ERROR WITH Python 2.7, will not fix it
return
else:
password = "******" * 2
temp = get_temp_folder(__file__, "temp_encrypt")
temp2 = get_temp_folder(__file__, "temp_encrypt2")
tempmm = get_temp_folder(__file__, "temp_encrypt_status")
cstatus = os.path.join(tempmm, "crypt_status.txt")
cmap = os.path.join(tempmm, "crypt_map.txt")
srcf = os.path.abspath(os.path.join(temp, ".."))
sys.argv = ["", srcf, temp, password,
"--status", cstatus,
"--map", cmap]
encrypt(fLOG=fLOG)
this = __file__
sys.argv = ["", temp, temp2, password]
decrypt(fLOG=fLOG)
with open(__file__, "rb") as f:
c1 = f.read()
with open(os.path.join(temp2, os.path.split(this)[-1]), "rb") as f:
c2 = f.read()
self.assertEqual(c1, c2)
fLOG("end")
def test_jconvert_sequence_into_batch_file_split2(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
self.zz_st_jconvert_sequence_into_batch_file_split2("win")
def test_pandas_tennis(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
days = ["lundi", "mardi", "mercredi",
"jeudi", "vendredi", "samedi", "dimanche"]
temp = get_temp_folder(__file__, "temp_pandas_tennis")
rows = []
dt = datetime.datetime.now()
for i in range(0, 360):
row = dict(date="%04d-%02d-%02d" % (dt.year, dt.month, dt.day),
jour=dt.weekday(),
journ=days[dt.weekday()])
dt += datetime.timedelta(1)
rows.append(row)
df = pandas.DataFrame(rows)
df["equipe"] = 0
df.ix[(df.jour == 2) | (df.jour == 4), "equipe"] = 1
df["equipe_sum"] = (df.equipe.cumsum() *
df["equipe"] + df["equipe"]) % 3
df["equipe_sum"] += df["equipe"]
fLOG(df.head(n=10))
df.to_excel(os.path.join(temp, "tennis.xlsx"), index=False)
def test_transfer_ftp_true(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if sys.version_info[0] < 3:
warnings.warn(
"No testing transfer FTP on Pyghon 2.7 (issue with str and bytes)")
return
import keyring
machine = os.environ.get(
"COMPUTERNAME", os.environ.get("HOSTNAME", "CI"))
try:
user = keyring.get_password("web", machine + "user")
pwd = keyring.get_password("web", machine + "pwd")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
raise Exception("user password is empty, machine='{0}', username='******'".format(
machine, os.environ.get("USERNAME", None)))
else:
return
web = TransferFTP("ftp.xavierdupre.fr", user, pwd, fLOG=fLOG)
r = web.ls(".")
fLOG(r)
assert isinstance(r, list)
def test_filename(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
pos = ["c:\\test",
"c:\\test.txt",
"..",
".txt",
"r.",
"r",
"mqldkfnqmodnsc/\\y"]
for p in pos:
if not is_file_string(p):
raise Exception(p)
neg = ["h\ng",
"r\tr",
"cd:ggd.h"]
for p in neg:
if is_file_string(p):
raise Exception(p)
def zz_st_jconvert_sequence_into_batch_file_split(self, platform):
this = os.path.abspath(os.path.dirname(__file__))
yml = os.path.abspath(os.path.join(
this, "data", "local.yml"))
if not os.path.exists(yml):
raise FileNotFoundError(yml)
context = dict(Python35="fake", Python36="C:\\Python35_x64",
project_name="pyquickhelper", root_path="ROOT")
obj, name = load_yaml(yml, context=context, platform=platform)
assert name is not None
res = list(enumerate_convert_yaml_into_instructions(
obj, add_environ=False))
convs = []
for r, v in res:
conv = convert_sequence_into_batch_file(
r, variables=v, platform=platform)
if not isinstance(conv, list):
raise TypeError(type(conv))
convs.append(conv)
assert len(res) > 0
self.assertEqual(len(convs), 2)
conv = convs[0]
assert conv
assert isinstance(conv, list)
fr = 0
for c in conv:
if "JENKINS_SPLIT" in c:
raise Exception(c)
if "pip freeze" in c:
fr += 1
fLOG("-------------------------------")
fLOG(c)
self.assertEqual(fr, 1)
