def test_notebook_runner_operation(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # written in Python 3 return temp = get_temp_folder(__file__, "temp_notebook_operation") nbfile = os.path.join(temp, "..", "data", "simple_example.ipynb") nbfile2 = os.path.join( temp, "..", "data", "td2a_cenonce_session_4B.ipynb") nb1 = read_nb(nbfile, kernel=False) n1 = len(nb1) nb2 = read_nb(nbfile2, kernel=False) n2 = len(nb2) add = nb1 + nb2 nb1.merge_notebook([nb2]) n3a = len(add) n3 = len(nb1) if n1 + n2 != n3: raise Exception("{0} + {1} != {2}".format(n1, n2, n3)) if n3a != n3: raise Exception("{0} != {1}".format(n3a, n3)) fLOG(n1, n2, n3, n3a) outfile = os.path.join(temp, "merge_nb.ipynb") nb1.to_json(outfile) assert os.path.exists(outfile) outfile = os.path.join(temp, "merge_nb_add.ipynb") add.to_json(outfile) assert os.path.exists(outfile)
def test_convert_slides_api(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: return path = os.path.abspath(os.path.split(__file__)[0]) fold = os.path.normpath( os.path.join( path, "..", "..", "_doc", "notebooks")) nb = os.path.join(fold, "example_pyquickhelper.ipynb") assert os.path.exists(nb) nbr = read_nb(nb, kernel=False) temp = get_temp_folder(__file__, "temp_nb_api") outfile = os.path.join(temp, "out_nb_slides.slides.html") res = nb2slides(nbr, outfile) assert len(res) > 1 for r in res: assert os.path.exists(r) outfile = os.path.join(temp, "out_nb_slides.html") res = nb2html(nbr, outfile) assert len(res) == 1 for r in res: assert os.path.exists(r)
def test_regex(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") exp = re.compile(r"(.{3}[\\]\$)") s = ": [ ['$'," fLOG(s) r = exp.finditer(s) nb = 0 for _ in r: fLOG("1", _.groups()) nb += 1 nb1 = nb s = r"\def\PYZdl{\char`\$}" fLOG(s) r = exp.finditer(s) nb = 0 for _ in r: fLOG("2", _.groups()) nb += 1 assert nb1 == 0 assert nb > 0
def test_pandas_rst_right(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") df = pandas.DataFrame([{"A": "x", "AA": "xx", "AAA": "xxx"}, {"AA": "xxxxxxx", "AAA": "xxx"}]) rst = df2rst(df, align="r") exp = """+-----+---------+-----+ | A | AA | AAA | +=====+=========+=====+ | x | xx | xxx | +-----+---------+-----+ | | xxxxxxx | xxx | +-----+---------+-----+ """.replace(" ", "") self.assertEqual(rst, exp) rst = df2rst(df, align="c") exp = """+-----+---------+-----+ | A | AA | AAA | +=====+=========+=====+ | x | xx | xxx | +-----+---------+-----+ | | xxxxxxx | xxx | +-----+---------+-----+ """.replace(" ", "") self.assertEqual(rst, exp)
def test_encrypt_decrypt_file(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # the module returns the following error # ENCODING ERROR WITH Python 2.7, will not fix it return else: password = "******" * 2 temp = get_temp_folder(__file__, "temp_encrypt_file") dest = os.path.join(temp, "__file__.enc") sys.argv = ["", __file__, dest, password] encrypt_file(fLOG=fLOG) dest2 = os.path.join(temp, "__file__.py") sys.argv = ["", dest, dest2, password] decrypt_file(fLOG=fLOG) with open(__file__, "rb") as f: c1 = f.read() with open(dest2, "rb") as f: c2 = f.read() self.assertEqual(c1, c2) fLOG("end")
def test_jinja2_exception_not_here(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") tmpl = """ {% for i in range(0, len(lll)) %} print({{ll[i]}}) {% endfor %} """ exp = """ print(0) print(2) """ try: res = apply_template(tmpl, dict( lll=[0, 2], len=len), engine="jinja2") assert False except CustomTemplateException as e: if "Some parameters are missing or mispelled" not in str(e): raise Exception(str(e)) return assert False fLOG(res) self.assertEqual(res.replace(" ", "").replace("\n", ""), exp.replace(" ", "").replace("\n", ""))
def test_notebook_iter(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # written in Python 3 return temp = get_temp_folder(__file__, "temp_notebook_add_slides_metadata") nbfile = os.path.join( temp, "..", "data", "having_a_form_in_a_notebook.ipynb") assert os.path.exists(nbfile) nb = read_nb(nbfile, kernel=False) for i, cell in enumerate(nb.iter_cells()): ls = nb.cell_height(cell) #fLOG(i,"-",nb.cell_type(cell), l) #if l > 400: fLOG(cell) assert ls > 0 return new_tags = nb.add_tag_slide() for k, v in sorted(new_tags.items()): a, b, c = v fLOG(k, a, b) assert len(new_tags) > 0
def test_encryption_stream_fernet_chunck_size(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: return infile = StreamIO(bytes(list(i % 255 for i in range(0, 10000)))) outst = StreamIO() r = encrypt_stream("key0" * 8, infile, outst, algo="fernet", chunksize=256) assert r is None enc = StreamIO(outst.getvalue()) enc2 = StreamIO(outst.getvalue()) outdec = StreamIO() r2 = decrypt_stream("key0" * 8, enc, outdec, algo="fernet", chunksize=256) assert r2 is None self.assertEqual(infile.getvalue(), outdec.getvalue()) outdec2 = StreamIO() try: r3 = decrypt_stream("key1" * 8, enc2, outdec2, algo="fernet", chunksize=256) except: return assert r3 is None self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_walk(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") this = os.path.abspath(os.path.dirname(__file__)) ut = os.path.join(this, "..") fLOG(this) nb = 0 for root, dirs, files in walk(ut): nb += 1 if sys.version_info[0] >= 3: nb2 = 0 for root, dirs, files in walk(ut, neg_filter="*ut_loghelper*"): nb2 += 1 assert nb2 > 0 assert nb2 < nb else: nb2 = 1e6 def filter(d): return "loghelper" in d or "helpgen" in d nb3 = 0 for root, dirs, files in walk(ut, neg_filter=filter): nb3 += 1 assert nb3 > 0 assert nb3 < nb2
def test_encryption_stream(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: return try: import Cryptodome as skip___ except ImportError: warnings.warn("pycryptodomex is not installed") return infile = StreamIO(bytes([0, 1, 2, 3, 4])) outst = StreamIO() r = encrypt_stream(b"key0" * 4, infile, outst) assert r is None enc = StreamIO(outst.getvalue()) enc2 = StreamIO(outst.getvalue()) outdec = StreamIO() r2 = decrypt_stream(b"key0" * 4, enc, outdec) assert r2 is None self.assertEqual(infile.getvalue(), outdec.getvalue()) outdec2 = StreamIO() r3 = decrypt_stream(b"key1" * 4, enc2, outdec2) assert r3 is None self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_stream_fernet(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: return infile = StreamIO(bytes([0, 1, 2, 3, 4])) outst = StreamIO() r = encrypt_stream("key0" * 8, infile, outst, algo="fernet") assert r is None enc = StreamIO(outst.getvalue()) enc2 = StreamIO(outst.getvalue()) outdec = StreamIO() r2 = decrypt_stream("key0" * 8, enc, outdec, algo="fernet") assert r2 is None self.assertEqual(infile.getvalue(), outdec.getvalue()) outdec2 = StreamIO() try: r3 = decrypt_stream("key1" * 8, enc2, outdec2, algo="fernet") except: return assert r3 is None self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_bytes(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: return try: import Cryptodome as skip__ except ImportError: warnings.warn("pycryptodomex is not installed") return infile = bytes([0, 1, 2, 3, 4]) r = encrypt_stream(b"key0" * 4, infile) assert r is not None r2 = decrypt_stream(b"key0" * 4, r) assert r2 is not None self.assertEqual(infile, r2) r3 = decrypt_stream(b"key1" * 4, r) assert r3 is not None self.assertNotEqual(infile, r3)
def zz_st_jenkins_job_verif(self, platform): engines_default = dict(anaconda2="c:\\Anaconda", anaconda3="c:\\Anaconda3", py35="c:\\Python35_x64", default="c:\\Python34_x64", winpython="c:\\APythonENSAE\\python") srv = JenkinsExt( "http://localhost:8080/", "user", "password", mock=True, fLOG=fLOG, engines=engines_default, platform=platform) if platform == "win": job = "standalone [conda_update] [anaconda3]" cmd = srv.get_cmd_standalone(job) assert "Anaconda3" in cmd if platform == "win": job = "pyrsslocal [py35] <-- pyquickhelper, pyensae" cmd = "\n".join(srv.get_jenkins_script(job)) assert "Python34" not in cmd else: try: job = "pyrsslocal [py35] <-- pyquickhelper, pyensae" except NotImplementedError: fLOG("OK FOR", platform)
def test_notebook_runner_empty(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # written in Python 3 return temp = get_temp_folder(__file__, "temp_notebook_empty") nbfile = os.path.join( temp, "..", "data", "td2a_cenonce_session_4B.ipynb") assert os.path.exists(nbfile) addpath = os.path.normpath(os.path.join(temp, "..", "..", "..", "src")) assert os.path.exists(addpath) kernel_name = None if is_travis_or_appveyor() is not None else install_python_kernel_for_unittest( "pyquickhelper") outfile = os.path.join(temp, "out_notebook.ipynb") assert not os.path.exists(outfile) out = run_notebook(nbfile, working_dir=temp, outfilename=outfile, additional_path=[addpath], fLOG=fLOG, kernel_name=kernel_name) fLOG(out) assert os.path.exists(outfile) assert "No module named 'pyquickhelper'" not in out
def test_bug_exe(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") command = "dir" if sys.platform.startswith("win32") else "ls" yml = """ language: python python: - {{Python35}} before: - %s after_script: - %s {{PLATFORM}} script: - %s """ % (command, command, command) context = dict(Python34="fake", Python35=os.path.dirname(sys.executable), Python27=None, Anaconda3=None, Anaconda2=None, WinPython35=None, project_name="pyquickhelper", root_path="ROOT", PLATFORM="win") obj, name = load_yaml(yml, context=context) assert name is not None try: res = list(enumerate_convert_yaml_into_instructions( obj, variables=context)) assert False assert res except ValueError as e: assert "'before'" in str(e)
def test_read_content_ufs(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") url = "https://raw.githubusercontent.com/sdpython/pyquickhelper/master/src/pyquickhelper/ipythonhelper/magic_parser.py" content = read_content_ufs(url, encoding="utf8") assert "MagicCommandParser" in content assert isinstance(content, str # unicode# ) typstr = str # unicode# file = typstr(__file__) file_, ext = os.path.splitext(file) if ext != ".py": file = file_ + ".py" content = read_content_ufs(file, encoding="utf8") assert "TestDownloadContent" in content content2 = read_content_ufs(content, encoding="utf8") self.assertEqual(content2, content) st = StringIO(content) content2 = read_content_ufs(st, encoding="utf8") self.assertEqual(content2, content) if sys.version_info[0] > 2: by = BytesIO(content.encode("utf8")) content2 = read_content_ufs(by, encoding="utf8") self.assertEqual(content2, content)
def test_notebook_gallery(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: return temp = get_temp_folder(__file__, "temp_gallery") fold = os.path.normpath(os.path.join( temp, "..", "data_gallery", "notebooks")) assert os.path.exists(fold) file = os.path.join(temp, "all_notebooks.rst") build_notebooks_gallery(fold, file, fLOG=fLOG) if not os.path.exists(file): raise FileNotFoundError(file) with open(file, "r", encoding="utf8") as f: text = f.read() if "GitHub/pyquickhelper" in text.replace("\\", "/"): raise Exception(text) spl = text.split("notebooks/td2a_eco_competition_modeles_logistiques") if len(spl) != 3: raise Exception(text)
def test_url_more_recent(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") this = os.path.abspath(os.path.dirname(__file__)) name = "test_syncho.py" temp = get_temp_folder(__file__, "temp_url_more_recent") shutil.copy(os.path.join(this, name), temp) dest = os.path.join(temp, name) url = "https://github.com/sdpython/pyquickhelper/tree/master/_unittests/ut_loghelper/" + name f1 = URL.urlopen(url) r = _first_more_recent(f1, dest) f1.close() assert isinstance(r, bool) url = "http://www.lemonde.fr/" f1 = URL.urlopen(url) r = _first_more_recent(f1, dest) f1.close() assert isinstance(r, bool) fLOG(r)
def test_mako_exceptions_not_here(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") tmpl = """ % for i in range(0, len(lll)): print(${ll[i]}) % endfor """ exp = """ print(0) print(2) """ try: res = apply_template(tmpl, dict(lll=[0, 2])) assert False except CustomTemplateException as e: # fLOG(str(e)) assert "Undefined" in str(e) return assert False fLOG(res) self.assertEqual(res.replace(" ", "").replace("\n", ""), exp.replace(" ", "").replace("\n", ""))
def test_clean_pep8(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_pep8_clean") name = os.path.join(temp, "python_try.py") with open(name, "w") as f: f.write(""" import sys import os def f1 () : #g return [2,3] """.replace(" ", "")) r = remove_extra_spaces_and_pep8(name) assert r > 0 with open(name, "r") as f: content = f.read() self.assertEqual(content.strip(), """ import sys import os def f1(): # g return [2, 3] """.replace(" ", "").strip()) r = remove_extra_spaces_and_pep8(name) self.assertEqual(r, 0)
def test_build_script(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.platform.startswith("win") and sys.version_info[0] != 2: sc = get_build_script("pyquickhelper") lines = sc.split("\n") for line in lines: if "__" in line and _default_nofolder not in line: raise Exception("issue with __ in:\n" + line + "\nFULL\n" + sc) scc = get_script_command( "unittest", "pyquickhelper", requirements=[]) assert "setup.py" in scc assert "__" not in scc sccc = get_extra_script_command( "local_pypi", "pyquickhelper", port=8067, requirements=[]) assert "python" in sccc if "__" in sccc: raise Exception(sccc) else: # not yet implemented for this platform return
def test_version(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") url = "http://www.xavierdupre.fr/enseignement/complements/marathon.txt" repeat = 0 while True: try: df = read_url( url, sep="\t", names=[ "ville", "annee", "temps", "secondes"]) break except ConnectionResetError as e: if repeat >= 2: raise e repeat += 1 assert len(df) > 0 assert len(df.columns) == 4 fLOG(df.head())
def test_jenkins_job_verif(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") this = os.path.abspath(os.path.dirname(__file__)) yml = os.path.abspath(os.path.join( this, "..", "..", ".local.jenkins.win.yml")) if not os.path.exists(yml): yml = os.path.abspath(os.path.join( this, "..", "..", "..", ".local.jenkins.win.yml")) if not os.path.exists(yml): raise FileNotFoundError(yml) context = dict(Python34=None, Python35=os.path.dirname(sys.executable), Python36=os.path.dirname(sys.executable), Python27=None, Anaconda3=None, Anaconda2=None, WinPython35=None, project_name="pyquickhelper", root_path="ROOT") obj, name = load_yaml(yml, context=context) for k, v in obj.items(): fLOG(k, type(v), v) assert "python" in obj assert isinstance(obj["python"], list) assert name is not None
def test_missing_funcwin_file_split(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") assert _clean_name_variable("s-6") == "s_6" assert _get_format_zero_nb_integer(5006) == "%04d" ioout = [io.StringIO(), io.StringIO()] if sys.version_info[0] == 2: warnings.warn("skip testing Python 2.7") return file_split(os.path.abspath(__file__), out=ioout, header=True) ioout = [io.StringIO(), io.StringIO()] s1 = ioout[0].getvalue() s2 = ioout[0].getvalue() self.assertEqual(s1[:5], s2[:5]) nb = file_split(os.path.abspath(__file__), out=ioout, header=False) size = os.stat(os.path.abspath(__file__)).st_size sa = 0 for l in ioout: s = l.getvalue() assert len(s) > 0 sa += len(s) assert 0 < sa <= size if sa not in (size, size - nb - 1, size - nb, size - 1): raise Exception("{0} != {1}, nb={2}".format(sa, size, nb))
def test_encrypt_decrypt(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] == 2: # the module returns the following error # ENCODING ERROR WITH Python 2.7, will not fix it return else: password = "******" * 2 temp = get_temp_folder(__file__, "temp_encrypt") temp2 = get_temp_folder(__file__, "temp_encrypt2") tempmm = get_temp_folder(__file__, "temp_encrypt_status") cstatus = os.path.join(tempmm, "crypt_status.txt") cmap = os.path.join(tempmm, "crypt_map.txt") srcf = os.path.abspath(os.path.join(temp, "..")) sys.argv = ["", srcf, temp, password, "--status", cstatus, "--map", cmap] encrypt(fLOG=fLOG) this = __file__ sys.argv = ["", temp, temp2, password] decrypt(fLOG=fLOG) with open(__file__, "rb") as f: c1 = f.read() with open(os.path.join(temp2, os.path.split(this)[-1]), "rb") as f: c2 = f.read() self.assertEqual(c1, c2) fLOG("end")
def test_jconvert_sequence_into_batch_file_split2(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") self.zz_st_jconvert_sequence_into_batch_file_split2("win")
def test_pandas_tennis(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") days = ["lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi", "dimanche"] temp = get_temp_folder(__file__, "temp_pandas_tennis") rows = [] dt = datetime.datetime.now() for i in range(0, 360): row = dict(date="%04d-%02d-%02d" % (dt.year, dt.month, dt.day), jour=dt.weekday(), journ=days[dt.weekday()]) dt += datetime.timedelta(1) rows.append(row) df = pandas.DataFrame(rows) df["equipe"] = 0 df.ix[(df.jour == 2) | (df.jour == 4), "equipe"] = 1 df["equipe_sum"] = (df.equipe.cumsum() * df["equipe"] + df["equipe"]) % 3 df["equipe_sum"] += df["equipe"] fLOG(df.head(n=10)) df.to_excel(os.path.join(temp, "tennis.xlsx"), index=False)
def test_transfer_ftp_true(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if sys.version_info[0] < 3: warnings.warn( "No testing transfer FTP on Pyghon 2.7 (issue with str and bytes)") return import keyring machine = os.environ.get( "COMPUTERNAME", os.environ.get("HOSTNAME", "CI")) try: user = keyring.get_password("web", machine + "user") pwd = keyring.get_password("web", machine + "pwd") except RuntimeError: user = None pwd = None if user is None: if not is_travis_or_appveyor(): raise Exception("user password is empty, machine='{0}', username='******'".format( machine, os.environ.get("USERNAME", None))) else: return web = TransferFTP("ftp.xavierdupre.fr", user, pwd, fLOG=fLOG) r = web.ls(".") fLOG(r) assert isinstance(r, list)
def test_filename(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") pos = ["c:\\test", "c:\\test.txt", "..", ".txt", "r.", "r", "mqldkfnqmodnsc/\\y"] for p in pos: if not is_file_string(p): raise Exception(p) neg = ["h\ng", "r\tr", "cd:ggd.h"] for p in neg: if is_file_string(p): raise Exception(p)
def zz_st_jconvert_sequence_into_batch_file_split(self, platform): this = os.path.abspath(os.path.dirname(__file__)) yml = os.path.abspath(os.path.join( this, "data", "local.yml")) if not os.path.exists(yml): raise FileNotFoundError(yml) context = dict(Python35="fake", Python36="C:\\Python35_x64", project_name="pyquickhelper", root_path="ROOT") obj, name = load_yaml(yml, context=context, platform=platform) assert name is not None res = list(enumerate_convert_yaml_into_instructions( obj, add_environ=False)) convs = [] for r, v in res: conv = convert_sequence_into_batch_file( r, variables=v, platform=platform) if not isinstance(conv, list): raise TypeError(type(conv)) convs.append(conv) assert len(res) > 0 self.assertEqual(len(convs), 2) conv = convs[0] assert conv assert isinstance(conv, list) fr = 0 for c in conv: if "JENKINS_SPLIT" in c: raise Exception(c) if "pip freeze" in c: fr += 1 fLOG("-------------------------------") fLOG(c) self.assertEqual(fr, 1)