def ftp_list_modules(ftp_location="/home/ftpuser/ftp/web/enseignement/setup", http_location="http://www.xavierdupre.fr/enseignement/setup", filename="index_modules_list.html", ftps='SFTP'): """ Updates the list of backuped modules assuming they are stored on a FTP website. It gets the list of wheels in a folder and creates a HTML pages. It then uploads the final pages. @param ftp_location location on the website @param http_location same location but on http protocol @param filename name of the file to produce @return list of modules The module uses *keyring* to retrieve the credentials. You can set them up with: :: from pyquickhelper.loghelper import get_password get_password("ftp_list_modules", "ensae_teaching_cs2,site", "...") get_password("ftp_list_modules", "ensae_teaching_cs2,login", "...") get_password("ftp_list_modules", "ensae_teaching_cs2,password", "...") """ ftp_site = get_password( "ftp_list_modules", "ensae_teaching_cs2,site") login = get_password( "ftp_list_modules", "ensae_teaching_cs2,login") password = get_password( "ftp_list_modules", "ensae_teaching_cs2,password") if not ftp_site: raise ValueError("ftp_site is empty, some missing keyring?") if not login: raise ValueError("login is empty") if not password: raise ValueError("password is empty") ftp = TransferFTP(ftp_site, login, password, ftps=ftps) res = ftp.ls(ftp_location) rows = ["<html><body><h1>storage for unit test</h1>\n<ul>"] ret = [] for i, v in enumerate(sorted(_["name"] for _ in res)): if v in ('.', '..'): continue ret.append(v) line = '<li>{1} - <a href="{2}/{0}">{0}</a></li>'.format( v, i, http_location) rows.append(line) rows.append("</ul></body></html>") content = "\n".join(rows) bstr = content.encode('ascii') ftp.transfer(bstr, ftp_location + "/", filename) ftp.close() return ret
def test_ensae_2016_answers(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no stored password return password = get_password("cpt", "ensae_teaching_cs,pwd") r = data_cpt_ENSAE_2016_11_blind_set(password) self.assertEqual(len(r), 7500) truth = r r = data_cpt_ENSAE_2016_11_blind_set("dummy") self.assertEqual(len(r), 7500) s = sum(r) assert 0 <= s <= len(r) auc = AUC(truth, r) fLOG(auc) assert 0 <= auc <= 1 temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers") out = os.path.join(temp, "answer.txt") with open(out, "w") as f: f.write("\n".join(str(_) for _ in r)) zip_files(os.path.join("submission.zip"), [out])
def test_transfer_ftp_true(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") prefix = "pyquickhelper," try: user = get_password("web", prefix + "user", ask=False) pwd = get_password("web", prefix + "pwd", ask=False) ftpsite = get_password("web", prefix + "ftp", ask=False) except (RuntimeError, AttributeError): user = None pwd = None if user is None: if not is_travis_or_appveyor(): zoo = [] for k, v in sorted(os.environ.items()): zoo.append("{0}={1}".format(k, v)) raise Exception( "user password is empty, prefix='{0}', username='******'\n{2}" .format(prefix, get_user(), "\n".join(zoo))) return try: web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG) except ftplib.error_perm as e: if "Non-anonymous sessions must use encryption." in str(e): return raise e except ftplib.error_temp as e: if "421 Home directory not available" in str(e): return raise e except socket.gaierror as ee: if "Name or service not known" in str(ee): return if "getaddrinfo failed" in str(ee): return raise ee r = web.ls(".") fLOG(r) self.assertTrue(isinstance(r, list)) web.close()
def test_password_keyring(self): pwd = 'bibi' try: set_password('pyq', 'jj', pwd, lib='keyring') except Exception as e: if "Prompt dismissed" in str(e): return raise e pwd2 = get_password('pyq', 'jj', lib='keyring') self.assertEqual(pwd, pwd2)
def get_web(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) import keyring prefix = "pyquickhelper2," try: user = get_password("web", prefix + "user", ask=False) pwd = get_password("web", prefix + "pwd", ask=False) ftpsite = get_password("web", prefix + "ftp", ask=False) except (RuntimeError, AttributeError): user = None pwd = None if user is None: if not is_travis_or_appveyor(): zoo = [] for k, v in sorted(os.environ.items()): zoo.append("{0}={1}".format(k, v)) raise Exception( "user password is empty, prefix='{0}', username='******'\n{2}" .format(prefix, get_user(), "\n".join(zoo))) return None web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG, ftps='SFTP') return web
def test_geocoding(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") temp = get_temp_folder(__file__, "temp_geocoding") from actuariat_python.data import geocode data = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "bureau.txt") df = pandas.read_csv(data, sep="\t", encoding="utf-8") he = df.head(n=5) every = os.path.join(temp, "every.csv") # we retrieve an encrypted key bing_key = get_password("bing", "actuariat_python,key") self.assertNotEmpty(bing_key) fLOG(bing_key) coders = ["Nominatim"] if bing_key: coders.append(("bing", bing_key)) fLOG("geocoding 1", len(he)) # test res = geocode(he, save_every=every, every=1, index=False, encoding="utf-8", coders=coders, fLOG=fLOG) self.assertExists(every) # fLOG(res) out = os.path.join(temp, "geo.csv") res.to_csv(out, sep="\t", encoding="utf-8", index=False) res.to_excel(out + ".xlsx", index=False) fLOG("geocoding 2", len(res)) res = geocode(he, save_every=every, every=1, index=False, encoding="utf-8", coders=coders, fLOG=fLOG) self.assertExists(every) fLOG(res)
def test_import_exam_vigenere(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return from ensae_teaching_cs.td_1a.vigenere import code_vigenere temp = get_temp_folder(__file__, "temp_import_exam_vigenere") pwd = get_password("exam", "ensae_teaching_cs,key") pwd = pwd.encode("ascii") this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere") assert os.path.exists(dst) with open(dst, "rb") as f: content = f.read() content = code_vigenere(content, pwd, binary=True, decode=True) decr = os.path.join(temp, "onemod_vigenere.py") with open(decr, "wb") as f: f.write(content) assert os.path.exists(decr) fLOG("importing") sys.path.append(temp) import onemod as temp_module del sys.path[-1] answer = temp_module.build(120) exp = """*** H H *** H *** H *** H H *** H *** H *** H H *** H *** H *** H H *** H *** H""".replace(" ", "") self.assertEqual(answer, exp)
def test_crypt_file(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted") if not os.path.exists(dst) or os.stat(dst).st_size < 10: fLOG("crypt") pwd = get_password("exam", "ensae_teaching_cs,key") if pwd is None: raise ValueError("pwd cannot be None") pwd += "*" * (16 - len(pwd)) pwd = pwd.encode("ascii") fLOG(type(pwd)) this = os.path.join(this, "cryptcode.py") assert os.path.exists(this) encrypt_stream(pwd, this, dst) fLOG(os.stat(dst).st_size)
def test_import_exam(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return temp = get_temp_folder(__file__, "temp_import_exam") pwd = get_password("exam", "ensae_teaching_cs,key") pwd += "*" * (16 - len(pwd)) pwd = pwd.encode("ascii") this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted") assert os.path.exists(dst) decr = os.path.join(temp, "onemod.py") decrypt_stream(pwd, dst, decr) assert os.path.exists(decr) fLOG("importing") sys.path.append(temp) import onemod as temp_module del sys.path[-1] answer = temp_module.build(120) exp = """*** H H *** H *** H *** H H *** H *** H *** H H *** H *** H *** H H *** H *** H""".replace(" ", "") self.assertEqual(answer, exp)
def test_crypt_file_vigenere(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): # no password return from ensae_teaching_cs.td_1a.vigenere import code_vigenere this = os.path.abspath(os.path.dirname(module_file)) dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere") if not os.path.exists(dst) or os.stat(dst).st_size < 10: fLOG("crypt") pwd = get_password("exam", "ensae_teaching_cs,key") pwd = pwd.encode("ascii") this = os.path.join(this, "cryptcode.py") assert os.path.exists(this) with open(this, "rb") as f: content = f.read() code = code_vigenere(content, pwd, binary=True) with open(dst, "wb") as f: f.write(code)
def test_transfer_ftp_start_transfering(self): fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__") prefix = "pyquickhelper," try: user = get_password("web", prefix + "user", ask=False) pwd = get_password("web", prefix + "pwd", ask=False) ftpsite = get_password("web", prefix + "ftp", ask=False) except (RuntimeError, AttributeError): user = None pwd = None if user is None: if not is_travis_or_appveyor(): raise Exception( "user password is empty, prefix='{0}', username='******'". format(prefix, get_user())) return # Transfering now = datetime.datetime.now() temp = get_temp_folder(__file__, "temp_transfer_ftp_true") with open(os.path.join(temp, "essai.txt"), 'w') as f: f.write(str(now)) sfile = os.path.join(temp, "status_ut.txt") ftn = FileTreeNode(temp) # one try: ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG) except ftplib.error_perm as e: if "Non-anonymous sessions must use encryption." in str(e): return raise e except ftplib.error_temp as e: if "421 Home directory not available" in str(e): return raise e except socket.gaierror as ee: if "Name or service not known" in str(ee): return if "getaddrinfo failed" in str(ee): return raise ee web = FolderTransferFTP(ftn, ftp, sfile, root_web="/www/htdocs/apptest/", fLOG=fLOG) done = web.start_transfering(delay=0.1) ftp.close() names = [os.path.split(f.filename)[-1] for f in done] self.assertEqual(names, ['essai.txt']) # two, same file, should not be done again ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG) web = FolderTransferFTP(ftn, ftp, sfile, root_web="/www/htdocs/apptest/", fLOG=fLOG) done = web.start_transfering(delay=0.1) ftp.close() self.assertEmpty(done)
import sys import os import warnings from pyquickhelper.loghelper import get_password, fLOG # publish_lectures from pyquickhelper.jenkinshelper import JenkinsExt from ensae_teaching_cs.automation.jenkins_helper import ( setup_jenkins_server, engines_default) ######################################### # logging fLOG(OutputPrint=True) ######################################### # récupération des identifiants Jenkins user = get_password("jenkins", "_automation,user", ask=False) pwd = get_password("jenkins", "_automation,pwd", ask=False) host = get_password("jenkins", "_automation,host", ask=False) platform = "linux" if pwd is None: raise RuntimeError("Password is missing (None).") fLOG('password has been recovered.') ######################################### # instantiation d'une classe faisant l'interface avec le service if platform.startswith("win"): letter = "d" if os.path.exists("d:") else "c" location = letter + ":\\jenkins\\pymy" else: location = "/var/lib/jenkins/workspace"
def test_password_cryptfile(self): os.environ['UTTESTPYQ'] = 'bypass' pwd = 'bibi' set_password('pyq', 'jj', pwd, env='UTTESTPYQ') pwd2 = get_password('pyq', 'jj', env='UTTESTPYQ') self.assertEqual(pwd, pwd2)
def test_exc(self): self.assertRaise(lambda: set_password('pyq', 'jj', 'aa', 'keyring2'), RuntimeError) self.assertRaise(lambda: get_password('pyq', 'jj', 'keyring2'), RuntimeError)
fLOG(OutputPrint=True) ######################################### # import des fonctions dont on a besoin from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP from ensae_teaching_cs.automation.ftp_publish_helper import publish_teachings_to_web from ensae_teaching_cs.automation.teaching_modules import get_teaching_modules ################## # accès au site web # on utilise keyring pour stocker les identifiants # à commenter ou décommenter au besoin if "2" in sys.argv: user = get_password("web", "_automation2,user", ask=False) pwd = get_password("web", "_automation2,pwd", ask=False) ftpsite = get_password("web", "_automation2,ftp", ask=False) ftps = "SFTP" root_template2 = "/home/ftpuser/ftp/web/app/%s/%s" import paramiko import socket sock = socket.socket() sock.connect((ftpsite, 22)) trans = paramiko.transport.Transport(sock) trans.start_client() k = trans.get_remote_server_key() hk = paramiko.hostkeys.HostKeys() hk.add(ftpsite, 'ssh-rsa', k)