def ftp_list_modules(ftp_location="/home/ftpuser/ftp/web/enseignement/setup",
http_location="http://www.xavierdupre.fr/enseignement/setup",
filename="index_modules_list.html",
ftps='SFTP'):
"""
Updates the list of backuped modules assuming they are stored on a FTP website.
It gets the list of wheels in a folder and creates a HTML pages.
It then uploads the final pages.
@param ftp_location location on the website
@param http_location same location but on http protocol
@param filename name of the file to produce
@return list of modules
The module uses *keyring* to retrieve the credentials.
You can set them up with:
::
from pyquickhelper.loghelper import get_password
get_password("ftp_list_modules", "ensae_teaching_cs2,site", "...")
get_password("ftp_list_modules", "ensae_teaching_cs2,login", "...")
get_password("ftp_list_modules", "ensae_teaching_cs2,password", "...")
"""
ftp_site = get_password(
"ftp_list_modules", "ensae_teaching_cs2,site")
login = get_password(
"ftp_list_modules", "ensae_teaching_cs2,login")
password = get_password(
"ftp_list_modules", "ensae_teaching_cs2,password")
if not ftp_site:
raise ValueError("ftp_site is empty, some missing keyring?")
if not login:
raise ValueError("login is empty")
if not password:
raise ValueError("password is empty")
ftp = TransferFTP(ftp_site, login, password, ftps=ftps)
res = ftp.ls(ftp_location)
rows = ["<html><body><h1>storage for unit test</h1>\n<ul>"]
ret = []
for i, v in enumerate(sorted(_["name"] for _ in res)):
if v in ('.', '..'):
continue
ret.append(v)
line = '<li>{1} - <a href="{2}/{0}">{0}</a></li>'.format(
v, i, http_location)
rows.append(line)
rows.append("</ul></body></html>")
content = "\n".join(rows)
bstr = content.encode('ascii')
ftp.transfer(bstr, ftp_location + "/", filename)
ftp.close()
return ret
def test_ensae_2016_answers(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no stored password
return
password = get_password("cpt", "ensae_teaching_cs,pwd")
r = data_cpt_ENSAE_2016_11_blind_set(password)
self.assertEqual(len(r), 7500)
truth = r
r = data_cpt_ENSAE_2016_11_blind_set("dummy")
self.assertEqual(len(r), 7500)
s = sum(r)
assert 0 <= s <= len(r)
auc = AUC(truth, r)
fLOG(auc)
assert 0 <= auc <= 1
temp = get_temp_folder(__file__, "temp_cpt_ensae_2016_answers")
out = os.path.join(temp, "answer.txt")
with open(out, "w") as f:
f.write("\n".join(str(_) for _ in r))
zip_files(os.path.join("submission.zip"), [out])
def test_transfer_ftp_true(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
prefix = "pyquickhelper,"
try:
user = get_password("web", prefix + "user", ask=False)
pwd = get_password("web", prefix + "pwd", ask=False)
ftpsite = get_password("web", prefix + "ftp", ask=False)
except (RuntimeError, AttributeError):
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
zoo = []
for k, v in sorted(os.environ.items()):
zoo.append("{0}={1}".format(k, v))
raise Exception(
"user password is empty, prefix='{0}', username='******'\n{2}"
.format(prefix, get_user(), "\n".join(zoo)))
return
try:
web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
except ftplib.error_perm as e:
if "Non-anonymous sessions must use encryption." in str(e):
return
raise e
except ftplib.error_temp as e:
if "421 Home directory not available" in str(e):
return
raise e
except socket.gaierror as ee:
if "Name or service not known" in str(ee):
return
if "getaddrinfo failed" in str(ee):
return
raise ee
r = web.ls(".")
fLOG(r)
self.assertTrue(isinstance(r, list))
web.close()
def test_password_keyring(self):
pwd = 'bibi'
try:
set_password('pyq', 'jj', pwd, lib='keyring')
except Exception as e:
if "Prompt dismissed" in str(e):
return
raise e
pwd2 = get_password('pyq', 'jj', lib='keyring')
self.assertEqual(pwd, pwd2)
def get_web(self):
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper2,"
try:
user = get_password("web", prefix + "user", ask=False)
pwd = get_password("web", prefix + "pwd", ask=False)
ftpsite = get_password("web", prefix + "ftp", ask=False)
except (RuntimeError, AttributeError):
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
zoo = []
for k, v in sorted(os.environ.items()):
zoo.append("{0}={1}".format(k, v))
raise Exception(
"user password is empty, prefix='{0}', username='******'\n{2}"
.format(prefix, get_user(), "\n".join(zoo)))
return None
web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG, ftps='SFTP')
return web
def test_geocoding(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_geocoding")
from actuariat_python.data import geocode
data = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data",
"bureau.txt")
df = pandas.read_csv(data, sep="\t", encoding="utf-8")
he = df.head(n=5)
every = os.path.join(temp, "every.csv")
# we retrieve an encrypted key
bing_key = get_password("bing", "actuariat_python,key")
self.assertNotEmpty(bing_key)
fLOG(bing_key)
coders = ["Nominatim"]
if bing_key:
coders.append(("bing", bing_key))
fLOG("geocoding 1", len(he))
# test
res = geocode(he,
save_every=every,
every=1,
index=False,
encoding="utf-8",
coders=coders,
fLOG=fLOG)
self.assertExists(every)
# fLOG(res)
out = os.path.join(temp, "geo.csv")
res.to_csv(out, sep="\t", encoding="utf-8", index=False)
res.to_excel(out + ".xlsx", index=False)
fLOG("geocoding 2", len(res))
res = geocode(he,
save_every=every,
every=1,
index=False,
encoding="utf-8",
coders=coders,
fLOG=fLOG)
self.assertExists(every)
fLOG(res)
def test_import_exam_vigenere(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
from ensae_teaching_cs.td_1a.vigenere import code_vigenere
temp = get_temp_folder(__file__, "temp_import_exam_vigenere")
pwd = get_password("exam", "ensae_teaching_cs,key")
pwd = pwd.encode("ascii")
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere")
assert os.path.exists(dst)
with open(dst, "rb") as f:
content = f.read()
content = code_vigenere(content, pwd, binary=True, decode=True)
decr = os.path.join(temp, "onemod_vigenere.py")
with open(decr, "wb") as f:
f.write(content)
assert os.path.exists(decr)
fLOG("importing")
sys.path.append(temp)
import onemod as temp_module
del sys.path[-1]
answer = temp_module.build(120)
exp = """*** H
H *** H
*** H
*** H
H *** H
*** H
*** H
H *** H
*** H
*** H
H *** H
*** H""".replace(" ", "")
self.assertEqual(answer, exp)
def test_crypt_file(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = get_password("exam", "ensae_teaching_cs,key")
if pwd is None:
raise ValueError("pwd cannot be None")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def test_import_exam(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
temp = get_temp_folder(__file__, "temp_import_exam")
pwd = get_password("exam", "ensae_teaching_cs,key")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
assert os.path.exists(dst)
decr = os.path.join(temp, "onemod.py")
decrypt_stream(pwd, dst, decr)
assert os.path.exists(decr)
fLOG("importing")
sys.path.append(temp)
import onemod as temp_module
del sys.path[-1]
answer = temp_module.build(120)
exp = """*** H
H *** H
*** H
*** H
H *** H
*** H
*** H
H *** H
*** H
*** H
H *** H
*** H""".replace(" ", "")
self.assertEqual(answer, exp)
def test_crypt_file_vigenere(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
from ensae_teaching_cs.td_1a.vigenere import code_vigenere
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.vigenere")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = get_password("exam", "ensae_teaching_cs,key")
pwd = pwd.encode("ascii")
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
with open(this, "rb") as f:
content = f.read()
code = code_vigenere(content, pwd, binary=True)
with open(dst, "wb") as f:
f.write(code)
def test_transfer_ftp_start_transfering(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
prefix = "pyquickhelper,"
try:
user = get_password("web", prefix + "user", ask=False)
pwd = get_password("web", prefix + "pwd", ask=False)
ftpsite = get_password("web", prefix + "ftp", ask=False)
except (RuntimeError, AttributeError):
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
raise Exception(
"user password is empty, prefix='{0}', username='******'".
format(prefix, get_user()))
return
# Transfering
now = datetime.datetime.now()
temp = get_temp_folder(__file__, "temp_transfer_ftp_true")
with open(os.path.join(temp, "essai.txt"), 'w') as f:
f.write(str(now))
sfile = os.path.join(temp, "status_ut.txt")
ftn = FileTreeNode(temp)
# one
try:
ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
except ftplib.error_perm as e:
if "Non-anonymous sessions must use encryption." in str(e):
return
raise e
except ftplib.error_temp as e:
if "421 Home directory not available" in str(e):
return
raise e
except socket.gaierror as ee:
if "Name or service not known" in str(ee):
return
if "getaddrinfo failed" in str(ee):
return
raise ee
web = FolderTransferFTP(ftn,
ftp,
sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
names = [os.path.split(f.filename)[-1] for f in done]
self.assertEqual(names, ['essai.txt'])
# two, same file, should not be done again
ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
web = FolderTransferFTP(ftn,
ftp,
sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
self.assertEmpty(done)
import sys
import os
import warnings
from pyquickhelper.loghelper import get_password, fLOG # publish_lectures
from pyquickhelper.jenkinshelper import JenkinsExt
from ensae_teaching_cs.automation.jenkins_helper import (
setup_jenkins_server, engines_default)
#########################################
# logging
fLOG(OutputPrint=True)
#########################################
# récupération des identifiants Jenkins
user = get_password("jenkins", "_automation,user", ask=False)
pwd = get_password("jenkins", "_automation,pwd", ask=False)
host = get_password("jenkins", "_automation,host", ask=False)
platform = "linux"
if pwd is None:
raise RuntimeError("Password is missing (None).")
fLOG('password has been recovered.')
#########################################
# instantiation d'une classe faisant l'interface avec le service
if platform.startswith("win"):
letter = "d" if os.path.exists("d:") else "c"
location = letter + ":\\jenkins\\pymy"
else:
location = "/var/lib/jenkins/workspace"
def test_password_cryptfile(self):
os.environ['UTTESTPYQ'] = 'bypass'
pwd = 'bibi'
set_password('pyq', 'jj', pwd, env='UTTESTPYQ')
pwd2 = get_password('pyq', 'jj', env='UTTESTPYQ')
self.assertEqual(pwd, pwd2)
def test_exc(self):
self.assertRaise(lambda: set_password('pyq', 'jj', 'aa', 'keyring2'),
RuntimeError)
self.assertRaise(lambda: get_password('pyq', 'jj', 'keyring2'),
RuntimeError)
fLOG(OutputPrint=True)
#########################################
# import des fonctions dont on a besoin
from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP
from ensae_teaching_cs.automation.ftp_publish_helper import publish_teachings_to_web
from ensae_teaching_cs.automation.teaching_modules import get_teaching_modules
##################
# accès au site web
# on utilise keyring pour stocker les identifiants
# à commenter ou décommenter au besoin
if "2" in sys.argv:
user = get_password("web", "_automation2,user", ask=False)
pwd = get_password("web", "_automation2,pwd", ask=False)
ftpsite = get_password("web", "_automation2,ftp", ask=False)
ftps = "SFTP"
root_template2 = "/home/ftpuser/ftp/web/app/%s/%s"
import paramiko
import socket
sock = socket.socket()
sock.connect((ftpsite, 22))
trans = paramiko.transport.Transport(sock)
trans.start_client()
k = trans.get_remote_server_key()
hk = paramiko.hostkeys.HostKeys()
hk.add(ftpsite, 'ssh-rsa', k)