def test_crypt_file(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = keyring.get_password(
"exam", "ensae_teaching_cs,key")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def do_main(source, dest, password, encrypt, fLOG=None):
"""
Encrypt or decrypt of a file
@param source source of files to encrypt or decrypt
@param dest destination
@param password password
@param encrypt boolean, True to encrypt
@param fLOG logging function
"""
if not os.path.exists(source):
raise FileNotFoundError(source)
try:
from pyquickhelper.filehelper import encrypt_stream, decrypt_stream
except ImportError:
folder = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", ".."))
sys.path.append(folder)
from pyquickhelper.filehelper import encrypt_stream, decrypt_stream
if isinstance(password, str):
password = bytes(password, encoding="ascii")
if encrypt:
encrypt_stream(key=password,
filename=source,
out_filename=dest,
chunksize=os.stat(source).st_size * 2 + 1)
else:
decrypt_stream(key=password,
filename=source,
out_filename=dest,
chunksize=os.stat(source).st_size * 2 + 1)
def do_main(source, dest, password, encrypt, fLOG=None):
"""
Encrypt or decrypt of a file
@param source source of files to encrypt or decrypt
@param dest destination
@param password password
@param encrypt boolean, True to encrypt
@param fLOG logging function
"""
if not os.path.exists(source):
raise FileNotFoundError(source)
try:
from pyquickhelper.filehelper import encrypt_stream, decrypt_stream
except ImportError:
folder = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "..",
".."))
sys.path.append(folder)
from pyquickhelper.filehelper import encrypt_stream, decrypt_stream
if sys.version_info[0] >= 3 and isinstance(password, str):
password = bytes(password, encoding="ascii")
if encrypt:
encrypt_stream(key=password,
filename=source,
out_filename=dest,
chunksize=os.stat(source).st_size * 2 + 1)
else:
decrypt_stream(key=password,
filename=source,
out_filename=dest,
chunksize=os.stat(source).st_size * 2 + 1)
def test_crypt_file(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = keyring.get_password(
"exam", "ensae_teaching_cs,key")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def test_encryption_file_size(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Cryptodome as skip__
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
temp = get_temp_folder(__file__, "temp_encryption1")
infile = os.path.abspath(__file__).replace(".pyc", ".py")
outfile = os.path.join(temp, "out_crypted.enc")
r = encrypt_stream(b"key0" * 4, infile, outfile, chunksize=16)
assert r is None
outfile2 = os.path.join(temp, "out_decrypted.enc")
r = decrypt_stream(b"key0" * 4, outfile, outfile2, chunksize=16)
assert r is None
with open(infile, "rb") as f:
inc = f.read()
with open(outfile2, "rb") as f:
ouc = f.read()
self.assertEqual(inc, ouc)
outfile3 = os.path.join(temp, "out_decrypted2.enc")
r = decrypt_stream(b"key1" * 4, outfile, outfile3, chunksize=16)
assert r is None
with open(outfile3, "rb") as f:
ouc3 = f.read()
self.assertNotEqual(inc, ouc3)
def test_encryption_stream_fernet(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
infile = StreamIO(bytes([0, 1, 2, 3, 4]))
outst = StreamIO()
r = encrypt_stream("key0" * 8, infile, outst, algo="fernet")
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream("key0" * 8, enc, outdec, algo="fernet")
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
try:
r3 = decrypt_stream("key1" * 8, enc2, outdec2, algo="fernet")
except Exception:
return
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_stream_fernet_chunck_size(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
infile = StreamIO(bytes(list(i % 255 for i in range(0, 10000))))
outst = StreamIO()
r = encrypt_stream("key0" * 8, infile, outst,
algo="fernet", chunksize=256)
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream("key0" * 8, enc, outdec,
algo="fernet", chunksize=256)
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
try:
r3 = decrypt_stream("key1" * 8, enc2, outdec2,
algo="fernet", chunksize=256)
except Exception:
return
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_file_size(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Cryptodome as skip__
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
temp = get_temp_folder(__file__, "temp_encryption1")
infile = os.path.abspath(__file__).replace(".pyc", ".py")
outfile = os.path.join(temp, "out_crypted.enc")
r = encrypt_stream(b"key0" * 4, infile, outfile, chunksize=16)
assert r is None
outfile2 = os.path.join(temp, "out_decrypted.enc")
r = decrypt_stream(b"key0" * 4, outfile, outfile2, chunksize=16)
assert r is None
with open(infile, "rb") as f:
inc = f.read()
with open(outfile2, "rb") as f:
ouc = f.read()
self.assertEqual(inc, ouc)
outfile3 = os.path.join(temp, "out_decrypted2.enc")
r = decrypt_stream(b"key1" * 4, outfile, outfile3, chunksize=16)
assert r is None
with open(outfile3, "rb") as f:
ouc3 = f.read()
self.assertNotEqual(inc, ouc3)
def test_encryption_stream(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Cryptodome as skip___
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
infile = StreamIO(bytes([0, 1, 2, 3, 4]))
outst = StreamIO()
r = encrypt_stream(b"key0" * 4, infile, outst)
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream(b"key0" * 4, enc, outdec)
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
r3 = decrypt_stream(b"key1" * 4, enc2, outdec2)
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_stream_fernet(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
infile = StreamIO(bytes([0, 1, 2, 3, 4]))
outst = StreamIO()
r = encrypt_stream("key0" * 8, infile, outst, algo="fernet")
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream("key0" * 8, enc, outdec, algo="fernet")
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
try:
r3 = decrypt_stream("key1" * 8, enc2, outdec2, algo="fernet")
except Exception:
return
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_stream(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Cryptodome as skip___
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
infile = StreamIO(bytes([0, 1, 2, 3, 4]))
outst = StreamIO()
r = encrypt_stream(b"key0" * 4, infile, outst)
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream(b"key0" * 4, enc, outdec)
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
r3 = decrypt_stream(b"key1" * 4, enc2, outdec2)
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def encrypt_file(infile, outfile, password=None):
"""
Encrypt a file with a specific password.
@param password password for the hackathon, if None, look into
``os.environ["PWDCROIXROUGE"]``
@param infile input file
@param outfile output file
@return outfile
"""
password = get_password_from_keyring_or_env(password)
return encrypt_stream(password, infile, outfile)
def test_crypt_file(self):
fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
import keyring
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = keyring.get_password("exam", os.environ["COMPUTERNAME"] + "key")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def test_crypt_file(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = get_password("exam", "ensae_teaching_cs,key")
if pwd is None:
raise ValueError("pwd cannot be None")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def test_crypt_file(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# no password
return
import keyring
this = os.path.abspath(os.path.dirname(module_file))
dst = os.path.join(this, "encrypted", "cryptcode_exam_2016.crypted")
if not os.path.exists(dst) or os.stat(dst).st_size < 10:
fLOG("crypt")
pwd = keyring.get_password("exam",
os.environ["COMPUTERNAME"] + "key")
pwd += "*" * (16 - len(pwd))
pwd = pwd.encode("ascii")
fLOG(type(pwd))
this = os.path.join(this, "cryptcode.py")
assert os.path.exists(this)
encrypt_stream(pwd, this, dst)
fLOG(os.stat(dst).st_size)
def test_encryption_stream_fernet_chunck_size(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
infile = StreamIO(bytes(list(i % 255 for i in range(0, 10000))))
outst = StreamIO()
r = encrypt_stream("key0" * 8,
infile,
outst,
algo="fernet",
chunksize=256)
assert r is None
enc = StreamIO(outst.getvalue())
enc2 = StreamIO(outst.getvalue())
outdec = StreamIO()
r2 = decrypt_stream("key0" * 8,
enc,
outdec,
algo="fernet",
chunksize=256)
assert r2 is None
self.assertEqual(infile.getvalue(), outdec.getvalue())
outdec2 = StreamIO()
try:
r3 = decrypt_stream("key1" * 8,
enc2,
outdec2,
algo="fernet",
chunksize=256)
except Exception:
return
assert r3 is None
self.assertNotEqual(infile.getvalue(), outdec2.getvalue())
def test_encryption_bytes(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Cryptodome as skip__
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
infile = bytes([0, 1, 2, 3, 4])
r = encrypt_stream(b"key0" * 4, infile)
assert r is not None
r2 = decrypt_stream(b"key0" * 4, r)
assert r2 is not None
self.assertEqual(infile, r2)
r3 = decrypt_stream(b"key1" * 4, r)
assert r3 is not None
self.assertNotEqual(infile, r3)
def test_encryption_bytes(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Cryptodome as skip__
except ImportError:
warnings.warn("pycryptodomex is not installed")
return
infile = bytes([0, 1, 2, 3, 4])
r = encrypt_stream(b"key0" * 4, infile)
assert r is not None
r2 = decrypt_stream(b"key0" * 4, r)
assert r2 is not None
self.assertEqual(infile, r2)
r3 = decrypt_stream(b"key1" * 4, r)
assert r3 is not None
self.assertNotEqual(infile, r3)
# write summary
if True:
fLOG("summary")
index = os.path.join(dest_folder, "index.html")
if os.path.exists(index):
os.remove(index)
proj.write_summary()
#################
# zip everything
if True:
if os.path.exists(filename_zip):
os.remove(filename_zip)
proj.zip_group(None,
filename_zip,
addition=[
index,
os.path.join(dest_folder, "mail_style.css"),
filename_excel, filename_mails
])
############
# encryption
if True:
fLOG("encryption")
enc = filename_zip.replace(".zip", ".enc")
encrypt_stream(password, filename_zip, enc, chunksize=2**30)
def extract_students_mails_from_gmail_and_stores_in_folders(folder=".", filemails="emails.txt",
user=None, pwd=None, server="imap.gmail.com",
mailfolder=[
"ensae/ENSAE_2016_3A"],
date="1-Jan-2016", zipfilename="projet_3A_2016.zip",
zipencpwd=b"sixteenbyteskeys", dataframe=None,
columns={
"name": "nom_prenom", "group": "groupe", "subject": "sujet"},
skip_names=None, process_name=None,
title="List of emails", nolink_if=None, fLOG=fLOG):
"""
The scenario is the following:
* You are the teacher.
* Students started their projects at date *t*.
* They can work alone or by group.
* They send mails, you reply.
* Their address mail follows the convention: ``<first name>.<last name>@anything``
so it is to associate a mail address to a student name.
* You move every mail you received in a separate folder in your inbox.
* Sometime, you send a mail to everybody.
* Finally they send their project with attachments.
* You want to store everything (mails and attachements) in folders, one per group.
* You want a summary of what was received.
* You want to build a zip file to share their work with others teachers.
* You want to update the folder if a new mail was sent.
This function looks into a folder of your inbox and grabs every mails and
attachements from a groups of students.
@param folder where to store the results
@param filemails files used to store students address,
the operation is done once, remove the file
to force the function to rebuild the information.
@param user user of the gmail inbox
@param pwd password of the gmail inbox
@param server gmail server, it should be ``"imap.gmail.com"``,
it works with others mail servers using the *IMAP* protocol
@param mailfolder folder in your inbox to look into,
there can be several
@param date when to start looking (do not change the format,
look at the default value)
@param zipfilename name of the zip file to create
@param zipencpwd the zip file is also encrypted for a safer share with this key
and function `encrypt_stream <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/
pyquickhelper/filehelper/encryption.html#pyquickhelper.filehelper.encryption.encrypt_stream>`_.
@param dataframe dataframe which contains the definition of students groups
@param columns columns the function will look into, students names, group definition
(a unique number for all students in the same group), subject
@param skip_names list of names to skip
@param process_name to operate a transformation before matching students names with
their emails
@param title each group folder contains a html file connecting them,
this is its title
@param nolink_if The summary extracts links from url, it skips the urls which
contains on the substrings included in that list (None to use a default set)
@param fLOG logging function
@return @see cl ProjectsRepository
By default, Gmail does not let you programmatically access you own inbox,
you need to modify your gmail parameters to let this function do so.
"""
folder = os.path.abspath(".")
filemails = os.path.join(folder, filemails)
zipfilename = os.path.join(folder, zipfilename)
zipfilenameenc = zipfilename + ".enc"
# load the groups
if isinstance(dataframe, pandas.DataFrame):
df = dataframe
elif dataframe.endswith("xlsx"):
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] read dataframe", dataframe)
df = pandas.read_excel(dataframe)
else:
df = pandas.read_csv(dataframe, sep="\t", encoding="utf8")
# check mails
if "mail" not in columns:
if os.path.exists(filemails):
fLOG(
"[extract_students_mails_from_gmail_and_stores_in_folders] read addresses from ", filemails)
with open(filemails, "r", encoding="utf8") as f:
lines = f.readlines()
emails = [li.strip("\r\t\n ") for li in lines]
else:
fLOG(
"[extract_students_mails_from_gmail_and_stores_in_folders] mine address ")
box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
box.login()
emails = grab_addresses(box, mailfolder, date, fLOG=fLOG)
box.logout()
with open(filemails, "w", encoding="utf8") as f:
f.write("\n".join(emails))
else:
# nothing to do mail already present
emails = set(df[columns["mail"]])
# we remove empty names
df = df[~df[columns["name"]].isnull()].copy()
if process_name:
df[columns["name"]] = df[columns["name"]].apply(
lambda f: process_name(f))
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] create groups folders in", folder)
proj = ProjectsRepository(folder, fLOG=fLOG)
proj = ProjectsRepository.create_folders_from_dataframe(df, folder,
col_subject=columns[
"subject"], fLOG=fLOG, col_group=columns["group"],
col_student=columns[
"name"], email_function=emails, skip_if_nomail=False,
col_mail=columns["mail"], must_have_email=True, skip_names=skip_names)
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] nb groups", len(
proj.Groups))
# gathers mails
email_renderer = EmailMessageRenderer(tmpl=template_email_html_short,
fLOG=fLOG)
renderer = EmailMessageListRenderer(title=title, email_renderer=email_renderer,
fLOG=fLOG)
box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
box.login()
proj.dump_group_mails(renderer, group=None, mailbox=box, subfolder=mailfolder,
date=date, overwrite=False, skip_if_empty=True)
box.logout()
# cleaning files
for group in proj.Groups:
files = list(proj.enumerate_group_files(group))
att = [_ for _ in files if ".html" in _]
if len(att) <= 1:
fLOG(
"[extract_students_mails_from_gmail_and_stores_in_folders] remove '{}'".format(group))
proj.remove_group(group)
# unzip files and convert notebooks
for group in proj.Groups:
proj.unzip_convert(group)
summary = os.path.join(folder, "index.html")
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] write summary '{}'".format(summary))
if os.path.exists(summary):
os.remove(summary)
proj.write_run_command()
proj.write_summary(nolink_if=nolink_if)
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] zip everything in", zipfilename)
if os.path.exists(zipfilename):
os.remove(zipfilename)
proj.zip_group(None, zipfilename,
addition=["index.html", "mail_style.css", "emails.txt"])
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] encrypt the zip file in '{}'.".format(
zipfilenameenc))
if os.path.exists(zipfilenameenc):
os.remove(zipfilenameenc)
encrypt_stream(zipencpwd, zipfilename, zipfilenameenc, chunksize=2 ** 30)
return proj
box.logout()
################
# write summary
if True:
fLOG("summary")
index = os.path.join(dest_folder, "index.html")
if os.path.exists(index):
os.remove(index)
proj.write_summary()
#################
# zip everything
if True:
if os.path.exists(filename_zip):
os.remove(filename_zip)
proj.zip_group(None, filename_zip,
addition=[index, os.path.join(dest_folder, "mail_style.css"),
filename_excel, filename_mails])
############
# encryption
if True:
fLOG("encryption")
enc = filename_zip.replace(".zip", ".enc")
encrypt_stream(password, filename_zip, enc, chunksize=2**30)
def extract_students_mails_from_gmail_and_stores_in_folders(folder=".", filemails="emails.txt",
user=None, pwd=None, server="imap.gmail.com",
mailfolder=[
"ensae/ENSAE_2016_3A"],
date="1-Jan-2016", zipfilename="projet_3A_2016.zip",
zipencpwd=b"sixteenbyteskeys", dataframe=None,
columns={
"name": "nom_prenom", "group": "groupe", "subject": "sujet"},
skip_names=None, process_name=None,
title="List of emails", nolink_if=None, fLOG=fLOG):
"""
The scenario is the following:
* You are the teacher.
* Students started their projects at date *t*.
* They can work alone or by group.
* They send mails, you reply.
* Their address mail follows the convention: ``<first name>.<last name>@anything``
so it is to associate a mail address to a student name.
* You move every mail you received in a separate folder in your inbox.
* Sometime, you send a mail to everybody.
* Finally they send their project with attachments.
* You want to store everything (mails and attachements) in folders, one per group.
* You want a summary of what was received.
* You want to build a zip file to share their work with others teachers.
* You want to update the folder if a new mail was sent.
This function looks into a folder of your inbox and grabs every mails and
attachements from a groups of students.
@param folder where to store the results
@param filemails files used to store students address,
the operation is done once, remove the file
to force the function to rebuild the information.
@param user user of the gmail inbox
@param pwd password of the gmail inbox
@param server gmail server, it should be ``"imap.gmail.com"``,
it works with others mail servers using the *IMAP* protocol
@param mailfolder folder in your inbox to look into,
there can be several
@param date when to start looking (do not change the format,
look at the default value)
@param zipfilename name of the zip file to create
@param zipencpwd the zip file is also encrypted for a safer share with this key
and function `encrypt_stream <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/
pyquickhelper/filehelper/encryption.html#pyquickhelper.filehelper.encryption.encrypt_stream>`_.
@param dataframe dataframe which contains the definition of students groups
@param columns columns the function will look into, students names, group definition
(a unique number for all students in the same group), subject
@param skip_names list of names to skip
@param process_name to operate a transformation before matching students names with
their emails
@param title each group folder contains a html file connecting them,
this is its title
@param nolink_if The summary extracts links from url, it skips the urls which
contains on the substrings included in that list (None to use a default set)
@param fLOG logging function
@return @see cl ProjectsRepository
By default, Gmail does not let you programmatically access you own inbox,
you need to modify your gmail parameters to let this function do so.
"""
folder = os.path.abspath(".")
filemails = os.path.join(folder, filemails)
zipfilename = os.path.join(folder, zipfilename)
zipfilenameenc = zipfilename + ".enc"
# load the groups
if isinstance(dataframe, pandas.DataFrame):
df = dataframe
elif dataframe.endswith("xlsx"):
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] read dataframe", dataframe)
df = pandas.read_excel(dataframe)
else:
df = pandas.read_csv(dataframe, sep="\t", encoding="utf8")
# check mails
if "mail" not in columns:
if os.path.exists(filemails):
fLOG(
"[extract_students_mails_from_gmail_and_stores_in_folders] read addresses from ", filemails)
with open(filemails, "r", encoding="utf8") as f:
lines = f.readlines()
emails = [l.strip("\r\t\n ") for l in lines]
else:
fLOG(
"[extract_students_mails_from_gmail_and_stores_in_folders] mine address ")
box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
box.login()
emails = grab_addresses(box, mailfolder, date, fLOG=fLOG)
box.logout()
with open(filemails, "w", encoding="utf8") as f:
f.write("\n".join(emails))
else:
# nothing to do mail already present
emails = set(df[columns["mail"]])
# we remove empty names
df = df[~df[columns["name"]].isnull()].copy()
if process_name:
df[columns["name"]] = df[columns["name"]].apply(
lambda f: process_name(f))
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] create groups folders in", folder)
proj = ProjectsRepository(folder, fLOG=fLOG)
proj = ProjectsRepository.create_folders_from_dataframe(df, folder,
col_subject=columns[
"subject"], fLOG=fLOG, col_group=columns["group"],
col_student=columns[
"name"], email_function=emails, skip_if_nomail=False,
col_mail=columns["mail"], must_have_email=True, skip_names=skip_names)
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] nb groups", len(
proj.Groups))
# gathers mails
email_renderer = EmailMessageRenderer(tmpl=template_email_html_short,
fLOG=fLOG)
renderer = EmailMessageListRenderer(title=title, email_renderer=email_renderer,
fLOG=fLOG)
box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
box.login()
proj.dump_group_mails(renderer, group=None, mailbox=box, subfolder=mailfolder,
date=date, overwrite=False, skip_if_empty=True)
box.logout()
# cleaning files
for group in proj.Groups:
files = list(proj.enumerate_group_files(group))
att = [_ for _ in files if ".html" in _]
if len(att) <= 1:
fLOG(
"[extract_students_mails_from_gmail_and_stores_in_folders] remove ", group)
proj.remove_group(group)
# unzip files and convert notebooks
for group in proj.Groups:
proj.unzip_convert(group)
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] summary ")
summary = os.path.join(folder, "index.html")
if os.path.exists(summary):
os.remove(summary)
proj.write_summary(nolink_if=nolink_if)
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] zip everything in", zipfilename)
if os.path.exists(zipfilename):
os.remove(zipfilename)
proj.zip_group(None, zipfilename,
addition=["index.html", "mail_style.css", "emails.txt"])
fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] encrypt the zip file in", zipfilenameenc)
if os.path.exists(zipfilenameenc):
os.remove(zipfilenameenc)
encrypt_stream(zipencpwd, zipfilename, zipfilenameenc, chunksize=2 ** 30)
return proj
