def test_box_mock_render(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        data = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))

        temp = get_temp_folder(__file__, "temp_render_mock_list_mail")
        box = MailBoxMock(data, b"unittestunittest", fLOG)
        box.login()
        folders = box.folders()
        assert len(folders) == 1
        fLOG(folders)
        mails = list(box.enumerate_mails_in_folder("trav"))
        box.logout()

        email_render = EmailMessageRenderer()

        def tempf(message, location, prev_mail, next_mail):
            email_render.render(location, message, None,
                                file_css="mail_style.css",
                                prev_mail=prev_mail, next_mail=next_mail)
            return ""

        mails = list((m, tempf) for m in mails)
        render = EmailMessageListRenderer(
            title="list of mails", email_renderer=email_render, fLOG=fLOG)
        res = render.render(iter=mails, location=temp)
        render.flush()
        # fLOG(res[0])
        exp = ('<a href="d_2015-08-01_p_noreply-at-voyages-sncf-com_ii_8de6a63addb7c03407bc6f0caabd967e.html">' +
               '2015/08/01 -\n Voyages-sncf.com</a>')
        if exp not in res[0]:
            raise Exception(res[0])
    def test_box_mock_write(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        data = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))

        temp = get_temp_folder(__file__, "temp_write_mock_list_mail")
        box = MailBoxMock(data, b"unittestunittest", fLOG)
        box.login()
        folders = box.folders()
        assert len(folders) == 1
        fLOG(folders)
        mails = list(box.enumerate_mails_in_folder("trav"))
        box.logout()

        email_render = EmailMessageRenderer()
        render = EmailMessageListRenderer(
            title="list of mails", email_renderer=email_render, fLOG=fLOG)
        res = render.write(iter=mails, location=temp, filename="essai.html")
        render.flush()

        with open(res[0], "r", encoding="utf8") as f:
            content = f.read()
        exp = ('<a href="d_2015-12-20_p_noreply-at-voyages-sncf-com_ii_1bb6fa70421145bed927e00c5e292277.html">' +
               '2015/12/20 -\n Voyages-sncf.com</a>')
        if exp not in content:
            raise Exception(content)
        if 'list of mails</h1>' not in content:
            raise Exception(content)
        allfiles = render.BufferWrite.listfiles()
        assert len(allfiles) > 0

        allfiles.sort()
        with open(allfiles[0], "r", encoding="utf8") as f:
            content = f.read()
        if '<a href="d_2015-08-01_p_noreply-at-voyages-sncf-com_ii_8de6a63addb7c03407bc6f0caabd967e.html">&lt;--</a>' not in content:
            raise Exception(content)
        fLOG=fLOG,
        col_group="groupe",
        col_student="nom_prenom",
        email_function=emails,
        skip_if_nomail=False,
        must_have_email=True)
fLOG("nb groups", len(proj.Groups))

#############
# dump mails

if do_mail:
    email_render = EmailMessageRenderer(tmpl=template_email_html_short,
                                        fLOG=fLOG)
    render = EmailMessageListRenderer(title="list of mails",
                                      email_renderer=email_render,
                                      fLOG=fLOG)

    box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
    box.login()
    mails = proj.dump_group_mails(render,
                                  group=None,
                                  mailbox=box,
                                  subfolder=mailfolder,
                                  date=date,
                                  overwrite=False,
                                  convert_files=True)
    box.logout()

################
# write summary
Beispiel #4
0
def extract_students_mails_from_gmail_and_stores_in_folders(folder=".", filemails="emails.txt",
                                                            user=None, pwd=None, server="imap.gmail.com",
                                                            mailfolder=[
                                                                "ensae/ENSAE_2016_3A"],
                                                            date="1-Jan-2016", zipfilename="projet_3A_2016.zip",
                                                            zipencpwd=b"sixteenbyteskeys", dataframe=None,
                                                            columns={
                                                                "name": "nom_prenom", "group": "groupe", "subject": "sujet"},
                                                            skip_names=None, process_name=None,
                                                            title="List of emails", nolink_if=None, fLOG=fLOG):
    """
    The scenario is the following:

    * You are the teacher.
    * Students started their projects at date *t*.
    * They can work alone or by group.
    * They send mails, you reply.
    * Their address mail follows the convention: ``<first name>.<last name>@anything``
      so it is to associate a mail address to a student name.
    * You move every mail you received in a separate folder in your inbox.
    * Sometime, you send a mail to everybody.
    * Finally they send their project with attachments.
    * You want to store everything (mails and attachements) in folders, one per group.
    * You want a summary of what was received.
    * You want to build a zip file to share their work with others teachers.
    * You want to update the folder if a new mail was sent.

    This function looks into a folder of your inbox and grabs every mails and
    attachements from a groups of students.

    @param      folder              where to store the results
    @param      filemails           files used to store students address,
                                    the operation is done once, remove the file
                                    to force the function to rebuild the information.
    @param      user                user of the gmail inbox
    @param      pwd                 password of the gmail inbox
    @param      server              gmail server, it should be ``"imap.gmail.com"``,
                                    it works with others mail servers using the *IMAP* protocol
    @param      mailfolder          folder in your inbox to look into,
                                    there can be several
    @param      date                when to start looking (do not change the format,
                                    look at the default value)
    @param      zipfilename         name of the zip file to create
    @param      zipencpwd           the zip file is also encrypted for a safer share with this key
                                    and function `encrypt_stream <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/
                                    pyquickhelper/filehelper/encryption.html#pyquickhelper.filehelper.encryption.encrypt_stream>`_.
    @param      dataframe           dataframe which contains the definition of students groups
    @param      columns             columns the function will look into, students names, group definition
                                    (a unique number for all students in the same group), subject
    @param      skip_names          list of names to skip
    @param      process_name        to operate a transformation before matching students names with
                                    their emails
    @param      title               each group folder contains a html file connecting them,
                                    this is its title
    @param      nolink_if           The summary extracts links from url, it skips the urls which
                                    contains on the substrings included in that list (None to use a default set)
    @param      fLOG                logging function
    @return                         @see cl ProjectsRepository

    By default, Gmail does not let you programmatically access you own inbox,
    you need to modify your gmail parameters to let this function do so.
    """
    folder = os.path.abspath(".")
    filemails = os.path.join(folder, filemails)
    zipfilename = os.path.join(folder, zipfilename)
    zipfilenameenc = zipfilename + ".enc"

    # load the groups
    if isinstance(dataframe, pandas.DataFrame):
        df = dataframe
    elif dataframe.endswith("xlsx"):
        fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] read dataframe", dataframe)
        df = pandas.read_excel(dataframe)
    else:
        df = pandas.read_csv(dataframe, sep="\t", encoding="utf8")

    # check mails
    if "mail" not in columns:
        if os.path.exists(filemails):
            fLOG(
                "[extract_students_mails_from_gmail_and_stores_in_folders] read addresses from ", filemails)
            with open(filemails, "r", encoding="utf8") as f:
                lines = f.readlines()
            emails = [li.strip("\r\t\n ") for li in lines]
        else:
            fLOG(
                "[extract_students_mails_from_gmail_and_stores_in_folders] mine address ")
            box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
            box.login()
            emails = grab_addresses(box, mailfolder, date, fLOG=fLOG)
            box.logout()

            with open(filemails, "w", encoding="utf8") as f:
                f.write("\n".join(emails))
    else:
        # nothing to do mail already present
        emails = set(df[columns["mail"]])

    # we remove empty names
    df = df[~df[columns["name"]].isnull()].copy()

    if process_name:
        df[columns["name"]] = df[columns["name"]].apply(
            lambda f: process_name(f))

    fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] create groups folders in", folder)
    proj = ProjectsRepository(folder, fLOG=fLOG)

    proj = ProjectsRepository.create_folders_from_dataframe(df, folder,
                                                            col_subject=columns[
                                                                "subject"], fLOG=fLOG, col_group=columns["group"],
                                                            col_student=columns[
                                                                "name"], email_function=emails, skip_if_nomail=False,
                                                            col_mail=columns["mail"], must_have_email=True, skip_names=skip_names)
    fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] nb groups", len(
        proj.Groups))

    # gathers mails
    email_renderer = EmailMessageRenderer(tmpl=template_email_html_short,
                                          fLOG=fLOG)
    renderer = EmailMessageListRenderer(title=title, email_renderer=email_renderer,
                                        fLOG=fLOG)

    box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
    box.login()
    proj.dump_group_mails(renderer, group=None, mailbox=box, subfolder=mailfolder,
                          date=date, overwrite=False, skip_if_empty=True)

    box.logout()

    # cleaning files
    for group in proj.Groups:
        files = list(proj.enumerate_group_files(group))
        att = [_ for _ in files if ".html" in _]
        if len(att) <= 1:
            fLOG(
                "[extract_students_mails_from_gmail_and_stores_in_folders] remove '{}'".format(group))
            proj.remove_group(group)

    # unzip files and convert notebooks
    for group in proj.Groups:
        proj.unzip_convert(group)

    summary = os.path.join(folder, "index.html")
    fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] write summary '{}'".format(summary))
    if os.path.exists(summary):
        os.remove(summary)
    proj.write_run_command()
    proj.write_summary(nolink_if=nolink_if)

    fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] zip everything in", zipfilename)
    if os.path.exists(zipfilename):
        os.remove(zipfilename)
    proj.zip_group(None, zipfilename,
                   addition=["index.html", "mail_style.css", "emails.txt"])

    fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] encrypt the zip file in '{}'.".format(
        zipfilenameenc))
    if os.path.exists(zipfilenameenc):
        os.remove(zipfilenameenc)
    encrypt_stream(zipencpwd, zipfilename, zipfilenameenc, chunksize=2 ** 30)

    return proj
Beispiel #5
0
    def test_sections(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        data = os.path.abspath(os.path.dirname(__file__))
        data = os.path.join(data, "data")
        dfile = os.path.join(data, "notes_eleves_2104_2015.xlsx")
        df = pandas.read_excel(dfile, skiprows=5, engine='openpyxl')
        df = df[df["Groupe"] != "moyenne"].copy()
        df = df[~df["Eleves"].isna()].copy()
        fLOG(df.columns)
        fLOG(df.tail())
        fLOG(df.shape)
        emails = ["*****@*****.**".lower(),
                  "*****@*****.**"]
        temp = get_temp_folder(__file__, "temp_repository")
        try:
            proj = ProjectsRepository.create_folders_from_dataframe(
                df, temp, col_subject="sujet", fLOG=fLOG, col_group=None,
                col_student="Eleves", col_mail=None, email_function=emails,
                skip_if_nomail=True)
        except ProjectsRepository.MailNotFound:
            pass

        emails = ["*****@*****.**".lower(),
                  "*****@*****.**",
                  "*****@*****.**"]

        proj = ProjectsRepository.create_folders_from_dataframe(
            df, temp, col_subject="sujet", fLOG=fLOG, col_group=None,
            col_student="Eleves", col_mail=None, email_function=emails,
            must_have_email=False)

        do_test = True
        if do_test:
            data = os.path.abspath(os.path.join(
                os.path.dirname(__file__), "data"))
            box = MailBoxMock(data, b"unittestunittest", fLOG)
            box.login()

            email_render = EmailMessageRenderer(
                tmpl=template_email_html_short, fLOG=fLOG)
            render = EmailMessageListRenderer(title="list of mails",
                                              email_renderer=email_render, fLOG=fLOG)

            mails = proj.dump_group_mails(render, group=None,
                                          mailbox=box, subfolder="trav",
                                          date=datetime.datetime(2015, 1, 9))

            box.logout()

        suivi = os.path.join(temp, "ABOUT.firstname", "suivi.rst")
        with open(suivi, "r", encoding="utf8") as f:
            content = f.read()
        self.assertIn("* mails: [email protected]", content)

        self.assertEqual(len(proj.Groups), 3)
        mails = proj.get_emails(proj.Groups[0])
        self.assertEqual(len(mails), 1)
        self.assertIn(mails[0], emails)
        fLOG("------", os.path.exists(os.path.join(temp, "mail_style.css")))
        proj.write_run_command()
        proj.write_summary()
        fLOG("------")
        files = [os.path.join(temp, "index.html"),
                 os.path.join(
                     temp, "ABOUT.firstname", "d_2015-08-01_p_noreply-at-voyages-sncf-com_ii_8de6a63addb7c03407bc6f0caabd967e.html"),
                 os.path.join(temp, "mail_style.css")]
        nb = 0
        for name in files:
            if not os.path.exists(name):
                raise FileNotFoundError(name)
            nb += 1
            with open(name, "r", encoding="utf8") as f:
                content = f.read()
            self.assertNotIn("ut_automation_students", content)
        self.assertEqual(nb, len(files))