Exemple #1
0
    def test_tohtml(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")
        data = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))
        mesf = os.path.join(data, "message.pickle")

        if "anaconda" in sys.executable.lower() or "anaconda" in sys.base_prefix.lower():
            # issue with Anaconda about module pickle
            # pickle has issues when getting a file saved by pickle on another
            # distribution
            return

        with open(mesf, "rb") as f:
            try:
                import pymmails
                assert pymmails is not None
                obj = pickle.load(f)
            except ImportError:
                path = os.path.normpath(
                    os.path.abspath(
                        os.path.join(
                            os.path.split(__file__)[0],
                            "..",
                            "..",
                            "src")))
                if path not in sys.path:
                    sys.path.append(path)
                import pymmails
                assert pymmails is not None
                obj = pickle.load(f)
                del sys.path[-1]

        temp = get_temp_folder(__file__, "temp_dump_html")
        render = EmailMessageRenderer()
        ff = obj.dump(render, location=temp, fLOG=fLOG)
        render.flush()
        fLOG("ff=", type(ff), ff)
        with open(ff[0][0], "r", encoding="utf8") as f:
            content = f.read()
        if '<link rel="stylesheet" type="text/css" href="mail_style.css">' not in content:
            raise Exception(content)
        if "d_2014-12-15_p_yyyyy-matthieu-at-xxxxx-xxx_ii_48bdbc9f9fd180ab917cec5bed8ca529.html" not in ff[0][0]:
            raise Exception(ff[0][0])
        if "<h1>2014/12/15 - projet 3A - élément logiciel</h1>" not in content:
            raise Exception(content)
Exemple #2
0
    def test_mailbox_dump(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor():
            warnings.warn("requires a password")
            return
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', DeprecationWarning)
            import keyring
        code = keyring.get_password("sdut", "pymmails")
        temp = get_temp_folder(__file__, "temp_dump")
        box = MailBoxImap("unittest.sdpython", code,
                          "imap.gmail.com", ssl=True, fLOG=fLOG)
        render = EmailMessageRenderer()
        box.login()
        mails = box.enumerate_mails_in_folder("test4", date="1-Jan-2016")
        for mail in mails:
            mail.dump(render, location=temp, fLOG=fLOG)
        render.flush()
        box.logout()
Exemple #3
0
    def write_summary(self,
                      renderer=None,
                      link="index_mails.html",
                      outfile="index.html",
                      title="summary",
                      nolink_if=None):
        """
        Produces a summary and uses a :epkg:`Jinja2` template.

        @param      renderer    instance of `EmailMessageRenderer
                                <http://www.xavierdupre.fr/app/pymmails/
                                helpsphinx//pymmails/render/email_message_renderer.html>`_),
                                can be None
        @param      link        look for this file in each folder
        @param      outfile     output file
        @param      nolink_if   link containing those strings will be removed (if None, a default set will be assigned)
        @param      title       title
        @return                 summary

        The current default template is::

        .. runpython::

            from ensae_teaching_cs.automation_students.projects_repository import _default_template_summary_template
            print(_default_template_summary)
        """
        if nolink_if is None:
            nolink_if = ProjectsRepository._known_strings

        def filter_in(url):
            if "\n" in url or "\r" in url or "\t" in url:
                return False
            if url.endswith("&quot;"):
                return False
            for _ in nolink_if:
                if _ in url:
                    return False
            if ".ipynb_checkpoints" in url:
                return False
            return True

        def clean_url(u):
            u = u.replace("&#43;", "+").strip(".#'/ \r\n\t ")
            if u.endswith("&nbsp;"):
                u = u[:-6]
            return u

        def url_domain_name(url):
            r = urlparse(url)
            domain = r.netloc
            name = [_ for _ in url.split("/") if _]
            last = name[-1] if len(name) > 0 else domain
            if len(last) > 30:
                last = last[-30:]
            return domain, clean_url(last)

        def format_size(s):
            if s <= 2**11:
                return "{0} bytes".format(s)
            elif s <= 2**21:
                return "{0} Kb".format(s // (2**10))
            elif s <= 2**31:
                return "{0} Mb".format(s // (2**20))
            else:
                return "{0} Gb".format(s // (2**30))

        groups = []
        for group in self.Groups:
            lp = os.path.join(self.get_group_location(group), link)
            if os.path.exists(lp):
                c = os.path.relpath(lp, self._location), group
            else:
                c = "file:///{0}".format(group), group
            nb_files = 0
            size = 0
            atts = []
            emails = []
            links = []
            created_files = []
            for name in self.enumerate_group_files(group):
                if name.endswith(".metadata"):
                    continue
                loc = self.get_group_location(group)
                nb_files += 1
                tn = name
                size += os.stat(tn).st_size
                folder = os.path.split(name)[0]
                splf = folder.replace("\\", "/").split("/")
                if folder.endswith("attachments"):
                    meta = name + ".metadata"
                    if os.path.exists(meta):
                        data = EmailMessage.read_metadata(meta)
                        day = data["date"].strftime("%Y-%m-%d")
                    else:
                        data = None
                        day = ""
                    atts.append((day, os.path.relpath(name,
                                                      self._location), data))
                elif "attachments" in splf:
                    rel = os.path.relpath(name, loc)
                    dest = os.path.relpath(name, self._location)
                    if rel == dest:
                        raise Exception("weird\n{0}\n{1}".format(rel, dest))
                    ssize = format_size(os.stat(name).st_size)
                    if "__MACOSX" not in rel and "__MACOSX" not in dest and \
                            ".ipynb_checkpoints" not in dest and ".ipynb_checkpoints" not in rel:
                        created_files.append((rel, dest, ssize))
                else:
                    mail = os.path.split(name)[-1]
                    res = EmailMessage.interpret_default_filename(mail)
                    if "date" in res and "uid" in res and "from" in res:
                        emails.append(
                            (res["date"], res["from"], res["uid"], res))
                        with open(os.path.join(loc, mail),
                                  "r",
                                  encoding="utf8") as f:
                            content = f.read()
                        urls = ProjectsRepository._link_regex.findall(content)
                        if urls:
                            for u in set(urls):
                                u = clean_url(u)
                                if not filter_in(u):
                                    continue
                                domain, last = url_domain_name(u)
                                links.append((res["date"], res["from"],
                                              clean_url(u), domain, last))

            # we sort
            atts.sort()
            links.sort()

            # we clean duplicated links
            mlinks = links
            links = []
            done = {}
            for date, from_, url, domain, last in mlinks:
                if url in done:
                    continue
                if "__MACOSX" in url or "__MACOSX" in last or \
                        ".ipynb_checkpoints" in last or ".ipynb_checkpoints" in url:
                    continue
                links.append((date, from_, url, domain, last))
                done[url] = True

            # we create the variable for the template
            emails = [_[-1] for _ in sorted(emails)]
            c = dict(link=c[0].replace("\\", "/"),
                     group=c[1],
                     nb=nb_files,
                     size=size,
                     attachments=atts,
                     emails=emails,
                     links=links,
                     created_files=created_files)

            groups.append(c)

        # final summary
        if renderer is None:
            tmpl = ProjectsRepository._default_template_summary
            renderer = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG)
            dof = True
        else:
            dof = False
        res = renderer.write(filename=outfile,
                             location=self.Location,
                             mail=None,
                             attachments=None,
                             groups=groups,
                             title=title,
                             len=len,
                             os=os,
                             format_size=format_size)
        if dof:
            renderer.flush()
        return res
    def write_summary(self,
                      render=None,
                      link="index_mails.html",
                      outfile="index.html",
                      title="summary",
                      nolink_if=None):
        """
        produces a summary and uses a Jinja2 template

        @param      render      instance of `EmailMessageRenderer <http://www.xavierdupre.fr/app/pymmails/helpsphinx//pymmails/render/email_message_renderer.html>`_),
                                can be None
        @param      link        look for this file in each folder
        @param      outfile     output file
        @param      nolink_if   link containing those strings will be removed (if None, a default set will be assigned)
        @return                 summary

        the current default template is::

            <?xml version="1.0" encoding="utf-8"?>
            <head>
            <meta http-equiv="content-type" content="text/html; charset=utf-8" />
            </head>
            <body>
            <html>
            <head>
            <title>{{ title }}</title>
            <link rel="stylesheet" type="text/css" href="{{ css }}">
            </head>
            <body>
            <h1>{{ title }}</h1>
            <ol type="1">
            {% for ps in groups %}
                <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i>
                    {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} -
                    last mail {{ ps["emails"][-1]["date"] }} ---
                    {{ len(ps["attachments"]) }} attachments</i></small>
                {% if len(ps["attachments"]) > 0 %}
                    <ul>
                    {% for day, att, data in ps["attachments"] %}
                        <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li>
                    {% endfor %}
                    {% for date, from_, url, domain, last in ps["links"] %}
                        <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li>
                    {% endfor %}
                    </ul>
                {% endif %}
                {% if len(ps["created_files"]) > 0 %}
                    <ul>
                    {% for name, relpath, size in ps["created_files"] %}
                        <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li>
                    {% endfor %}
                    </ul>
                {% endif %}
                </li>
            {% endfor %}
            </ol>
            </body>
            </html>

        """
        if nolink_if is None:
            nolink_if = ProjectsRepository._known_strings

        def filter_in(url):
            if "\n" in url or "\r" in u or "\t" in u:
                return False
            if url.endswith("&quot;"):
                return False
            for _ in nolink_if:
                if _ in url:
                    return False
            return True

        def clean_url(u):
            u = u.replace("&#43;", "+").strip(".#'/ \r\n\t ")
            if u.endswith("&nbsp;"):
                u = u[:-6]
            return u

        def url_domain_name(url):
            r = urlparse(url)
            domain = r.netloc
            name = [_ for _ in url.split("/") if _]
            last = name[-1] if len(name) > 0 else domain
            if len(last) > 30:
                last = last[-30:]
            return domain, clean_url(last)

        def format_size(s):
            if s <= 2**11:
                return "{0} bytes".format(s)
            elif s <= 2**21:
                return "{0} Kb".format(s // (2**10))
            elif s <= 2**31:
                return "{0} Mb".format(s // (2**20))
            else:
                return "{0} Gb".format(s // (2**30))

        groups = []
        for group in self.Groups:
            lp = os.path.join(self.get_group_location(group), link)
            if os.path.exists(lp):
                c = os.path.relpath(lp, self._location), group
            else:
                c = "file:///{0}".format(group), group
            nb_files = 0
            size = 0
            atts = []
            emails = []
            links = []
            created_files = []
            for name in self.enumerate_group_files(group):
                if name.endswith(".metadata"):
                    continue
                loc = self.get_group_location(group)
                nb_files += 1
                tn = name
                size += os.stat(tn).st_size
                folder = os.path.split(name)[0]
                splf = folder.replace("\\", "/").split("/")
                if folder.endswith("attachments"):
                    meta = name + ".metadata"
                    if os.path.exists(meta):
                        data = EmailMessage.read_metadata(meta)
                        day = data["date"].strftime("%Y-%m-%d")
                    else:
                        data = None
                        day = ""
                    atts.append((day, os.path.relpath(name,
                                                      self._location), data))
                elif "attachments" in splf:
                    rel = os.path.relpath(name, loc)
                    dest = os.path.relpath(name, self._location)
                    if rel == dest:
                        raise Exception("weird\n{0}\n{1}".format(rel, dest))
                    ssize = format_size(os.stat(name).st_size)
                    created_files.append((rel, dest, ssize))
                else:
                    mail = os.path.split(name)[-1]
                    res = EmailMessage.interpret_default_filename(mail)
                    if "date" in res and "uid" in res and "from" in res:
                        emails.append(
                            (res["date"], res["from"], res["uid"], res))
                        with open(os.path.join(loc, mail),
                                  "r",
                                  encoding="utf8") as f:
                            content = f.read()
                        urls = ProjectsRepository._link_regex.findall(content)
                        if urls:
                            for u in set(urls):
                                u = clean_url(u)
                                if not filter_in(u):
                                    continue
                                domain, last = url_domain_name(u)
                                links.append((res["date"], res["from"],
                                              clean_url(u), domain, last))

            # we sort
            atts.sort()
            links.sort()

            # we clean dupicated links
            mlinks = links
            links = []
            done = {}
            for date, from_, url, domain, last in mlinks:
                if url in done:
                    continue
                links.append((date, from_, url, domain, last))
                done[url] = True

            # we create the variable for the template
            emails = [_[-1] for _ in sorted(emails)]
            c = dict(link=c[0].replace("\\", "/"),
                     group=c[1],
                     nb=nb_files,
                     size=size,
                     attachments=atts,
                     emails=emails,
                     links=links,
                     created_files=created_files)

            groups.append(c)

        if render is None:
            tmpl = """<?xml version="1.0" encoding="utf-8"?>
                    <head>
                    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
                    </head>
                    <body>
                    <html>
                    <head>
                    <title>{{ title }}</title>
                    <link rel="stylesheet" type="text/css" href="{{ css }}">
                    </head>
                    <body>
                    <h1>{{ title }}</h1>
                    <ol type="1">
                    {% for ps in groups %}
                        <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i>
                            {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} -
                            last mail {{ ps["emails"][-1]["date"] }} ---
                            {{ len(ps["attachments"]) }} attachments</i></small>
                        {% if len(ps["attachments"]) > 0 %}
                            <ul>
                            {% for day, att, data in ps["attachments"] %}
                                <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li>
                            {% endfor %}
                            {% for date, from_, url, domain, last in ps["links"] %}
                                <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li>
                            {% endfor %}
                            </ul>
                        {% endif %}
                        {% if len(ps["created_files"]) > 0 %}
                            <ul>
                            {% for name, relpath, size in ps["created_files"] %}
                                <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li>
                            {% endfor %}
                            </ul>
                        {% endif %}
                        </li>
                    {% endfor %}
                    </ol>
                    </body>
                    </html>
                    """.replace("                    ", "")
            render = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG)
            dof = True
        else:
            dof = False
        res = render.write(filename=outfile,
                           location=self.Location,
                           mail=None,
                           attachments=None,
                           groups=groups,
                           title=title,
                           len=len,
                           os=os,
                           format_size=format_size)
        if dof:
            render.flush()
        return res
    def write_summary(self, renderer=None, link="index_mails.html",
                      outfile="index.html", title="summary",
                      nolink_if=None):
        """
        Produces a summary and uses a :epkg:`Jinja2` template.

        @param      renderer    instance of `EmailMessageRenderer
                                <http://www.xavierdupre.fr/app/pymmails/
                                helpsphinx//pymmails/render/email_message_renderer.html>`_),
                                can be None
        @param      link        look for this file in each folder
        @param      outfile     output file
        @param      nolink_if   link containing those strings will be removed (if None, a default set will be assigned)
        @param      title       title
        @return                 summary

        The current default template is::

        .. runpython::

            from ensae_teaching_cs.automation_students.projects_repository import _default_template_summary_template
            print(_default_template_summary)
        """
        if nolink_if is None:
            nolink_if = ProjectsRepository._known_strings

        def filter_in(url):
            if "\n" in url or "\r" in url or "\t" in url:
                return False
            if url.endswith("&quot;"):
                return False
            for _ in nolink_if:
                if _ in url:
                    return False
            if ".ipynb_checkpoints" in url:
                return False
            return True

        def clean_url(u):
            u = u.replace("&#43;", "+").strip(".#'/ \r\n\t ")
            if u.endswith("&nbsp;"):
                u = u[:-6]
            return u

        def url_domain_name(url):
            r = urlparse(url)
            domain = r.netloc
            name = [_ for _ in url.split("/") if _]
            last = name[-1] if len(name) > 0 else domain
            if len(last) > 30:
                last = last[-30:]
            return domain, clean_url(last)

        def format_size(s):
            if s <= 2 ** 11:
                return "{0} bytes".format(s)
            elif s <= 2 ** 21:
                return "{0} Kb".format(s // (2 ** 10))
            elif s <= 2 ** 31:
                return "{0} Mb".format(s // (2 ** 20))
            else:
                return "{0} Gb".format(s // (2 ** 30))

        groups = []
        for group in self.Groups:
            lp = os.path.join(self.get_group_location(group), link)
            if os.path.exists(lp):
                c = os.path.relpath(lp, self._location), group
            else:
                c = "file:///{0}".format(group), group
            nb_files = 0
            size = 0
            atts = []
            emails = []
            links = []
            created_files = []
            for name in self.enumerate_group_files(group):
                if name.endswith(".metadata"):
                    continue
                loc = self.get_group_location(group)
                nb_files += 1
                tn = name
                size += os.stat(tn).st_size
                folder = os.path.split(name)[0]
                splf = folder.replace("\\", "/").split("/")
                if folder.endswith("attachments"):
                    meta = name + ".metadata"
                    if os.path.exists(meta):
                        data = EmailMessage.read_metadata(meta)
                        day = data["date"].strftime("%Y-%m-%d")
                    else:
                        data = None
                        day = ""
                    atts.append((day, os.path.relpath(
                        name, self._location), data))
                elif "attachments" in splf:
                    rel = os.path.relpath(name, loc)
                    dest = os.path.relpath(name, self._location)
                    if rel == dest:
                        raise Exception("weird\n{0}\n{1}".format(rel, dest))
                    ssize = format_size(os.stat(name).st_size)
                    if "__MACOSX" not in rel and "__MACOSX" not in dest and \
                            ".ipynb_checkpoints" not in dest and ".ipynb_checkpoints" not in rel:
                        created_files.append((rel, dest, ssize))
                else:
                    mail = os.path.split(name)[-1]
                    res = EmailMessage.interpret_default_filename(mail)
                    if "date" in res and "uid" in res and "from" in res:
                        emails.append(
                            (res["date"], res["from"], res["uid"], res))
                        with open(os.path.join(loc, mail), "r", encoding="utf8") as f:
                            content = f.read()
                        urls = ProjectsRepository._link_regex.findall(content)
                        if urls:
                            for u in set(urls):
                                u = clean_url(u)
                                if not filter_in(u):
                                    continue
                                domain, last = url_domain_name(u)
                                links.append(
                                    (res["date"], res["from"], clean_url(u), domain, last))

            # we sort
            atts.sort()
            links.sort()

            # we clean duplicated links
            mlinks = links
            links = []
            done = {}
            for date, from_, url, domain, last in mlinks:
                if url in done:
                    continue
                if "__MACOSX" in url or "__MACOSX" in last or \
                        ".ipynb_checkpoints" in last or ".ipynb_checkpoints" in url:
                    continue
                links.append((date, from_, url, domain, last))
                done[url] = True

            # we create the variable for the template
            emails = [_[-1] for _ in sorted(emails)]
            c = dict(link=c[0].replace("\\", "/"), group=c[1], nb=nb_files,
                     size=size, attachments=atts, emails=emails, links=links,
                     created_files=created_files)

            groups.append(c)

        # final summary
        if renderer is None:
            tmpl = ProjectsRepository._default_template_summary
            renderer = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG)
            dof = True
        else:
            dof = False
        res = renderer.write(filename=outfile, location=self.Location,
                             mail=None, attachments=None, groups=groups,
                             title=title, len=len, os=os,
                             format_size=format_size)
        if dof:
            renderer.flush()
        return res
Exemple #6
0
    def write_summary(self, render=None, link="index_mails.html",
                      outfile="index.html", title="summary",
                      nolink_if=None):
        """
        produces a summary and uses a Jinja2 template

        @param      render      instance of `EmailMessageRenderer <http://www.xavierdupre.fr/app/pymmails/helpsphinx//pymmails/render/email_message_renderer.html>`_),
                                can be None
        @param      link        look for this file in each folder
        @param      outfile     output file
        @param      nolink_if   link containing those strings will be removed (if None, a default set will be assigned)
        @return                 summary

        the current default template is::

            <?xml version="1.0" encoding="utf-8"?>
            <head>
            <meta http-equiv="content-type" content="text/html; charset=utf-8" />
            </head>
            <body>
            <html>
            <head>
            <title>{{ title }}</title>
            <link rel="stylesheet" type="text/css" href="{{ css }}">
            </head>
            <body>
            <h1>{{ title }}</h1>
            <ol type="1">
            {% for ps in groups %}
                <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i>
                    {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} -
                    last mail {{ ps["emails"][-1]["date"] }} ---
                    {{ len(ps["attachments"]) }} attachments</i></small>
                {% if len(ps["attachments"]) > 0 %}
                    <ul>
                    {% for day, att, data in ps["attachments"] %}
                        <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li>
                    {% endfor %}
                    {% for date, from_, url, domain, last in ps["links"] %}
                        <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li>
                    {% endfor %}
                    </ul>
                {% endif %}
                {% if len(ps["created_files"]) > 0 %}
                    <ul>
                    {% for name, relpath, size in ps["created_files"] %}
                        <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li>
                    {% endfor %}
                    </ul>
                {% endif %}
                </li>
            {% endfor %}
            </ol>
            </body>
            </html>

        """
        if nolink_if is None:
            nolink_if = ProjectsRepository._known_strings

        def filter_in(url):
            if "\n" in url or "\r" in u or "\t" in u:
                return False
            if url.endswith("&quot;"):
                return False
            for _ in nolink_if:
                if _ in url:
                    return False
            return True

        def clean_url(u):
            u = u.replace("&#43;", "+").strip(".#'/ \r\n\t ")
            if u.endswith("&nbsp;"):
                u = u[:-6]
            return u

        def url_domain_name(url):
            r = urlparse(url)
            domain = r.netloc
            name = [_ for _ in url.split("/") if _]
            last = name[-1] if len(name) > 0 else domain
            if len(last) > 30:
                last = last[-30:]
            return domain, clean_url(last)

        def format_size(s):
            if s <= 2 ** 11:
                return "{0} bytes".format(s)
            elif s <= 2 ** 21:
                return "{0} Kb".format(s // (2 ** 10))
            elif s <= 2 ** 31:
                return "{0} Mb".format(s // (2 ** 20))
            else:
                return "{0} Gb".format(s // (2 ** 30))

        groups = []
        for group in self.Groups:
            lp = os.path.join(self.get_group_location(group), link)
            if os.path.exists(lp):
                c = os.path.relpath(lp, self._location), group
            else:
                c = "file:///{0}".format(group), group
            nb_files = 0
            size = 0
            atts = []
            emails = []
            links = []
            created_files = []
            for name in self.enumerate_group_files(group):
                if name.endswith(".metadata"):
                    continue
                loc = self.get_group_location(group)
                nb_files += 1
                tn = name
                size += os.stat(tn).st_size
                folder = os.path.split(name)[0]
                splf = folder.replace("\\", "/").split("/")
                if folder.endswith("attachments"):
                    meta = name + ".metadata"
                    if os.path.exists(meta):
                        data = EmailMessage.read_metadata(meta)
                        day = data["date"].strftime("%Y-%m-%d")
                    else:
                        data = None
                        day = ""
                    atts.append((day, os.path.relpath(
                        name, self._location), data))
                elif "attachments" in splf:
                    rel = os.path.relpath(name, loc)
                    dest = os.path.relpath(name, self._location)
                    if rel == dest:
                        raise Exception("weird\n{0}\n{1}".format(rel, dest))
                    ssize = format_size(os.stat(name).st_size)
                    created_files.append((rel, dest, ssize))
                else:
                    mail = os.path.split(name)[-1]
                    res = EmailMessage.interpret_default_filename(mail)
                    if "date" in res and "uid" in res and "from" in res:
                        emails.append(
                            (res["date"], res["from"], res["uid"], res))
                        with open(os.path.join(loc, mail), "r", encoding="utf8") as f:
                            content = f.read()
                        urls = ProjectsRepository._link_regex.findall(content)
                        if urls:
                            for u in set(urls):
                                u = clean_url(u)
                                if not filter_in(u):
                                    continue
                                domain, last = url_domain_name(u)
                                links.append(
                                    (res["date"], res["from"], clean_url(u), domain, last))

            # we sort
            atts.sort()
            links.sort()

            # we clean dupicated links
            mlinks = links
            links = []
            done = {}
            for date, from_, url, domain, last in mlinks:
                if url in done:
                    continue
                links.append((date, from_, url, domain, last))
                done[url] = True

            # we create the variable for the template
            emails = [_[-1] for _ in sorted(emails)]
            c = dict(link=c[0].replace("\\", "/"), group=c[1], nb=nb_files,
                     size=size, attachments=atts, emails=emails, links=links,
                     created_files=created_files)

            groups.append(c)

        if render is None:
            tmpl = """<?xml version="1.0" encoding="utf-8"?>
                    <head>
                    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
                    </head>
                    <body>
                    <html>
                    <head>
                    <title>{{ title }}</title>
                    <link rel="stylesheet" type="text/css" href="{{ css }}">
                    </head>
                    <body>
                    <h1>{{ title }}</h1>
                    <ol type="1">
                    {% for ps in groups %}
                        <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i>
                            {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} -
                            last mail {{ ps["emails"][-1]["date"] }} ---
                            {{ len(ps["attachments"]) }} attachments</i></small>
                        {% if len(ps["attachments"]) > 0 %}
                            <ul>
                            {% for day, att, data in ps["attachments"] %}
                                <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li>
                            {% endfor %}
                            {% for date, from_, url, domain, last in ps["links"] %}
                                <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li>
                            {% endfor %}
                            </ul>
                        {% endif %}
                        {% if len(ps["created_files"]) > 0 %}
                            <ul>
                            {% for name, relpath, size in ps["created_files"] %}
                                <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li>
                            {% endfor %}
                            </ul>
                        {% endif %}
                        </li>
                    {% endfor %}
                    </ol>
                    </body>
                    </html>
                    """.replace("                    ", "")
            render = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG)
            dof = True
        else:
            dof = False
        res = render.write(filename=outfile, location=self.Location,
                           mail=None, attachments=None, groups=groups,
                           title=title, len=len, os=os,
                           format_size=format_size)
        if dof:
            render.flush()
        return res