def test_tohtml(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") data = os.path.abspath(os.path.join(os.path.dirname(__file__), "data")) mesf = os.path.join(data, "message.pickle") if "anaconda" in sys.executable.lower() or "anaconda" in sys.base_prefix.lower(): # issue with Anaconda about module pickle # pickle has issues when getting a file saved by pickle on another # distribution return with open(mesf, "rb") as f: try: import pymmails assert pymmails is not None obj = pickle.load(f) except ImportError: path = os.path.normpath( os.path.abspath( os.path.join( os.path.split(__file__)[0], "..", "..", "src"))) if path not in sys.path: sys.path.append(path) import pymmails assert pymmails is not None obj = pickle.load(f) del sys.path[-1] temp = get_temp_folder(__file__, "temp_dump_html") render = EmailMessageRenderer() ff = obj.dump(render, location=temp, fLOG=fLOG) render.flush() fLOG("ff=", type(ff), ff) with open(ff[0][0], "r", encoding="utf8") as f: content = f.read() if '<link rel="stylesheet" type="text/css" href="mail_style.css">' not in content: raise Exception(content) if "d_2014-12-15_p_yyyyy-matthieu-at-xxxxx-xxx_ii_48bdbc9f9fd180ab917cec5bed8ca529.html" not in ff[0][0]: raise Exception(ff[0][0]) if "<h1>2014/12/15 - projet 3A - élément logiciel</h1>" not in content: raise Exception(content)
def test_mailbox_dump(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") if is_travis_or_appveyor(): warnings.warn("requires a password") return with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) import keyring code = keyring.get_password("sdut", "pymmails") temp = get_temp_folder(__file__, "temp_dump") box = MailBoxImap("unittest.sdpython", code, "imap.gmail.com", ssl=True, fLOG=fLOG) render = EmailMessageRenderer() box.login() mails = box.enumerate_mails_in_folder("test4", date="1-Jan-2016") for mail in mails: mail.dump(render, location=temp, fLOG=fLOG) render.flush() box.logout()
def write_summary(self, renderer=None, link="index_mails.html", outfile="index.html", title="summary", nolink_if=None): """ Produces a summary and uses a :epkg:`Jinja2` template. @param renderer instance of `EmailMessageRenderer <http://www.xavierdupre.fr/app/pymmails/ helpsphinx//pymmails/render/email_message_renderer.html>`_), can be None @param link look for this file in each folder @param outfile output file @param nolink_if link containing those strings will be removed (if None, a default set will be assigned) @param title title @return summary The current default template is:: .. runpython:: from ensae_teaching_cs.automation_students.projects_repository import _default_template_summary_template print(_default_template_summary) """ if nolink_if is None: nolink_if = ProjectsRepository._known_strings def filter_in(url): if "\n" in url or "\r" in url or "\t" in url: return False if url.endswith("""): return False for _ in nolink_if: if _ in url: return False if ".ipynb_checkpoints" in url: return False return True def clean_url(u): u = u.replace("+", "+").strip(".#'/ \r\n\t ") if u.endswith(" "): u = u[:-6] return u def url_domain_name(url): r = urlparse(url) domain = r.netloc name = [_ for _ in url.split("/") if _] last = name[-1] if len(name) > 0 else domain if len(last) > 30: last = last[-30:] return domain, clean_url(last) def format_size(s): if s <= 2**11: return "{0} bytes".format(s) elif s <= 2**21: return "{0} Kb".format(s // (2**10)) elif s <= 2**31: return "{0} Mb".format(s // (2**20)) else: return "{0} Gb".format(s // (2**30)) groups = [] for group in self.Groups: lp = os.path.join(self.get_group_location(group), link) if os.path.exists(lp): c = os.path.relpath(lp, self._location), group else: c = "file:///{0}".format(group), group nb_files = 0 size = 0 atts = [] emails = [] links = [] created_files = [] for name in self.enumerate_group_files(group): if name.endswith(".metadata"): continue loc = self.get_group_location(group) nb_files += 1 tn = name size += os.stat(tn).st_size folder = os.path.split(name)[0] splf = folder.replace("\\", "/").split("/") if folder.endswith("attachments"): meta = name + ".metadata" if os.path.exists(meta): data = EmailMessage.read_metadata(meta) day = data["date"].strftime("%Y-%m-%d") else: data = None day = "" atts.append((day, os.path.relpath(name, self._location), data)) elif "attachments" in splf: rel = os.path.relpath(name, loc) dest = os.path.relpath(name, self._location) if rel == dest: raise Exception("weird\n{0}\n{1}".format(rel, dest)) ssize = format_size(os.stat(name).st_size) if "__MACOSX" not in rel and "__MACOSX" not in dest and \ ".ipynb_checkpoints" not in dest and ".ipynb_checkpoints" not in rel: created_files.append((rel, dest, ssize)) else: mail = os.path.split(name)[-1] res = EmailMessage.interpret_default_filename(mail) if "date" in res and "uid" in res and "from" in res: emails.append( (res["date"], res["from"], res["uid"], res)) with open(os.path.join(loc, mail), "r", encoding="utf8") as f: content = f.read() urls = ProjectsRepository._link_regex.findall(content) if urls: for u in set(urls): u = clean_url(u) if not filter_in(u): continue domain, last = url_domain_name(u) links.append((res["date"], res["from"], clean_url(u), domain, last)) # we sort atts.sort() links.sort() # we clean duplicated links mlinks = links links = [] done = {} for date, from_, url, domain, last in mlinks: if url in done: continue if "__MACOSX" in url or "__MACOSX" in last or \ ".ipynb_checkpoints" in last or ".ipynb_checkpoints" in url: continue links.append((date, from_, url, domain, last)) done[url] = True # we create the variable for the template emails = [_[-1] for _ in sorted(emails)] c = dict(link=c[0].replace("\\", "/"), group=c[1], nb=nb_files, size=size, attachments=atts, emails=emails, links=links, created_files=created_files) groups.append(c) # final summary if renderer is None: tmpl = ProjectsRepository._default_template_summary renderer = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG) dof = True else: dof = False res = renderer.write(filename=outfile, location=self.Location, mail=None, attachments=None, groups=groups, title=title, len=len, os=os, format_size=format_size) if dof: renderer.flush() return res
def write_summary(self, render=None, link="index_mails.html", outfile="index.html", title="summary", nolink_if=None): """ produces a summary and uses a Jinja2 template @param render instance of `EmailMessageRenderer <http://www.xavierdupre.fr/app/pymmails/helpsphinx//pymmails/render/email_message_renderer.html>`_), can be None @param link look for this file in each folder @param outfile output file @param nolink_if link containing those strings will be removed (if None, a default set will be assigned) @return summary the current default template is:: <?xml version="1.0" encoding="utf-8"?> <head> <meta http-equiv="content-type" content="text/html; charset=utf-8" /> </head> <body> <html> <head> <title>{{ title }}</title> <link rel="stylesheet" type="text/css" href="{{ css }}"> </head> <body> <h1>{{ title }}</h1> <ol type="1"> {% for ps in groups %} <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i> {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} - last mail {{ ps["emails"][-1]["date"] }} --- {{ len(ps["attachments"]) }} attachments</i></small> {% if len(ps["attachments"]) > 0 %} <ul> {% for day, att, data in ps["attachments"] %} <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li> {% endfor %} {% for date, from_, url, domain, last in ps["links"] %} <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li> {% endfor %} </ul> {% endif %} {% if len(ps["created_files"]) > 0 %} <ul> {% for name, relpath, size in ps["created_files"] %} <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li> {% endfor %} </ul> {% endif %} </li> {% endfor %} </ol> </body> </html> """ if nolink_if is None: nolink_if = ProjectsRepository._known_strings def filter_in(url): if "\n" in url or "\r" in u or "\t" in u: return False if url.endswith("""): return False for _ in nolink_if: if _ in url: return False return True def clean_url(u): u = u.replace("+", "+").strip(".#'/ \r\n\t ") if u.endswith(" "): u = u[:-6] return u def url_domain_name(url): r = urlparse(url) domain = r.netloc name = [_ for _ in url.split("/") if _] last = name[-1] if len(name) > 0 else domain if len(last) > 30: last = last[-30:] return domain, clean_url(last) def format_size(s): if s <= 2**11: return "{0} bytes".format(s) elif s <= 2**21: return "{0} Kb".format(s // (2**10)) elif s <= 2**31: return "{0} Mb".format(s // (2**20)) else: return "{0} Gb".format(s // (2**30)) groups = [] for group in self.Groups: lp = os.path.join(self.get_group_location(group), link) if os.path.exists(lp): c = os.path.relpath(lp, self._location), group else: c = "file:///{0}".format(group), group nb_files = 0 size = 0 atts = [] emails = [] links = [] created_files = [] for name in self.enumerate_group_files(group): if name.endswith(".metadata"): continue loc = self.get_group_location(group) nb_files += 1 tn = name size += os.stat(tn).st_size folder = os.path.split(name)[0] splf = folder.replace("\\", "/").split("/") if folder.endswith("attachments"): meta = name + ".metadata" if os.path.exists(meta): data = EmailMessage.read_metadata(meta) day = data["date"].strftime("%Y-%m-%d") else: data = None day = "" atts.append((day, os.path.relpath(name, self._location), data)) elif "attachments" in splf: rel = os.path.relpath(name, loc) dest = os.path.relpath(name, self._location) if rel == dest: raise Exception("weird\n{0}\n{1}".format(rel, dest)) ssize = format_size(os.stat(name).st_size) created_files.append((rel, dest, ssize)) else: mail = os.path.split(name)[-1] res = EmailMessage.interpret_default_filename(mail) if "date" in res and "uid" in res and "from" in res: emails.append( (res["date"], res["from"], res["uid"], res)) with open(os.path.join(loc, mail), "r", encoding="utf8") as f: content = f.read() urls = ProjectsRepository._link_regex.findall(content) if urls: for u in set(urls): u = clean_url(u) if not filter_in(u): continue domain, last = url_domain_name(u) links.append((res["date"], res["from"], clean_url(u), domain, last)) # we sort atts.sort() links.sort() # we clean dupicated links mlinks = links links = [] done = {} for date, from_, url, domain, last in mlinks: if url in done: continue links.append((date, from_, url, domain, last)) done[url] = True # we create the variable for the template emails = [_[-1] for _ in sorted(emails)] c = dict(link=c[0].replace("\\", "/"), group=c[1], nb=nb_files, size=size, attachments=atts, emails=emails, links=links, created_files=created_files) groups.append(c) if render is None: tmpl = """<?xml version="1.0" encoding="utf-8"?> <head> <meta http-equiv="content-type" content="text/html; charset=utf-8" /> </head> <body> <html> <head> <title>{{ title }}</title> <link rel="stylesheet" type="text/css" href="{{ css }}"> </head> <body> <h1>{{ title }}</h1> <ol type="1"> {% for ps in groups %} <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i> {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} - last mail {{ ps["emails"][-1]["date"] }} --- {{ len(ps["attachments"]) }} attachments</i></small> {% if len(ps["attachments"]) > 0 %} <ul> {% for day, att, data in ps["attachments"] %} <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li> {% endfor %} {% for date, from_, url, domain, last in ps["links"] %} <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li> {% endfor %} </ul> {% endif %} {% if len(ps["created_files"]) > 0 %} <ul> {% for name, relpath, size in ps["created_files"] %} <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li> {% endfor %} </ul> {% endif %} </li> {% endfor %} </ol> </body> </html> """.replace(" ", "") render = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG) dof = True else: dof = False res = render.write(filename=outfile, location=self.Location, mail=None, attachments=None, groups=groups, title=title, len=len, os=os, format_size=format_size) if dof: render.flush() return res
def write_summary(self, renderer=None, link="index_mails.html", outfile="index.html", title="summary", nolink_if=None): """ Produces a summary and uses a :epkg:`Jinja2` template. @param renderer instance of `EmailMessageRenderer <http://www.xavierdupre.fr/app/pymmails/ helpsphinx//pymmails/render/email_message_renderer.html>`_), can be None @param link look for this file in each folder @param outfile output file @param nolink_if link containing those strings will be removed (if None, a default set will be assigned) @param title title @return summary The current default template is:: .. runpython:: from ensae_teaching_cs.automation_students.projects_repository import _default_template_summary_template print(_default_template_summary) """ if nolink_if is None: nolink_if = ProjectsRepository._known_strings def filter_in(url): if "\n" in url or "\r" in url or "\t" in url: return False if url.endswith("""): return False for _ in nolink_if: if _ in url: return False if ".ipynb_checkpoints" in url: return False return True def clean_url(u): u = u.replace("+", "+").strip(".#'/ \r\n\t ") if u.endswith(" "): u = u[:-6] return u def url_domain_name(url): r = urlparse(url) domain = r.netloc name = [_ for _ in url.split("/") if _] last = name[-1] if len(name) > 0 else domain if len(last) > 30: last = last[-30:] return domain, clean_url(last) def format_size(s): if s <= 2 ** 11: return "{0} bytes".format(s) elif s <= 2 ** 21: return "{0} Kb".format(s // (2 ** 10)) elif s <= 2 ** 31: return "{0} Mb".format(s // (2 ** 20)) else: return "{0} Gb".format(s // (2 ** 30)) groups = [] for group in self.Groups: lp = os.path.join(self.get_group_location(group), link) if os.path.exists(lp): c = os.path.relpath(lp, self._location), group else: c = "file:///{0}".format(group), group nb_files = 0 size = 0 atts = [] emails = [] links = [] created_files = [] for name in self.enumerate_group_files(group): if name.endswith(".metadata"): continue loc = self.get_group_location(group) nb_files += 1 tn = name size += os.stat(tn).st_size folder = os.path.split(name)[0] splf = folder.replace("\\", "/").split("/") if folder.endswith("attachments"): meta = name + ".metadata" if os.path.exists(meta): data = EmailMessage.read_metadata(meta) day = data["date"].strftime("%Y-%m-%d") else: data = None day = "" atts.append((day, os.path.relpath( name, self._location), data)) elif "attachments" in splf: rel = os.path.relpath(name, loc) dest = os.path.relpath(name, self._location) if rel == dest: raise Exception("weird\n{0}\n{1}".format(rel, dest)) ssize = format_size(os.stat(name).st_size) if "__MACOSX" not in rel and "__MACOSX" not in dest and \ ".ipynb_checkpoints" not in dest and ".ipynb_checkpoints" not in rel: created_files.append((rel, dest, ssize)) else: mail = os.path.split(name)[-1] res = EmailMessage.interpret_default_filename(mail) if "date" in res and "uid" in res and "from" in res: emails.append( (res["date"], res["from"], res["uid"], res)) with open(os.path.join(loc, mail), "r", encoding="utf8") as f: content = f.read() urls = ProjectsRepository._link_regex.findall(content) if urls: for u in set(urls): u = clean_url(u) if not filter_in(u): continue domain, last = url_domain_name(u) links.append( (res["date"], res["from"], clean_url(u), domain, last)) # we sort atts.sort() links.sort() # we clean duplicated links mlinks = links links = [] done = {} for date, from_, url, domain, last in mlinks: if url in done: continue if "__MACOSX" in url or "__MACOSX" in last or \ ".ipynb_checkpoints" in last or ".ipynb_checkpoints" in url: continue links.append((date, from_, url, domain, last)) done[url] = True # we create the variable for the template emails = [_[-1] for _ in sorted(emails)] c = dict(link=c[0].replace("\\", "/"), group=c[1], nb=nb_files, size=size, attachments=atts, emails=emails, links=links, created_files=created_files) groups.append(c) # final summary if renderer is None: tmpl = ProjectsRepository._default_template_summary renderer = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG) dof = True else: dof = False res = renderer.write(filename=outfile, location=self.Location, mail=None, attachments=None, groups=groups, title=title, len=len, os=os, format_size=format_size) if dof: renderer.flush() return res
def write_summary(self, render=None, link="index_mails.html", outfile="index.html", title="summary", nolink_if=None): """ produces a summary and uses a Jinja2 template @param render instance of `EmailMessageRenderer <http://www.xavierdupre.fr/app/pymmails/helpsphinx//pymmails/render/email_message_renderer.html>`_), can be None @param link look for this file in each folder @param outfile output file @param nolink_if link containing those strings will be removed (if None, a default set will be assigned) @return summary the current default template is:: <?xml version="1.0" encoding="utf-8"?> <head> <meta http-equiv="content-type" content="text/html; charset=utf-8" /> </head> <body> <html> <head> <title>{{ title }}</title> <link rel="stylesheet" type="text/css" href="{{ css }}"> </head> <body> <h1>{{ title }}</h1> <ol type="1"> {% for ps in groups %} <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i> {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} - last mail {{ ps["emails"][-1]["date"] }} --- {{ len(ps["attachments"]) }} attachments</i></small> {% if len(ps["attachments"]) > 0 %} <ul> {% for day, att, data in ps["attachments"] %} <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li> {% endfor %} {% for date, from_, url, domain, last in ps["links"] %} <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li> {% endfor %} </ul> {% endif %} {% if len(ps["created_files"]) > 0 %} <ul> {% for name, relpath, size in ps["created_files"] %} <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li> {% endfor %} </ul> {% endif %} </li> {% endfor %} </ol> </body> </html> """ if nolink_if is None: nolink_if = ProjectsRepository._known_strings def filter_in(url): if "\n" in url or "\r" in u or "\t" in u: return False if url.endswith("""): return False for _ in nolink_if: if _ in url: return False return True def clean_url(u): u = u.replace("+", "+").strip(".#'/ \r\n\t ") if u.endswith(" "): u = u[:-6] return u def url_domain_name(url): r = urlparse(url) domain = r.netloc name = [_ for _ in url.split("/") if _] last = name[-1] if len(name) > 0 else domain if len(last) > 30: last = last[-30:] return domain, clean_url(last) def format_size(s): if s <= 2 ** 11: return "{0} bytes".format(s) elif s <= 2 ** 21: return "{0} Kb".format(s // (2 ** 10)) elif s <= 2 ** 31: return "{0} Mb".format(s // (2 ** 20)) else: return "{0} Gb".format(s // (2 ** 30)) groups = [] for group in self.Groups: lp = os.path.join(self.get_group_location(group), link) if os.path.exists(lp): c = os.path.relpath(lp, self._location), group else: c = "file:///{0}".format(group), group nb_files = 0 size = 0 atts = [] emails = [] links = [] created_files = [] for name in self.enumerate_group_files(group): if name.endswith(".metadata"): continue loc = self.get_group_location(group) nb_files += 1 tn = name size += os.stat(tn).st_size folder = os.path.split(name)[0] splf = folder.replace("\\", "/").split("/") if folder.endswith("attachments"): meta = name + ".metadata" if os.path.exists(meta): data = EmailMessage.read_metadata(meta) day = data["date"].strftime("%Y-%m-%d") else: data = None day = "" atts.append((day, os.path.relpath( name, self._location), data)) elif "attachments" in splf: rel = os.path.relpath(name, loc) dest = os.path.relpath(name, self._location) if rel == dest: raise Exception("weird\n{0}\n{1}".format(rel, dest)) ssize = format_size(os.stat(name).st_size) created_files.append((rel, dest, ssize)) else: mail = os.path.split(name)[-1] res = EmailMessage.interpret_default_filename(mail) if "date" in res and "uid" in res and "from" in res: emails.append( (res["date"], res["from"], res["uid"], res)) with open(os.path.join(loc, mail), "r", encoding="utf8") as f: content = f.read() urls = ProjectsRepository._link_regex.findall(content) if urls: for u in set(urls): u = clean_url(u) if not filter_in(u): continue domain, last = url_domain_name(u) links.append( (res["date"], res["from"], clean_url(u), domain, last)) # we sort atts.sort() links.sort() # we clean dupicated links mlinks = links links = [] done = {} for date, from_, url, domain, last in mlinks: if url in done: continue links.append((date, from_, url, domain, last)) done[url] = True # we create the variable for the template emails = [_[-1] for _ in sorted(emails)] c = dict(link=c[0].replace("\\", "/"), group=c[1], nb=nb_files, size=size, attachments=atts, emails=emails, links=links, created_files=created_files) groups.append(c) if render is None: tmpl = """<?xml version="1.0" encoding="utf-8"?> <head> <meta http-equiv="content-type" content="text/html; charset=utf-8" /> </head> <body> <html> <head> <title>{{ title }}</title> <link rel="stylesheet" type="text/css" href="{{ css }}"> </head> <body> <h1>{{ title }}</h1> <ol type="1"> {% for ps in groups %} <li><a href="{{ ps["link"] }}">{{ ps["group"] }}</a><small><i> {{ ps["nb"] }} files - {{ format_size(ps["size"]) }} - last mail {{ ps["emails"][-1]["date"] }} --- {{ len(ps["attachments"]) }} attachments</i></small> {% if len(ps["attachments"]) > 0 %} <ul> {% for day, att, data in ps["attachments"] %} <li>att: {{ day }} - <a href="{{ att }}">{{ os.path.split(att)[-1] }}</a></li> {% endfor %} {% for date, from_, url, domain, last in ps["links"] %} <li>link: {{ date }} <a href="{{ url }}">{{ domain }} // {{ last }}</a> from {{ from_ }}</li> {% endfor %} </ul> {% endif %} {% if len(ps["created_files"]) > 0 %} <ul> {% for name, relpath, size in ps["created_files"] %} <li>added: <a href="{{ relpath }}">{{ name }}</a> {{ size }}</li> {% endfor %} </ul> {% endif %} </li> {% endfor %} </ol> </body> </html> """.replace(" ", "") render = EmailMessageRenderer(tmpl=tmpl, fLOG=self.fLOG) dof = True else: dof = False res = render.write(filename=outfile, location=self.Location, mail=None, attachments=None, groups=groups, title=title, len=len, os=os, format_size=format_size) if dof: render.flush() return res