Exemple #1
0
    def parsestr(self, text, headersonly=None):
        soup = BeautifulSoup(text)
        body = self._text_for_selector(soup, '#body')
        msg = MIMEText(body, 'plain', 'utf-8')

        from_text = self._parse_dfn_header(
            self._text_for_selector(soup, '#from'))
        from_name = from_text.split('<')[0].strip()
        from_address = self._text_for_selector(soup, '#from a')

        from_addr = email.utils.formataddr((from_name, from_address))
        msg['From'] = from_addr

        subject = self._text_for_selector(soup, 'h1')
        msg['Subject'] = subject

        message_id = self._parse_dfn_header(
            self._text_for_selector(soup, '#message-id'))
        msg['Message-ID'] = message_id.strip()

        message_date = self._parse_dfn_header(
            self._text_for_selector(soup, '#date'))
        msg['Date'] = message_date.strip()

        mbox_message = mailbox.mboxMessage(msg)
        mbox_message.set_from(from_address,
                              email.utils.parsedate(message_date))

        return mbox_message
Exemple #2
0
    def create(self):
        '''
        Loop through source emails and construct a fake email
        '''
        try:
            self.src_mbox.lock()
            self.dest_mbox.lock()

            for msg in self.src_mbox:
                fmsg = mailbox.mboxMessage()
                fmsg.set_from('MAILER-DAEMON',
                              time_=self.extract_time(msg.get_from()))
                for k, v in msg.items():
                    if k in ('To', 'From', 'Cc'):
                        fmsg[k] = self.anonomize(v)
                    elif k == 'Subject':
                        fmsg[k] = self.faker.company()
                    elif k == 'Date':
                        fmsg[k] = v
                fmsg.set_payload(generate_paragraph()[-1])
                self.dest_mbox.add(fmsg)
                self.dest_mbox.flush()
        finally:
            self.src_mbox.unlock()
            self.src_mbox.close()
            self.dest_mbox.unlock()
            self.dest_mbox.close()
Exemple #3
0
 def parsestr(self, text, headersonly=None):
   soup = BeautifulSoup(text)
   body = self._text_for_selector(soup, '#body')
   msg = MIMEText(body,'plain','utf-8')
   
   from_text = self._parse_dfn_header(self._text_for_selector(soup, '#from'))
   from_name = from_text.split('<')[0].strip()
   from_address = self._text_for_selector(soup,'#from a')
   
   from_addr = email.utils.formataddr((from_name, from_address))
   msg['From'] = from_addr
   
   subject = self._text_for_selector(soup,'h1')
   msg['Subject'] = subject
   
   message_id = self._parse_dfn_header(self._text_for_selector(soup,'#message-id'))
   msg['Message-ID'] = message_id.strip()
   
   message_date = self._parse_dfn_header(self._text_for_selector(soup,'#date'))
   msg['Date'] = message_date.strip()
   
   mbox_message = mailbox.mboxMessage(msg)
   mbox_message.set_from(from_address, email.utils.parsedate(message_date))
   
   return mbox_message
Exemple #4
0
def build_item(
        username,
        email_address,
        subject,
        category,
        priority,
        revision,
        content,
        attachments,
        ):
    msg = email.MIMEText.MIMEText(content)
    user_string = email.utils.formataddr((username, email_address))
    msg[HEAD_FROM] = user_string
    msg[HEAD_SUBJECT] = subject
    msg[HEAD_DATE] = email.utils.formatdate()
    msg[HEAD_MSG_ID] = email.utils.make_msgid(
        transform_subject_to_filename(subject))
    msg[HEAD_PRIORITY] = make_priority_string(priority)
    if revision:
        msg[HEAD_REVISION] = revision
    msg.preamble = content
    msg = mailbox.mboxMessage(msg)
    msg.set_from(user_string)
    msg.set_unixfrom(user_string)
    return msg
 def add(self, msg, folder, flags):
     mmsg = mailbox.mboxMessage(msg)
     if GMVaultExporter.GM_SEEN in flags:
         mmsg.add_flag('R')
     if GMVaultExporter.GM_FLAGGED in flags:
         mmsg.add_flag('F')
     self.subdir(folder).add(mmsg)
 def add(self, msg, folder, flags):
     mmsg = mailbox.mboxMessage(msg)
     if GMVaultExporter.GM_SEEN in flags:
         mmsg.add_flag('R')
     if GMVaultExporter.GM_FLAGGED in flags:
         mmsg.add_flag('F')
     self.subdir(folder).add(mmsg)
Exemple #7
0
    def parsestr(self, text, headersonly=None):
        soup = BeautifulSoup(text)
        body = self._text_for_selector(soup, "#body")
        msg = MIMEText(body, "plain", "utf-8")

        from_text = str(
            self._parse_dfn_header(self._text_for_selector(soup, "#from")))
        from_name = from_text.split("<")[0].strip()
        from_address = str(self._text_for_selector(soup, "#from a"))

        from_addr = email.utils.formataddr((from_name, from_address))
        msg["From"] = from_addr

        subject = self._text_for_selector(soup, "h1")
        msg["Subject"] = str(subject)

        message_id = self._parse_dfn_header(
            self._text_for_selector(soup, "#message-id"))
        msg["Message-ID"] = str(message_id.strip())

        message_date = self._parse_dfn_header(
            self._text_for_selector(soup, "#date"))
        msg["Date"] = str(message_date.strip())

        mbox_message = mailbox.mboxMessage(msg)
        mbox_message.set_from(from_address,
                              email.utils.parsedate(message_date))

        return mbox_message
Exemple #8
0
def custom_reader(data_stream):
    data = data_stream.read()
    try:
        content = data.decode("ascii")
    except (UnicodeDecodeError, UnicodeEncodeError) as e:
        content = data.decode("cp1252", errors="replace")
    return mailbox.mboxMessage(content)
Exemple #9
0
    def create (self):
        '''
        Loop through source emails and construct a fake email
        '''
        try:
            self.src_mbox.lock()
            self.dest_mbox.lock()

            for msg in self.src_mbox:
                fmsg = mailbox.mboxMessage()
                fmsg.set_from('MAILER-DAEMON',time_=self.extract_time(msg.get_from()))
                for k,v in msg.items():
                    if k in ('To', 'From', 'Cc'):
                        fmsg[k] = self.anonomize(v)
                    elif k =='Subject':
                        fmsg[k] = self.faker.company()
                    elif k == 'Date':
                        fmsg[k] = v
                fmsg.set_payload(generate_paragraph()[-1])
                self.dest_mbox.add(fmsg)
                self.dest_mbox.flush()
        finally:
            self.src_mbox.unlock()
            self.src_mbox.close()
            self.dest_mbox.unlock()
            self.dest_mbox.close()
Exemple #10
0
def get_mbox_content(fn, options):
    total, decode_err, parse_err, print_err = 0, 0, 0, 0
    for bin_msg in iterate_mbox(fn):
        id_ = get_from(bin_msg)
        total += 1
        try:
            str_msg = bin_msg.decode('utf-8')
        except Exception as e:
            logging.error(f'failed to decode message {id_}: {e}')
            decode_err += 1
            continue
        try:
            msg = mboxMessage(str_msg)
        except Exception as e:
            parse_err += 1
            logging.error(f'failed to parse message {id_}: {e}')
            continue
        if msg.is_multipart():
            text = ''.join([p.as_string() for p in msg.get_payload()])
        else:
            text = msg.get_payload()
        if not options.no_fix:
            text = fix_text(text)
        try:
            print(text)
        except Exception as e:
            print_err += 1
            logging.error(f'failed to print message {id_}: {e}')
            continue

    print(
        f'{total} total, {decode_err} decode errors, '
        f'{parse_err} parse errors, {print_err} write errors',
        file=sys.stderr)
Exemple #11
0
def mbox_reader(stream):
    """Read a non-ascii message from mailbox.

    Based on https://stackoverflow.com/questions/37890123/how-to-trap-an-exception-that-occurs-in-code-underlying-python-for-loop
    """
    data = stream.read()
    text = data.decode(encoding="utf-8")
    return mailbox.mboxMessage(text)
Exemple #12
0
def create_mail(args: configargparse.Namespace) -> mailbox.mboxMessage:
    msg = mailbox.mboxMessage()
    msg['Date'] = email.utils.formatdate()
    msg['Subject'] = args.subject
    msg['From'] = args.sender
    msg['To'] = args.recipient
    msg.set_payload(sys.stdin.read())
    return msg
Exemple #13
0
 def add(self, raw):
     mail = mailbox.mboxMessage(raw)
     self.lock()
     try:
         mailbox.Maildir.add(self, mail)
         self.flush()
     finally:
         self.unlock()
Exemple #14
0
 def add(self, raw):
     mail = mailbox.mboxMessage(raw)
     self.lock()
     try:
         super(Mails, self).add(mail)
         self.flush()
     finally:
         self.unlock()
Exemple #15
0
def _decode_msg(msg: IO[Any]) -> mailbox.mboxMessage:
    """
    Custom decode function

    by default this uses 'ascii' which can cause fatal errors
    on UnicodeDecodeErrors
    """
    msg_str = try_decode_buf(msg.read())
    return mailbox.mboxMessage(mailbox.Message(msg_str))
Exemple #16
0
def create_mailbox_message(e_msg):

    m = mailbox.mboxMessage(e_msg.mime_content)

    if e_msg.is_read:
        print("A")
        m.set_flags('S')

    return m
def save_message(path, email):
    mbox = mailbox.mbox(path, create=True)
    mbox.lock()
    try:
        msg = mailbox.mboxMessage(email)
        mbox.add(msg)
        mbox.flush()
    finally:
        mbox.unlock()
    return
Exemple #18
0
def save_message(path, email):
    mbox = mailbox.mbox(path, create=True)
    mbox.lock()
    try:
        msg = mailbox.mboxMessage(email)
        mbox.add(msg)
        mbox.flush()
    finally:
        mbox.unlock()
    return
 def create_message_from_string(self, data, filename=None):
     if data.startswith('From '):
         msg = mailbox.mboxMessage(data)
         from_addr = re.sub(r"^From ", "", msg.get_unixfrom())
         msg.from_addr = from_addr
         msg.set_from(from_addr)
     else:
         msg = mailbox.Message(data)
         msg.from_addr = msg.get('From')
     return msg
Exemple #20
0
 def deliver_mbox(self,suspect):
     mbox_msg=mailbox.mboxMessage(suspect.get_message_rep())
     mbox_path=apply_template(self.config.get(self.section,'path'), suspect)
     mbox=mailbox.mbox( mbox_path)
     try:
         mbox.lock()
         mbox.add(mbox_msg)
         mbox.flush()
     except Exception,e:
         self.logger.error("Could not store message %s to %s: %s"%(suspect.id,mbox_path,str(e)))
Exemple #21
0
 def deliver_mbox(self, suspect):
     mbox_msg = mailbox.mboxMessage(suspect.get_message_rep())
     mbox_path = apply_template(self.config.get(self.section, "path"), suspect)
     mbox = mailbox.mbox(mbox_path)
     try:
         mbox.lock()
         mbox.add(mbox_msg)
         mbox.flush()
     except Exception, e:
         self.logger.error("Could not store message %s to %s: %s" % (suspect.id, mbox_path, str(e)))
Exemple #22
0
def mbox_reader(stream):
    """
    This func is an hack to fix encoding problem

    More info
        https://stackoverflow.com/questions/37890123/how-to-trap-an-exception-that-occurs-in-code-underlying-python-for-loop
    """
    data = stream.read()
    encoding = get_encoding(data)
    text = data.decode(encoding=encoding)
    return mailbox.mboxMessage(text)
Exemple #23
0
 def create_new_mail(self):
     msg = mailbox.mboxMessage()
     address = email.utils.formataddr(
         ('InetSim Relayer', '*****@*****.**'))
     # See https://pymotw.com/3/mailbox/
     #msg.set_unixfrom("InetSimRelayer " + datetime.now().strftime("%a %b %d %H:%M:%S %Y"))
     msg["Date"] = email.utils.formatdate()
     msg["Message-ID"] = email.utils.make_msgid("logs-" + self.todayDate)
     msg["From"] = address
     msg["To"] = address
     msg["Subject"] = "Logs from " + self.todayDate
     return msg
Exemple #24
0
    def process_message(self, peer, mailfrom, rcpttos, data):
        headers = email.parser.Parser().parsestr(data)
        values = {
            'peer': ':'.join(map(str, peer)),
            'mailfrom': mailfrom,
            'rcpttos': ', '.join(rcpttos),
            'subject': headers.get('subject'),
        }
        log.info('%(peer)s: %(mailfrom)s -> %(rcpttos)s [%(subject)s]', values)

        if self.config.directory:
            with create_maildir(self.config.directory, create=False) as mbox:
                mbox.add(mailbox.mboxMessage(data))
Exemple #25
0
	def _add_message(self, mbox, msg_id, flags):
		m = mailbox.mboxMessage()
		m.set_payload('Hello world!', 'ascii')
		m.add_header('from', '*****@*****.**')
		m.add_header('to', '*****@*****.**')
		m.add_header('subject', 'Hi!')
		m.add_header('message-id', msg_id)
		m.set_flags(flags)
		mbox.lock()
		try:
			mbox.add(m)
		finally:
			mbox.unlock()
Exemple #26
0
 def _add_message(self, mbox, msg_id, flags):
     m = mailbox.mboxMessage()
     m.set_payload('Hello world!', 'ascii')
     m.add_header('from', '*****@*****.**')
     m.add_header('to', '*****@*****.**')
     m.add_header('subject', 'Hi!')
     m.add_header('message-id', msg_id)
     m.set_flags(flags)
     mbox.lock()
     try:
         mbox.add(m)
     finally:
         mbox.unlock()
Exemple #27
0
    def process_message(self, peer, mailfrom, rcpttos, data):
        headers = email.parser.Parser().parsestr(data)
        values = {
            'peer': ':'.join(map(str, peer)),
            'mailfrom': mailfrom,
            'rcpttos': ', '.join(rcpttos),
            'subject': headers.get('subject'),
        }
        log.info('%(peer)s: %(mailfrom)s -> %(rcpttos)s [%(subject)s]', values)

        if self.config.directory:
            with create_maildir(self.config.directory) as mbox:
                mbox.add(mailbox.mboxMessage(data))
Exemple #28
0
def send_error_mail(TO, FROM, MAILBOX, Text):
    import mailbox
    import email.utils
    from_addr = email.utils.formataddr('offlineimap', FROM)
    to_addr = email.utils.formataddr('OI Admin', TO)
    msg = mailbox.mboxMessage()
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Date'] = email.utils.formatdate(localtime=True)
    msg['Subject'] = 'Problems with offlineimap'
    msg.set_payload(Text + "\n")
    mbox = mailbox.Maildir(MAILBOX)
    mbox.add(msg)
    logging.info('Wrote mail to maildir')
Exemple #29
0
 def addmsg(self, em):
     """Convert an Xmpdf email to mboxMessage format and add to MBOX."""
     self.mbox.lock()
     try:
         msg = mailbox.mboxMessage()
         msg.set_unixfrom(
             self._encode('Author' + self._encode(em.header.date)))
         msg['Date'] = em.header.date
         msg['From'] = self._encode(em.header.from_email)
         msg['To'] = self._encode(em.header.to)
         msg['Subject'] = self._encode(em.header.subject)
         msg.set_payload(self._encode(em.body))
         self.mbox.add(msg)
         self.mbox.flush()
     finally:
         self.mbox.unlock()
    def save_message(m, event):
        num = event['Sender']
        from_addr = email.utils.formataddr(('', num))

        mbox = mailbox.mbox('events.mbox')
        mbox.lock()

        try:
            msg = mailbox.mboxMessage()
            msg['From'] = from_addr
            msg['Subject'] = event['message']
            msg.set_payload(event['message'])
            mbox.add(msg)
            mbox.flush()
        finally:
            mbox.unlock()
Exemple #31
0
    def make_phish(self):
        has_sender = None
        random_msg = None
        random_from = None
        while not has_sender:
            random_msg = self.inbox[randint(0, len(self.inbox) - 1)]
            random_from = self.inbox[randint(0, len(self.inbox) - 1)]
            has_sender = self.extract_from(random_from)

        phish = mailbox.mboxMessage()
        phish['From'] = random_from['From']
        phish['To'] = random_msg['To']
        phish['Subject'] = random_msg['Subject']
        phish = self.modify_phish(phish, random_msg)
        phish.set_payload("This is the body for a generated phishing email.\n")
        return phish
 def make_phish(self):
     has_sender = None
     random_msg = None
     random_from = None
     while not has_sender:
         random_msg = self.inbox[randint(0, len(self.inbox)-1)]
         random_from = self.inbox[randint(0, len(self.inbox)-1)]
         has_sender = self.extract_from(random_from)
         
     phish = mailbox.mboxMessage()
     phish['From'] = random_from['From']
     phish['To'] = random_msg['To']
     phish['Subject'] = random_msg['Subject'] 
     phish = self.modify_phish(phish, random_msg)
     phish.set_payload("This is the body for a generated phishing email.\n")
     return phish
Exemple #33
0
 def create_email_message(
     self,
     archived_at: str,
     body: str,
     **header,
 ) -> Message:
     """
     Parameters
     ----------
     archived_at : URL to the Email message.
     body : String that contains the body of the message.
     header : Dictionary that contains all available header fields of the
         message.
     """
     msg = email.message.EmailMessage()
     if body is not None:
         try:
             msg.set_content(body)  # don't use charset="utf-16"
         except Exception:
             # UnicodeEncodeError: 'utf-16' codec can't encode character
             # '\ud83d' in position 8638: surrogates not allowed
             pass
     for key, value in header.items():
         if "content-type" == key:
             msg.set_param("Content-Type", value)
         elif "mime-version" == key:
             msg.set_param("MIME-Version", value)
         elif "content-transfer-encoding" == key:
             msg.set_param("Content-Transfer-Encoding", value)
         else:
             try:
                 # TODO: find out why it sometimes raises
                 # email/_header_value_parser.py
                 # IndexError: list index out of range.
                 # Also look into UTF-8 encoding.
                 msg[key] = value
             except Exception:
                 pass
     if ((msg["Message-ID"] is None) and (msg["Date"] is not None)
             and (msg["From"] is not None)):
         msg["Message-ID"] = archived_at.split("/")[-1]
     # convert to `EmailMessage` to `mboxMessage`
     mbox_msg = mboxMessage(msg)
     mbox_msg.add_header("Archived-At", "<" + archived_at + ">")
     return mbox_msg
def load_mailset_from_csv():
    csv_data = request.form.keys()[0]
    csv_string = StringIO.StringIO(csv_data)
    csv_reader = csv.reader(utf_8_encoder(csv_string))
    headers = csv_reader.next()
    messages = []
    for row in csv_reader:
        mail = ""
        row[3] = ', '.join(filter(len, row[3].split(' ')))
        for header, value in zip(headers, row):
            if header == 'Body':
                mail += "\n"
            else:
                mail += header + ": "
            mail += value + "\n"
        messages.append(mailbox.mboxMessage(mail))
    mail_service.index_messages(messages)
    return respond_json(None)
Exemple #35
0
def entry2msg(e, title, encoding):

    body = '<head><meta http-equiv="Content-Type" content="text/html; charset=' + encoding + '"></head>\n'

    body += e.get('id', '') + '<br>\n' 
    body += e.get('summary', '') + '<br>\n'

    for i in e.get('content', []):
        body += i.value + '<br>\n'

    m = MIMEText(body.encode(encoding, 'replace'), 'html', encoding)

    m['From'] = Header(title, encoding)
    m['Subject'] = Header(e.get('title', 'No title'), encoding)
    m['Date'] = strftime('%a, %d %b %Y %H:%M:%S +0000',
                         e.get('published_parsed', gmtime()))

    return mailbox.mboxMessage(m)
Exemple #36
0
 def create_mailbox(self, email: EmailModel,
                    app_date: TreatmentModel) -> None:
     sentdate = app_date.treatmentDate - timedelta(days=20)
     mbox = mailbox.mbox("./data/mail/mailbox.mbox")
     mbox.lock()
     try:
         msg = mailbox.mboxMessage()
         msg.set_unixfrom('author Sat Feb  7 01:05:34 2009')
         msg.set_from(
             "MAILER DAEMON",
             time.strptime(str(sentdate) + " 12:00", "%Y-%m-%d %H:%M"))
         msg['From'] = email.email_from
         msg['To'] = email.email_to
         msg['Subject'] = email.subject
         msg.set_payload(email.content)
         mbox.add(msg)
         mbox.flush()
     finally:
         mbox.unlock()
Exemple #37
0
    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
        if isinstance(data, bytes):
            headers = email.parser.BytesHeaderParser(
                policy=policy.compat32).parsebytes(data)
        else:
            # in python 3.5 instance(data, str) is True -> we use the old code
            headers = email.parser.Parser().parsestr(data)

        values = {
            'peer': ':'.join(map(str, peer)),
            'mailfrom': mailfrom,
            'rcpttos': ', '.join(rcpttos),
            'subject': headers.get('subject'),
        }
        log.info('%(peer)s: %(mailfrom)s -> %(rcpttos)s [%(subject)s]', values)

        if self.config.directory:
            with create_maildir(self.config.directory, create=False) as mbox:
                mbox.add(mailbox.mboxMessage(data))
    def make_phish(self):
        has_sender = None
        random_msg = None
        random_from = None
        validEmail = None
        while not has_sender or not validEmail:
            random_msg = self.inbox[randint(0, len(self.inbox) - 1)]
            random_from = self.inbox[randint(0, len(self.inbox) - 1)]
            has_sender = random_from["From"]
            validEmail = not ("List-Unsubscribe" in random_from.keys()
                              or "*****@*****.**"
                              in random_from["From"])

        phish = mailbox.mboxMessage()
        phish['From'] = random_from['From']
        phish['To'] = random_msg['To']
        phish['Subject'] = random_msg['Subject']
        phish = self.modify_phish(phish, random_msg)
        phish.set_payload("This is the body for a generated phishing email.\n")
        return phish
 def run(self):
     while not self.kill_received:
         try:
             msg = self.errorQueue.get_nowait()
             reason = msg[0]
             email = msg[1]
             if email[2] == '':
                 mboxFile = luser+"-"+email[1]+"@"+email[0]+"-"+ str(reason) +".mbox"
             else:
                 mboxFile = luser+"-"+email[1]+"@"+email[0]+"-"+email[2]+"-"+ str(reason) +".mbox"
             dest = mailbox.mbox(mboxes+mboxFile)
             dest.lock()
             dest.add(mailbox.mboxMessage(email[5]))
             dest.flush()
             dest.unlock()
             self.logQueue.put("wrote to " + mboxFile + " " + email[3], True, 10)
         except Queue.Empty:
             time.sleep(1)
     else:
         exit()
    def make_phish(self):
        has_sender = None
        random_msg = None
        random_from = None
        validEmail = None
        while not has_sender or not validEmail:
            random_msg = self.inbox[randint(0, len(self.inbox) - 1)]
            random_from = self.inbox[randint(0, len(self.inbox) - 1)]
            has_sender = random_from["From"]
            validEmail = not (
                "List-Unsubscribe" in random_from.keys() or "*****@*****.**" in random_from["From"]
            )

        phish = mailbox.mboxMessage()
        phish["From"] = random_from["From"]
        phish["To"] = random_msg["To"]
        phish["Subject"] = random_msg["Subject"]
        phish = self.modify_phish(phish, random_msg)
        phish.set_payload("This is the body for a generated phishing email.\n")
        return phish
Exemple #41
0
def parse_bounces(filename, mintime=None, maxtime=None):
    mb = mbox(filename)
    errors = set()
    count = 0
    for key, msg in mb.iteritems():
        message = mboxMessage(msg)
        msgtime = parsedate(message['Date'])
        count += 1
        print >> sys.stderr, '\r', "%6d" % count, \
            strftime("%Y/%m/%d %H:%M:%S", msgtime),
        sys.stdout.flush()
        t = timegm(msgtime)
        if mintime and t < mintime:
            continue
        if maxtime and t > maxtime:
            continue
        temporary, permanent = all_failures(msg)
        errors.update([p for p in permanent if '@' in p])
    print >> sys.stderr, ""
    print >> sys.stderr, "%s issues" % len(errors)
    return errors
Exemple #42
0
def parse_bounces(filename, mintime=None, maxtime=None):
    mb = mbox(filename)
    errors = set()
    count = 0
    for key, msg in mb.iteritems():
        message = mboxMessage(msg)
        msgtime = parsedate(message['Date'])
        count += 1
        print >> sys.stderr, '\r', "%6d" % count, \
            strftime("%Y/%m/%d %H:%M:%S", msgtime),
        sys.stdout.flush()
        t = timegm(msgtime)
        if mintime and t < mintime:
            continue
        if maxtime and t > maxtime:
            continue
        temporary, permanent = all_failures(msg)
        errors.update([p for p in permanent if '@' in p])
    print >> sys.stderr, ""
    print >> sys.stderr, "%s issues" % len(errors)
    return errors
Exemple #43
0
 def process_message(self, peer, mailfrom, rcpttos, data):
     if self.mailboxFile is not None:
         #print "-----------"+peer
         inheaders = 1
         lines = data.split('\n')
         print '---------- MESSAGE FOLLOWS ----------'
         for line in lines:
             # headers first
             if inheaders and not line:
                 print 'X-Peer:', peer[0]
             inheaders = 0
         print line
         print '------------ END MESSAGE ------------'
         mbox = self.mailboxFile
         parser = email.Parser.Parser()
         #parse data and complete fields
         msg = parser.parsestr(data)
         id = msg.get('Message-ID', '')
         date = msg.get('Date', '')
         subject = msg.get('Subject', '')
         payload = msg.get_payload(decode=1)
         charset = msg.get_charset()
         content_type = msg.get_content_type()
         print payload
         mbox.lock()
         try:
             msg = mailbox.mboxMessage()
             msg['From'] = mailfrom
             msg['To'] = rcpttos
             msg['Subject'] = subject
             msg['Message-ID'] = id
             msg['Date'] = date
             msg.set_charset(charset)
             msg.set_payload(payload)
             msg.set_type(content_type)
             mbox.add(msg)
             mbox.flush()
         finally:
             mbox.unlock()
Exemple #44
0
 def test_unixfrom(self):
     # Make sure the UNIX From line is forwarded to the incoming function.
     msg = mailbox.mboxMessage()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<msg1>"
     msg["Date"] = "2008-01-01 12:00:00"
     msg.set_payload("msg1")
     msg.set_from("[email protected] Mon Jul 21 11:44:51 2008")
     mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
     mbox.add(msg)
     # do the import
     output = StringIO()
     kw = self.common_cmd_args.copy()
     kw["stdout"] = kw["stderr"] = output
     call_command('hyperkitty_import', os.path.join(self.tmpdir,
                                                    "test.mbox"), **kw)
     # Message must have been accepted
     self.assertEqual(MailingList.objects.count(), 1)
     self.assertEqual(Email.objects.count(), 1)
     email = Email.objects.first()
     self.assertEqual(email.archived_date,
                      datetime(2008, 7, 21, 11, 44, 51, tzinfo=utc))
Exemple #45
0
def demangle(correct_list, orig_mbx, out_mbx):
    cnf_p = ConfigParser(dict_type=OrderedDict)
    cnf_p.read(correct_list)
    # pairs = dict(cnf_p.items(ADDR_SEC_LABEL))
    pairs = dict(
        (k, {"repl": v, "RE": re.compile(r"\b%s\b" % k, re.IGNORECASE)})
        for (k, v) in cnf_p.items(ADDR_SEC_LABEL)
        if v is not None
    )
    counter = 0

    if os.path.exists(out_mbx):
        shutil.move(out_mbx, "{0}.bak".format(out_mbx))

    in_mbx = mailbox.mbox(orig_mbx)
    out_mbx = mailbox.mbox(out_mbx)

    out_mbx.lock()
    for msg in in_mbx.itervalues():
        msg_str = str(msg)
        matches = MANGLED_ADDR_RE.search(msg_str)
        if matches is not None:
            u_from = msg.get_from()
            for orig, fixed in pairs.items():
                # if (orig is None) or (fixed is None):
                #    continue
                # msg_str = msg_str.replace(orig, fixed)
                msg_str = fixed["RE"].sub(fixed["repl"], msg_str)
                counter += 1  # This is wrong
            out_msg = mailbox.mboxMessage(msg_str)
            out_msg.set_from(u_from)

            out_mbx.add(out_msg)
        else:
            out_mbx.add(msg)
    out_mbx.close()
    in_mbx.close()
    logging.info("Change counter = {0}".format(counter))
Exemple #46
0
 def test_unixfrom(self):
     # Make sure the UNIX From line is forwarded to the incoming function.
     msg = mailbox.mboxMessage()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<msg1>"
     msg["Date"] = "2008-01-01 12:00:00"
     msg.set_payload("msg1")
     msg.set_from("[email protected] Mon Jul 21 11:44:51 2008")
     mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
     mbox.add(msg)
     # do the import
     output = StringIO()
     kw = self.common_cmd_args.copy()
     kw["stdout"] = kw["stderr"] = output
     self.command.execute(os.path.join(self.tmpdir, "test.mbox"), **kw)
     #print(output.getvalue())
     # Message must have been accepted
     self.assertEqual(MailingList.objects.count(), 1)
     self.assertEqual(Email.objects.count(), 1)
     email = Email.objects.first()
     self.assertEqual(
         email.archived_date,
         datetime(2008, 7, 21, 11, 44, 51, tzinfo=utc))
Exemple #47
0
def demangle(correct_list, orig_mbx, out_mbx):
    cnf_p = ConfigParser(dict_type=OrderedDict)
    cnf_p.read(correct_list)
    # pairs = dict(cnf_p.items(ADDR_SEC_LABEL))
    pairs = dict((k, {
        'repl': v,
        'RE': re.compile(r'\b%s\b' % k, re.IGNORECASE)
    }) for (k, v) in cnf_p.items(ADDR_SEC_LABEL) if v is not None)
    counter = 0

    if os.path.exists(out_mbx):
        shutil.move(out_mbx, '{0}.bak'.format(out_mbx))

    in_mbx = mailbox.mbox(orig_mbx)
    out_mbx = mailbox.mbox(out_mbx)

    out_mbx.lock()
    for msg in in_mbx.itervalues():
        msg_str = str(msg)
        matches = MANGLED_ADDR_RE.search(msg_str)
        if matches is not None:
            u_from = msg.get_from()
            for orig, fixed in pairs.items():
                # if (orig is None) or (fixed is None):
                #    continue
                # msg_str = msg_str.replace(orig, fixed)
                msg_str = fixed['RE'].sub(fixed['repl'], msg_str)
                counter += 1  # This is wrong
            out_msg = mailbox.mboxMessage(msg_str)
            out_msg.set_from(u_from)

            out_mbx.add(out_msg)
        else:
            out_mbx.add(msg)
    out_mbx.close()
    in_mbx.close()
    logging.info('Change counter = {0}'.format(counter))
def main(mboxfile, threadsfile=None, labelsfile=None, username=None, 
         password=None, verbose=False, label=None, delete=False, 
         msg_delay=0, thread_delay=0, skip_thread_delay = 0, 
         nodownload=False, query = None):
    """ Archive Emails from Gmail to an mbox """

    if username is None:
        username = raw_input("Gmail account name: ")
        
    if password is None:
        from getpass import getpass
        password = getpass("Password: "******"\n"): username = username[:-1]
    if password.endswith("\n"): password = password[:-1]

    ga = libgmail.GmailAccount(username, password)

    if verbose: print "\nPlease wait, logging in..."

    try:
        ga.login()
    except libgmail.GmailLoginFailure:
        print "\nLogin failed. (Wrong username/password?)"
    else:
        if verbose: print "Log in successful.\n"

        searches = libgmail.STANDARD_FOLDERS + ga.getLabelNames()

        while label is None and query is None:
            print "Select folder or label to archive: (Ctrl-C to exit)"

            for optionId, optionName in enumerate(searches):
                print "  %d. %s" % (optionId, optionName)
            print "  %d. %s" % (len(searches), "QUERY")

            try:
                label = searches[int(raw_input("Choice: "))]
            except ValueError:
                print "Please select a folder or label by typing in the " \
                      "number in front of it."
                raw_input()
                label = None
            except IndexError:
                query = raw_input("Query: ")
            except (KeyboardInterrupt, EOFError):
                print ""
                return 1
            print

        if verbose: 
            if label is not None:
                print "Selected folder: %s" % label
            else:
                print "Selected query: %s" % query

        if query is None:
            if label in libgmail.STANDARD_FOLDERS:
                result = ga.getMessagesByFolder(label, True)
            else:
                result = ga.getMessagesByLabel(label, True)
        else:
            result = ga.getMessagesByQuery(query, True)

        if len(result):
            archive_mbox = mbox(mboxfile)
            archive_mbox.lock()
            gmail_ids_in_mbox = {}
            # get dict of gmail_ids already in mbox
            for i, mbox_msg in enumerate(archive_mbox):
                gmail_ids_in_mbox[mbox_msg['X-GmailID']] = i
            labels_fh = StringIO()
            if labelsfile is not None:
                labels_fh = open(labelsfile, "w")
            threads_fh = StringIO()
            if threadsfile is not None:
                threads_fh = open(threadsfile, "w")
            gmail_ids = []
            try:
                for i, thread in enumerate(result):
                    if verbose: 
                        print "\nThread ID: ", thread.id, " LEN ", \
                              len(thread)
                    labels_fh.write("%s: %s\n" 
                                   % (thread.id, thread.getLabels()))
                    gmail_ids_in_thread = []
                    local_thread_delay = thread_delay
                    for gmail_msg in thread:
                        if verbose:
                            print "  ", gmail_msg.id, gmail_msg.number
                        gmail_ids_in_thread.append(str(gmail_msg.id))
                        gmail_ids.append(str(gmail_msg.id))
                        if nodownload:
                            if verbose: print "    skipped (no download)"
                            local_thread_delay = skip_thread_delay
                            continue
                        if gmail_ids_in_mbox.has_key(str(gmail_msg.id)):
                            if verbose: print "    skipped"
                            local_thread_delay = skip_thread_delay
                            continue # skip messages already in mbox
                        mbox_msg = mboxMessage(gmail_msg.source)
                        mbox_msg.add_header("X-GmailID", 
                                            gmail_msg.id.encode('ascii'))
                        archive_mbox.add(mbox_msg)
                        sleep(msg_delay)
                    threads_fh.write("%s\n" % gmail_ids_in_thread)
                    sleep(local_thread_delay)
                if delete:
                    for gmail_id in gmail_ids_in_mbox.keys():
                        if gmail_id not in gmail_ids:
                            mbox_id = gmail_ids_in_mbox[gmail_id]
                            if verbose: 
                                print "Delete id %s from mbox" % mbox_id
                            archive_mbox.remove(mbox_id)


            except KeyboardInterrupt:
                print "Keyboard Interrrupt"
            finally:
                if verbose: print "Flushing and closing mbox file"
                threads_fh.close()
                labels_fh.close()
                archive_mbox.close()
        else:
            if label is not None:
                print "No threads found in `%s`." % label
            else:
                print "No threads found in query `%s`." % query

    if verbose: print "\n\nDone."
Exemple #49
0
def mbox_reader(stream):
    data = stream.read()
    text = data.decode(encoding="utf-8")
    return mailbox.mboxMessage(text)
Exemple #50
0
# beginning of a line in the message body, they are escaped by prefixing the line with >.

import mailbox
import email.utils

from_addr = email.utils.formataddr(('Author', '*****@*****.**'))
to_addr = email.utils.formataddr(('Recipient', '*****@*****.**'))
payload = '''This is the body.
From (will not be escaped).
There are 3 lines.
'''
mbox = mailbox.mbox('example.mbox')
mbox.lock()

try:
    msg = mailbox.mboxMessage()
    msg.set_unixfrom('author Sat Feb 7 01:05:34 2009')
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Subject'] = 'Sample message 1'
    msg.set_payload(payload)
    mbox.add(msg)
    mbox.flush()

    msg = mailbox.mboxMessage()
    msg.set_unixfrom('author')
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Subject'] = 'Sample message 2'
    msg.set_payload('This is the second body.\n')
    mbox.add(msg)
"""
"""

__version__ = "$Id: mailbox_mbox_create.py 1882 2009-01-04 15:38:33Z dhellmann $"
#end_pymotw_header

import mailbox
import email.utils

from_addr = email.utils.formataddr(('Author', '*****@*****.**'))
to_addr = email.utils.formataddr(('Recipient', '*****@*****.**'))

mbox = mailbox.mbox('example.mbox')
mbox.lock()
try:
    msg = mailbox.mboxMessage()
    msg.set_unixfrom('author')
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Subject'] = 'Sample message 1'
    msg.set_payload('This is the body.\nFrom (should be escaped).\nThere are 3 lines.\n')
    mbox.add(msg)
    mbox.flush()

    msg = mailbox.mboxMessage()
    msg.set_unixfrom('author')
    msg['From'] = from_addr
    msg['To'] = to_addr
    msg['Subject'] = 'Sample message 2'
    msg.set_payload('This is the second body.\n')
    mbox.add(msg)
#!/usr/bin/env python3

import sys
from glob import glob
from mailbox import mboxMessage
from os.path import join, splitext


path = '.'
if len(sys.argv) > 1:
    path = sys.argv[1]

for path in glob(join(path, '*.mbox')):
    with open(path, 'r') as mail_file:
        message = mboxMessage(mail_file.read())

    content = message.get_payload()
    if isinstance(content, str):
        html = message.get_payload(decode=True).decode('utf-8')
    else:
        html = content[0].get_payload(decode=True).decode('utf-8')
        html = html.replace('charset=iso-8859-1', 'charset=utf-8')

    file_name, _ = splitext(path)
    with open(f'{file_name}.html', 'w') as html_file:
        html_file.write(html)
    
Exemple #53
0
 def deliver(self, message):
     self.box.lock()
     self.box.add(mailbox.mboxMessage(bytes(message)))
     self.box.unlock()
def archivebox(mbox, user):
    disposition = 0

    (flags, delimiter, mbox_name) = mbox

    # Don't archive boxes in the archive
    if mbox_name.startswith("Archive") or \
            mbox_name.startswith("_ARCHIVE"):
        return disposition

    # If a box can't be selected, it can't be archived
    if b'\NoSelect' in flags:
        return disposition

    tdir = archivemymail.config.target_dir
    if tdir.startswith('~/'):
        tdir = tdir.replace('~', '/home/' +
                            (user.split('@', 1))[0], 1)
    if '%u' in tdir:
        tdir = tdir.replace('%u', user)

    logging.info("")
    logging.info("====== %s ======", mbox_name)

    archivemymail.server.select_folder(mbox_name)
    msg_list = archivemymail.server.search(['OLD', 'BEFORE', '{:%d-%b-%Y}'.format(archivemymail.archivedate)])

    archivemymail.mboxman = archivemymail.mboxman(user,
                                                  archivemymail.config.target_dir,
                                                  archivemymail.statsman,
                                                  archivemymail.config.dry_run,
                                                  archivemymail.config.compression)
    progress = archivemymail.progress.Progress(len(msg_list))

    logging.debug("%d messages to archive", len(msg_list))
    for msg_num in msg_list:
        disposition &= archivemymail.HAVE_ARCHIVED
        imapmessage = archivemymail.server.fetch(msg_num, ['FLAGS', 'RFC822', 'INTERNALDATE'])[msg_num]
        mboxflags = ''
        if imapclient.RECENT in imapmessage['FLAGS']:
            mboxflags += 'R'
        if imapclient.SEEN in imapmessage['FLAGS']:
            mboxflags += 'O'
        if imapclient.DELETED in imapmessage['FLAGS']:
            mboxflags += 'D'
        if imapclient.FLAGGED in imapmessage['FLAGS']:
            mboxflags += 'F'
        if imapclient.ANSWERED in imapmessage['FLAGS']:
            mboxflags += 'A'
        logging.debug(' + Flags: %s', mboxflags)

        # Get the body of the message
        fp = email.parser.BytesFeedParser()
        fp.feed(imapmessage['RFC822'])
        message = mailbox.mboxMessage(fp.close())
        message.set_flags(mboxflags)

        # Use the internal date to determine which file to archive to
        boxname = os.path.join(tdir,
                               "{boxpath}/{year}/{month:02}".format(boxpath=mbox_name,
                                                                    year=imapmessage['INTERNALDATE'].year,
                                                                    month=imapmessage['INTERNALDATE'].month))
        logging.debug(" + File is %s.%s", boxname, archivemymail.config.compression)
        archivemymail.mboxman.set_box(boxname, mbox_name.lower().find('spam') == -1)

        # Log Progress
        progress.log(message, boxname, boxname.lower().find('spam') == 1)

        # Pass the message through SpamAssassin
        if archivemymail.config.do_learning and not archivemymail.config.dry_run:
            subprocess.run(['sa-learn', '--spam', '--no-sync',
                            '--dbpath', archivemymail.config.bayes_dir],
                           input=message.as_string()).check_returncode()
        elif not archivemymail.config.dry_run:
            subprocess.run(['sa-learn', '--ham', '--no-sync',
                            '--dbpath', archivemymail.config.bayes_dir],
                           input=message.as_string()).check_returncode()

        # Add the message to the mbox
        archivemymail.mboxman.add(message)

        # And remove it from the mailbox
        archivemymail.server.delete_messages(msg_num)

    # Archived all the necessary messages
    # Delete the folder, too, if there's nothing left
    if mbox_name.lower() not in ('inbox',  # Don't delete the special folders
                                 'drafts',
                                 'sent',
                                 'spam',
                                 'trash',
                                 'learnspam',
                                 'queue'):
        msg_list = archivemymail.server.search(['NOT', 'DELETED'])
        if len(msg_list) <= 0:
            disposition &= archivemymail.MBOX_DELETED
            if archivemymail.config.dry_run:
                logging.info("Would delete now-empty folder %s", mbox_name)
            else:
                logging.debug("Deleting now-empty folder %s")
                archivemymail.server.delete_folder(mbox_name)

    archivemymail.server.close_folder()

    return disposition
Exemple #55
0
                './Mail/friend1/in/': ['*****@*****.**'],
                './Mail/friend2/in/': ['*****@*****.**'],
                }
            if not mail.get('To', None):
                del mail["To"]
                for p in folder_to_map:
                    if filepath.startswith(p):
                        for m in folder_to_map[p]:
                            mail.add_header('To', m)
                        break

            try:
                assert mail.get('From', None) is not None
                assert mail.get('To', None)
                assert mail.get('Subject', None) is not None
                assert mail.get('Date', None) is not None
            except:
                import pdb; pdb.set_trace()

            mbox.add(mailbox.mboxMessage(mail))
            # The mail was successfuly migrated, remove its source
            os.remove(filepath)

    # Remove empty folders
    for root, dirs, files in os.walk(ARCHIVE_FOLDER, topdown=False):
        for name in dirs:
            fname = os.path.join(root, name)
            if os.path.exists(fname) and not os.listdir(fname):
                print 'Remove %s ...' % fname
                os.removedirs(fname)