Exemple #1
0
        def _actualize_ephemeral(self, ephemeral_mid):
            if isinstance(ephemeral_mid, int):
                # Not actually ephemeral, just return a normal Email
                return Email(self._idx(), ephemeral_mid)

            etype, mid = ephemeral_mid.split(':')
            etype = etype.lower()

            if etype in ('forward', 'forward-att'):
                refs = [Email(self._idx(), int(mid, 36))]
                e = Forward.CreateForward(self._idx(),
                                          self.session,
                                          refs,
                                          with_atts=('att' in etype))[0]
                self._track_action('fwded', refs)

            elif etype in ('reply', 'reply-all'):
                refs = [Email(self._idx(), int(mid, 36))]
                e = Reply.CreateReply(self._idx(),
                                      self.session,
                                      refs,
                                      reply_all=('all' in etype))[0]
                self._track_action('replied', refs)

            else:
                e = Compose.CreateMessage(self._idx(), self.session)[0]

            self._tag_blank([e])
            self.session.ui.debug('Actualized: %s' % e.msg_mid())

            return Email(self._idx(), e.msg_idx_pos)
Exemple #2
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        delay = play_nice_with_threads()
        if delay > 0:
            session.ui.notify(
                (_("Note: periodic delay is %ss, run from shell to " "speed up: mp --rescan=...")) % delay
            )

        if self.args and self.args[0].lower() == "vcards":
            return self._rescan_vcards(session, config)
        elif self.args and self.args[0].lower() == "mailboxes":
            return self._rescan_mailboxes(session, config)
        elif self.args and self.args[0].lower() == "all":
            self.args.pop(0)

        msg_idxs = self._choose_messages(self.args)
        if msg_idxs:
            for msg_idx_pos in msg_idxs:
                e = Email(idx, msg_idx_pos)
                session.ui.mark("Re-indexing %s" % e.msg_mid())
                idx.index_email(self.session, e)
            return {"messages": len(msg_idxs)}
        else:
            # FIXME: Need a lock here?
            if "rescan" in config._running:
                return True
            config._running["rescan"] = True
            try:
                return dict_merge(self._rescan_vcards(session, config), self._rescan_mailboxes(session, config))
            finally:
                del config._running["rescan"]
Exemple #3
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        delay = play_nice_with_threads()
        if delay > 0:
            session.ui.notify((
                _('Note: periodic delay is %ss, run from shell to '
                  'speed up: mp --rescan=...')
            ) % delay)

        if self.args and self.args[0].lower() == 'vcards':
            return self._rescan_vcards(session, config)
        elif self.args and self.args[0].lower() == 'all':
            self.args.pop(0)

        msg_idxs = self._choose_messages(self.args)
        if msg_idxs:
            session.ui.warning(_('FIXME: rescan messages: %s') % msg_idxs)
            for msg_idx_pos in msg_idxs:
                e = Email(idx, msg_idx_pos)
                session.ui.mark('Re-indexing %s' % e.msg_mid())
                idx.index_email(self.session, e)
            return {'messages': len(msg_idxs)}
        else:
            # FIXME: Need a lock here?
            if 'rescan' in config._running:
                return True
            config._running['rescan'] = True
            try:
                return dict_merge(
                    self._rescan_vcards(session, config),
                    self._rescan_mailboxes(session, config)
                )
            finally:
                del config._running['rescan']
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)
        if args and args[-1][0] == "#":
            attid = args.pop()
        else:
            attid = self.data.get("att", "application/pgp-keys")
        args.extend(["=%s" % x for x in self.data.get("mid", [])])
        eids = self._choose_messages(args)
        if len(eids) < 0:
            return self._error("No messages selected", None)
        elif len(eids) > 1:
            return self._error("One message at a time, please", None)

        email = Email(idx, list(eids)[0])
        fn, attr = email.extract_attachment(session, attid, mode="inline")
        if attr and attr["data"]:
            res = self._gnupg().import_keys(attr["data"])

            # Previous crypto evaluations may now be out of date, so we
            # clear the cache so users can see results right away.
            ClearParseCache(pgpmime=True)

            return self._success("Imported key", res)

        return self._error("No results found", None)
Exemple #5
0
    def command(self, save=True):
        session, config, idx = self.session, self.session.config, self._idx()
        mbox_type = config.prefs.export_format

        args = list(self.args)
        if args and ':' in args[-1]:
            mbox_type, path = args.pop(-1).split(':', 1)
        else:
            path = self.export_path(mbox_type)

        if os.path.exists(path):
            return self._error('Already exists: %s' % path)

        msg_idxs = self._choose_messages(args)
        if not msg_idxs:
            session.ui.warning('No messages selected')
            return False

        mbox = self.create_mailbox(mbox_type, path)
        for msg_idx in msg_idxs:
            e = Email(idx, msg_idx)
            session.ui.mark('Exporting =%s ...' % e.msg_mid())
            m = e.get_msg()
            # FIXME: This doesn't work
            #tags = [t.slug for t in e.get_message_tags()]
            #print 'Tags: %s' % tags
            #m['X-Mailpile-Tags'] = ', '.join(tags)
            mbox.add(m)
        mbox.flush()

        session.ui.mark('Exported %d messages to %s' % (len(msg_idxs), path))
        return {'exported': len(msg_idxs), 'created': path}
Exemple #6
0
    def command(self, save=True):
        session, config, idx = self.session, self.session.config, self._idx()
        mbox_type = config.prefs.export_format

        args = list(self.args)
        if args and ":" in args[-1]:
            mbox_type, path = args.pop(-1).split(":", 1)
        else:
            path = self.export_path(mbox_type)

        if os.path.exists(path):
            return self._error("Already exists: %s" % path)

        msg_idxs = self._choose_messages(args)
        if not msg_idxs:
            session.ui.warning("No messages selected")
            return False

        mbox = self.create_mailbox(mbox_type, path)
        for msg_idx in msg_idxs:
            e = Email(idx, msg_idx)
            session.ui.mark("Exporting =%s ..." % e.msg_mid())
            m = e.get_msg()
            # FIXME: This doesn't work
            # tags = [t.slug for t in e.get_message_tags()]
            # print 'Tags: %s' % tags
            # m['X-Mailpile-Tags'] = ', '.join(tags)
            mbox.add(m)
        mbox.flush()

        session.ui.mark("Exported %d messages to %s" % (len(msg_idxs), path))
        return {"exported": len(msg_idxs), "created": path}
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)
        if args and args[-1][0] == "#":
            attid = args.pop()
        else:
            attid = self.data.get("att", 'application/pgp-keys')
        args.extend(["=%s" % x for x in self.data.get("mid", [])])
        eids = self._choose_messages(args)
        if len(eids) < 0:
            return self._error("No messages selected", None)
        elif len(eids) > 1:
            return self._error("One message at a time, please", None)

        email = Email(idx, list(eids)[0])
        fn, attr = email.extract_attachment(session, attid, mode='inline')
        if attr and attr["data"]:
            res = self._gnupg().import_keys(attr["data"])

            # Previous crypto evaluations may now be out of date, so we
            # clear the cache so users can see results right away.
            ClearParseCache(pgpmime=True)

            return self._success("Imported key", res)

        return self._error("No results found", None)
Exemple #8
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        if self.args and self.args[0].lower() == 'all':
            reply_all = self.args.pop(0) or True
        else:
            reply_all = False

        refs = [Email(idx, i) for i in self._choose_messages(self.args)]
        if refs:
            trees = [
                m.evaluate_pgp(m.get_message_tree(), decrypt=True)
                for m in refs
            ]
            ref_ids = [t['headers_lc'].get('message-id') for t in trees]
            ref_subjs = [t['headers_lc'].get('subject') for t in trees]
            msg_to = [
                t['headers_lc'].get('reply-to', t['headers_lc']['from'])
                for t in trees
            ]
            msg_cc = []
            if reply_all:
                msg_cc += [t['headers_lc'].get('to', '') for t in trees]
                msg_cc += [t['headers_lc'].get('cc', '') for t in trees]
            msg_bodies = []
            for t in trees:
                # FIXME: Templates/settings for how we quote replies?
                text = (('%s wrote:\n' % t['headers_lc']['from']) + ''.join([
                    p['data'] for p in t['text_parts']
                    if p['type'] in ('text', 'quote', 'pgpsignedtext',
                                     'pgpsecuretext', 'pgpverifiedtext')
                ]))
                msg_bodies.append(text.replace('\n', '\n> '))

            local_id, lmbox = config.open_local_mailbox(session)
            try:
                email = Email.Create(idx,
                                     local_id,
                                     lmbox,
                                     msg_text='\n\n'.join(msg_bodies),
                                     msg_subject=('Re: %s' % ref_subjs[-1]),
                                     msg_to=msg_to,
                                     msg_cc=[r for r in msg_cc if r],
                                     msg_references=[i for i in ref_ids if i])
                try:
                    idx.add_tag(
                        session,
                        session.config.get_tag_id('Drafts'),
                        msg_idxs=[int(email.get_msg_info(idx.MSG_IDX), 36)],
                        conversation=False)
                except (TypeError, ValueError, IndexError):
                    self._ignore_exception()

            except NoFromAddressError:
                return self._error('You must configure a From address first.')

            return self._edit_new_messages(session, idx, [email])
        else:
            return self._error('No message found')
Exemple #9
0
    def command(self, save=True):
        session, config, idx = self.session, self.session.config, self._idx()
        mbox_type = config.prefs.export_format

        if self.session.config.sys.lockdown:
            return self._error(_('In lockdown, doing nothing.'))

        args = list(self.args)
        if args and ':' in args[-1]:
            mbox_type, path = args.pop(-1).split(':', 1)
        else:
            path = self.export_path(mbox_type)

        if args and args[-1] == 'flat':
            flat = True
            args.pop(-1)
        else:
            flat = False

        if os.path.exists(path):
            return self._error('Already exists: %s' % path)

        msg_idxs = list(self._choose_messages(args))
        if not msg_idxs:
            session.ui.warning('No messages selected')
            return False

        # Exporting messages without their threads barely makes any
        # sense.
        if not flat:
            for i in reversed(range(0, len(msg_idxs))):
                mi = msg_idxs[i]
                msg_idxs[i:i + 1] = [
                    int(m[idx.MSG_MID], 36)
                    for m in idx.get_conversation(msg_idx=mi)
                ]

        # Let's always export in the same order. Stability is nice.
        msg_idxs.sort()

        mbox = self.create_mailbox(mbox_type, path)
        exported = {}
        while msg_idxs:
            msg_idx = msg_idxs.pop(0)
            if msg_idx not in exported:
                e = Email(idx, msg_idx)
                session.ui.mark('Exporting =%s ...' % e.msg_mid())
                mbox.add(e.get_msg())
                exported[msg_idx] = 1

        mbox.flush()

        return self._success(
            _('Exported %d messages to %s') % (len(exported), path), {
                'exported': len(exported),
                'created': path
            })
Exemple #10
0
    def command(self, save=True):
        session, config, idx = self.session, self.session.config, self._idx()
        mbox_type = config.prefs.export_format

        if self.session.config.sys.lockdown:
            return self._error(_('In lockdown, doing nothing.'))

        args = list(self.args)
        if args and ':' in args[-1]:
            mbox_type, path = args.pop(-1).split(':', 1)
        else:
            path = self.export_path(mbox_type)

        if args and args[-1] == 'flat':
            flat = True
            args.pop(-1)
        else:
            flat = False

        if os.path.exists(path):
            return self._error('Already exists: %s' % path)

        msg_idxs = list(self._choose_messages(args))
        if not msg_idxs:
            session.ui.warning('No messages selected')
            return False

        # Exporting messages without their threads barely makes any
        # sense.
        if not flat:
            for i in reversed(range(0, len(msg_idxs))):
                mi = msg_idxs[i]
                msg_idxs[i:i+1] = [int(m[idx.MSG_MID], 36)
                                   for m in idx.get_conversation(msg_idx=mi)]

        # Let's always export in the same order. Stability is nice.
        msg_idxs.sort()

        mbox = self.create_mailbox(mbox_type, path)
        exported = {}
        while msg_idxs:
            msg_idx = msg_idxs.pop(0)
            if msg_idx not in exported:
                e = Email(idx, msg_idx)
                session.ui.mark('Exporting =%s ...' % e.msg_mid())
                mbox.add(e.get_msg())
                exported[msg_idx] = 1

        mbox.flush()

        return self._success(
            _('Exported %d messages to %s') % (len(exported), path),
            {
                'exported': len(exported),
                'created': path
            })
Exemple #11
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        if self.args and self.args[0].lower().startswith('att'):
            with_atts = self.args.pop(0) or True
        else:
            with_atts = False

        refs = [Email(idx, i) for i in self._choose_messages(self.args)]
        if refs:
            trees = [
                m.evaluate_pgp(m.get_message_tree(), decrypt=True)
                for m in refs
            ]
            ref_subjs = [t['headers_lc']['subject'] for t in trees]
            msg_bodies = []
            msg_atts = []
            for t in trees:
                # FIXME: Templates/settings for how we quote forwards?
                text = '-------- Original Message --------\n'
                for h in ('Date', 'Subject', 'From', 'To'):
                    v = t['headers_lc'].get(h.lower(), None)
                    if v:
                        text += '%s: %s\n' % (h, v)
                text += '\n'
                text += ''.join([
                    p['data'] for p in t['text_parts']
                    if p['type'] in self._TEXT_PARTTYPES
                ])
                msg_bodies.append(text)
                if with_atts:
                    for att in t['attachments']:
                        if att['mimetype'] not in self._ATT_MIMETYPES:
                            msg_atts.append(att['part'])

            local_id, lmbox = config.open_local_mailbox(session)
            email = Email.Create(idx,
                                 local_id,
                                 lmbox,
                                 msg_text='\n\n'.join(msg_bodies),
                                 msg_subject=('Fwd: %s' % ref_subjs[-1]))
            if msg_atts:
                msg = email.get_msg()
                for att in msg_atts:
                    msg.attach(att)
                email.update_from_msg(msg)

            if self.BLANK_TAG:
                self._tag_emails([email], self.BLANK_TAG)

            return self._edit_messages([email])
        else:
            return self._error('No message found')
Exemple #12
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        # Command-line arguments...
        msgs = list(self.args)
        timeout = -1
        with_header = False
        without_mid = False
        columns = []
        while msgs and msgs[0].lower() != '--':
            arg = msgs.pop(0)
            if arg.startswith('--timeout='):
                timeout = float(arg[10:])
            elif arg.startswith('--header'):
                with_header = True
            elif arg.startswith('--no-mid'):
                without_mid = True
            else:
                columns.append(msgs.pop(0))
        if msgs and msgs[0].lower() == '--':
            msgs.pop(0)

        # Form arguments...
        timeout = float(self.data.get('timeout', [timeout])[0])
        with_header |= self._truthy(self.data.get('header', [''])[0])
        without_mid |= self._truthy(self.data.get('no-mid', [''])[0])
        columns.extend(self.data.get('term', []))
        msgs.extend(['=%s' % mid.replace('=', '')
                     for mid in self.data.get('mid', [])])

        # Add a header to the CSV if requested
        if with_header:
            results = [[col.split('||')[0].split(':', 1)[0].split('=', 1)[0]
                        for col in columns]]
            if not without_mid:
                results[0] = ['MID'] + results[0]
        else:
            results = []

        deadline = (time.time() + timeout) if (timeout > 0) else None
        msg_idxs = self._choose_messages(msgs)
        for msg_idx in msg_idxs:
            e = Email(idx, msg_idx)
            session.ui.mark(_('Digging into =%s') % e.msg_mid())
            row = [] if without_mid else ['%s' % e.msg_mid()]
            for cellspec in columns:
                row.extend(self._cell(idx, e, cellspec))
            results.append(row)
            if deadline and deadline < time.time():
                break

        return self._success(_('Found %d rows in %d messages'
                               ) % (len(results), len(msg_idxs)), results)
Exemple #13
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        if self.args and self.args[0].lower() == 'all':
            reply_all = self.args.pop(0) or True
        else:
            reply_all = False

        refs = [Email(idx, i) for i in self._choose_messages(self.args)]
        if refs:
            trees = [
                m.evaluate_pgp(m.get_message_tree(), decrypt=True)
                for m in refs
            ]
            ref_ids = [t['headers_lc'].get('message-id') for t in trees]
            ref_subjs = [t['headers_lc'].get('subject') for t in trees]
            msg_to = [
                t['headers_lc'].get('reply-to', t['headers_lc']['from'])
                for t in trees
            ]
            msg_cc = []
            if reply_all:
                msg_cc += [t['headers_lc'].get('to', '') for t in trees]
                msg_cc += [t['headers_lc'].get('cc', '') for t in trees]
            msg_bodies = []
            for t in trees:
                # FIXME: Templates/settings for how we quote replies?
                text = (('%s wrote:\n' % t['headers_lc']['from']) + ''.join([
                    p['data'] for p in t['text_parts']
                    if p['type'] in self._TEXT_PARTTYPES
                ]))
                msg_bodies.append(text.replace('\n', '\n> '))

            local_id, lmbox = config.open_local_mailbox(session)
            try:
                email = Email.Create(idx,
                                     local_id,
                                     lmbox,
                                     msg_text='\n\n'.join(msg_bodies),
                                     msg_subject=('Re: %s' % ref_subjs[-1]),
                                     msg_to=msg_to,
                                     msg_cc=[r for r in msg_cc if r],
                                     msg_references=[i for i in ref_ids if i])
                if self.BLANK_TAG:
                    self._tag_emails([email], self.BLANK_TAG)

            except NoFromAddressError:
                return self._error('You must configure a From address first.')

            return self._edit_messages([email])
        else:
            return self._error('No message found')
Exemple #14
0
    def _lookup(self, address):
        results = {}
        terms = ["from:%s" % x for x in address.split('@')]
        terms.append("has:pgpkey")
        session, idx, _, _ = self._do_search(search=terms)
        for messageid in session.results:
            email = Email(self._idx(), messageid)
            attachments = email.get_message_tree("attachments")["attachments"]
            for part in attachments:
                if part["mimetype"] == "application/pgp-keys":
                    key = part["part"].get_payload(None, True)
                    results.update(self._get_keydata(key))

        return results
Exemple #15
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)

        if config.sys.lockdown:
            return self._error(_("In lockdown, doing nothing."))

        delay = play_nice_with_threads()
        if delay > 0:
            session.ui.notify(
                (_("Note: periodic delay is %ss, run from shell to " "speed up: mp --rescan=...")) % delay
            )

        if args and args[0].lower() == "vcards":
            return self._rescan_vcards(session, config)
        elif args and args[0].lower() == "mailboxes":
            return self._rescan_mailboxes(session, config)
        elif args and args[0].lower() == "all":
            args.pop(0)

        msg_idxs = self._choose_messages(args)
        if msg_idxs:
            for msg_idx_pos in msg_idxs:
                e = Email(idx, msg_idx_pos)
                try:
                    session.ui.mark("Re-indexing %s" % e.msg_mid())
                    idx.index_email(self.session, e)
                except KeyboardInterrupt:
                    raise
                except:
                    self._ignore_exception()
                    session.ui.warning(_("Failed to reindex: %s") % e.msg_mid())
            return self._success(_("Indexed %d messages") % len(msg_idxs), result={"messages": len(msg_idxs)})

        else:
            # FIXME: Need a lock here?
            if "rescan" in config._running:
                return self._success(_("Rescan already in progress"))
            config._running["rescan"] = True
            try:
                results = {}
                results.update(self._rescan_vcards(session, config))
                results.update(self._rescan_mailboxes(session, config))
                if "aborted" in results:
                    raise KeyboardInterrupt()
                return self._success(_("Rescanned vcards and mailboxes"), result=results)
            except (KeyboardInterrupt), e:
                return self._error(_("User aborted"), info=results)
            finally:
Exemple #16
0
 def _get_message_keys(self, messageid):
     keys = self.key_cache.get(messageid, [])
     if not keys:
         email = Email(self._idx(), messageid)
         attachments = email.get_message_tree(
             want=["attachments"])["attachments"]
         for part in attachments:
             if _might_be_pgp_key(part["filename"], part["mimetype"]):
                 key = part["part"].get_payload(None, True)
                 for keydata in _get_keydata(key):
                     keys.append((keydata, key))
                 if len(keys) > 5:  # Just to set some limit...
                     break
         self.key_cache[messageid] = keys
     return keys
Exemple #17
0
 def _get_message_keys(self, messageid):
     keys = self.key_cache.get(messageid, [])
     if not keys:
         email = Email(self._idx(), messageid)
         attachments = email.get_message_tree(want=["attachments"]
                                              )["attachments"]
         for part in attachments:
             if _might_be_pgp_key(part["filename"], part["mimetype"]):
                 key = part["part"].get_payload(None, True)
                 for keydata in _get_keydata(key):
                     keys.append((keydata, key))
                 if len(keys) > 5:  # Just to set some limit...
                     break
         self.key_cache[messageid] = keys
     return keys
Exemple #18
0
 def _get_keys(self, messageid):
     keys = self.key_cache.get(messageid, [])
     if not keys:
         email = Email(self._idx(), messageid)
         attachments = email.get_message_tree("attachments")["attachments"]
         for part in attachments:
             if part["mimetype"] == "application/pgp-keys":
                 key = part["part"].get_payload(None, True)
                 for keydata in self._get_keydata(key):
                     keys.append(keydata)
                     self.key_cache[keydata["fingerprint"]] = key
                 if len(keys) > 5:  # Just to set some limit...
                     break
         self.key_cache[messageid] = keys
     return keys
Exemple #19
0
 def _get_keys(self, messageid):
     keys = self.key_cache.get(messageid, [])
     if not keys:
         email = Email(self._idx(), messageid)
         attachments = email.get_message_tree("attachments")["attachments"]
         for part in attachments:
             if part["mimetype"] == "application/pgp-keys":
                 key = part["part"].get_payload(None, True)
                 for keydata in self._get_keydata(key):
                     keys.append(keydata)
                     self.key_cache[keydata["fingerprint"]] = key
                 if len(keys) > 5:  # Just to set some limit...
                     break
         self.key_cache[messageid] = keys
     return keys
Exemple #20
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        if self.args[0] in ('inline', 'inline-preview', 'preview', 'download'):
            mode = self.args.pop(0)
        else:
            mode = 'download'
        cid = self.args.pop(0)
        if len(self.args) > 0 and self.args[-1].startswith('>'):
            name_fmt = self.args.pop(-1)[1:]
        else:
            name_fmt = None

        emails = [Email(idx, i) for i in self._choose_messages(self.args)]
        results = []
        for email in emails:
            fn, info = email.extract_attachment(session,
                                                cid,
                                                name_fmt=name_fmt,
                                                mode=mode)
            if info:
                info['idx'] = email.msg_idx_pos
                if fn:
                    info['created_file'] = fn
                results.append(info)
        return results
Exemple #21
0
 def command(self):
     session, config, idx = self.session, self.session.config, self._idx()
     results = []
     if self.args and self.args[0].lower() == 'raw':
         raw = self.args.pop(0)
     else:
         raw = False
     emails = [Email(idx, mid) for mid in self._choose_messages(self.args)]
     idx.apply_filters(session,
                       '@read',
                       msg_idxs=[e.msg_idx_pos for e in emails])
     for email in emails:
         if raw:
             results.append(
                 self.RawResult({'data': email.get_file().read()}))
         else:
             conv = [
                 int(c[0], 36)
                 for c in idx.get_conversation(msg_idx=email.msg_idx_pos)
             ]
             if email.msg_idx_pos not in conv:
                 conv.append(email.msg_idx_pos)
             conv.reverse()
             results.append(
                 SearchResults(session,
                               idx,
                               results=conv,
                               num=len(conv),
                               expand=[email]))
     if len(results) == 1:
         return results[0]
     else:
         return results
Exemple #22
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        emails = [Email(idx, mid) for mid in self._choose_messages(self.args)]
        scores = self._classify(emails)
        tag = {}
        for mid in scores:
            for at_config in autotag_configs(config):
                at_tag = config.get_tag(at_config.match_tag)
                if not at_tag:
                    continue
                want = scores[mid].get(at_tag._key, (False, ))[0]

                if want is True:
                    if at_config.match_tag not in tag:
                        tag[at_config.match_tag] = [mid]
                    else:
                        tag[at_config.match_tag].append(mid)

                elif at_config.unsure_tag and want is None:
                    if at_config.unsure_tag not in tag:
                        tag[at_config.unsure_tag] = [mid]
                    else:
                        tag[at_config.unsure_tag].append(mid)

        for tid in tag:
            idx.add_tag(session, tid, msg_idxs=[int(i, 36) for i in tag[tid]])

        return self._success(_('Auto-tagged %d messages') % len(emails), tag)
Exemple #23
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        files = []
        while os.path.exists(self.args[-1]):
            files.append(self.args.pop(-1))
        if not files:
            return self._error('No files found')

        emails = [Email(idx, i) for i in self._choose_messages(self.args)]
        if not emails:
            return self._error('No messages selected')

        # FIXME: Using "say" here is rather lame.
        updated = []
        for email in emails:
            subject = email.get_msg_info(MailIndex.MSG_SUBJECT)
            try:
                email.add_attachments(files)
                updated.append(email)
            except NotEditableError:
                session.ui.error('Read-only message: %s' % subject)
            except:
                session.ui.error('Error attaching to %s' % subject)
                self._ignore_exception()

        session.ui.notify(
            ('Attached %s to %d messages') % (', '.join(files), len(updated)))
        return self._return_search_results(session, idx, updated, updated)
Exemple #24
0
    def command(self, emails=None):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)

        bounce_to = []
        while args and '@' in args[-1]:
            bounce_to.append(args.pop(-1))
        for rcpt in (self.data.get('to', []) +
                     self.data.get('cc', []) +
                     self.data.get('bcc', [])):
            bounce_to.extend(ExtractEmails(rcpt))

        if not emails:
            args.extend(['=%s' % mid for mid in self.data.get('mid', [])])
            mids = self._choose_messages(args)
            emails = [Email(idx, i) for i in mids]

        # Process one at a time so we don't eat too much memory
        sent = []
        missing_keys = []
        for email in emails:
            try:
                msg_mid = email.get_msg_info(idx.MSG_MID)
                # FIXME: We are failing to capture error states with sufficient
                #        granularity, messages may be delivered to some
                #        recipients but not all...
                SendMail(session, [PrepareMessage(config,
                                                  email.get_msg(pgpmime=False),
                                                  rcpts=(bounce_to or None))])
                sent.append(email)
            except KeyLookupError, kle:
                session.ui.warning(_('Missing keys %s') % kle.missing)
                missing_keys.extend(kle.missing)
                self._ignore_exception()
            except:
Exemple #25
0
 def _get_canned(cls, idx, cid):
     try:
         return Email(idx, int(cid,
                               36)).get_editing_strings().get('body', '')
     except (ValueError, IndexError, TypeError, OSError, IOError):
         traceback.print_exc()  # FIXME, ugly
         return ''
Exemple #26
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        with_atts = False
        ephemeral = False
        while self.args:
            if self.args[0].lower() == 'att':
                with_atts = self.args.pop(0) or True
            elif self.args[0].lower() == 'ephemeral':
                ephemeral = self.args.pop(0) or True
            else:
                break
        if ephemeral and with_atts:
            raise UsageError(
                _('Sorry, ephemeral messages cannot have '
                  'attachments at this time.'))

        refs = [Email(idx, i) for i in self._choose_messages(self.args)]
        if refs:
            email, ephemeral = self.CreateReply(idx,
                                                session,
                                                refs,
                                                with_atts=with_atts,
                                                ephemeral=ephemeral)

            if not ephemeral:
                self._track_action('fwded', refs)
                self._tag_blank([email])

            return self._edit_messages([email], ephemeral=ephemeral)
        else:
            return self._error(_('No message found'))
Exemple #27
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        reply_all = False
        ephemeral = False
        while self.args:
            if self.args[0].lower() == 'all':
                reply_all = self.args.pop(0) or True
            elif self.args[0].lower() == 'ephemeral':
                ephemeral = self.args.pop(0) or True
            else:
                break

        refs = [Email(idx, i) for i in self._choose_messages(self.args)]
        if refs:
            try:
                email, ephemeral = self.CreateReply(idx,
                                                    session,
                                                    refs,
                                                    reply_all=reply_all,
                                                    ephemeral=ephemeral)
            except NoFromAddressError:
                return self._error(
                    _('You must configure a '
                      'From address first.'))

            if not ephemeral:
                self._track_action('replied', refs)
                self._tag_blank([email])

            return self._edit_messages([email], ephemeral=ephemeral)
        else:
            return self._error(_('No message found'))
Exemple #28
0
    def command(self):
        session, index = self.session, self._idx()

        session.ui.mark('Checking index for duplicate MSG IDs...')
        found = {}
        for i in range(0, len(index.INDEX)):
            msg_id = index.get_msg_at_idx_pos(i)[index.MSG_ID]
            if msg_id in found:
                found[msg_id].append(i)
            else:
                found[msg_id] = [i]

        session.ui.mark('Attempting to fix dups with bad location...')
        for msg_id in found:
            if len(found[msg_id]) > 1:
                good, bad = [], []
                for idx_pos in found[msg_id]:
                    msg = Email(index, idx_pos).get_msg()
                    if msg:
                        good.append(idx_pos)
                    else:
                        bad.append(idx_pos)
                if good and bad:
                    good_info = index.get_msg_at_idx_pos(good[0])
                    for bad_idx in bad:
                        bad_info = index.get_msg_at_idx_pos(bad_idx)
                        bad_info[index.MSG_PTRS] = good_info[index.MSG_PTRS]
                        index.set_msg_at_idx_pos(bad_idx, bad_info)

        session.ui.mark('Done!')
        return True
Exemple #29
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        mode = 'download'
        name_fmt = None

        if self.args[0] in ('inline', 'inline-preview', 'preview', 'download'):
            mode = self.args.pop(0)

        if len(self.args) > 0 and self.args[-1].startswith('>'):
            name_fmt = self.args.pop(-1)[1:]

        if self.args[0].startswith('#') or self.args[0].startswith('part:'):
            cid = self.args.pop(0)
        else:
            cid = self.args.pop(-1)

        eids = self._choose_messages(self.args)
        print 'Download %s from %s as %s/%s' % (cid, eids, mode, name_fmt)

        emails = [Email(idx, i) for i in eids]
        results = []
        for e in emails:
            fn, info = e.extract_attachment(session,
                                            cid,
                                            name_fmt=name_fmt,
                                            mode=mode)
            if info:
                info['idx'] = email.msg_idx_pos
                if fn:
                    info['created_file'] = fn
                results.append(info)
        return results
Exemple #30
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        results = []
        args = list(self.args)
        if args and args[0].lower() == 'raw':
            raw = args.pop(0)
        else:
            raw = False
        emails = [Email(idx, mid) for mid in self._choose_messages(args)]

        rv = self._side_effects(emails)
        if rv is not None:
            # This is here so derived classes can do funky things.
            return rv

        for email in emails:
            if raw:
                subject = email.get_msg_info(idx.MSG_SUBJECT)
                results.append(
                    self.RawResult({
                        'summary': _('Raw message: %s') % subject,
                        'source': email.get_file().read()
                    }))
            else:
                old_result = None
                for result in results:
                    if email.msg_idx_pos in result.results:
                        old_result = result
                if old_result:
                    old_result.add_email(email)
                    continue

                conv = [
                    int(c[0], 36)
                    for c in idx.get_conversation(msg_idx=email.msg_idx_pos)
                ]
                if email.msg_idx_pos not in conv:
                    conv.append(email.msg_idx_pos)

                # FIXME: This is a hack. The indexer should just keep things
                #        in the right order on rescan. Fixing threading is a
                #        bigger problem though, so we do this for now.
                def sort_conv_key(msg_idx_pos):
                    info = idx.get_msg_at_idx_pos(msg_idx_pos)
                    return -int(info[idx.MSG_DATE], 36)

                conv.sort(key=sort_conv_key)

                results.append(
                    SearchResults(session,
                                  idx,
                                  results=conv,
                                  num=len(conv),
                                  emails=[email]))
        if len(results) == 1:
            return self._success(_('Displayed a single message'),
                                 result=results[0])
        else:
            return self._success(_('Displayed %d messages') % len(results),
                                 result=results)
Exemple #31
0
 def process_message(self, peer, mailfrom, rcpttos, data):
     # We can assume that the mailfrom and rcpttos have checked out
     # and this message is indeed intended for us. Spool it to disk
     # and add to the index!
     session, config = self.session, self.session.config
     blank_tid = config.get_tags(type='blank')[0]._key
     idx = config.index
     play_nice_with_threads()
     try:
         message = email.parser.Parser().parsestr(data)
         lid, lmbox = config.open_local_mailbox(session)
         e = Email.Create(idx, lid, lmbox, ephemeral_mid=False)
         idx.add_tag(session,
                     blank_tid,
                     msg_idxs=[e.msg_idx_pos],
                     conversation=False)
         e.update_from_msg(session, message)
         idx.remove_tag(session,
                        blank_tid,
                        msg_idxs=[e.msg_idx_pos],
                        conversation=False)
         return None
     except:
         traceback.print_exc()
         return '400 Oops wtf'
Exemple #32
0
    def command(self, emails=None):
        session, config, idx = self.session, self.session.config, self._idx()

        bounce_to = []
        while self.args and '@' in self.args[-1]:
            bounce_to.append(self.args.pop(-1))
        for rcpt in (self.data.get('to', []) + self.data.get('cc', []) +
                     self.data.get('bcc', [])):
            bounce_to.extend(ExtractEmails(rcpt))

        args = self.args[:]
        if not emails:
            args.extend(['=%s' % mid for mid in self.data.get('mid', [])])
            mids = self._choose_messages(args)
            emails = [Email(idx, i) for i in mids]

        # Process one at a time so we don't eat too much memory
        sent = []
        missing_keys = []
        for email in emails:
            try:
                msg_mid = email.get_msg_info(idx.MSG_MID)
                SendMail(session, [
                    PrepareMessage(config,
                                   email.get_msg(pgpmime=False),
                                   rcpts=(bounce_to or None))
                ])
                sent.append(email)
            except KeyLookupError, kle:
                missing_keys.extend(kle.missing)
                self._ignore_exception()
            except:
Exemple #33
0
    def CreateForward(cls,
                      idx,
                      session,
                      refs,
                      msgid,
                      with_atts=False,
                      cid=None,
                      ephemeral=False):
        trees = [
            m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs
        ]
        ref_subjs = [t['headers_lc']['subject'] for t in trees]
        msg_bodies = []
        msg_atts = []
        for t in trees:
            # FIXME: Templates/settings for how we quote forwards?
            text = '-------- Original Message --------\n'
            for h in ('Date', 'Subject', 'From', 'To'):
                v = t['headers_lc'].get(h.lower(), None)
                if v:
                    text += '%s: %s\n' % (h, v)
            text += '\n'
            text += ''.join([
                p['data'] for p in t['text_parts']
                if p['type'] in cls._TEXT_PARTTYPES
            ])
            msg_bodies.append(text)
            if with_atts:
                for att in t['attachments']:
                    if att['mimetype'] not in cls._ATT_MIMETYPES:
                        msg_atts.append(att['part'])

        if not ephemeral:
            local_id, lmbox = session.config.open_local_mailbox(session)
        else:
            local_id, lmbox = -1, None
            fmt = 'forward-att-%s-%s' if msg_atts else 'forward-%s-%s'
            ephemeral = [
                fmt % (msgid[1:-1].replace('@', '_'), refs[0].msg_mid())
            ]

        if cid:
            # FIXME: Instead, we should use placeholders in the template
            #        and insert the quoted bits in the right place (or
            #        nowhere if the template doesn't want them).
            msg_bodies[:0] = [cls._get_canned(idx, cid)]

        email = Email.Create(idx,
                             local_id,
                             lmbox,
                             msg_text='\n\n'.join(msg_bodies),
                             msg_subject=cls.prefix_subject(
                                 ref_subjs[-1], 'Fwd:', cls._FW_REGEXP),
                             msg_id=msgid,
                             msg_atts=msg_atts,
                             save=(not ephemeral),
                             ephemeral_mid=ephemeral and ephemeral[0])

        return email, ephemeral
Exemple #34
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)

        if config.sys.lockdown:
            return self._error(_('In lockdown, doing nothing.'))

        delay = play_nice_with_threads()
        if delay > 0:
            session.ui.notify((
                _('Note: periodic delay is %ss, run from shell to '
                  'speed up: mp --rescan=...')
            ) % delay)

        if args and args[0].lower() == 'vcards':
            return self._rescan_vcards(session, config)
        elif args and args[0].lower() == 'mailboxes':
            return self._rescan_mailboxes(session, config)
        elif args and args[0].lower() == 'all':
            args.pop(0)

        msg_idxs = self._choose_messages(args)
        if msg_idxs:
            for msg_idx_pos in msg_idxs:
                e = Email(idx, msg_idx_pos)
                session.ui.mark('Re-indexing %s' % e.msg_mid())
                idx.index_email(self.session, e)
            return self._success(_('Indexed %d messages') % len(msg_idxs),
                                 result={'messages': len(msg_idxs)})

        else:
            # FIXME: Need a lock here?
            if 'rescan' in config._running:
                return self._success(_('Rescan already in progress'))
            config._running['rescan'] = True
            try:
                results = {}
                results.update(self._rescan_vcards(session, config))
                results.update(self._rescan_mailboxes(session, config))
                if 'aborted' in results:
                    raise KeyboardInterrupt()
                return self._success(_('Rescanned vcards and mailboxes'),
                                     result=results)
            except (KeyboardInterrupt), e:
                return self._error(_('User aborted'), info=results)
            finally:
Exemple #35
0
  def command(self, search=None):
    session, idx, start = self._do_search(search=search)

    nodes = []
    links = []
    res = {}

    for messageid in session.results:
      message = Email(self._idx(), messageid)
      try:
        msgfrom = ExtractEmails(message.get("from"))[0].lower()
      except IndexError, e:
        print "No e-mail address in '%s'" % message.get("from")
        continue

      msgto = [x.lower() for x in ExtractEmails(message.get("to"))]
      msgcc = [x.lower() for x in ExtractEmails(message.get("cc"))]
      msgbcc = [x.lower() for x in ExtractEmails(message.get("bcc"))]

      if msgfrom not in [m["email"] for m in nodes]:
        nodes.append({"email": msgfrom})

      for msgset in [msgto, msgcc, msgbcc]:
        for address in msgset:
          if address not in [m["email"] for m in nodes]:
            nodes.append({"email": address})

        curnodes = [x["email"] for x in nodes]
        fromid = curnodes.index(msgfrom)
        searchspace = [m for m in links if m["source"] == fromid]
        for recipient in msgset:
          index = curnodes.index(recipient)
          link = [m for m in searchspace if m["target"] == index]
          if len(link) == 0:
            links.append({"source": fromid, "target": index, "value": 1})
          elif len(link) == 1:
            link[0]["value"] += 1
          else:
            raise ValueError("Too many links! - This should never happen.")

      if len(nodes) >= 200:
        # Let's put a hard upper limit on how many nodes we can have, for performance reasons.
        # There might be a better way to do this though...
        res["limit_hit"] = True
        break
Exemple #36
0
        def _actualize_ephemeral(self, ephemeral_mid):
            idx = self._idx()

            if isinstance(ephemeral_mid, int):
                # Not actually ephemeral, just return a normal Email
                return Email(idx, ephemeral_mid)

            msgid, mid = ephemeral_mid.rsplit('-', 1)
            etype, etarg, msgid = ephemeral_mid.split('-', 2)
            if etarg not in ('all', 'att'):
                msgid = etarg + '-' + msgid
            msgid = '<%s>' % msgid.replace('_', '@')
            etype = etype.lower()

            enc_msgid = idx.encode_msg_id(msgid)
            msg_idx = idx.MSGIDS.get(enc_msgid)
            if msg_idx is not None:
                # Already actualized, just return a normal Email
                return Email(idx, msg_idx)

            if etype == 'forward':
                refs = [Email(idx, int(mid, 36))]
                e = Forward.CreateForward(idx,
                                          self.session,
                                          refs,
                                          msgid,
                                          with_atts=(etarg == 'att'))[0]
                self._track_action('fwded', refs)

            elif etype == 'reply':
                refs = [Email(idx, int(mid, 36))]
                e = Reply.CreateReply(idx,
                                      self.session,
                                      refs,
                                      msgid,
                                      reply_all=(etarg == 'all'))[0]
                self._track_action('replied', refs)

            else:
                e = Compose.CreateMessage(idx, self.session, msgid)[0]

            self._tag_blank([e])
            self.session.ui.debug('Actualized: %s' % e.msg_mid())

            return Email(idx, e.msg_idx_pos)
Exemple #37
0
    def CreateReply(cls, idx, session, refs, msgid,
                    reply_all=False, cid=None, ephemeral=False):
        trees = [m.evaluate_pgp(m.get_message_tree(), decrypt=True)
                 for m in refs]

        headers = cls._create_from_to_cc(idx, session, trees)
        if not reply_all and 'cc' in headers:
            del headers['cc']

        ref_ids = [t['headers_lc'].get('message-id') for t in trees]
        ref_subjs = [t['headers_lc'].get('subject') for t in trees]
        msg_bodies = []
        for t in trees:
            # FIXME: Templates/settings for how we quote replies?
            quoted = ''.join([p['data'] for p in t['text_parts']
                              if p['type'] in cls._TEXT_PARTTYPES
                              and p['data']])
            if quoted:
                target_width = session.config.prefs.line_length
                if target_width > 40:
                    quoted = reflow_text(quoted, target_width=target_width-2)
                text = ((_('%s wrote:') % t['headers_lc']['from']) + '\n' +
                        quoted)
                msg_bodies.append('\n\n' + text.replace('\n', '\n> '))

        if not ephemeral:
            local_id, lmbox = session.config.open_local_mailbox(session)
        else:
            local_id, lmbox = -1, None
            fmt = 'reply-all-%s-%s' if reply_all else 'reply-%s-%s'
            ephemeral = [fmt % (msgid[1:-1].replace('@', '_'),
                                refs[0].msg_mid())]

        if 'cc' in headers:
            fmt = _('Composing a reply from %(from)s to %(to)s, cc %(cc)s')
        else:
            fmt = _('Composing a reply from %(from)s to %(to)s')
        session.ui.debug(fmt % headers)

        if cid:
            # FIXME: Instead, we should use placeholders in the template
            #        and insert the quoted bits in the right place (or
            #        nowhere if the template doesn't want them).
            msg_bodies[:0] = [cls._get_canned(idx, cid)]

        return (Email.Create(idx, local_id, lmbox,
                             msg_text='\n\n'.join(msg_bodies),
                             msg_subject=cls.prefix_subject(
                                 ref_subjs[-1], 'Re:', cls._RE_REGEXP),
                             msg_from=headers.get('from', None),
                             msg_to=headers.get('to', []),
                             msg_cc=headers.get('cc', []),
                             msg_references=[i for i in ref_ids if i],
                             msg_id=msgid,
                             save=(not ephemeral),
                             ephemeral_mid=ephemeral and ephemeral[0]),
                ephemeral)
Exemple #38
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        results = []
        args = list(self.args)
        args.extend(
            ['=%s' % mid.replace('=', '') for mid in self.data.get('mid', [])])
        if args and args[0].lower() == 'raw':
            raw = args.pop(0)
        else:
            raw = False
        emails = [Email(idx, mid) for mid in self._choose_messages(args)]

        rv = self._side_effects(emails)
        if rv is not None:
            # This is here so derived classes can do funky things.
            return rv

        for email in emails:
            if raw:
                subject = email.get_msg_info(idx.MSG_SUBJECT)
                results.append(
                    self.RawResult({
                        'summary': _('Raw message: %s') % subject,
                        'source': email.get_file().read()
                    }))
            else:
                old_result = None
                for result in results:
                    if email.msg_idx_pos in result.results:
                        old_result = result
                if old_result:
                    old_result.add_email(email)
                    continue

                # Get conversation
                conv = idx.get_conversation(msg_idx=email.msg_idx_pos)

                # Sort our results by date...
                def sort_conv_key(info):
                    return -int(info[idx.MSG_DATE], 36)

                conv.sort(key=sort_conv_key)

                # Convert to index positions only
                conv = [int(info[idx.MSG_MID], 36) for info in conv]

                session.results = conv
                results.append(
                    SearchResults(session, idx, emails=[email], num=len(conv)))
        if len(results) == 1:
            return self._success(_('Displayed a single message'),
                                 result=results[0])
        else:
            session.results = []
            return self._success(_('Displayed %d messages') % len(results),
                                 result=results)
Exemple #39
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        if self.args:
            emails = [Email(idx, i) for i in self._choose_messages(self.args)]
        else:
            local_id, lmbox = config.open_local_mailbox(session)
            emails = [Email.Create(idx, local_id, lmbox)]
            try:
                msg_idxs = [
                    int(e.get_msg_info(idx.MSG_IDX), 36) for e in emails
                ]
                idx.add_tag(session,
                            session.config.get_tag_id('Drafts'),
                            msg_idxs=msg_idxs,
                            conversation=False)
            except (TypeError, ValueError, IndexError):
                self._ignore_exception()

        return self._edit_new_messages(session, idx, emails)
Exemple #40
0
 def CreateMessage(cls, idx, session, ephemeral=False):
     if not ephemeral:
         local_id, lmbox = session.config.open_local_mailbox(session)
     else:
         local_id, lmbox = -1, None
         ephemeral = ['new:mail']
     return (Email.Create(idx, local_id, lmbox,
                          save=(not ephemeral),
                          ephemeral_mid=ephemeral and ephemeral[0]),
             ephemeral)
Exemple #41
0
    def command(self, emails=None):
        session, idx = self.session, self._idx()
        args = list(self.args)
        atts = []

        if '--' in args:
            atts = args[args.index('--') + 1:]
            args = args[:args.index('--')]
        elif args:
            atts = [args.pop(-1)]
        atts.extend(self.data.get('att', []))

        if not emails:
            args.extend(['=%s' % mid for mid in self.data.get('mid', [])])
            emails = [
                self._actualize_ephemeral(i)
                for i in self._choose_messages(args, allow_ephemeral=True)
            ]
        if not emails:
            return self._error(_('No messages selected'))
        else:
            emails = [Email(idx, i) for i in emails]

        updated = []
        errors = []

        def err(msg):
            errors.append(msg)
            session.ui.error(msg)

        for email in emails:
            subject = email.get_msg_info(MailIndex.MSG_SUBJECT)
            try:
                email.remove_attachments(session, *atts)
                updated.append(email)
            except KeyboardInterrupt:
                raise
            except NotEditableError:
                err(_('Read-only message: %s') % subject)
            except:
                err(_('Error removing from %s') % subject)
                self._ignore_exception()

        if errors:
            self.message = _('Removed %s from %d messages, failed %d') % (
                ', '.join(atts), len(updated), len(errors))
        else:
            self.message = _('Removed %s from %d messages') % (', '.join(atts),
                                                               len(updated))

        session.ui.notify(self.message)
        return self._return_search_results(self.message,
                                           updated,
                                           expand=updated,
                                           error=errors)
Exemple #42
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)
        if args and args[-1][0] == "#":
            attid = args.pop()
        else:
            attid = self.data.get("att", 'application/pgp-keys')
        args.extend(["=%s" % x for x in self.data.get("mid", [])])
        eids = self._choose_messages(args)
        if len(eids) < 0:
            return self._error("No messages selected", None)
        elif len(eids) > 1:
            return self._error("One message at a time, please", None)

        email = Email(idx, list(eids)[0])
        fn, attr = email.extract_attachment(session, attid, mode='inline')
        if attr and attr["data"]:
            res = self._gnupg().import_keys(attr["data"])
            return self._success("Imported key", res)

        return self._error("No results found", None)
Exemple #43
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        args = list(self.args)
        flags = []
        while args and args[0][:1] == '-':
            flags.append(args.pop(0))

        msg_idxs = list(self._choose_messages(args))
        if not msg_idxs:
            return self._error('No messages selected')

        wrote = []
        for msg_idx in msg_idxs:
            e = Email(idx, msg_idx)
            ts = long(e.get_msg_info(field=idx.MSG_DATE), 36)
            dt = datetime.datetime.fromtimestamp(ts)
            subject = e.get_msg_info(field=idx.MSG_SUBJECT)

            fn = ('%4.4d-%2.2d-%2.2d.%s.%s.html'
                  % (dt.year, dt.month, dt.day,
                     CleanText(subject,
                               banned=CleanText.NONDNS, replace='_'
                               ).clean.replace('____', '_')[:50],
                     e.msg_mid())
                  ).encode('ascii', 'ignore')

            session.ui.mark(_('Printing e-mail to %s') % fn)
            smv = SingleMessageView(session, arg=['=%s' % e.msg_mid()])
            html = smv.run().as_html()
            if '-sign' in flags:
                key = config.prefs.gpg_recipient
                html = '<printed ts=%d -->\n%s\n<!-- \n' % (time.time(), html)
                rc, signed = self._gnupg().sign(html.encode('utf-8'),
                                                fromkey=key,
                                                clearsign=True)
                if rc != 0:
                   return self._error('Failed to sign printout')
                html = '<!--\n%s\n-->\n' % signed.decode('utf-8')
            with open(fn, 'wb') as fd:
                fd.write(html.encode('utf-8'))
                wrote.append({'mid': e.msg_mid(), 'filename': fn})

        return self._success(_('Printed to %d files') % len(wrote), wrote)
Exemple #44
0
    def _retrain(self, tags=None):
        "Retrain autotaggers"
        session, config, idx = self.session, self.session.config, self._idx()
        tags = tags or [asb.match_tag for asb in autotag_configs(config)]
        tids = [config.get_tag(t)._key for t in tags if t]

        session.ui.mark(_('Retraining SpamBayes autotaggers'))
        if not config.real_hasattr('autotag'):
            config.real_setattr('autotag', {})

        # Find all the interesting messages! We don't look in the trash,
        # but we do look at interesting spam.
        #
        # Note: By specifically stating that we DON'T want trash, we
        #       disable the search engine's default result suppression
        #       and guarantee these results don't corrupt the somewhat
        #       lame/broken result cache.
        #
        no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')]
        interest = {}
        for ttype in ('replied', 'read', 'tagged'):
            interest[ttype] = set()
            for tag in config.get_tags(type=ttype):
                interest[ttype] |= idx.search(session,
                                              ['in:%s' % tag.slug] + no_trash
                                              ).as_set()
            session.ui.notify(_('Have %d interesting %s messages'
                                ) % (len(interest[ttype]), ttype))

        retrained, unreadable = [], []
        count_all = 0
        for at_config in autotag_configs(config):
            at_tag = config.get_tag(at_config.match_tag)
            if at_tag and at_tag._key in tids:
                session.ui.mark('Retraining: %s' % at_tag.name)

                yn = [(set(), set(), 'in:%s' % at_tag.slug, True),
                      (set(), set(), '-in:%s' % at_tag.slug, False)]

                # Get the current message sets: tagged and untagged messages
                # excluding trash.
                for tset, mset, srch, which in yn:
                    mset |= idx.search(session, [srch] + no_trash).as_set()

                # If we have any exclude_tags, they are particularly
                # interesting, so we'll look at them first.
                interesting = []
                for etagid in at_config.exclude_tags:
                    etag = config.get_tag(etagid)
                    if etag._key not in interest:
                        srch = ['in:%s' % etag._key] + no_trash
                        interest[etag._key] = idx.search(session, srch
                                                         ).as_set()
                    interesting.append(etag._key)
                interesting.extend(['replied', 'read', 'tagged', None])

                # Go through the interest types in order of preference and
                # while we still lack training data, add to the training set.
                for ttype in interesting:
                    for tset, mset, srch, which in yn:
                        # False positives are really annoying, and generally
                        # speaking any autotagged subset should be a small
                        # part of the Universe. So we divide the corpus
                        # budget 33% True, 67% False.
                        full_size = int(at_config.corpus_size *
                                        (0.33 if which else 0.67))
                        want = min(full_size // len(interesting),
                                   max(0, full_size - len(tset)))
                        # Make sure we always fully utilize our budget
                        if full_size > len(tset) and not ttype:
                            want = full_size - len(tset)

                        if want:
                            if ttype:
                                adding = sorted(list(mset & interest[ttype]))
                            else:
                                adding = sorted(list(mset))
                            adding = set(list(reversed(adding))[:want])
                            tset |= adding
                            mset -= adding

                # Load classifier, reset
                atagger = config.load_auto_tagger(at_config)
                atagger.reset(at_config)
                for tset, mset, srch, which in yn:
                    count = 0
                    # We go through the list of message in order, to avoid
                    # thrashing caches too badly.
                    for msg_idx in sorted(list(tset)):
                        try:
                            e = Email(idx, msg_idx)
                            count += 1
                            count_all += 1
                            session.ui.mark(
                                _('Reading %s (%d/%d, %s=%s)'
                                  ) % (e.msg_mid(), count, len(tset),
                                       at_tag.name, which))
                            atagger.learn(at_config,
                                          e.get_msg(),
                                          self._get_keywords(e),
                                          which)
                            play_nice_with_threads()
                            if mailpile.util.QUITTING:
                                return self._error('Aborted')
                        except (IndexError, TypeError, ValueError,
                                OSError, IOError):
                            if 'autotag' in session.config.sys.debug:
                                import traceback
                                traceback.print_exc()
                            unreadable.append(msg_idx)
                            session.ui.warning(
                                _('Failed to process message at =%s'
                                  ) % (b36(msg_idx)))

                # We got this far without crashing, so save the result.
                config.save_auto_tagger(at_config)
                retrained.append(at_tag.name)

        message = _('Retrained SpamBayes auto-tagging for %s'
                    ) % ', '.join(retrained)
        session.ui.mark(message)
        return self._success(message, result={
            'retrained': retrained,
            'unreadable': unreadable,
            'read_messages': count_all
        })
Exemple #45
0
    def read_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts,
                     mailbox=None):
        keywords = []
        snippet = ''
        payload = [None]
        for part in msg.walk():
            textpart = payload[0] = None
            ctype = part.get_content_type()
            charset = part.get_charset() or 'iso-8859-1'

            def _loader(p):
                if payload[0] is None:
                    payload[0] = self.try_decode(p.get_payload(None, True),
                                                 charset)
                return payload[0]

            if ctype == 'text/plain':
                textpart = _loader(part)
            elif ctype == 'text/html':
                _loader(part)
                if len(payload[0]) > 3:
                    try:
                        textpart = lxml.html.fromstring(payload[0]
                                                        ).text_content()
                    except:
                        session.ui.warning(_('=%s/%s has bogus HTML.'
                                             ) % (msg_mid, msg_id))
                        textpart = payload[0]
                else:
                    textpart = payload[0]
            elif 'pgp' in part.get_content_type():
                keywords.append('pgp:has')

            att = part.get_filename()
            if att:
                att = self.try_decode(att, charset)
                keywords.append('attachment:has')
                keywords.extend([t + ':att' for t
                                 in re.findall(WORD_REGEXP, att.lower())])
                textpart = (textpart or '') + ' ' + att

            if textpart:
                # FIXME: Does this lowercase non-ASCII characters correctly?
                keywords.extend(re.findall(WORD_REGEXP, textpart.lower()))

                # NOTE: As a side effect here, the cryptostate plugin will
                #       add a 'crypto:has' keyword which we check for below
                #       before performing further processing.
                for kwe in plugins.get_text_kw_extractors():
                    keywords.extend(kwe(self, msg, ctype, textpart))

                if len(snippet) < 1024:
                    snippet += ' ' + textpart

            for extract in plugins.get_data_kw_extractors():
                keywords.extend(extract(self, msg, ctype, att, part,
                                        lambda: _loader(part)))

        if 'crypto:has' in keywords:
            e = Email(self, -1)
            e.msg_parsed = msg
            e.msg_info = self.BOGUS_METADATA[:]
            tree = e.get_message_tree(want=(e.WANT_MSG_TREE_PGP +
                                            ('text_parts', )))

            # Look for inline PGP parts, update our status if found
            e.evaluate_pgp(tree, decrypt=session.config.prefs.index_encrypted)
            msg.signature_info = tree['crypto']['signature']
            msg.encryption_info = tree['crypto']['encryption']

            # Index the contents, if configured to do so
            if session.config.prefs.index_encrypted:
                for text in [t['data'] for t in tree['text_parts']]:
                    keywords.extend(re.findall(WORD_REGEXP, text.lower()))
                    for kwe in plugins.get_text_kw_extractors():
                        keywords.extend(kwe(self, msg, 'text/plain', text))

        keywords.append('%s:id' % msg_id)
        keywords.extend(re.findall(WORD_REGEXP,
                                   self.hdr(msg, 'subject').lower()))
        keywords.extend(re.findall(WORD_REGEXP,
                                   self.hdr(msg, 'from').lower()))
        if mailbox:
            keywords.append('%s:mailbox' % mailbox.lower())
        keywords.append('%s:hp' % HeaderPrint(msg))

        for key in msg.keys():
            key_lower = key.lower()
            if key_lower not in BORING_HEADERS:
                emails = ExtractEmails(self.hdr(msg, key).lower())
                words = set(re.findall(WORD_REGEXP,
                                       self.hdr(msg, key).lower()))
                words -= STOPLIST
                keywords.extend(['%s:%s' % (t, key_lower) for t in words])
                keywords.extend(['%s:%s' % (e, key_lower) for e in emails])
                keywords.extend(['%s:email' % e for e in emails])
                if 'list' in key_lower:
                    keywords.extend(['%s:list' % t for t in words])
        for key in EXPECTED_HEADERS:
            if not msg[key]:
                keywords.append('missing:%s' % key)

        for extract in plugins.get_meta_kw_extractors():
            keywords.extend(extract(self, msg_mid, msg, msg_size, msg_ts))

        snippet = snippet.replace('\n', ' '
                                  ).replace('\t', ' ').replace('\r', '')
        return (set(keywords) - STOPLIST), snippet.strip()
Exemple #46
0
    def command(self, save=True):
        session, config, idx = self.session, self.session.config, self._idx()
        mbox_type = config.prefs.export_format

        if self.session.config.sys.lockdown:
            return self._error(_('In lockdown, doing nothing.'))

        args = list(self.args)
        if args and ':' in args[-1]:
            mbox_type, path = args.pop(-1).split(':', 1)
        else:
            path = self.export_path(mbox_type)

        flat = notags = False
        while args and args[0][:1] == '-':
            option = args.pop(0).replace('-', '')
            if option == 'flat':
                flat = True
            elif option == 'notags':
                notags = True

        if os.path.exists(path):
            return self._error('Already exists: %s' % path)

        msg_idxs = list(self._choose_messages(args))
        if not msg_idxs:
            session.ui.warning('No messages selected')
            return False

        # Exporting messages without their threads barely makes any
        # sense.
        if not flat:
            for i in reversed(range(0, len(msg_idxs))):
                mi = msg_idxs[i]
                msg_idxs[i:i+1] = [int(m[idx.MSG_MID], 36)
                                   for m in idx.get_conversation(msg_idx=mi)]

        # Let's always export in the same order. Stability is nice.
        msg_idxs.sort()

        mbox = self.create_mailbox(mbox_type, path)
        exported = {}
        while msg_idxs:
            msg_idx = msg_idxs.pop(0)
            if msg_idx not in exported:
                e = Email(idx, msg_idx)
                session.ui.mark('Exporting =%s ...' % e.msg_mid())
                fd = e.get_file()
                try:
                    data = fd.read()
                    if not notags:
                        tags = [tag.slug for tag in
                                (self.session.config.get_tag(t) or t for t
                                 in e.get_msg_info(idx.MSG_TAGS).split(',')
                                 if t)
                                if hasattr(tag, 'slug')]
                        lf = '\r\n' if ('\r\n' in data[:200]) else '\n'
                        header, body = data.split(lf+lf, 1)
                        data = str(lf.join([
                            header,
                            'X-Mailpile-Tags: ' + '; '.join(sorted(tags)
                                                            ).encode('utf-8'),
                            '',
                            body
                        ]))
                    mbox.add(data.replace('\r\n', '\n'))
                    exported[msg_idx] = 1
                finally:
                    fd.close()

        mbox.flush()

        return self._success(
            _('Exported %d messages to %s') % (len(exported), path),
            {
                'exported': len(exported),
                'created': path
            })
Exemple #47
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()
        tags = self.args or [asb.match_tag for asb in config.prefs.autotag]
        tids = [config.get_tag(t)._key for t in tags if t]

        session.ui.mark(_('Retraining SpamBayes autotaggers'))
        if not hasattr(config, 'autotag'):
            config.autotag = {}

        # Find all the interesting messages! We don't look in the trash,
        # but we do look at interesting spam.
        #
        # Note: By specifically stating that we DON'T want trash, we
        #       disable the search engine's default result suppression
        #       and guarantee these results don't corrupt the somewhat
        #       lame/broken result cache.
        #
        no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')]
        interest = {}
        for ttype in ('replied', 'read', 'tagged'):
            interest[ttype] = set()
            for tag in config.get_tags(type=ttype):
                interest[ttype] |= idx.search(session,
                                              ['in:%s' % tag.slug] + no_trash
                                              ).as_set()
            session.ui.notify(_('Have %d interesting %s messages'
                                ) % (len(interest[ttype]), ttype))

        retrained = []
        count_all = 0
        for at_config in config.prefs.autotag:
            at_tag = config.get_tag(at_config.match_tag)
            if at_tag and at_tag._key in tids:
                session.ui.mark('Retraining: %s' % at_tag.name)

                yn = [(set(), set(), 'in:%s' % at_tag.slug, True),
                      (set(), set(), '-in:%s' % at_tag.slug, False)]

                # Get the current message sets: tagged and untagged messages
                # excluding trash.
                for tset, mset, srch, which in yn:
                    mset |= idx.search(session, [srch] + no_trash).as_set()

                # If we have any exclude_tags, they are particularly
                # interesting, so we'll look at them first.
                interesting = []
                for etagid in at_config.exclude_tags:
                    etag = config.get_tag(etagid)
                    if etag._key not in interest:
                        srch = ['in:%s' % etag._key] + no_trash
                        interest[etag._key] = idx.search(session, srch
                                                         ).as_set()
                    interesting.append(etag._key)
                interesting.extend(['replied', 'read', 'tagged', None])

                # Go through the interest types in order of preference and
                # while we still lack training data, add to the training set.
                for ttype in interesting:
                    for tset, mset, srch, which in yn:
                        # FIXME: Is this a good idea? No single data source
                        # is allowed to be more than 50% of the corpus, to
                        # try and encourage diversity.
                        want = min(at_config.corpus_size / 4,
                                   max(0,
                                       at_config.corpus_size / 2 - len(tset)))
                        if want:
                            if ttype:
                                adding = sorted(list(mset & interest[ttype]))
                            else:
                                adding = sorted(list(mset))
                            adding = set(list(reversed(adding))[:want])
                            tset |= adding
                            mset -= adding

                # Load classifier, reset
                atagger = config.load_auto_tagger(at_config)
                atagger.reset(at_config)
                for tset, mset, srch, which in yn:
                    count = 0
                    for msg_idx in tset:
                        e = Email(idx, msg_idx)
                        count += 1
                        count_all += 1
                        session.ui.mark(('Reading %s (%d/%d, %s=%s)'
                                         ) % (e.msg_mid(), count, len(tset),
                                              at_tag.name, which))
                        atagger.learn(at_config,
                                      e.get_msg(),
                                      self._get_keywords(e),
                                      which)

                # We got this far without crashing, so save the result.
                config.save_auto_tagger(at_config)
                retrained.append(at_tag.name)

        session.ui.mark(_('Retrained SpamBayes auto-tagging for %s'
                          ) % ', '.join(retrained))
        return {'retrained': retrained, 'read_messages': count_all}
Exemple #48
0
    def read_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts, mailbox=None):
        keywords = []
        snippet = ""
        payload = [None]
        for part in msg.walk():
            textpart = payload[0] = None
            ctype = part.get_content_type()
            charset = part.get_content_charset() or "iso-8859-1"

            def _loader(p):
                if payload[0] is None:
                    payload[0] = self.try_decode(p.get_payload(None, True), charset)
                return payload[0]

            if ctype == "text/plain":
                textpart = _loader(part)
            elif ctype == "text/html":
                _loader(part)
                if len(payload[0]) > 3:
                    try:
                        textpart = lxml.html.fromstring(payload[0]).text_content()
                    except:
                        session.ui.warning(_("=%s/%s has bogus HTML.") % (msg_mid, msg_id))
                        textpart = payload[0]
                else:
                    textpart = payload[0]
            elif "pgp" in part.get_content_type():
                keywords.append("pgp:has")

            att = part.get_filename()
            if att:
                att = self.try_decode(att, charset)
                keywords.append("attachment:has")
                keywords.extend([t + ":att" for t in re.findall(WORD_REGEXP, att.lower())])
                textpart = (textpart or "") + " " + att

            if textpart:
                # FIXME: Does this lowercase non-ASCII characters correctly?
                keywords.extend(re.findall(WORD_REGEXP, textpart.lower()))

                # NOTE: As a side effect here, the cryptostate plugin will
                #       add a 'crypto:has' keyword which we check for below
                #       before performing further processing.
                for kwe in plugins.get_text_kw_extractors():
                    keywords.extend(kwe(self, msg, ctype, textpart))

                if len(snippet) < 1024:
                    snippet += " " + textpart

            for extract in plugins.get_data_kw_extractors():
                keywords.extend(extract(self, msg, ctype, att, part, lambda: _loader(part)))

        if "crypto:has" in keywords:
            e = Email(self, -1)
            e.msg_parsed = msg
            e.msg_info = self.BOGUS_METADATA[:]
            tree = e.get_message_tree(want=(e.WANT_MSG_TREE_PGP + ("text_parts",)))

            # Look for inline PGP parts, update our status if found
            e.evaluate_pgp(tree, decrypt=session.config.prefs.index_encrypted)
            msg.signature_info = tree["crypto"]["signature"]
            msg.encryption_info = tree["crypto"]["encryption"]

            # Index the contents, if configured to do so
            if session.config.prefs.index_encrypted:
                for text in [t["data"] for t in tree["text_parts"]]:
                    keywords.extend(re.findall(WORD_REGEXP, text.lower()))
                    for kwe in plugins.get_text_kw_extractors():
                        keywords.extend(kwe(self, msg, "text/plain", text))

        keywords.append("%s:id" % msg_id)
        keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, "subject").lower()))
        keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, "from").lower()))
        if mailbox:
            keywords.append("%s:mailbox" % mailbox.lower())
        keywords.append("%s:hp" % HeaderPrint(msg))

        for key in msg.keys():
            key_lower = key.lower()
            if key_lower not in BORING_HEADERS:
                emails = ExtractEmails(self.hdr(msg, key).lower())
                words = set(re.findall(WORD_REGEXP, self.hdr(msg, key).lower()))
                words -= STOPLIST
                keywords.extend(["%s:%s" % (t, key_lower) for t in words])
                keywords.extend(["%s:%s" % (e, key_lower) for e in emails])
                keywords.extend(["%s:email" % e for e in emails])
                if "list" in key_lower:
                    keywords.extend(["%s:list" % t for t in words])
        for key in EXPECTED_HEADERS:
            if not msg[key]:
                keywords.append("%s:missing" % key)

        for extract in plugins.get_meta_kw_extractors():
            keywords.extend(extract(self, msg_mid, msg, msg_size, msg_ts))

        snippet = snippet.replace("\n", " ").replace("\t", " ").replace("\r", "")
        return (set(keywords) - STOPLIST), snippet.strip()
Exemple #49
0
    def command(self, save=True):
        session, config, idx = self.session, self.session.config, self._idx()
        mbox_type = config.prefs.export_format

        args = list(self.args)
        if args and ':' in args[-1]:
            mbox_type, path = args.pop(-1).split(':', 1)
        else:
            path = self.export_path(mbox_type)

        flat = notags = False
        while args and args[0][:1] == '-':
            option = args.pop(0).replace('-', '')
            if option == 'flat':
                flat = True
            elif option == 'notags':
                notags = True

        if os.path.exists(path):
            return self._error('Already exists: %s' % path)

        msg_idxs = list(self._choose_messages(args))
        if not msg_idxs:
            session.ui.warning('No messages selected')
            return False

        # Exporting messages without their threads barely makes any
        # sense.
        if not flat:
            for i in reversed(range(0, len(msg_idxs))):
                mi = msg_idxs[i]
                msg_idxs[i:i+1] = [int(m[idx.MSG_MID], 36)
                                   for m in idx.get_conversation(msg_idx=mi)]

        # Let's always export in the same order. Stability is nice.
        msg_idxs.sort()

        try:
            mbox = self.create_mailbox(mbox_type, path)
        except (IOError, OSError):
            mbox = None
        if mbox is None:
            if not os.path.exists(os.path.dirname(path)):
                reason = _('Parent directory does not exist.')
            else:
                reason = _('Is the disk full? Are permissions lacking?')
            return self._error(_('Failed to create mailbox: %s') % reason)

        exported = {}
        failed = []
        while msg_idxs:
            msg_idx = msg_idxs.pop(0)
            if msg_idx not in exported:
                e = Email(idx, msg_idx)
                session.ui.mark(_('Exporting message =%s ...') % e.msg_mid())
                fd = e.get_file()
                if fd is None:
                    failed.append(e.msg_mid())
                    session.ui.warning(_('Message =%s is unreadable! Skipping.'
                                         ) % e.msg_mid())
                    continue
                try:
                    data = fd.read()
                    if not notags:
                        tags = [tag.slug for tag in
                                (self.session.config.get_tag(t) or t for t
                                 in e.get_msg_info(idx.MSG_TAGS).split(',')
                                 if t)
                                if hasattr(tag, 'slug')]
                        lf = '\r\n' if ('\r\n' in data[:200]) else '\n'
                        header, body = data.split(lf+lf, 1)
                        data = str(lf.join([
                            header,
                            'X-Mailpile-Tags: ' + '; '.join(sorted(tags)
                                                            ).encode('utf-8'),
                            '',
                            body
                        ]))
                    mbox.add(data.replace('\r\n', '\n'))
                    exported[msg_idx] = 1
                finally:
                    fd.close()

        mbox.flush()
        result = {
            'exported': len(exported),
            'created': path
        }
        if failed:
            result['failed'] = failed
        return self._success(
            _('Exported %d messages to %s') % (len(exported), path),
            result)
Exemple #50
0
    def read_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts,
                     mailbox=None):
        keywords = []
        snippet = ''
        payload = [None]
        for part in msg.walk():
            textpart = payload[0] = None
            ctype = part.get_content_type()
            charset = part.get_charset() or 'iso-8859-1'

            def _loader(p):
                if payload[0] is None:
                    payload[0] = self.try_decode(p.get_payload(None, True),
                                                 charset)
                return payload[0]

            if ctype == 'text/plain':
                textpart = _loader(part)
            elif ctype == 'text/html':
                _loader(part)
                if len(payload[0]) > 3:
                    try:
                        textpart = lxml.html.fromstring(payload[0]
                                                        ).text_content()
                    except:
                        session.ui.warning(_('=%s/%s has bogus HTML.'
                                             ) % (msg_mid, msg_id))
                        textpart = payload[0]
                else:
                    textpart = payload[0]
            elif 'pgp' in part.get_content_type():
                keywords.append('pgp:has')

            att = part.get_filename()
            if att:
                att = self.try_decode(att, charset)
                keywords.append('attachment:has')
                keywords.extend([t + ':att' for t
                                 in re.findall(WORD_REGEXP, att.lower())])
                textpart = (textpart or '') + ' ' + att

            if textpart:
                # FIXME: Does this lowercase non-ASCII characters correctly?
                # FIXME: What about encrypted content?
                # FIXME: Do this better.
                if ('-----BEGIN PGP' in textpart and
                        '-----END PGP' in textpart):
                    keywords.append('pgp:has')
                    if '-----BEGIN PGP ENCRYPTED' in textpart:
                        keywords.append('pgp-encrypted-text:has')
                    else:
                        keywords.append('pgp-signed-text:has')
                keywords.extend(re.findall(WORD_REGEXP, textpart.lower()))
                for extract in plugins.get_text_kw_extractors():
                    keywords.extend(extract(self, msg, ctype,
                                            lambda: textpart))

                if len(snippet) < 1024:
                    snippet += ' ' + textpart

            for extract in plugins.get_data_kw_extractors():
                keywords.extend(extract(self, msg, ctype, att, part,
                                        lambda: _loader(part)))

        if (session.config.prefs.index_encrypted and
                'pgp-encrypted-text:has' in keywords):
            e = Email(None, -1)
            e.msg_parsed = msg
            e.msg_info = ['' for i in range(0, self.MSG_FIELDS_V2)]
            tree = e.get_message_tree(want=['text_parts'])
            for text in [t['data'] for t in tree['text_parts']]:
                print 'OOO, INLINE PGP, PARSING, WOOT'
                keywords.extend(re.findall(WORD_REGEXP, text.lower()))
                for extract in plugins.get_text_kw_extractors():
                    keywords.extend(extract(self, msg, 'text/plain',
                                            lambda: text))

        keywords.append('%s:id' % msg_id)
        keywords.extend(re.findall(WORD_REGEXP,
                                   self.hdr(msg, 'subject').lower()))
        keywords.extend(re.findall(WORD_REGEXP,
                                   self.hdr(msg, 'from').lower()))
        if mailbox:
            keywords.append('%s:mailbox' % mailbox.lower())
        keywords.append('%s:hp' % HeaderPrint(msg))

        for key in msg.keys():
            key_lower = key.lower()
            if key_lower not in BORING_HEADERS:
                emails = ExtractEmails(self.hdr(msg, key).lower())
                words = set(re.findall(WORD_REGEXP,
                                       self.hdr(msg, key).lower()))
                words -= STOPLIST
                keywords.extend(['%s:%s' % (t, key_lower) for t in words])
                keywords.extend(['%s:%s' % (e, key_lower) for e in emails])
                keywords.extend(['%s:email' % e for e in emails])
                if 'list' in key_lower:
                    keywords.extend(['%s:list' % t for t in words])
        for key in EXPECTED_HEADERS:
            if not msg[key]:
                keywords.append('missing:%s' % key)

        for extract in plugins.get_meta_kw_extractors():
            keywords.extend(extract(self, msg_mid, msg, msg_size, msg_ts))

        snippet = snippet.replace('\n', ' '
                                  ).replace('\t', ' ').replace('\r', '')
        return (set(keywords) - STOPLIST), snippet.strip()
Exemple #51
0
    def command(self):
        session, config, idx = self.session, self.session.config, self._idx()

        # Command-line arguments...
        msgs = list(self.args)
        timeout = -1
        tracking_id = None
        with_header = False
        without_mid = False
        columns = []
        while msgs and msgs[0].lower() != '--':
            arg = msgs.pop(0)
            if arg.startswith('--timeout='):
                timeout = float(arg[10:])
            elif arg.startswith('--header'):
                with_header = True
            elif arg.startswith('--no-mid'):
                without_mid = True
            else:
                columns.append(arg)
        if msgs and msgs[0].lower() == '--':
            msgs.pop(0)

        # Form arguments...
        timeout = float(self.data.get('timeout', [timeout])[0])
        with_header |= truthy(self.data.get('header', [''])[0])
        without_mid |= truthy(self.data.get('no-mid', [''])[0])
        tracking_id = self.data.get('track-id', [tracking_id])[0]
        columns.extend(self.data.get('term', []))
        msgs.extend(['={0!s}'.format(mid.replace('=', ''))
                     for mid in self.data.get('mid', [])])

        # Add a header to the CSV if requested
        if with_header:
            results = [[col.split('||')[0].split(':', 1)[0].split('=', 1)[0]
                        for col in columns]]
            if not without_mid:
                results[0] = ['MID'] + results[0]
        else:
            results = []

        deadline = (time.time() + timeout) if (timeout > 0) else None
        msg_idxs = self._choose_messages(msgs)
        progress = []
        for msg_idx in msg_idxs:
            e = Email(idx, msg_idx)
            if self.event and tracking_id:
                progress.append(msg_idx)
                self.event.private_data = {"progress": len(progress),
                                           "track-id": tracking_id,
                                           "total": len(msg_idxs),
                                           "reading": e.msg_mid()}
                self.event.message = _('Digging into =%s') % e.msg_mid()
                self._update_event_state(self.event.RUNNING, log=True)
            else:
                session.ui.mark(_('Digging into =%s') % e.msg_mid())
            row = [] if without_mid else ['{0!s}'.format(e.msg_mid())]
            for cellspec in columns:
                row.extend(self._cell(idx, e, cellspec))
            results.append(row)
            if deadline and deadline < time.time():
                break

        return self._success(_('Found %d rows in %d messages'
                               ) % (len(results), len(msg_idxs)), results)