def _actualize_ephemeral(self, ephemeral_mid): if isinstance(ephemeral_mid, int): # Not actually ephemeral, just return a normal Email return Email(self._idx(), ephemeral_mid) etype, mid = ephemeral_mid.split(':') etype = etype.lower() if etype in ('forward', 'forward-att'): refs = [Email(self._idx(), int(mid, 36))] e = Forward.CreateForward(self._idx(), self.session, refs, with_atts=('att' in etype))[0] self._track_action('fwded', refs) elif etype in ('reply', 'reply-all'): refs = [Email(self._idx(), int(mid, 36))] e = Reply.CreateReply(self._idx(), self.session, refs, reply_all=('all' in etype))[0] self._track_action('replied', refs) else: e = Compose.CreateMessage(self._idx(), self.session)[0] self._tag_blank([e]) self.session.ui.debug('Actualized: %s' % e.msg_mid()) return Email(self._idx(), e.msg_idx_pos)
def command(self): session, config, idx = self.session, self.session.config, self._idx() delay = play_nice_with_threads() if delay > 0: session.ui.notify( (_("Note: periodic delay is %ss, run from shell to " "speed up: mp --rescan=...")) % delay ) if self.args and self.args[0].lower() == "vcards": return self._rescan_vcards(session, config) elif self.args and self.args[0].lower() == "mailboxes": return self._rescan_mailboxes(session, config) elif self.args and self.args[0].lower() == "all": self.args.pop(0) msg_idxs = self._choose_messages(self.args) if msg_idxs: for msg_idx_pos in msg_idxs: e = Email(idx, msg_idx_pos) session.ui.mark("Re-indexing %s" % e.msg_mid()) idx.index_email(self.session, e) return {"messages": len(msg_idxs)} else: # FIXME: Need a lock here? if "rescan" in config._running: return True config._running["rescan"] = True try: return dict_merge(self._rescan_vcards(session, config), self._rescan_mailboxes(session, config)) finally: del config._running["rescan"]
def command(self): session, config, idx = self.session, self.session.config, self._idx() delay = play_nice_with_threads() if delay > 0: session.ui.notify(( _('Note: periodic delay is %ss, run from shell to ' 'speed up: mp --rescan=...') ) % delay) if self.args and self.args[0].lower() == 'vcards': return self._rescan_vcards(session, config) elif self.args and self.args[0].lower() == 'all': self.args.pop(0) msg_idxs = self._choose_messages(self.args) if msg_idxs: session.ui.warning(_('FIXME: rescan messages: %s') % msg_idxs) for msg_idx_pos in msg_idxs: e = Email(idx, msg_idx_pos) session.ui.mark('Re-indexing %s' % e.msg_mid()) idx.index_email(self.session, e) return {'messages': len(msg_idxs)} else: # FIXME: Need a lock here? if 'rescan' in config._running: return True config._running['rescan'] = True try: return dict_merge( self._rescan_vcards(session, config), self._rescan_mailboxes(session, config) ) finally: del config._running['rescan']
def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) if args and args[-1][0] == "#": attid = args.pop() else: attid = self.data.get("att", "application/pgp-keys") args.extend(["=%s" % x for x in self.data.get("mid", [])]) eids = self._choose_messages(args) if len(eids) < 0: return self._error("No messages selected", None) elif len(eids) > 1: return self._error("One message at a time, please", None) email = Email(idx, list(eids)[0]) fn, attr = email.extract_attachment(session, attid, mode="inline") if attr and attr["data"]: res = self._gnupg().import_keys(attr["data"]) # Previous crypto evaluations may now be out of date, so we # clear the cache so users can see results right away. ClearParseCache(pgpmime=True) return self._success("Imported key", res) return self._error("No results found", None)
def command(self, save=True): session, config, idx = self.session, self.session.config, self._idx() mbox_type = config.prefs.export_format args = list(self.args) if args and ':' in args[-1]: mbox_type, path = args.pop(-1).split(':', 1) else: path = self.export_path(mbox_type) if os.path.exists(path): return self._error('Already exists: %s' % path) msg_idxs = self._choose_messages(args) if not msg_idxs: session.ui.warning('No messages selected') return False mbox = self.create_mailbox(mbox_type, path) for msg_idx in msg_idxs: e = Email(idx, msg_idx) session.ui.mark('Exporting =%s ...' % e.msg_mid()) m = e.get_msg() # FIXME: This doesn't work #tags = [t.slug for t in e.get_message_tags()] #print 'Tags: %s' % tags #m['X-Mailpile-Tags'] = ', '.join(tags) mbox.add(m) mbox.flush() session.ui.mark('Exported %d messages to %s' % (len(msg_idxs), path)) return {'exported': len(msg_idxs), 'created': path}
def command(self, save=True): session, config, idx = self.session, self.session.config, self._idx() mbox_type = config.prefs.export_format args = list(self.args) if args and ":" in args[-1]: mbox_type, path = args.pop(-1).split(":", 1) else: path = self.export_path(mbox_type) if os.path.exists(path): return self._error("Already exists: %s" % path) msg_idxs = self._choose_messages(args) if not msg_idxs: session.ui.warning("No messages selected") return False mbox = self.create_mailbox(mbox_type, path) for msg_idx in msg_idxs: e = Email(idx, msg_idx) session.ui.mark("Exporting =%s ..." % e.msg_mid()) m = e.get_msg() # FIXME: This doesn't work # tags = [t.slug for t in e.get_message_tags()] # print 'Tags: %s' % tags # m['X-Mailpile-Tags'] = ', '.join(tags) mbox.add(m) mbox.flush() session.ui.mark("Exported %d messages to %s" % (len(msg_idxs), path)) return {"exported": len(msg_idxs), "created": path}
def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) if args and args[-1][0] == "#": attid = args.pop() else: attid = self.data.get("att", 'application/pgp-keys') args.extend(["=%s" % x for x in self.data.get("mid", [])]) eids = self._choose_messages(args) if len(eids) < 0: return self._error("No messages selected", None) elif len(eids) > 1: return self._error("One message at a time, please", None) email = Email(idx, list(eids)[0]) fn, attr = email.extract_attachment(session, attid, mode='inline') if attr and attr["data"]: res = self._gnupg().import_keys(attr["data"]) # Previous crypto evaluations may now be out of date, so we # clear the cache so users can see results right away. ClearParseCache(pgpmime=True) return self._success("Imported key", res) return self._error("No results found", None)
def command(self): session, config, idx = self.session, self.session.config, self._idx() if self.args and self.args[0].lower() == 'all': reply_all = self.args.pop(0) or True else: reply_all = False refs = [Email(idx, i) for i in self._choose_messages(self.args)] if refs: trees = [ m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs ] ref_ids = [t['headers_lc'].get('message-id') for t in trees] ref_subjs = [t['headers_lc'].get('subject') for t in trees] msg_to = [ t['headers_lc'].get('reply-to', t['headers_lc']['from']) for t in trees ] msg_cc = [] if reply_all: msg_cc += [t['headers_lc'].get('to', '') for t in trees] msg_cc += [t['headers_lc'].get('cc', '') for t in trees] msg_bodies = [] for t in trees: # FIXME: Templates/settings for how we quote replies? text = (('%s wrote:\n' % t['headers_lc']['from']) + ''.join([ p['data'] for p in t['text_parts'] if p['type'] in ('text', 'quote', 'pgpsignedtext', 'pgpsecuretext', 'pgpverifiedtext') ])) msg_bodies.append(text.replace('\n', '\n> ')) local_id, lmbox = config.open_local_mailbox(session) try: email = Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=('Re: %s' % ref_subjs[-1]), msg_to=msg_to, msg_cc=[r for r in msg_cc if r], msg_references=[i for i in ref_ids if i]) try: idx.add_tag( session, session.config.get_tag_id('Drafts'), msg_idxs=[int(email.get_msg_info(idx.MSG_IDX), 36)], conversation=False) except (TypeError, ValueError, IndexError): self._ignore_exception() except NoFromAddressError: return self._error('You must configure a From address first.') return self._edit_new_messages(session, idx, [email]) else: return self._error('No message found')
def command(self, save=True): session, config, idx = self.session, self.session.config, self._idx() mbox_type = config.prefs.export_format if self.session.config.sys.lockdown: return self._error(_('In lockdown, doing nothing.')) args = list(self.args) if args and ':' in args[-1]: mbox_type, path = args.pop(-1).split(':', 1) else: path = self.export_path(mbox_type) if args and args[-1] == 'flat': flat = True args.pop(-1) else: flat = False if os.path.exists(path): return self._error('Already exists: %s' % path) msg_idxs = list(self._choose_messages(args)) if not msg_idxs: session.ui.warning('No messages selected') return False # Exporting messages without their threads barely makes any # sense. if not flat: for i in reversed(range(0, len(msg_idxs))): mi = msg_idxs[i] msg_idxs[i:i + 1] = [ int(m[idx.MSG_MID], 36) for m in idx.get_conversation(msg_idx=mi) ] # Let's always export in the same order. Stability is nice. msg_idxs.sort() mbox = self.create_mailbox(mbox_type, path) exported = {} while msg_idxs: msg_idx = msg_idxs.pop(0) if msg_idx not in exported: e = Email(idx, msg_idx) session.ui.mark('Exporting =%s ...' % e.msg_mid()) mbox.add(e.get_msg()) exported[msg_idx] = 1 mbox.flush() return self._success( _('Exported %d messages to %s') % (len(exported), path), { 'exported': len(exported), 'created': path })
def command(self, save=True): session, config, idx = self.session, self.session.config, self._idx() mbox_type = config.prefs.export_format if self.session.config.sys.lockdown: return self._error(_('In lockdown, doing nothing.')) args = list(self.args) if args and ':' in args[-1]: mbox_type, path = args.pop(-1).split(':', 1) else: path = self.export_path(mbox_type) if args and args[-1] == 'flat': flat = True args.pop(-1) else: flat = False if os.path.exists(path): return self._error('Already exists: %s' % path) msg_idxs = list(self._choose_messages(args)) if not msg_idxs: session.ui.warning('No messages selected') return False # Exporting messages without their threads barely makes any # sense. if not flat: for i in reversed(range(0, len(msg_idxs))): mi = msg_idxs[i] msg_idxs[i:i+1] = [int(m[idx.MSG_MID], 36) for m in idx.get_conversation(msg_idx=mi)] # Let's always export in the same order. Stability is nice. msg_idxs.sort() mbox = self.create_mailbox(mbox_type, path) exported = {} while msg_idxs: msg_idx = msg_idxs.pop(0) if msg_idx not in exported: e = Email(idx, msg_idx) session.ui.mark('Exporting =%s ...' % e.msg_mid()) mbox.add(e.get_msg()) exported[msg_idx] = 1 mbox.flush() return self._success( _('Exported %d messages to %s') % (len(exported), path), { 'exported': len(exported), 'created': path })
def command(self): session, config, idx = self.session, self.session.config, self._idx() if self.args and self.args[0].lower().startswith('att'): with_atts = self.args.pop(0) or True else: with_atts = False refs = [Email(idx, i) for i in self._choose_messages(self.args)] if refs: trees = [ m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs ] ref_subjs = [t['headers_lc']['subject'] for t in trees] msg_bodies = [] msg_atts = [] for t in trees: # FIXME: Templates/settings for how we quote forwards? text = '-------- Original Message --------\n' for h in ('Date', 'Subject', 'From', 'To'): v = t['headers_lc'].get(h.lower(), None) if v: text += '%s: %s\n' % (h, v) text += '\n' text += ''.join([ p['data'] for p in t['text_parts'] if p['type'] in self._TEXT_PARTTYPES ]) msg_bodies.append(text) if with_atts: for att in t['attachments']: if att['mimetype'] not in self._ATT_MIMETYPES: msg_atts.append(att['part']) local_id, lmbox = config.open_local_mailbox(session) email = Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=('Fwd: %s' % ref_subjs[-1])) if msg_atts: msg = email.get_msg() for att in msg_atts: msg.attach(att) email.update_from_msg(msg) if self.BLANK_TAG: self._tag_emails([email], self.BLANK_TAG) return self._edit_messages([email]) else: return self._error('No message found')
def command(self): session, config, idx = self.session, self.session.config, self._idx() # Command-line arguments... msgs = list(self.args) timeout = -1 with_header = False without_mid = False columns = [] while msgs and msgs[0].lower() != '--': arg = msgs.pop(0) if arg.startswith('--timeout='): timeout = float(arg[10:]) elif arg.startswith('--header'): with_header = True elif arg.startswith('--no-mid'): without_mid = True else: columns.append(msgs.pop(0)) if msgs and msgs[0].lower() == '--': msgs.pop(0) # Form arguments... timeout = float(self.data.get('timeout', [timeout])[0]) with_header |= self._truthy(self.data.get('header', [''])[0]) without_mid |= self._truthy(self.data.get('no-mid', [''])[0]) columns.extend(self.data.get('term', [])) msgs.extend(['=%s' % mid.replace('=', '') for mid in self.data.get('mid', [])]) # Add a header to the CSV if requested if with_header: results = [[col.split('||')[0].split(':', 1)[0].split('=', 1)[0] for col in columns]] if not without_mid: results[0] = ['MID'] + results[0] else: results = [] deadline = (time.time() + timeout) if (timeout > 0) else None msg_idxs = self._choose_messages(msgs) for msg_idx in msg_idxs: e = Email(idx, msg_idx) session.ui.mark(_('Digging into =%s') % e.msg_mid()) row = [] if without_mid else ['%s' % e.msg_mid()] for cellspec in columns: row.extend(self._cell(idx, e, cellspec)) results.append(row) if deadline and deadline < time.time(): break return self._success(_('Found %d rows in %d messages' ) % (len(results), len(msg_idxs)), results)
def command(self): session, config, idx = self.session, self.session.config, self._idx() if self.args and self.args[0].lower() == 'all': reply_all = self.args.pop(0) or True else: reply_all = False refs = [Email(idx, i) for i in self._choose_messages(self.args)] if refs: trees = [ m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs ] ref_ids = [t['headers_lc'].get('message-id') for t in trees] ref_subjs = [t['headers_lc'].get('subject') for t in trees] msg_to = [ t['headers_lc'].get('reply-to', t['headers_lc']['from']) for t in trees ] msg_cc = [] if reply_all: msg_cc += [t['headers_lc'].get('to', '') for t in trees] msg_cc += [t['headers_lc'].get('cc', '') for t in trees] msg_bodies = [] for t in trees: # FIXME: Templates/settings for how we quote replies? text = (('%s wrote:\n' % t['headers_lc']['from']) + ''.join([ p['data'] for p in t['text_parts'] if p['type'] in self._TEXT_PARTTYPES ])) msg_bodies.append(text.replace('\n', '\n> ')) local_id, lmbox = config.open_local_mailbox(session) try: email = Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=('Re: %s' % ref_subjs[-1]), msg_to=msg_to, msg_cc=[r for r in msg_cc if r], msg_references=[i for i in ref_ids if i]) if self.BLANK_TAG: self._tag_emails([email], self.BLANK_TAG) except NoFromAddressError: return self._error('You must configure a From address first.') return self._edit_messages([email]) else: return self._error('No message found')
def _lookup(self, address): results = {} terms = ["from:%s" % x for x in address.split('@')] terms.append("has:pgpkey") session, idx, _, _ = self._do_search(search=terms) for messageid in session.results: email = Email(self._idx(), messageid) attachments = email.get_message_tree("attachments")["attachments"] for part in attachments: if part["mimetype"] == "application/pgp-keys": key = part["part"].get_payload(None, True) results.update(self._get_keydata(key)) return results
def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) if config.sys.lockdown: return self._error(_("In lockdown, doing nothing.")) delay = play_nice_with_threads() if delay > 0: session.ui.notify( (_("Note: periodic delay is %ss, run from shell to " "speed up: mp --rescan=...")) % delay ) if args and args[0].lower() == "vcards": return self._rescan_vcards(session, config) elif args and args[0].lower() == "mailboxes": return self._rescan_mailboxes(session, config) elif args and args[0].lower() == "all": args.pop(0) msg_idxs = self._choose_messages(args) if msg_idxs: for msg_idx_pos in msg_idxs: e = Email(idx, msg_idx_pos) try: session.ui.mark("Re-indexing %s" % e.msg_mid()) idx.index_email(self.session, e) except KeyboardInterrupt: raise except: self._ignore_exception() session.ui.warning(_("Failed to reindex: %s") % e.msg_mid()) return self._success(_("Indexed %d messages") % len(msg_idxs), result={"messages": len(msg_idxs)}) else: # FIXME: Need a lock here? if "rescan" in config._running: return self._success(_("Rescan already in progress")) config._running["rescan"] = True try: results = {} results.update(self._rescan_vcards(session, config)) results.update(self._rescan_mailboxes(session, config)) if "aborted" in results: raise KeyboardInterrupt() return self._success(_("Rescanned vcards and mailboxes"), result=results) except (KeyboardInterrupt), e: return self._error(_("User aborted"), info=results) finally:
def _get_message_keys(self, messageid): keys = self.key_cache.get(messageid, []) if not keys: email = Email(self._idx(), messageid) attachments = email.get_message_tree( want=["attachments"])["attachments"] for part in attachments: if _might_be_pgp_key(part["filename"], part["mimetype"]): key = part["part"].get_payload(None, True) for keydata in _get_keydata(key): keys.append((keydata, key)) if len(keys) > 5: # Just to set some limit... break self.key_cache[messageid] = keys return keys
def _get_message_keys(self, messageid): keys = self.key_cache.get(messageid, []) if not keys: email = Email(self._idx(), messageid) attachments = email.get_message_tree(want=["attachments"] )["attachments"] for part in attachments: if _might_be_pgp_key(part["filename"], part["mimetype"]): key = part["part"].get_payload(None, True) for keydata in _get_keydata(key): keys.append((keydata, key)) if len(keys) > 5: # Just to set some limit... break self.key_cache[messageid] = keys return keys
def _get_keys(self, messageid): keys = self.key_cache.get(messageid, []) if not keys: email = Email(self._idx(), messageid) attachments = email.get_message_tree("attachments")["attachments"] for part in attachments: if part["mimetype"] == "application/pgp-keys": key = part["part"].get_payload(None, True) for keydata in self._get_keydata(key): keys.append(keydata) self.key_cache[keydata["fingerprint"]] = key if len(keys) > 5: # Just to set some limit... break self.key_cache[messageid] = keys return keys
def command(self): session, config, idx = self.session, self.session.config, self._idx() if self.args[0] in ('inline', 'inline-preview', 'preview', 'download'): mode = self.args.pop(0) else: mode = 'download' cid = self.args.pop(0) if len(self.args) > 0 and self.args[-1].startswith('>'): name_fmt = self.args.pop(-1)[1:] else: name_fmt = None emails = [Email(idx, i) for i in self._choose_messages(self.args)] results = [] for email in emails: fn, info = email.extract_attachment(session, cid, name_fmt=name_fmt, mode=mode) if info: info['idx'] = email.msg_idx_pos if fn: info['created_file'] = fn results.append(info) return results
def command(self): session, config, idx = self.session, self.session.config, self._idx() results = [] if self.args and self.args[0].lower() == 'raw': raw = self.args.pop(0) else: raw = False emails = [Email(idx, mid) for mid in self._choose_messages(self.args)] idx.apply_filters(session, '@read', msg_idxs=[e.msg_idx_pos for e in emails]) for email in emails: if raw: results.append( self.RawResult({'data': email.get_file().read()})) else: conv = [ int(c[0], 36) for c in idx.get_conversation(msg_idx=email.msg_idx_pos) ] if email.msg_idx_pos not in conv: conv.append(email.msg_idx_pos) conv.reverse() results.append( SearchResults(session, idx, results=conv, num=len(conv), expand=[email])) if len(results) == 1: return results[0] else: return results
def command(self): session, config, idx = self.session, self.session.config, self._idx() emails = [Email(idx, mid) for mid in self._choose_messages(self.args)] scores = self._classify(emails) tag = {} for mid in scores: for at_config in autotag_configs(config): at_tag = config.get_tag(at_config.match_tag) if not at_tag: continue want = scores[mid].get(at_tag._key, (False, ))[0] if want is True: if at_config.match_tag not in tag: tag[at_config.match_tag] = [mid] else: tag[at_config.match_tag].append(mid) elif at_config.unsure_tag and want is None: if at_config.unsure_tag not in tag: tag[at_config.unsure_tag] = [mid] else: tag[at_config.unsure_tag].append(mid) for tid in tag: idx.add_tag(session, tid, msg_idxs=[int(i, 36) for i in tag[tid]]) return self._success(_('Auto-tagged %d messages') % len(emails), tag)
def command(self): session, config, idx = self.session, self.session.config, self._idx() files = [] while os.path.exists(self.args[-1]): files.append(self.args.pop(-1)) if not files: return self._error('No files found') emails = [Email(idx, i) for i in self._choose_messages(self.args)] if not emails: return self._error('No messages selected') # FIXME: Using "say" here is rather lame. updated = [] for email in emails: subject = email.get_msg_info(MailIndex.MSG_SUBJECT) try: email.add_attachments(files) updated.append(email) except NotEditableError: session.ui.error('Read-only message: %s' % subject) except: session.ui.error('Error attaching to %s' % subject) self._ignore_exception() session.ui.notify( ('Attached %s to %d messages') % (', '.join(files), len(updated))) return self._return_search_results(session, idx, updated, updated)
def command(self, emails=None): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) bounce_to = [] while args and '@' in args[-1]: bounce_to.append(args.pop(-1)) for rcpt in (self.data.get('to', []) + self.data.get('cc', []) + self.data.get('bcc', [])): bounce_to.extend(ExtractEmails(rcpt)) if not emails: args.extend(['=%s' % mid for mid in self.data.get('mid', [])]) mids = self._choose_messages(args) emails = [Email(idx, i) for i in mids] # Process one at a time so we don't eat too much memory sent = [] missing_keys = [] for email in emails: try: msg_mid = email.get_msg_info(idx.MSG_MID) # FIXME: We are failing to capture error states with sufficient # granularity, messages may be delivered to some # recipients but not all... SendMail(session, [PrepareMessage(config, email.get_msg(pgpmime=False), rcpts=(bounce_to or None))]) sent.append(email) except KeyLookupError, kle: session.ui.warning(_('Missing keys %s') % kle.missing) missing_keys.extend(kle.missing) self._ignore_exception() except:
def _get_canned(cls, idx, cid): try: return Email(idx, int(cid, 36)).get_editing_strings().get('body', '') except (ValueError, IndexError, TypeError, OSError, IOError): traceback.print_exc() # FIXME, ugly return ''
def command(self): session, config, idx = self.session, self.session.config, self._idx() with_atts = False ephemeral = False while self.args: if self.args[0].lower() == 'att': with_atts = self.args.pop(0) or True elif self.args[0].lower() == 'ephemeral': ephemeral = self.args.pop(0) or True else: break if ephemeral and with_atts: raise UsageError( _('Sorry, ephemeral messages cannot have ' 'attachments at this time.')) refs = [Email(idx, i) for i in self._choose_messages(self.args)] if refs: email, ephemeral = self.CreateReply(idx, session, refs, with_atts=with_atts, ephemeral=ephemeral) if not ephemeral: self._track_action('fwded', refs) self._tag_blank([email]) return self._edit_messages([email], ephemeral=ephemeral) else: return self._error(_('No message found'))
def command(self): session, config, idx = self.session, self.session.config, self._idx() reply_all = False ephemeral = False while self.args: if self.args[0].lower() == 'all': reply_all = self.args.pop(0) or True elif self.args[0].lower() == 'ephemeral': ephemeral = self.args.pop(0) or True else: break refs = [Email(idx, i) for i in self._choose_messages(self.args)] if refs: try: email, ephemeral = self.CreateReply(idx, session, refs, reply_all=reply_all, ephemeral=ephemeral) except NoFromAddressError: return self._error( _('You must configure a ' 'From address first.')) if not ephemeral: self._track_action('replied', refs) self._tag_blank([email]) return self._edit_messages([email], ephemeral=ephemeral) else: return self._error(_('No message found'))
def command(self): session, index = self.session, self._idx() session.ui.mark('Checking index for duplicate MSG IDs...') found = {} for i in range(0, len(index.INDEX)): msg_id = index.get_msg_at_idx_pos(i)[index.MSG_ID] if msg_id in found: found[msg_id].append(i) else: found[msg_id] = [i] session.ui.mark('Attempting to fix dups with bad location...') for msg_id in found: if len(found[msg_id]) > 1: good, bad = [], [] for idx_pos in found[msg_id]: msg = Email(index, idx_pos).get_msg() if msg: good.append(idx_pos) else: bad.append(idx_pos) if good and bad: good_info = index.get_msg_at_idx_pos(good[0]) for bad_idx in bad: bad_info = index.get_msg_at_idx_pos(bad_idx) bad_info[index.MSG_PTRS] = good_info[index.MSG_PTRS] index.set_msg_at_idx_pos(bad_idx, bad_info) session.ui.mark('Done!') return True
def command(self): session, config, idx = self.session, self.session.config, self._idx() mode = 'download' name_fmt = None if self.args[0] in ('inline', 'inline-preview', 'preview', 'download'): mode = self.args.pop(0) if len(self.args) > 0 and self.args[-1].startswith('>'): name_fmt = self.args.pop(-1)[1:] if self.args[0].startswith('#') or self.args[0].startswith('part:'): cid = self.args.pop(0) else: cid = self.args.pop(-1) eids = self._choose_messages(self.args) print 'Download %s from %s as %s/%s' % (cid, eids, mode, name_fmt) emails = [Email(idx, i) for i in eids] results = [] for e in emails: fn, info = e.extract_attachment(session, cid, name_fmt=name_fmt, mode=mode) if info: info['idx'] = email.msg_idx_pos if fn: info['created_file'] = fn results.append(info) return results
def command(self): session, config, idx = self.session, self.session.config, self._idx() results = [] args = list(self.args) if args and args[0].lower() == 'raw': raw = args.pop(0) else: raw = False emails = [Email(idx, mid) for mid in self._choose_messages(args)] rv = self._side_effects(emails) if rv is not None: # This is here so derived classes can do funky things. return rv for email in emails: if raw: subject = email.get_msg_info(idx.MSG_SUBJECT) results.append( self.RawResult({ 'summary': _('Raw message: %s') % subject, 'source': email.get_file().read() })) else: old_result = None for result in results: if email.msg_idx_pos in result.results: old_result = result if old_result: old_result.add_email(email) continue conv = [ int(c[0], 36) for c in idx.get_conversation(msg_idx=email.msg_idx_pos) ] if email.msg_idx_pos not in conv: conv.append(email.msg_idx_pos) # FIXME: This is a hack. The indexer should just keep things # in the right order on rescan. Fixing threading is a # bigger problem though, so we do this for now. def sort_conv_key(msg_idx_pos): info = idx.get_msg_at_idx_pos(msg_idx_pos) return -int(info[idx.MSG_DATE], 36) conv.sort(key=sort_conv_key) results.append( SearchResults(session, idx, results=conv, num=len(conv), emails=[email])) if len(results) == 1: return self._success(_('Displayed a single message'), result=results[0]) else: return self._success(_('Displayed %d messages') % len(results), result=results)
def process_message(self, peer, mailfrom, rcpttos, data): # We can assume that the mailfrom and rcpttos have checked out # and this message is indeed intended for us. Spool it to disk # and add to the index! session, config = self.session, self.session.config blank_tid = config.get_tags(type='blank')[0]._key idx = config.index play_nice_with_threads() try: message = email.parser.Parser().parsestr(data) lid, lmbox = config.open_local_mailbox(session) e = Email.Create(idx, lid, lmbox, ephemeral_mid=False) idx.add_tag(session, blank_tid, msg_idxs=[e.msg_idx_pos], conversation=False) e.update_from_msg(session, message) idx.remove_tag(session, blank_tid, msg_idxs=[e.msg_idx_pos], conversation=False) return None except: traceback.print_exc() return '400 Oops wtf'
def command(self, emails=None): session, config, idx = self.session, self.session.config, self._idx() bounce_to = [] while self.args and '@' in self.args[-1]: bounce_to.append(self.args.pop(-1)) for rcpt in (self.data.get('to', []) + self.data.get('cc', []) + self.data.get('bcc', [])): bounce_to.extend(ExtractEmails(rcpt)) args = self.args[:] if not emails: args.extend(['=%s' % mid for mid in self.data.get('mid', [])]) mids = self._choose_messages(args) emails = [Email(idx, i) for i in mids] # Process one at a time so we don't eat too much memory sent = [] missing_keys = [] for email in emails: try: msg_mid = email.get_msg_info(idx.MSG_MID) SendMail(session, [ PrepareMessage(config, email.get_msg(pgpmime=False), rcpts=(bounce_to or None)) ]) sent.append(email) except KeyLookupError, kle: missing_keys.extend(kle.missing) self._ignore_exception() except:
def CreateForward(cls, idx, session, refs, msgid, with_atts=False, cid=None, ephemeral=False): trees = [ m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs ] ref_subjs = [t['headers_lc']['subject'] for t in trees] msg_bodies = [] msg_atts = [] for t in trees: # FIXME: Templates/settings for how we quote forwards? text = '-------- Original Message --------\n' for h in ('Date', 'Subject', 'From', 'To'): v = t['headers_lc'].get(h.lower(), None) if v: text += '%s: %s\n' % (h, v) text += '\n' text += ''.join([ p['data'] for p in t['text_parts'] if p['type'] in cls._TEXT_PARTTYPES ]) msg_bodies.append(text) if with_atts: for att in t['attachments']: if att['mimetype'] not in cls._ATT_MIMETYPES: msg_atts.append(att['part']) if not ephemeral: local_id, lmbox = session.config.open_local_mailbox(session) else: local_id, lmbox = -1, None fmt = 'forward-att-%s-%s' if msg_atts else 'forward-%s-%s' ephemeral = [ fmt % (msgid[1:-1].replace('@', '_'), refs[0].msg_mid()) ] if cid: # FIXME: Instead, we should use placeholders in the template # and insert the quoted bits in the right place (or # nowhere if the template doesn't want them). msg_bodies[:0] = [cls._get_canned(idx, cid)] email = Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=cls.prefix_subject( ref_subjs[-1], 'Fwd:', cls._FW_REGEXP), msg_id=msgid, msg_atts=msg_atts, save=(not ephemeral), ephemeral_mid=ephemeral and ephemeral[0]) return email, ephemeral
def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) if config.sys.lockdown: return self._error(_('In lockdown, doing nothing.')) delay = play_nice_with_threads() if delay > 0: session.ui.notify(( _('Note: periodic delay is %ss, run from shell to ' 'speed up: mp --rescan=...') ) % delay) if args and args[0].lower() == 'vcards': return self._rescan_vcards(session, config) elif args and args[0].lower() == 'mailboxes': return self._rescan_mailboxes(session, config) elif args and args[0].lower() == 'all': args.pop(0) msg_idxs = self._choose_messages(args) if msg_idxs: for msg_idx_pos in msg_idxs: e = Email(idx, msg_idx_pos) session.ui.mark('Re-indexing %s' % e.msg_mid()) idx.index_email(self.session, e) return self._success(_('Indexed %d messages') % len(msg_idxs), result={'messages': len(msg_idxs)}) else: # FIXME: Need a lock here? if 'rescan' in config._running: return self._success(_('Rescan already in progress')) config._running['rescan'] = True try: results = {} results.update(self._rescan_vcards(session, config)) results.update(self._rescan_mailboxes(session, config)) if 'aborted' in results: raise KeyboardInterrupt() return self._success(_('Rescanned vcards and mailboxes'), result=results) except (KeyboardInterrupt), e: return self._error(_('User aborted'), info=results) finally:
def command(self, search=None): session, idx, start = self._do_search(search=search) nodes = [] links = [] res = {} for messageid in session.results: message = Email(self._idx(), messageid) try: msgfrom = ExtractEmails(message.get("from"))[0].lower() except IndexError, e: print "No e-mail address in '%s'" % message.get("from") continue msgto = [x.lower() for x in ExtractEmails(message.get("to"))] msgcc = [x.lower() for x in ExtractEmails(message.get("cc"))] msgbcc = [x.lower() for x in ExtractEmails(message.get("bcc"))] if msgfrom not in [m["email"] for m in nodes]: nodes.append({"email": msgfrom}) for msgset in [msgto, msgcc, msgbcc]: for address in msgset: if address not in [m["email"] for m in nodes]: nodes.append({"email": address}) curnodes = [x["email"] for x in nodes] fromid = curnodes.index(msgfrom) searchspace = [m for m in links if m["source"] == fromid] for recipient in msgset: index = curnodes.index(recipient) link = [m for m in searchspace if m["target"] == index] if len(link) == 0: links.append({"source": fromid, "target": index, "value": 1}) elif len(link) == 1: link[0]["value"] += 1 else: raise ValueError("Too many links! - This should never happen.") if len(nodes) >= 200: # Let's put a hard upper limit on how many nodes we can have, for performance reasons. # There might be a better way to do this though... res["limit_hit"] = True break
def _actualize_ephemeral(self, ephemeral_mid): idx = self._idx() if isinstance(ephemeral_mid, int): # Not actually ephemeral, just return a normal Email return Email(idx, ephemeral_mid) msgid, mid = ephemeral_mid.rsplit('-', 1) etype, etarg, msgid = ephemeral_mid.split('-', 2) if etarg not in ('all', 'att'): msgid = etarg + '-' + msgid msgid = '<%s>' % msgid.replace('_', '@') etype = etype.lower() enc_msgid = idx.encode_msg_id(msgid) msg_idx = idx.MSGIDS.get(enc_msgid) if msg_idx is not None: # Already actualized, just return a normal Email return Email(idx, msg_idx) if etype == 'forward': refs = [Email(idx, int(mid, 36))] e = Forward.CreateForward(idx, self.session, refs, msgid, with_atts=(etarg == 'att'))[0] self._track_action('fwded', refs) elif etype == 'reply': refs = [Email(idx, int(mid, 36))] e = Reply.CreateReply(idx, self.session, refs, msgid, reply_all=(etarg == 'all'))[0] self._track_action('replied', refs) else: e = Compose.CreateMessage(idx, self.session, msgid)[0] self._tag_blank([e]) self.session.ui.debug('Actualized: %s' % e.msg_mid()) return Email(idx, e.msg_idx_pos)
def CreateReply(cls, idx, session, refs, msgid, reply_all=False, cid=None, ephemeral=False): trees = [m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs] headers = cls._create_from_to_cc(idx, session, trees) if not reply_all and 'cc' in headers: del headers['cc'] ref_ids = [t['headers_lc'].get('message-id') for t in trees] ref_subjs = [t['headers_lc'].get('subject') for t in trees] msg_bodies = [] for t in trees: # FIXME: Templates/settings for how we quote replies? quoted = ''.join([p['data'] for p in t['text_parts'] if p['type'] in cls._TEXT_PARTTYPES and p['data']]) if quoted: target_width = session.config.prefs.line_length if target_width > 40: quoted = reflow_text(quoted, target_width=target_width-2) text = ((_('%s wrote:') % t['headers_lc']['from']) + '\n' + quoted) msg_bodies.append('\n\n' + text.replace('\n', '\n> ')) if not ephemeral: local_id, lmbox = session.config.open_local_mailbox(session) else: local_id, lmbox = -1, None fmt = 'reply-all-%s-%s' if reply_all else 'reply-%s-%s' ephemeral = [fmt % (msgid[1:-1].replace('@', '_'), refs[0].msg_mid())] if 'cc' in headers: fmt = _('Composing a reply from %(from)s to %(to)s, cc %(cc)s') else: fmt = _('Composing a reply from %(from)s to %(to)s') session.ui.debug(fmt % headers) if cid: # FIXME: Instead, we should use placeholders in the template # and insert the quoted bits in the right place (or # nowhere if the template doesn't want them). msg_bodies[:0] = [cls._get_canned(idx, cid)] return (Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=cls.prefix_subject( ref_subjs[-1], 'Re:', cls._RE_REGEXP), msg_from=headers.get('from', None), msg_to=headers.get('to', []), msg_cc=headers.get('cc', []), msg_references=[i for i in ref_ids if i], msg_id=msgid, save=(not ephemeral), ephemeral_mid=ephemeral and ephemeral[0]), ephemeral)
def command(self): session, config, idx = self.session, self.session.config, self._idx() results = [] args = list(self.args) args.extend( ['=%s' % mid.replace('=', '') for mid in self.data.get('mid', [])]) if args and args[0].lower() == 'raw': raw = args.pop(0) else: raw = False emails = [Email(idx, mid) for mid in self._choose_messages(args)] rv = self._side_effects(emails) if rv is not None: # This is here so derived classes can do funky things. return rv for email in emails: if raw: subject = email.get_msg_info(idx.MSG_SUBJECT) results.append( self.RawResult({ 'summary': _('Raw message: %s') % subject, 'source': email.get_file().read() })) else: old_result = None for result in results: if email.msg_idx_pos in result.results: old_result = result if old_result: old_result.add_email(email) continue # Get conversation conv = idx.get_conversation(msg_idx=email.msg_idx_pos) # Sort our results by date... def sort_conv_key(info): return -int(info[idx.MSG_DATE], 36) conv.sort(key=sort_conv_key) # Convert to index positions only conv = [int(info[idx.MSG_MID], 36) for info in conv] session.results = conv results.append( SearchResults(session, idx, emails=[email], num=len(conv))) if len(results) == 1: return self._success(_('Displayed a single message'), result=results[0]) else: session.results = [] return self._success(_('Displayed %d messages') % len(results), result=results)
def command(self): session, config, idx = self.session, self.session.config, self._idx() if self.args: emails = [Email(idx, i) for i in self._choose_messages(self.args)] else: local_id, lmbox = config.open_local_mailbox(session) emails = [Email.Create(idx, local_id, lmbox)] try: msg_idxs = [ int(e.get_msg_info(idx.MSG_IDX), 36) for e in emails ] idx.add_tag(session, session.config.get_tag_id('Drafts'), msg_idxs=msg_idxs, conversation=False) except (TypeError, ValueError, IndexError): self._ignore_exception() return self._edit_new_messages(session, idx, emails)
def CreateMessage(cls, idx, session, ephemeral=False): if not ephemeral: local_id, lmbox = session.config.open_local_mailbox(session) else: local_id, lmbox = -1, None ephemeral = ['new:mail'] return (Email.Create(idx, local_id, lmbox, save=(not ephemeral), ephemeral_mid=ephemeral and ephemeral[0]), ephemeral)
def command(self, emails=None): session, idx = self.session, self._idx() args = list(self.args) atts = [] if '--' in args: atts = args[args.index('--') + 1:] args = args[:args.index('--')] elif args: atts = [args.pop(-1)] atts.extend(self.data.get('att', [])) if not emails: args.extend(['=%s' % mid for mid in self.data.get('mid', [])]) emails = [ self._actualize_ephemeral(i) for i in self._choose_messages(args, allow_ephemeral=True) ] if not emails: return self._error(_('No messages selected')) else: emails = [Email(idx, i) for i in emails] updated = [] errors = [] def err(msg): errors.append(msg) session.ui.error(msg) for email in emails: subject = email.get_msg_info(MailIndex.MSG_SUBJECT) try: email.remove_attachments(session, *atts) updated.append(email) except KeyboardInterrupt: raise except NotEditableError: err(_('Read-only message: %s') % subject) except: err(_('Error removing from %s') % subject) self._ignore_exception() if errors: self.message = _('Removed %s from %d messages, failed %d') % ( ', '.join(atts), len(updated), len(errors)) else: self.message = _('Removed %s from %d messages') % (', '.join(atts), len(updated)) session.ui.notify(self.message) return self._return_search_results(self.message, updated, expand=updated, error=errors)
def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) if args and args[-1][0] == "#": attid = args.pop() else: attid = self.data.get("att", 'application/pgp-keys') args.extend(["=%s" % x for x in self.data.get("mid", [])]) eids = self._choose_messages(args) if len(eids) < 0: return self._error("No messages selected", None) elif len(eids) > 1: return self._error("One message at a time, please", None) email = Email(idx, list(eids)[0]) fn, attr = email.extract_attachment(session, attid, mode='inline') if attr and attr["data"]: res = self._gnupg().import_keys(attr["data"]) return self._success("Imported key", res) return self._error("No results found", None)
def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) flags = [] while args and args[0][:1] == '-': flags.append(args.pop(0)) msg_idxs = list(self._choose_messages(args)) if not msg_idxs: return self._error('No messages selected') wrote = [] for msg_idx in msg_idxs: e = Email(idx, msg_idx) ts = long(e.get_msg_info(field=idx.MSG_DATE), 36) dt = datetime.datetime.fromtimestamp(ts) subject = e.get_msg_info(field=idx.MSG_SUBJECT) fn = ('%4.4d-%2.2d-%2.2d.%s.%s.html' % (dt.year, dt.month, dt.day, CleanText(subject, banned=CleanText.NONDNS, replace='_' ).clean.replace('____', '_')[:50], e.msg_mid()) ).encode('ascii', 'ignore') session.ui.mark(_('Printing e-mail to %s') % fn) smv = SingleMessageView(session, arg=['=%s' % e.msg_mid()]) html = smv.run().as_html() if '-sign' in flags: key = config.prefs.gpg_recipient html = '<printed ts=%d -->\n%s\n<!-- \n' % (time.time(), html) rc, signed = self._gnupg().sign(html.encode('utf-8'), fromkey=key, clearsign=True) if rc != 0: return self._error('Failed to sign printout') html = '<!--\n%s\n-->\n' % signed.decode('utf-8') with open(fn, 'wb') as fd: fd.write(html.encode('utf-8')) wrote.append({'mid': e.msg_mid(), 'filename': fn}) return self._success(_('Printed to %d files') % len(wrote), wrote)
def _retrain(self, tags=None): "Retrain autotaggers" session, config, idx = self.session, self.session.config, self._idx() tags = tags or [asb.match_tag for asb in autotag_configs(config)] tids = [config.get_tag(t)._key for t in tags if t] session.ui.mark(_('Retraining SpamBayes autotaggers')) if not config.real_hasattr('autotag'): config.real_setattr('autotag', {}) # Find all the interesting messages! We don't look in the trash, # but we do look at interesting spam. # # Note: By specifically stating that we DON'T want trash, we # disable the search engine's default result suppression # and guarantee these results don't corrupt the somewhat # lame/broken result cache. # no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')] interest = {} for ttype in ('replied', 'read', 'tagged'): interest[ttype] = set() for tag in config.get_tags(type=ttype): interest[ttype] |= idx.search(session, ['in:%s' % tag.slug] + no_trash ).as_set() session.ui.notify(_('Have %d interesting %s messages' ) % (len(interest[ttype]), ttype)) retrained, unreadable = [], [] count_all = 0 for at_config in autotag_configs(config): at_tag = config.get_tag(at_config.match_tag) if at_tag and at_tag._key in tids: session.ui.mark('Retraining: %s' % at_tag.name) yn = [(set(), set(), 'in:%s' % at_tag.slug, True), (set(), set(), '-in:%s' % at_tag.slug, False)] # Get the current message sets: tagged and untagged messages # excluding trash. for tset, mset, srch, which in yn: mset |= idx.search(session, [srch] + no_trash).as_set() # If we have any exclude_tags, they are particularly # interesting, so we'll look at them first. interesting = [] for etagid in at_config.exclude_tags: etag = config.get_tag(etagid) if etag._key not in interest: srch = ['in:%s' % etag._key] + no_trash interest[etag._key] = idx.search(session, srch ).as_set() interesting.append(etag._key) interesting.extend(['replied', 'read', 'tagged', None]) # Go through the interest types in order of preference and # while we still lack training data, add to the training set. for ttype in interesting: for tset, mset, srch, which in yn: # False positives are really annoying, and generally # speaking any autotagged subset should be a small # part of the Universe. So we divide the corpus # budget 33% True, 67% False. full_size = int(at_config.corpus_size * (0.33 if which else 0.67)) want = min(full_size // len(interesting), max(0, full_size - len(tset))) # Make sure we always fully utilize our budget if full_size > len(tset) and not ttype: want = full_size - len(tset) if want: if ttype: adding = sorted(list(mset & interest[ttype])) else: adding = sorted(list(mset)) adding = set(list(reversed(adding))[:want]) tset |= adding mset -= adding # Load classifier, reset atagger = config.load_auto_tagger(at_config) atagger.reset(at_config) for tset, mset, srch, which in yn: count = 0 # We go through the list of message in order, to avoid # thrashing caches too badly. for msg_idx in sorted(list(tset)): try: e = Email(idx, msg_idx) count += 1 count_all += 1 session.ui.mark( _('Reading %s (%d/%d, %s=%s)' ) % (e.msg_mid(), count, len(tset), at_tag.name, which)) atagger.learn(at_config, e.get_msg(), self._get_keywords(e), which) play_nice_with_threads() if mailpile.util.QUITTING: return self._error('Aborted') except (IndexError, TypeError, ValueError, OSError, IOError): if 'autotag' in session.config.sys.debug: import traceback traceback.print_exc() unreadable.append(msg_idx) session.ui.warning( _('Failed to process message at =%s' ) % (b36(msg_idx))) # We got this far without crashing, so save the result. config.save_auto_tagger(at_config) retrained.append(at_tag.name) message = _('Retrained SpamBayes auto-tagging for %s' ) % ', '.join(retrained) session.ui.mark(message) return self._success(message, result={ 'retrained': retrained, 'unreadable': unreadable, 'read_messages': count_all })
def read_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts, mailbox=None): keywords = [] snippet = '' payload = [None] for part in msg.walk(): textpart = payload[0] = None ctype = part.get_content_type() charset = part.get_charset() or 'iso-8859-1' def _loader(p): if payload[0] is None: payload[0] = self.try_decode(p.get_payload(None, True), charset) return payload[0] if ctype == 'text/plain': textpart = _loader(part) elif ctype == 'text/html': _loader(part) if len(payload[0]) > 3: try: textpart = lxml.html.fromstring(payload[0] ).text_content() except: session.ui.warning(_('=%s/%s has bogus HTML.' ) % (msg_mid, msg_id)) textpart = payload[0] else: textpart = payload[0] elif 'pgp' in part.get_content_type(): keywords.append('pgp:has') att = part.get_filename() if att: att = self.try_decode(att, charset) keywords.append('attachment:has') keywords.extend([t + ':att' for t in re.findall(WORD_REGEXP, att.lower())]) textpart = (textpart or '') + ' ' + att if textpart: # FIXME: Does this lowercase non-ASCII characters correctly? keywords.extend(re.findall(WORD_REGEXP, textpart.lower())) # NOTE: As a side effect here, the cryptostate plugin will # add a 'crypto:has' keyword which we check for below # before performing further processing. for kwe in plugins.get_text_kw_extractors(): keywords.extend(kwe(self, msg, ctype, textpart)) if len(snippet) < 1024: snippet += ' ' + textpart for extract in plugins.get_data_kw_extractors(): keywords.extend(extract(self, msg, ctype, att, part, lambda: _loader(part))) if 'crypto:has' in keywords: e = Email(self, -1) e.msg_parsed = msg e.msg_info = self.BOGUS_METADATA[:] tree = e.get_message_tree(want=(e.WANT_MSG_TREE_PGP + ('text_parts', ))) # Look for inline PGP parts, update our status if found e.evaluate_pgp(tree, decrypt=session.config.prefs.index_encrypted) msg.signature_info = tree['crypto']['signature'] msg.encryption_info = tree['crypto']['encryption'] # Index the contents, if configured to do so if session.config.prefs.index_encrypted: for text in [t['data'] for t in tree['text_parts']]: keywords.extend(re.findall(WORD_REGEXP, text.lower())) for kwe in plugins.get_text_kw_extractors(): keywords.extend(kwe(self, msg, 'text/plain', text)) keywords.append('%s:id' % msg_id) keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, 'subject').lower())) keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, 'from').lower())) if mailbox: keywords.append('%s:mailbox' % mailbox.lower()) keywords.append('%s:hp' % HeaderPrint(msg)) for key in msg.keys(): key_lower = key.lower() if key_lower not in BORING_HEADERS: emails = ExtractEmails(self.hdr(msg, key).lower()) words = set(re.findall(WORD_REGEXP, self.hdr(msg, key).lower())) words -= STOPLIST keywords.extend(['%s:%s' % (t, key_lower) for t in words]) keywords.extend(['%s:%s' % (e, key_lower) for e in emails]) keywords.extend(['%s:email' % e for e in emails]) if 'list' in key_lower: keywords.extend(['%s:list' % t for t in words]) for key in EXPECTED_HEADERS: if not msg[key]: keywords.append('missing:%s' % key) for extract in plugins.get_meta_kw_extractors(): keywords.extend(extract(self, msg_mid, msg, msg_size, msg_ts)) snippet = snippet.replace('\n', ' ' ).replace('\t', ' ').replace('\r', '') return (set(keywords) - STOPLIST), snippet.strip()
def command(self, save=True): session, config, idx = self.session, self.session.config, self._idx() mbox_type = config.prefs.export_format if self.session.config.sys.lockdown: return self._error(_('In lockdown, doing nothing.')) args = list(self.args) if args and ':' in args[-1]: mbox_type, path = args.pop(-1).split(':', 1) else: path = self.export_path(mbox_type) flat = notags = False while args and args[0][:1] == '-': option = args.pop(0).replace('-', '') if option == 'flat': flat = True elif option == 'notags': notags = True if os.path.exists(path): return self._error('Already exists: %s' % path) msg_idxs = list(self._choose_messages(args)) if not msg_idxs: session.ui.warning('No messages selected') return False # Exporting messages without their threads barely makes any # sense. if not flat: for i in reversed(range(0, len(msg_idxs))): mi = msg_idxs[i] msg_idxs[i:i+1] = [int(m[idx.MSG_MID], 36) for m in idx.get_conversation(msg_idx=mi)] # Let's always export in the same order. Stability is nice. msg_idxs.sort() mbox = self.create_mailbox(mbox_type, path) exported = {} while msg_idxs: msg_idx = msg_idxs.pop(0) if msg_idx not in exported: e = Email(idx, msg_idx) session.ui.mark('Exporting =%s ...' % e.msg_mid()) fd = e.get_file() try: data = fd.read() if not notags: tags = [tag.slug for tag in (self.session.config.get_tag(t) or t for t in e.get_msg_info(idx.MSG_TAGS).split(',') if t) if hasattr(tag, 'slug')] lf = '\r\n' if ('\r\n' in data[:200]) else '\n' header, body = data.split(lf+lf, 1) data = str(lf.join([ header, 'X-Mailpile-Tags: ' + '; '.join(sorted(tags) ).encode('utf-8'), '', body ])) mbox.add(data.replace('\r\n', '\n')) exported[msg_idx] = 1 finally: fd.close() mbox.flush() return self._success( _('Exported %d messages to %s') % (len(exported), path), { 'exported': len(exported), 'created': path })
def command(self): session, config, idx = self.session, self.session.config, self._idx() tags = self.args or [asb.match_tag for asb in config.prefs.autotag] tids = [config.get_tag(t)._key for t in tags if t] session.ui.mark(_('Retraining SpamBayes autotaggers')) if not hasattr(config, 'autotag'): config.autotag = {} # Find all the interesting messages! We don't look in the trash, # but we do look at interesting spam. # # Note: By specifically stating that we DON'T want trash, we # disable the search engine's default result suppression # and guarantee these results don't corrupt the somewhat # lame/broken result cache. # no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')] interest = {} for ttype in ('replied', 'read', 'tagged'): interest[ttype] = set() for tag in config.get_tags(type=ttype): interest[ttype] |= idx.search(session, ['in:%s' % tag.slug] + no_trash ).as_set() session.ui.notify(_('Have %d interesting %s messages' ) % (len(interest[ttype]), ttype)) retrained = [] count_all = 0 for at_config in config.prefs.autotag: at_tag = config.get_tag(at_config.match_tag) if at_tag and at_tag._key in tids: session.ui.mark('Retraining: %s' % at_tag.name) yn = [(set(), set(), 'in:%s' % at_tag.slug, True), (set(), set(), '-in:%s' % at_tag.slug, False)] # Get the current message sets: tagged and untagged messages # excluding trash. for tset, mset, srch, which in yn: mset |= idx.search(session, [srch] + no_trash).as_set() # If we have any exclude_tags, they are particularly # interesting, so we'll look at them first. interesting = [] for etagid in at_config.exclude_tags: etag = config.get_tag(etagid) if etag._key not in interest: srch = ['in:%s' % etag._key] + no_trash interest[etag._key] = idx.search(session, srch ).as_set() interesting.append(etag._key) interesting.extend(['replied', 'read', 'tagged', None]) # Go through the interest types in order of preference and # while we still lack training data, add to the training set. for ttype in interesting: for tset, mset, srch, which in yn: # FIXME: Is this a good idea? No single data source # is allowed to be more than 50% of the corpus, to # try and encourage diversity. want = min(at_config.corpus_size / 4, max(0, at_config.corpus_size / 2 - len(tset))) if want: if ttype: adding = sorted(list(mset & interest[ttype])) else: adding = sorted(list(mset)) adding = set(list(reversed(adding))[:want]) tset |= adding mset -= adding # Load classifier, reset atagger = config.load_auto_tagger(at_config) atagger.reset(at_config) for tset, mset, srch, which in yn: count = 0 for msg_idx in tset: e = Email(idx, msg_idx) count += 1 count_all += 1 session.ui.mark(('Reading %s (%d/%d, %s=%s)' ) % (e.msg_mid(), count, len(tset), at_tag.name, which)) atagger.learn(at_config, e.get_msg(), self._get_keywords(e), which) # We got this far without crashing, so save the result. config.save_auto_tagger(at_config) retrained.append(at_tag.name) session.ui.mark(_('Retrained SpamBayes auto-tagging for %s' ) % ', '.join(retrained)) return {'retrained': retrained, 'read_messages': count_all}
def read_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts, mailbox=None): keywords = [] snippet = "" payload = [None] for part in msg.walk(): textpart = payload[0] = None ctype = part.get_content_type() charset = part.get_content_charset() or "iso-8859-1" def _loader(p): if payload[0] is None: payload[0] = self.try_decode(p.get_payload(None, True), charset) return payload[0] if ctype == "text/plain": textpart = _loader(part) elif ctype == "text/html": _loader(part) if len(payload[0]) > 3: try: textpart = lxml.html.fromstring(payload[0]).text_content() except: session.ui.warning(_("=%s/%s has bogus HTML.") % (msg_mid, msg_id)) textpart = payload[0] else: textpart = payload[0] elif "pgp" in part.get_content_type(): keywords.append("pgp:has") att = part.get_filename() if att: att = self.try_decode(att, charset) keywords.append("attachment:has") keywords.extend([t + ":att" for t in re.findall(WORD_REGEXP, att.lower())]) textpart = (textpart or "") + " " + att if textpart: # FIXME: Does this lowercase non-ASCII characters correctly? keywords.extend(re.findall(WORD_REGEXP, textpart.lower())) # NOTE: As a side effect here, the cryptostate plugin will # add a 'crypto:has' keyword which we check for below # before performing further processing. for kwe in plugins.get_text_kw_extractors(): keywords.extend(kwe(self, msg, ctype, textpart)) if len(snippet) < 1024: snippet += " " + textpart for extract in plugins.get_data_kw_extractors(): keywords.extend(extract(self, msg, ctype, att, part, lambda: _loader(part))) if "crypto:has" in keywords: e = Email(self, -1) e.msg_parsed = msg e.msg_info = self.BOGUS_METADATA[:] tree = e.get_message_tree(want=(e.WANT_MSG_TREE_PGP + ("text_parts",))) # Look for inline PGP parts, update our status if found e.evaluate_pgp(tree, decrypt=session.config.prefs.index_encrypted) msg.signature_info = tree["crypto"]["signature"] msg.encryption_info = tree["crypto"]["encryption"] # Index the contents, if configured to do so if session.config.prefs.index_encrypted: for text in [t["data"] for t in tree["text_parts"]]: keywords.extend(re.findall(WORD_REGEXP, text.lower())) for kwe in plugins.get_text_kw_extractors(): keywords.extend(kwe(self, msg, "text/plain", text)) keywords.append("%s:id" % msg_id) keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, "subject").lower())) keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, "from").lower())) if mailbox: keywords.append("%s:mailbox" % mailbox.lower()) keywords.append("%s:hp" % HeaderPrint(msg)) for key in msg.keys(): key_lower = key.lower() if key_lower not in BORING_HEADERS: emails = ExtractEmails(self.hdr(msg, key).lower()) words = set(re.findall(WORD_REGEXP, self.hdr(msg, key).lower())) words -= STOPLIST keywords.extend(["%s:%s" % (t, key_lower) for t in words]) keywords.extend(["%s:%s" % (e, key_lower) for e in emails]) keywords.extend(["%s:email" % e for e in emails]) if "list" in key_lower: keywords.extend(["%s:list" % t for t in words]) for key in EXPECTED_HEADERS: if not msg[key]: keywords.append("%s:missing" % key) for extract in plugins.get_meta_kw_extractors(): keywords.extend(extract(self, msg_mid, msg, msg_size, msg_ts)) snippet = snippet.replace("\n", " ").replace("\t", " ").replace("\r", "") return (set(keywords) - STOPLIST), snippet.strip()
def command(self, save=True): session, config, idx = self.session, self.session.config, self._idx() mbox_type = config.prefs.export_format args = list(self.args) if args and ':' in args[-1]: mbox_type, path = args.pop(-1).split(':', 1) else: path = self.export_path(mbox_type) flat = notags = False while args and args[0][:1] == '-': option = args.pop(0).replace('-', '') if option == 'flat': flat = True elif option == 'notags': notags = True if os.path.exists(path): return self._error('Already exists: %s' % path) msg_idxs = list(self._choose_messages(args)) if not msg_idxs: session.ui.warning('No messages selected') return False # Exporting messages without their threads barely makes any # sense. if not flat: for i in reversed(range(0, len(msg_idxs))): mi = msg_idxs[i] msg_idxs[i:i+1] = [int(m[idx.MSG_MID], 36) for m in idx.get_conversation(msg_idx=mi)] # Let's always export in the same order. Stability is nice. msg_idxs.sort() try: mbox = self.create_mailbox(mbox_type, path) except (IOError, OSError): mbox = None if mbox is None: if not os.path.exists(os.path.dirname(path)): reason = _('Parent directory does not exist.') else: reason = _('Is the disk full? Are permissions lacking?') return self._error(_('Failed to create mailbox: %s') % reason) exported = {} failed = [] while msg_idxs: msg_idx = msg_idxs.pop(0) if msg_idx not in exported: e = Email(idx, msg_idx) session.ui.mark(_('Exporting message =%s ...') % e.msg_mid()) fd = e.get_file() if fd is None: failed.append(e.msg_mid()) session.ui.warning(_('Message =%s is unreadable! Skipping.' ) % e.msg_mid()) continue try: data = fd.read() if not notags: tags = [tag.slug for tag in (self.session.config.get_tag(t) or t for t in e.get_msg_info(idx.MSG_TAGS).split(',') if t) if hasattr(tag, 'slug')] lf = '\r\n' if ('\r\n' in data[:200]) else '\n' header, body = data.split(lf+lf, 1) data = str(lf.join([ header, 'X-Mailpile-Tags: ' + '; '.join(sorted(tags) ).encode('utf-8'), '', body ])) mbox.add(data.replace('\r\n', '\n')) exported[msg_idx] = 1 finally: fd.close() mbox.flush() result = { 'exported': len(exported), 'created': path } if failed: result['failed'] = failed return self._success( _('Exported %d messages to %s') % (len(exported), path), result)
def read_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts, mailbox=None): keywords = [] snippet = '' payload = [None] for part in msg.walk(): textpart = payload[0] = None ctype = part.get_content_type() charset = part.get_charset() or 'iso-8859-1' def _loader(p): if payload[0] is None: payload[0] = self.try_decode(p.get_payload(None, True), charset) return payload[0] if ctype == 'text/plain': textpart = _loader(part) elif ctype == 'text/html': _loader(part) if len(payload[0]) > 3: try: textpart = lxml.html.fromstring(payload[0] ).text_content() except: session.ui.warning(_('=%s/%s has bogus HTML.' ) % (msg_mid, msg_id)) textpart = payload[0] else: textpart = payload[0] elif 'pgp' in part.get_content_type(): keywords.append('pgp:has') att = part.get_filename() if att: att = self.try_decode(att, charset) keywords.append('attachment:has') keywords.extend([t + ':att' for t in re.findall(WORD_REGEXP, att.lower())]) textpart = (textpart or '') + ' ' + att if textpart: # FIXME: Does this lowercase non-ASCII characters correctly? # FIXME: What about encrypted content? # FIXME: Do this better. if ('-----BEGIN PGP' in textpart and '-----END PGP' in textpart): keywords.append('pgp:has') if '-----BEGIN PGP ENCRYPTED' in textpart: keywords.append('pgp-encrypted-text:has') else: keywords.append('pgp-signed-text:has') keywords.extend(re.findall(WORD_REGEXP, textpart.lower())) for extract in plugins.get_text_kw_extractors(): keywords.extend(extract(self, msg, ctype, lambda: textpart)) if len(snippet) < 1024: snippet += ' ' + textpart for extract in plugins.get_data_kw_extractors(): keywords.extend(extract(self, msg, ctype, att, part, lambda: _loader(part))) if (session.config.prefs.index_encrypted and 'pgp-encrypted-text:has' in keywords): e = Email(None, -1) e.msg_parsed = msg e.msg_info = ['' for i in range(0, self.MSG_FIELDS_V2)] tree = e.get_message_tree(want=['text_parts']) for text in [t['data'] for t in tree['text_parts']]: print 'OOO, INLINE PGP, PARSING, WOOT' keywords.extend(re.findall(WORD_REGEXP, text.lower())) for extract in plugins.get_text_kw_extractors(): keywords.extend(extract(self, msg, 'text/plain', lambda: text)) keywords.append('%s:id' % msg_id) keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, 'subject').lower())) keywords.extend(re.findall(WORD_REGEXP, self.hdr(msg, 'from').lower())) if mailbox: keywords.append('%s:mailbox' % mailbox.lower()) keywords.append('%s:hp' % HeaderPrint(msg)) for key in msg.keys(): key_lower = key.lower() if key_lower not in BORING_HEADERS: emails = ExtractEmails(self.hdr(msg, key).lower()) words = set(re.findall(WORD_REGEXP, self.hdr(msg, key).lower())) words -= STOPLIST keywords.extend(['%s:%s' % (t, key_lower) for t in words]) keywords.extend(['%s:%s' % (e, key_lower) for e in emails]) keywords.extend(['%s:email' % e for e in emails]) if 'list' in key_lower: keywords.extend(['%s:list' % t for t in words]) for key in EXPECTED_HEADERS: if not msg[key]: keywords.append('missing:%s' % key) for extract in plugins.get_meta_kw_extractors(): keywords.extend(extract(self, msg_mid, msg, msg_size, msg_ts)) snippet = snippet.replace('\n', ' ' ).replace('\t', ' ').replace('\r', '') return (set(keywords) - STOPLIST), snippet.strip()
def command(self): session, config, idx = self.session, self.session.config, self._idx() # Command-line arguments... msgs = list(self.args) timeout = -1 tracking_id = None with_header = False without_mid = False columns = [] while msgs and msgs[0].lower() != '--': arg = msgs.pop(0) if arg.startswith('--timeout='): timeout = float(arg[10:]) elif arg.startswith('--header'): with_header = True elif arg.startswith('--no-mid'): without_mid = True else: columns.append(arg) if msgs and msgs[0].lower() == '--': msgs.pop(0) # Form arguments... timeout = float(self.data.get('timeout', [timeout])[0]) with_header |= truthy(self.data.get('header', [''])[0]) without_mid |= truthy(self.data.get('no-mid', [''])[0]) tracking_id = self.data.get('track-id', [tracking_id])[0] columns.extend(self.data.get('term', [])) msgs.extend(['={0!s}'.format(mid.replace('=', '')) for mid in self.data.get('mid', [])]) # Add a header to the CSV if requested if with_header: results = [[col.split('||')[0].split(':', 1)[0].split('=', 1)[0] for col in columns]] if not without_mid: results[0] = ['MID'] + results[0] else: results = [] deadline = (time.time() + timeout) if (timeout > 0) else None msg_idxs = self._choose_messages(msgs) progress = [] for msg_idx in msg_idxs: e = Email(idx, msg_idx) if self.event and tracking_id: progress.append(msg_idx) self.event.private_data = {"progress": len(progress), "track-id": tracking_id, "total": len(msg_idxs), "reading": e.msg_mid()} self.event.message = _('Digging into =%s') % e.msg_mid() self._update_event_state(self.event.RUNNING, log=True) else: session.ui.mark(_('Digging into =%s') % e.msg_mid()) row = [] if without_mid else ['{0!s}'.format(e.msg_mid())] for cellspec in columns: row.extend(self._cell(idx, e, cellspec)) results.append(row) if deadline and deadline < time.time(): break return self._success(_('Found %d rows in %d messages' ) % (len(results), len(msg_idxs)), results)