def _get_body_doc(self): """ Return the document that keeps the body for this message. """ hdoc_content = self.hdoc.content body_phash = hdoc_content.get( fields.BODY_KEY, None) if not body_phash: logger.warning("No body phash for this document!") return None # XXX get from memstore too... # if memstore: memstore.get_phrash # memstore should keep a dict with weakrefs to the # phash doc... if self._container is not None: bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) if not empty(bdoc) and not empty(bdoc.content): return bdoc # no memstore, or no body doc found there if self._soledad: body_docs = self._soledad.get_from_index( fields.TYPE_P_HASH_IDX, fields.TYPE_CONTENT_VAL, str(body_phash)) return first(body_docs) else: logger.error("No phash in container, and no soledad found!")
def _get_msg_copy(self, message): """ Get a copy of the fdoc for this message, and check whether it already exists. :param message: an IMessage implementor :type message: LeapMessage :return: exist, new_fdoc :rtype: tuple """ # XXX for clarity, this could be delegated to a # MessageCollection mixin that implements copy too, and # moved out of here. msg = message memstore = self._memstore if empty(msg.fdoc): logger.warning("Tried to copy a MSG with no fdoc") return new_fdoc = copy.deepcopy(msg.fdoc.content) fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] dest_fdoc = memstore.get_fdoc_from_chash( fdoc_chash, self.mbox) exist = not empty(dest_fdoc) return exist, new_fdoc
def walk(self): """ Generator that iterates through all the parts, returning MessagePartDoc. Used for writing to SoledadStore. :rtype: generator """ if self._dirty: try: mbox = self.fdoc.content[fields.MBOX_KEY] uid = self.fdoc.content[fields.UID_KEY] docid_dict = self._dict[self.DOCS_ID] docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc( mbox, uid) except Exception as exc: logger.debug("Error while walking message...") logger.exception(exc) if not empty(self.fdoc.content) and 'uid' in self.fdoc.content: yield self.fdoc if not empty(self.hdoc.content): yield self.hdoc for cdoc in self.cdocs.values(): if not empty(cdoc): content_ref = weakref.proxy(cdoc) yield MessagePartDoc(new=self.new, dirty=self.dirty, store=self._storetype, part=MessagePartType.cdoc, content=content_ref, doc_id=None)
def getBodyFile(self): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading :rtype: StringIO """ fd = StringIO.StringIO() if not empty(self._pmap): multi = self._pmap.get('multi') if not multi: phash = self._pmap.get("phash", None) else: pmap = self._pmap.get('part_map') first_part = pmap.get('1', None) if not empty(first_part): phash = first_part['phash'] else: phash = None if phash is None: logger.warning("Could not find phash for this subpart!") payload = "" else: payload = self._get_payload_from_document_memoized(phash) if empty(payload): payload = self._get_payload_from_document(phash) else: logger.warning("Message with no part_map!") payload = "" if payload: content_type = self._get_ctype_from_document(phash) charset = find_charset(content_type) if charset is None: charset = self._get_charset(payload) try: if isinstance(payload, unicode): payload = payload.encode(charset) except UnicodeError as exc: logger.error( "Unicode error, using 'replace'. {0!r}".format(exc)) payload = payload.encode(charset, 'replace') fd.write(payload) fd.seek(0) return fd
def _add_message_locally(self, result): """ Adds a message to local inbox and delete it from the incoming db in soledad. # XXX this comes from a gatherresult... :param msgtuple: a tuple consisting of a SoledadDocument instance containing the incoming message and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) """ from twisted.internet import reactor msgtuple = first(result) doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) if isinstance(data, list): if empty(data): return False data = data[0] def msgSavedCallback(result): if not empty(result): leap_events.signal(IMAP_MSG_SAVED_LOCALLY) deferLater(reactor, 0, self._delete_incoming_message, doc) leap_events.signal(IMAP_MSG_DELETED_INCOMING) d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), notify_on_disk=True) d.addCallbacks(msgSavedCallback, self._errback)
def all_rdocs_iter(self): """ Return an iterator through all in-memory recent flag dicts, wrapped under a RecentFlagsDoc namedtuple. Used for saving to disk. :return: a generator of RecentFlagDoc :rtype: generator """ # XXX use enums DOC_ID = "doc_id" SET = "set" rflags_store = self._rflags_store def get_rdoc(mbox, rdict): mbox_rflag_set = rdict[SET] recent_set = copy(mbox_rflag_set) # zero it! mbox_rflag_set.difference_update(mbox_rflag_set) return RecentFlagsDoc( doc_id=rflags_store[mbox][DOC_ID], content={ fields.TYPE_KEY: fields.TYPE_RECENT_VAL, fields.MBOX_KEY: mbox, fields.RECENTFLAGS_KEY: list(recent_set) }) return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items() if not empty(rdict[SET]))
def _get_fdoc_from_chash(self, chash): """ Return a flags document for this mailbox with a given chash. :return: A SoledadDocument containing the Flags Document, or None if the query failed. :rtype: SoledadDocument or None. """ curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_C_HASH_IDX, fields.TYPE_FLAGS_VAL, self.mbox, chash) curried.expected = "fdoc" fdoc = try_unique_query(curried) if fdoc is not None: return fdoc else: # probably this should be the other way round, # ie, try fist on memstore... cf = self.memstore._chash_fdoc_store fdoc = cf[chash][self.mbox] # hey, I just needed to wrap fdoc thing into # a "content" attribute, look a better way... if not empty(fdoc): return MessagePartDoc( new=None, dirty=None, part=None, store=None, doc_id=None, content=fdoc)
def _soledad_write_document_parts(self, items): """ Write the document parts to soledad in a separate thread. :param items: the iterator through the different document wrappers payloads. :type items: iterator :return: whether the write was successful or not :rtype: bool """ failed = False for item, call in items: if empty(item): continue try: self._try_call(call, item) except Exception as exc: logger.debug("ITEM WAS: %s" % repr(item)) if hasattr(item, 'content'): logger.debug("ITEM CONTENT WAS: %s" % repr(item.content)) logger.exception(exc) failed = True continue return failed
def write_messages(self, store): """ Write the message documents in this MemoryStore to a different store. :param store: the IMessageStore to write to :rtype: False if queue is not empty, None otherwise. """ # For now, we pass if the queue is not empty, to avoid duplicate # queuing. # We would better use a flag to know when we've already enqueued an # item. # XXX this could return the deferred for all the enqueued operations if not self.producer.is_queue_empty(): return False if any(map(lambda i: not empty(i), (self._new, self._dirty))): logger.info("Writing messages to Soledad...") # TODO change for lock, and make the property access # is accquired with set_bool_flag(self, self.WRITING_FLAG): for rflags_doc_wrapper in self.all_rdocs_iter(): self.producer.push(rflags_doc_wrapper, state=self.producer.STATE_DIRTY) for msg_wrapper in self.all_new_msg_iter(): self.producer.push(msg_wrapper, state=self.producer.STATE_NEW) for msg_wrapper in self.all_dirty_msg_iter(): self.producer.push(msg_wrapper, state=self.producer.STATE_DIRTY)
def hdoc(self): """ An accessor to the headers document. """ container = self._container if container is not None: hdoc = self._container.hdoc if hdoc and not empty(hdoc.content): return hdoc hdoc = self._get_headers_doc() if container and not empty(hdoc.content): # mem-cache it hdoc_content = hdoc.content chash = hdoc_content.get(fields.CONTENT_HASH_KEY) hdocs = {chash: hdoc_content} container.memstore.load_header_docs(hdocs) return hdoc
def isMultipart(self): """ Return True if this message is multipart. """ if empty(self._pmap): logger.warning("Could not get part map!") return False multi = self._pmap.get("multi", False) return multi
def all_soledad_uid_iter(self): """ Return an iterator through the UIDs of all messages, sorted in ascending order. """ db_uids = set([doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) if not empty(doc)]) return db_uids
def get_docid_for_fdoc(self, mbox, uid): """ Return Soledad document id for the flags-doc for a given mbox and uid, or None of no flags document could be found. :param mbox: the mailbox :type mbox: str or unicode :param uid: the message UID :type uid: int :rtype: unicode or None """ with self._fdoc_docid_lock: doc_id = self._fdoc_id_store[mbox][uid] if empty(doc_id): fdoc = self._permanent_store.get_flags_doc(mbox, uid) if empty(fdoc) or empty(fdoc.content): return None doc_id = fdoc.doc_id self._fdoc_id_store[mbox][uid] = doc_id return doc_id
def getSize(self): """ Return the total size, in octets, of this message part. :return: size of the message, in octets :rtype: int """ if empty(self._pmap): return 0 size = self._pmap.get('size', None) if size is None: logger.error("Message part cannot find size in the partmap") size = 0 return size
def msgSavedCallback(result): if empty(result): return for listener in self._listeners: listener(result) def signal_deleted(doc_id): emit(catalog.MAIL_MSG_DELETED_INCOMING) return doc_id emit(catalog.MAIL_MSG_SAVED_LOCALLY) d = self._delete_incoming_message(doc) d.addCallback(signal_deleted) return d
def getBodyFile(self): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading :rtype: StringIO """ def write_fd(body): fd.write(body) fd.seek(0) return fd # TODO refactor with getBodyFile in MessagePart fd = StringIO.StringIO() if self.bdoc is not None: bdoc_content = self.bdoc.content if empty(bdoc_content): logger.warning("No BDOC content found for message!!!") return write_fd("") body = bdoc_content.get(self.RAW_KEY, "") content_type = bdoc_content.get('content-type', "") charset = find_charset(content_type) if charset is None: charset = self._get_charset(body) try: if isinstance(body, unicode): body = body.encode(charset) except UnicodeError as exc: logger.error( "Unicode error, using 'replace'. {0!r}".format(exc)) logger.debug("Attempted to encode with: %s" % charset) body = body.encode(charset, 'replace') finally: return write_fd(body) # We are still returning funky characters from here. else: logger.warning("No BDOC found for message.") return write_fd("")
def purge_fdoc_store(self, mbox): """ Purge the empty documents from a fdoc store. Called during initialization of the SoledadMailbox :param mbox: the mailbox :type mbox: str or unicode """ # XXX This is really a workaround until I find the conditions # that are making the empty items remain there. # This happens, for instance, after running several times # the regression test, that issues a store deleted + expunge + select # The items are being correclty deleted, but in succesive appends # the empty items with previously deleted uids reappear as empty # documents. I suspect it's a timing condition with a previously # evaluated sequence being used after the items has been removed. for uid, value in self._fdoc_store[mbox].items(): if empty(value): del self._fdoc_store[mbox][uid]
def all_headers(self, mbox): """ Return a dictionary with all the header docs for a given mbox. :param mbox: the mailbox :type mbox: str or unicode :rtype: dict """ headers_dict = {} uids = self.get_uids(mbox) fdoc_store = self._fdoc_store[mbox] hdoc_store = self._hdoc_store for uid in uids: try: chash = fdoc_store[uid][fields.CONTENT_HASH_KEY] hdoc = hdoc_store[chash] if not empty(hdoc): headers_dict[uid] = hdoc except KeyError: continue return headers_dict
def get_all_soledad_flag_docs(self): """ Return a dict with the content of all the flag documents in soledad store for the given mbox. :param mbox: the mailbox :type mbox: str or unicode :rtype: dict """ # XXX we really could return a reduced version with # just {'uid': (flags-tuple,) since the prefetch is # only oriented to get the flag tuples. all_docs = [( doc.content[self.UID_KEY], dict(doc.content)) for doc in self._soledad.get_from_index( fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) if not empty(doc.content)] all_flags = dict(all_docs) return all_flags
def get_fdoc_from_chash(self, chash, mbox): """ Return a flags-document by its content-hash and a given mailbox. Used during content-duplication detection while copying or adding a message. :param chash: the content hash to check against :type chash: str or unicode :param mbox: the mailbox :type mbox: str or unicode :return: MessagePartDoc. It will return None if the flags document has empty content or it is flagged as \\Deleted. """ fdoc = self._chash_fdoc_store[chash][mbox] # a couple of special cases. # 1. We might have a doc with empty content... if empty(fdoc): return None # 2. ...Or the message could exist, but being flagged for deletion. # We want to create a new one in this case. # Hmmm what if the deletion is un-done?? We would end with a # duplicate... if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []): return None uid = fdoc[fields.UID_KEY] key = mbox, uid new = key in self._new dirty = key in self._dirty return MessagePartDoc( new=new, dirty=dirty, store="mem", part=MessagePartType.fdoc, content=fdoc, doc_id=None)
def get_message(self, mbox, uid, dirtystate=DirtyState.none, flags_only=False): """ Get a MessageWrapper for the given mbox and uid combination. :param mbox: the mailbox :type mbox: str or unicode :param uid: the message UID :type uid: int :param dirtystate: DirtyState enum: one of `dirty`, `new` or `none` (default) :type dirtystate: enum :param flags_only: whether the message should carry only a reference to the flags document. :type flags_only: bool : :return: MessageWrapper or None """ if dirtystate == DirtyState.dirty: flags_only = True key = mbox, uid fdoc = self._fdoc_store[mbox][uid] if empty(fdoc): return None new, dirty = False, False if dirtystate == DirtyState.none: new, dirty = self._get_new_dirty_state(key) if dirtystate == DirtyState.dirty: new, dirty = False, True if dirtystate == DirtyState.new: new, dirty = True, False if flags_only: return MessageWrapper(fdoc=fdoc, new=new, dirty=dirty, memstore=weakref.proxy(self)) else: chash = fdoc.get(fields.CONTENT_HASH_KEY) hdoc = self._hdoc_store[chash] if empty(hdoc): hdoc = self._permanent_store.get_headers_doc(chash) if empty(hdoc): return None if not empty(hdoc.content): self._hdoc_store[chash] = hdoc.content hdoc = hdoc.content cdocs = None pmap = hdoc.get(fields.PARTS_MAP_KEY, None) if new and pmap is not None: # take the different cdocs for write... cdoc_store = self._cdoc_store cdocs_list = phash_iter(hdoc) cdocs = dict(enumerate( [cdoc_store[phash] for phash in cdocs_list], 1)) return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs, new=new, dirty=dirty, memstore=weakref.proxy(self))
def msgSavedCallback(result): if not empty(result): leap_events.signal(IMAP_MSG_SAVED_LOCALLY) deferLater(reactor, 0, self._delete_incoming_message, doc) leap_events.signal(IMAP_MSG_DELETED_INCOMING)
def does_exist(self): """ Return True if there is actually a flags document for this UID and mbox. """ return not empty(self.fdoc)