def _get_body_doc(self):
"""
Return the document that keeps the body for this
message.
"""
hdoc_content = self.hdoc.content
body_phash = hdoc_content.get(
fields.BODY_KEY, None)
if not body_phash:
logger.warning("No body phash for this document!")
return None
# XXX get from memstore too...
# if memstore: memstore.get_phrash
# memstore should keep a dict with weakrefs to the
# phash doc...
if self._container is not None:
bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
if not empty(bdoc) and not empty(bdoc.content):
return bdoc
# no memstore, or no body doc found there
if self._soledad:
body_docs = self._soledad.get_from_index(
fields.TYPE_P_HASH_IDX,
fields.TYPE_CONTENT_VAL, str(body_phash))
return first(body_docs)
else:
logger.error("No phash in container, and no soledad found!")
def _get_msg_copy(self, message):
"""
Get a copy of the fdoc for this message, and check whether
it already exists.
:param message: an IMessage implementor
:type message: LeapMessage
:return: exist, new_fdoc
:rtype: tuple
"""
# XXX for clarity, this could be delegated to a
# MessageCollection mixin that implements copy too, and
# moved out of here.
msg = message
memstore = self._memstore
if empty(msg.fdoc):
logger.warning("Tried to copy a MSG with no fdoc")
return
new_fdoc = copy.deepcopy(msg.fdoc.content)
fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
dest_fdoc = memstore.get_fdoc_from_chash(
fdoc_chash, self.mbox)
exist = not empty(dest_fdoc)
return exist, new_fdoc
def walk(self):
"""
Generator that iterates through all the parts, returning
MessagePartDoc. Used for writing to SoledadStore.
:rtype: generator
"""
if self._dirty:
try:
mbox = self.fdoc.content[fields.MBOX_KEY]
uid = self.fdoc.content[fields.UID_KEY]
docid_dict = self._dict[self.DOCS_ID]
docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc(
mbox, uid)
except Exception as exc:
logger.debug("Error while walking message...")
logger.exception(exc)
if not empty(self.fdoc.content) and 'uid' in self.fdoc.content:
yield self.fdoc
if not empty(self.hdoc.content):
yield self.hdoc
for cdoc in self.cdocs.values():
if not empty(cdoc):
content_ref = weakref.proxy(cdoc)
yield MessagePartDoc(new=self.new,
dirty=self.dirty,
store=self._storetype,
part=MessagePartType.cdoc,
content=content_ref,
doc_id=None)
def getBodyFile(self):
"""
Retrieve a file object containing only the body of this message.
:return: file-like object opened for reading
:rtype: StringIO
"""
fd = StringIO.StringIO()
if not empty(self._pmap):
multi = self._pmap.get('multi')
if not multi:
phash = self._pmap.get("phash", None)
else:
pmap = self._pmap.get('part_map')
first_part = pmap.get('1', None)
if not empty(first_part):
phash = first_part['phash']
else:
phash = None
if phash is None:
logger.warning("Could not find phash for this subpart!")
payload = ""
else:
payload = self._get_payload_from_document_memoized(phash)
if empty(payload):
payload = self._get_payload_from_document(phash)
else:
logger.warning("Message with no part_map!")
payload = ""
if payload:
content_type = self._get_ctype_from_document(phash)
charset = find_charset(content_type)
if charset is None:
charset = self._get_charset(payload)
try:
if isinstance(payload, unicode):
payload = payload.encode(charset)
except UnicodeError as exc:
logger.error(
"Unicode error, using 'replace'. {0!r}".format(exc))
payload = payload.encode(charset, 'replace')
fd.write(payload)
fd.seek(0)
return fd
def _add_message_locally(self, result):
"""
Adds a message to local inbox and delete it from the incoming db
in soledad.
# XXX this comes from a gatherresult...
:param msgtuple: a tuple consisting of a SoledadDocument
instance containing the incoming message
and data, the json-encoded, decrypted content of the
incoming message
:type msgtuple: (SoledadDocument, str)
"""
from twisted.internet import reactor
msgtuple = first(result)
doc, data = msgtuple
log.msg('adding message %s to local db' % (doc.doc_id,))
if isinstance(data, list):
if empty(data):
return False
data = data[0]
def msgSavedCallback(result):
if not empty(result):
leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
deferLater(reactor, 0, self._delete_incoming_message, doc)
leap_events.signal(IMAP_MSG_DELETED_INCOMING)
d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,),
notify_on_disk=True)
d.addCallbacks(msgSavedCallback, self._errback)
def all_rdocs_iter(self):
"""
Return an iterator through all in-memory recent flag dicts, wrapped
under a RecentFlagsDoc namedtuple.
Used for saving to disk.
:return: a generator of RecentFlagDoc
:rtype: generator
"""
# XXX use enums
DOC_ID = "doc_id"
SET = "set"
rflags_store = self._rflags_store
def get_rdoc(mbox, rdict):
mbox_rflag_set = rdict[SET]
recent_set = copy(mbox_rflag_set)
# zero it!
mbox_rflag_set.difference_update(mbox_rflag_set)
return RecentFlagsDoc(
doc_id=rflags_store[mbox][DOC_ID],
content={
fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
fields.MBOX_KEY: mbox,
fields.RECENTFLAGS_KEY: list(recent_set)
})
return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items()
if not empty(rdict[SET]))
def _get_fdoc_from_chash(self, chash):
"""
Return a flags document for this mailbox with a given chash.
:return: A SoledadDocument containing the Flags Document, or None if
the query failed.
:rtype: SoledadDocument or None.
"""
curried = partial(
self._soledad.get_from_index,
fields.TYPE_MBOX_C_HASH_IDX,
fields.TYPE_FLAGS_VAL, self.mbox, chash)
curried.expected = "fdoc"
fdoc = try_unique_query(curried)
if fdoc is not None:
return fdoc
else:
# probably this should be the other way round,
# ie, try fist on memstore...
cf = self.memstore._chash_fdoc_store
fdoc = cf[chash][self.mbox]
# hey, I just needed to wrap fdoc thing into
# a "content" attribute, look a better way...
if not empty(fdoc):
return MessagePartDoc(
new=None, dirty=None, part=None,
store=None, doc_id=None,
content=fdoc)
def _soledad_write_document_parts(self, items):
"""
Write the document parts to soledad in a separate thread.
:param items: the iterator through the different document wrappers
payloads.
:type items: iterator
:return: whether the write was successful or not
:rtype: bool
"""
failed = False
for item, call in items:
if empty(item):
continue
try:
self._try_call(call, item)
except Exception as exc:
logger.debug("ITEM WAS: %s" % repr(item))
if hasattr(item, 'content'):
logger.debug("ITEM CONTENT WAS: %s" %
repr(item.content))
logger.exception(exc)
failed = True
continue
return failed
def write_messages(self, store):
"""
Write the message documents in this MemoryStore to a different store.
:param store: the IMessageStore to write to
:rtype: False if queue is not empty, None otherwise.
"""
# For now, we pass if the queue is not empty, to avoid duplicate
# queuing.
# We would better use a flag to know when we've already enqueued an
# item.
# XXX this could return the deferred for all the enqueued operations
if not self.producer.is_queue_empty():
return False
if any(map(lambda i: not empty(i), (self._new, self._dirty))):
logger.info("Writing messages to Soledad...")
# TODO change for lock, and make the property access
# is accquired
with set_bool_flag(self, self.WRITING_FLAG):
for rflags_doc_wrapper in self.all_rdocs_iter():
self.producer.push(rflags_doc_wrapper,
state=self.producer.STATE_DIRTY)
for msg_wrapper in self.all_new_msg_iter():
self.producer.push(msg_wrapper,
state=self.producer.STATE_NEW)
for msg_wrapper in self.all_dirty_msg_iter():
self.producer.push(msg_wrapper,
state=self.producer.STATE_DIRTY)
def hdoc(self):
"""
An accessor to the headers document.
"""
container = self._container
if container is not None:
hdoc = self._container.hdoc
if hdoc and not empty(hdoc.content):
return hdoc
hdoc = self._get_headers_doc()
if container and not empty(hdoc.content):
# mem-cache it
hdoc_content = hdoc.content
chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
hdocs = {chash: hdoc_content}
container.memstore.load_header_docs(hdocs)
return hdoc
def isMultipart(self):
"""
Return True if this message is multipart.
"""
if empty(self._pmap):
logger.warning("Could not get part map!")
return False
multi = self._pmap.get("multi", False)
return multi
def all_soledad_uid_iter(self):
"""
Return an iterator through the UIDs of all messages, sorted in
ascending order.
"""
db_uids = set([doc.content[self.UID_KEY] for doc in
self._soledad.get_from_index(
fields.TYPE_MBOX_IDX,
fields.TYPE_FLAGS_VAL, self.mbox)
if not empty(doc)])
return db_uids
def get_docid_for_fdoc(self, mbox, uid):
"""
Return Soledad document id for the flags-doc for a given mbox and uid,
or None of no flags document could be found.
:param mbox: the mailbox
:type mbox: str or unicode
:param uid: the message UID
:type uid: int
:rtype: unicode or None
"""
with self._fdoc_docid_lock:
doc_id = self._fdoc_id_store[mbox][uid]
if empty(doc_id):
fdoc = self._permanent_store.get_flags_doc(mbox, uid)
if empty(fdoc) or empty(fdoc.content):
return None
doc_id = fdoc.doc_id
self._fdoc_id_store[mbox][uid] = doc_id
return doc_id
def getSize(self):
"""
Return the total size, in octets, of this message part.
:return: size of the message, in octets
:rtype: int
"""
if empty(self._pmap):
return 0
size = self._pmap.get('size', None)
if size is None:
logger.error("Message part cannot find size in the partmap")
size = 0
return size
def msgSavedCallback(result):
if empty(result):
return
for listener in self._listeners:
listener(result)
def signal_deleted(doc_id):
emit(catalog.MAIL_MSG_DELETED_INCOMING)
return doc_id
emit(catalog.MAIL_MSG_SAVED_LOCALLY)
d = self._delete_incoming_message(doc)
d.addCallback(signal_deleted)
return d
def getBodyFile(self):
"""
Retrieve a file object containing only the body of this message.
:return: file-like object opened for reading
:rtype: StringIO
"""
def write_fd(body):
fd.write(body)
fd.seek(0)
return fd
# TODO refactor with getBodyFile in MessagePart
fd = StringIO.StringIO()
if self.bdoc is not None:
bdoc_content = self.bdoc.content
if empty(bdoc_content):
logger.warning("No BDOC content found for message!!!")
return write_fd("")
body = bdoc_content.get(self.RAW_KEY, "")
content_type = bdoc_content.get('content-type', "")
charset = find_charset(content_type)
if charset is None:
charset = self._get_charset(body)
try:
if isinstance(body, unicode):
body = body.encode(charset)
except UnicodeError as exc:
logger.error(
"Unicode error, using 'replace'. {0!r}".format(exc))
logger.debug("Attempted to encode with: %s" % charset)
body = body.encode(charset, 'replace')
finally:
return write_fd(body)
# We are still returning funky characters from here.
else:
logger.warning("No BDOC found for message.")
return write_fd("")
def purge_fdoc_store(self, mbox):
"""
Purge the empty documents from a fdoc store.
Called during initialization of the SoledadMailbox
:param mbox: the mailbox
:type mbox: str or unicode
"""
# XXX This is really a workaround until I find the conditions
# that are making the empty items remain there.
# This happens, for instance, after running several times
# the regression test, that issues a store deleted + expunge + select
# The items are being correclty deleted, but in succesive appends
# the empty items with previously deleted uids reappear as empty
# documents. I suspect it's a timing condition with a previously
# evaluated sequence being used after the items has been removed.
for uid, value in self._fdoc_store[mbox].items():
if empty(value):
del self._fdoc_store[mbox][uid]
def all_headers(self, mbox):
"""
Return a dictionary with all the header docs for a given mbox.
:param mbox: the mailbox
:type mbox: str or unicode
:rtype: dict
"""
headers_dict = {}
uids = self.get_uids(mbox)
fdoc_store = self._fdoc_store[mbox]
hdoc_store = self._hdoc_store
for uid in uids:
try:
chash = fdoc_store[uid][fields.CONTENT_HASH_KEY]
hdoc = hdoc_store[chash]
if not empty(hdoc):
headers_dict[uid] = hdoc
except KeyError:
continue
return headers_dict
def get_all_soledad_flag_docs(self):
"""
Return a dict with the content of all the flag documents
in soledad store for the given mbox.
:param mbox: the mailbox
:type mbox: str or unicode
:rtype: dict
"""
# XXX we really could return a reduced version with
# just {'uid': (flags-tuple,) since the prefetch is
# only oriented to get the flag tuples.
all_docs = [(
doc.content[self.UID_KEY],
dict(doc.content))
for doc in
self._soledad.get_from_index(
fields.TYPE_MBOX_IDX,
fields.TYPE_FLAGS_VAL, self.mbox)
if not empty(doc.content)]
all_flags = dict(all_docs)
return all_flags
def get_fdoc_from_chash(self, chash, mbox):
"""
Return a flags-document by its content-hash and a given mailbox.
Used during content-duplication detection while copying or adding a
message.
:param chash: the content hash to check against
:type chash: str or unicode
:param mbox: the mailbox
:type mbox: str or unicode
:return: MessagePartDoc. It will return None if the flags document
has empty content or it is flagged as \\Deleted.
"""
fdoc = self._chash_fdoc_store[chash][mbox]
# a couple of special cases.
# 1. We might have a doc with empty content...
if empty(fdoc):
return None
# 2. ...Or the message could exist, but being flagged for deletion.
# We want to create a new one in this case.
# Hmmm what if the deletion is un-done?? We would end with a
# duplicate...
if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []):
return None
uid = fdoc[fields.UID_KEY]
key = mbox, uid
new = key in self._new
dirty = key in self._dirty
return MessagePartDoc(
new=new, dirty=dirty, store="mem",
part=MessagePartType.fdoc,
content=fdoc,
doc_id=None)
def _soledad_write_document_parts(self, items):
"""
Write the document parts to soledad in a separate thread.
:param items: the iterator through the different document wrappers
payloads.
:type items: iterator
:return: whether the write was successful or not
:rtype: bool
"""
failed = False
for item, call in items:
if empty(item):
continue
try:
self._try_call(call, item)
except Exception as exc:
logger.debug("ITEM WAS: %s" % repr(item))
if hasattr(item, 'content'):
logger.debug("ITEM CONTENT WAS: %s" % repr(item.content))
logger.exception(exc)
failed = True
continue
return failed
def get_message(self, mbox, uid, dirtystate=DirtyState.none,
flags_only=False):
"""
Get a MessageWrapper for the given mbox and uid combination.
:param mbox: the mailbox
:type mbox: str or unicode
:param uid: the message UID
:type uid: int
:param dirtystate: DirtyState enum: one of `dirty`, `new`
or `none` (default)
:type dirtystate: enum
:param flags_only: whether the message should carry only a reference
to the flags document.
:type flags_only: bool
:
:return: MessageWrapper or None
"""
if dirtystate == DirtyState.dirty:
flags_only = True
key = mbox, uid
fdoc = self._fdoc_store[mbox][uid]
if empty(fdoc):
return None
new, dirty = False, False
if dirtystate == DirtyState.none:
new, dirty = self._get_new_dirty_state(key)
if dirtystate == DirtyState.dirty:
new, dirty = False, True
if dirtystate == DirtyState.new:
new, dirty = True, False
if flags_only:
return MessageWrapper(fdoc=fdoc,
new=new, dirty=dirty,
memstore=weakref.proxy(self))
else:
chash = fdoc.get(fields.CONTENT_HASH_KEY)
hdoc = self._hdoc_store[chash]
if empty(hdoc):
hdoc = self._permanent_store.get_headers_doc(chash)
if empty(hdoc):
return None
if not empty(hdoc.content):
self._hdoc_store[chash] = hdoc.content
hdoc = hdoc.content
cdocs = None
pmap = hdoc.get(fields.PARTS_MAP_KEY, None)
if new and pmap is not None:
# take the different cdocs for write...
cdoc_store = self._cdoc_store
cdocs_list = phash_iter(hdoc)
cdocs = dict(enumerate(
[cdoc_store[phash] for phash in cdocs_list], 1))
return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs,
new=new, dirty=dirty,
memstore=weakref.proxy(self))
def msgSavedCallback(result):
if not empty(result):
leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
deferLater(reactor, 0, self._delete_incoming_message, doc)
leap_events.signal(IMAP_MSG_DELETED_INCOMING)
def does_exist(self):
"""
Return True if there is actually a flags document for this
UID and mbox.
"""
return not empty(self.fdoc)
