def start_client(self):
soledad_test_folder = self._generate_soledad_test_folder_name()
SearchEngine.DEFAULT_INDEX_HOME = soledad_test_folder
self.cleanup = lambda: shutil.rmtree(soledad_test_folder)
self.soledad = yield initialize_soledad(tempdir=soledad_test_folder)
self.keymanager = mock()
self.search_engine = SearchEngine(self.INDEX_KEY,
agent_home=soledad_test_folder)
self.mail_sender = self._create_mail_sender()
self.mail_store = SearchableMailStore(LeapMailStore(self.soledad),
self.search_engine)
account_ready_cb = defer.Deferred()
self.account = IMAPAccount(self.ACCOUNT, self.soledad,
account_ready_cb)
yield account_ready_cb
self.draft_service = DraftService(self.mail_store)
self.leap_session = mock()
self.feedback_service = FeedbackService(self.leap_session)
self.mail_service = self._create_mail_service(self.mail_sender,
self.mail_store,
self.search_engine)
mails = yield self.mail_service.all_mails()
self.search_engine.index_mails(mails)
self.resource = RootResource()
self.resource.initialize(self.keymanager, self.search_engine,
self.mail_service, self.draft_service,
self.feedback_service)
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
self.acct = IMAPAccount(self.userid, self.sol)
return True
def setup_account(_):
self.parser = parser.Parser()
# XXX this should be fixed in soledad.
# Soledad sync makes trial block forever. The sync it's mocked to
# fix this problem. _mock_soledad_get_from_index can be used from
# the tests to provide documents.
# TODO see here, possibly related? -- http://www.pythoneye.com/83_20424875/
self._soledad.sync = Mock()
d = defer.Deferred()
self.acc = IMAPAccount(USERID, self._soledad, d=d)
return d
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(
self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
self.acct = IMAPAccount(self.sol, self.userid)
return True
def __init__(self, uuid, userid, soledad):
"""
Initializes the server factory.
:param uuid: user uuid
:type uuid: str
:param userid: user id ([email protected])
:type userid: str
:param soledad: soledad instance
:type soledad: Soledad
"""
self._uuid = uuid
self._userid = userid
self._soledad = soledad
theAccount = IMAPAccount(uuid, soledad)
self.theAccount = theAccount
self._connections = defaultdict()
def _create_account(self, user_mail, soledad):
return IMAPAccount(user_mail, soledad, defer.Deferred())
def _create_account(self, user_mail, soledad_session):
account = IMAPAccount(user_mail, soledad_session.soledad)
return account
def _initialize_imap_account(self):
account_ready_cb = defer.Deferred()
self.account = IMAPAccount(self._user_id, self.soledad, account_ready_cb)
return account_ready_cb
class MBOXPlumber(object):
"""
An class that can fix things inside a soledadbacked account.
The idea is to gather in this helper different fixes for mailboxes
that can be invoked when data migration in the client is needed.
"""
def __init__(self, userid, passwd, mdir=None):
"""
Initialize the plumber with all that's needed to authenticate
against the provider.
:param userid: user identifier, foo@bar
:type userid: basestring
:param passwd: the soledad passphrase
:type passwd: basestring
:param mdir: a path to a maildir to import
:type mdir: str or None
"""
self.userid = userid
self.passwd = passwd
user, provider = userid.split('@')
self.user = user
self.mdir = mdir
self.sol = None
self._settings = Settings()
provider_config_path = os.path.join(get_path_prefix(),
get_provider_path(provider))
provider_config = ProviderConfig()
loaded = provider_config.load(provider_config_path)
if not loaded:
print "could not load provider config!"
return self.exit()
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
self.acct = IMAPAccount(self.userid, self.sol)
return True
#
# Account repairing
#
def repair_account(self, *args):
"""
Repair mbox uids for all mboxes in this account.
"""
init = self._init_local_soledad()
if not init:
return self.exit()
for mbox_name in self.acct.mailboxes:
self.repair_mbox_uids(mbox_name)
print "done."
self.exit()
def repair_mbox_uids(self, mbox_name):
"""
Repair indexes for a given mbox.
:param mbox_name: mailbox to repair
:type mbox_name: basestring
"""
print
print "REPAIRING INDEXES FOR MAILBOX %s" % (mbox_name, )
print "----------------------------------------------"
mbox = self.acct.getMailbox(mbox_name)
len_mbox = mbox.getMessageCount()
print "There are %s messages" % (len_mbox, )
last_ok = True if mbox.last_uid == len_mbox else False
uids_iter = mbox.messages.all_msg_iter()
dupes = self._has_dupes(uids_iter)
if last_ok and not dupes:
print "Mbox does not need repair."
return
# XXX CHANGE? ----
msgs = mbox.messages.get_all()
for zindex, doc in enumerate(msgs):
mindex = zindex + 1
old_uid = doc.content['uid']
doc.content['uid'] = mindex
self.sol.put_doc(doc)
if mindex != old_uid:
print "%s -> %s (%s)" % (mindex, doc.content['uid'], old_uid)
old_last_uid = mbox.last_uid
mbox.last_uid = len_mbox
print "LAST UID: %s (%s)" % (mbox.last_uid, old_last_uid)
def _has_dupes(self, sequence):
"""
Return True if the given sequence of ints has duplicates.
:param sequence: a sequence of ints
:type sequence: sequence
:rtype: bool
"""
d = defaultdict(lambda: 0)
for uid in sequence:
d[uid] += 1
if d[uid] != 1:
return True
return False
#
# Maildir import
#
def import_mail(self, mail_filename):
"""
Import a single mail into a mailbox.
:param mbox: the Mailbox instance to save in.
:type mbox: SoledadMailbox
:param mail_filename: the filename to the mail file to save
:type mail_filename: basestring
:return: a deferred
"""
def saved(_):
print "message added"
with open(mail_filename) as f:
mail_string = f.read()
# uid = self._mbox.getUIDNext()
# print "saving with UID: %s" % uid
d = self._mbox.messages.add_msg(mail_string, notify_on_disk=True)
return d
def import_maildir(self, mbox_name="INBOX"):
"""
Import all mails in a maildir.
We will process all subfolders as beloging
to the same mailbox (cur, new, tmp).
"""
# TODO parse hierarchical subfolders into
# inferior mailboxes.
if not os.path.isdir(self.mdir):
print "ERROR: maildir path does not exist."
return
init = self._init_local_soledad()
if not init:
return self.exit()
mbox = self.acct.getMailbox(mbox_name)
self._mbox = mbox
len_mbox = mbox.getMessageCount()
mail_files_g = flatten(
map(partial(os.path.join, f), files)
for f, _, files in os.walk(self.mdir))
# we only coerce the generator to give the
# len, but we could skip than and inform at the end.
mail_files = list(mail_files_g)
print "Got %s mails to import into %s (%s)" % (len(mail_files),
mbox_name, len_mbox)
def all_saved(_):
print "all messages imported"
deferreds = []
for f_name in mail_files:
deferreds.append(self.import_mail(f_name))
print "deferreds: ", deferreds
d1 = defer.gatherResults(deferreds, consumeErrors=False)
d1.addCallback(all_saved)
d1.addCallback(self._cbExit)
def _cbExit(self, ignored):
return self.exit()
def exit(self):
from twisted.internet import reactor
try:
if self.sol:
self.sol.close()
reactor.stop()
except Exception:
pass
return
def getInbox(_):
d = defer.Deferred()
theAccount = IMAPAccount(ADDRESS, self._soledad, d=d)
d.addCallback(
lambda _: theAccount.getMailbox(INBOX_NAME))
return d
class MBOXPlumber(object):
"""
An class that can fix things inside a soledadbacked account.
The idea is to gather in this helper different fixes for mailboxes
that can be invoked when data migration in the client is needed.
"""
def __init__(self, userid, passwd, mdir=None):
"""
Initialize the plumber with all that's needed to authenticate
against the provider.
:param userid: user identifier, foo@bar
:type userid: basestring
:param passwd: the soledad passphrase
:type passwd: basestring
:param mdir: a path to a maildir to import
:type mdir: str or None
"""
self.userid = userid
self.passwd = passwd
user, provider = userid.split('@')
self.user = user
self.mdir = mdir
self.sol = None
self._settings = Settings()
provider_config_path = os.path.join(get_path_prefix(),
get_provider_path(provider))
provider_config = ProviderConfig()
loaded = provider_config.load(provider_config_path)
if not loaded:
print "could not load provider config!"
return self.exit()
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(
self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
self.acct = IMAPAccount(self.sol, self.userid)
return True
#
# Account repairing
#
def repair_account(self, *args):
"""
Repair mbox uids for all mboxes in this account.
"""
init = self._init_local_soledad()
if not init:
return self.exit()
for mbox_name in self.acct.mailboxes:
self.repair_mbox_uids(mbox_name)
print "done."
self.exit()
def repair_mbox_uids(self, mbox_name):
"""
Repair indexes for a given mbox.
:param mbox_name: mailbox to repair
:type mbox_name: basestring
"""
print
print "REPAIRING INDEXES FOR MAILBOX %s" % (mbox_name,)
print "----------------------------------------------"
mbox = self.acct.getMailbox(mbox_name)
len_mbox = mbox.getMessageCount()
print "There are %s messages" % (len_mbox,)
last_ok = True if mbox.last_uid == len_mbox else False
uids_iter = mbox.messages.all_msg_iter()
dupes = self._has_dupes(uids_iter)
if last_ok and not dupes:
print "Mbox does not need repair."
return
# XXX CHANGE? ----
msgs = mbox.messages.get_all()
for zindex, doc in enumerate(msgs):
mindex = zindex + 1
old_uid = doc.content['uid']
doc.content['uid'] = mindex
self.sol.put_doc(doc)
if mindex != old_uid:
print "%s -> %s (%s)" % (mindex, doc.content['uid'], old_uid)
old_last_uid = mbox.last_uid
mbox.last_uid = len_mbox
print "LAST UID: %s (%s)" % (mbox.last_uid, old_last_uid)
def _has_dupes(self, sequence):
"""
Return True if the given sequence of ints has duplicates.
:param sequence: a sequence of ints
:type sequence: sequence
:rtype: bool
"""
d = defaultdict(lambda: 0)
for uid in sequence:
d[uid] += 1
if d[uid] != 1:
return True
return False
#
# Maildir import
#
def import_mail(self, mail_filename):
"""
Import a single mail into a mailbox.
:param mbox: the Mailbox instance to save in.
:type mbox: SoledadMailbox
:param mail_filename: the filename to the mail file to save
:type mail_filename: basestring
:return: a deferred
"""
def saved(_):
print "message added"
with open(mail_filename) as f:
mail_string = f.read()
# uid = self._mbox.getUIDNext()
# print "saving with UID: %s" % uid
d = self._mbox.messages.add_msg(
mail_string, notify_on_disk=True)
return d
def import_maildir(self, mbox_name="INBOX"):
"""
Import all mails in a maildir.
We will process all subfolders as beloging
to the same mailbox (cur, new, tmp).
"""
# TODO parse hierarchical subfolders into
# inferior mailboxes.
if not os.path.isdir(self.mdir):
print "ERROR: maildir path does not exist."
return
init = self._init_local_soledad()
if not init:
return self.exit()
mbox = self.acct.getMailbox(mbox_name)
self._mbox = mbox
len_mbox = mbox.getMessageCount()
mail_files_g = flatten(
map(partial(os.path.join, f), files)
for f, _, files in os.walk(self.mdir))
# we only coerce the generator to give the
# len, but we could skip than and inform at the end.
mail_files = list(mail_files_g)
print "Got %s mails to import into %s (%s)" % (
len(mail_files), mbox_name, len_mbox)
def all_saved(_):
print "all messages imported"
deferreds = []
for f_name in mail_files:
deferreds.append(self.import_mail(f_name))
print "deferreds: ", deferreds
d1 = defer.gatherResults(deferreds, consumeErrors=False)
d1.addCallback(all_saved)
d1.addCallback(self._cbExit)
def _cbExit(self, ignored):
return self.exit()
def exit(self):
from twisted.internet import reactor
try:
if self.sol:
self.sol.close()
reactor.stop()
except Exception:
pass
return
def getInbox(_):
d = defer.Deferred()
theAccount = IMAPAccount(ADDRESS, self._soledad, d=d)
d.addCallback(
lambda _: theAccount.getMailbox(INBOX_NAME))
return d
