def __init__(self, soledad_test_folder='/tmp/soledad-test/test'):
self.soledad = initialize_soledad(tempdir=soledad_test_folder)
self.mail_address = "*****@*****.**"
# setup app
PixelatedMail.from_email_address = self.mail_address
SearchEngine.DEFAULT_INDEX_HOME = soledad_test_folder
self.app = App()
self.soledad_querier = SoledadQuerier(self.soledad)
self.soledad_querier.get_index_masterkey = lambda: self.INDEX_KEY
self.account = SoledadBackedAccount('test', self.soledad, MagicMock())
self.mailboxes = Mailboxes(self.account, self.soledad_querier)
self.mail_sender = Mock()
self.tag_service = TagService()
self.draft_service = DraftService(self.mailboxes)
self.mail_service = MailService(self.mailboxes, self.mail_sender,
self.tag_service, self.soledad_querier)
self.search_engine = SearchEngine(self.soledad_querier)
self.search_engine.index_mails(self.mail_service.all_mails())
self.app.resource = RootResource()
self.app.resource.initialize(self.soledad_querier, self.search_engine,
self.mail_service, self.draft_service)
def start_client(self):
soledad_test_folder = self._generate_soledad_test_folder_name()
SearchEngine.DEFAULT_INDEX_HOME = soledad_test_folder
self.cleanup = lambda: shutil.rmtree(soledad_test_folder)
PixelatedMail.from_email_address = self.MAIL_ADDRESS
self.soledad = initialize_soledad(tempdir=soledad_test_folder)
self.soledad_querier = self._create_soledad_querier(
self.soledad, self.INDEX_KEY)
self.keymanager = mock()
self.search_engine = SearchEngine(self.soledad_querier,
agent_home=soledad_test_folder)
self.mail_sender = self._create_mail_sender()
self.account = SoledadBackedAccount(self.ACCOUNT, self.soledad,
MagicMock())
self.mailboxes = Mailboxes(self.account, self.soledad_querier,
self.search_engine)
self.draft_service = DraftService(self.mailboxes)
self.mail_service = self._create_mail_service(self.mailboxes,
self.mail_sender,
self.soledad_querier,
self.search_engine)
self.search_engine.index_mails(self.mail_service.all_mails())
self.app = App()
self.app.resource = RootResource()
self.app.resource.initialize(self.soledad_querier, self.keymanager,
self.search_engine, self.mail_service,
self.draft_service)
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
memstore = MemoryStore(permanent_store=SoledadStore(self.sol),
write_period=5)
self.acct = SoledadBackedAccount(self.userid,
self.sol,
memstore=memstore)
return True
def __init__(self, soledad_test_folder='soledad-test'):
self.soledad = initialize_soledad(tempdir=soledad_test_folder)
self.mail_address = "*****@*****.**"
# setup app
PixelatedMail.from_email_address = self.mail_address
SearchEngine.INDEX_FOLDER = soledad_test_folder + '/search_index'
self.app = pixelated.runserver.app
self.soledad_querier = SoledadQuerier(self.soledad)
self.soledad_querier.get_index_masterkey = lambda: '_yg2oG_5ELM8_-sQYcsxI37WesI0dOtZQXpwAqjvhR4='
self.account = SoledadBackedAccount('test', self.soledad, MagicMock())
self.mailboxes = Mailboxes(self.account, self.soledad_querier)
self.mail_sender = Mock()
self.tag_service = TagService()
self.draft_service = DraftService(self.mailboxes)
self.mail_service = MailService(self.mailboxes, self.mail_sender,
self.tag_service, self.soledad_querier)
self.search_engine = SearchEngine(self.soledad_querier)
self.search_engine.index_mails(self.mail_service.all_mails())
features_controller = FeaturesController()
features_controller.DISABLED_FEATURES.append('autoReload')
home_controller = HomeController()
mails_controller = MailsController(mail_service=self.mail_service,
draft_service=self.draft_service,
search_engine=self.search_engine)
tags_controller = TagsController(search_engine=self.search_engine)
contacts_controller = ContactsController(
search_engine=self.search_engine)
sync_info_controller = SyncInfoController()
attachments_controller = AttachmentsController(self.soledad_querier)
setup_routes(self.app, home_controller, mails_controller,
tags_controller, features_controller,
sync_info_controller, attachments_controller,
contacts_controller)
def setUp(self):
"""
Setup method for each test.
Initializes and run a LEAP IMAP4 Server,
but passing the same Soledad instance (it's costly to initialize),
so we have to be sure to restore state across tests.
"""
UUID = 'deadbeef',
USERID = TEST_USER
memstore = MemoryStore()
d = defer.Deferred()
self.server = LeapIMAPServer(
uuid=UUID, userid=USERID,
contextFactory=self.serverCTX,
# XXX do we really need this??
soledad=self._soledad)
self.client = SimpleClient(d, contextFactory=self.clientCTX)
self.connected = d
# XXX REVIEW-ME.
# We're adding theAccount here to server
# but it was also passed to initialization
# as it was passed to realm.
# I THINK we ONLY need to do it at one place now.
theAccount = SoledadBackedAccount(
USERID,
soledad=self._soledad,
memstore=memstore)
LeapIMAPServer.theAccount = theAccount
# in case we get something from previous tests...
for mb in self.server.theAccount.mailboxes:
self.server.theAccount.delete(mb)
# email parser
self.parser = parser.Parser()
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(
self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
memstore = MemoryStore(
permanent_store=SoledadStore(self.sol),
write_period=5)
self.acct = SoledadBackedAccount(self.userid, self.sol,
memstore=memstore)
return True
def __init__(self, uuid, userid, soledad):
"""
Initializes the server factory.
:param uuid: user uuid
:type uuid: str
:param userid: user id ([email protected])
:type userid: str
:param soledad: soledad instance
:type soledad: Soledad
"""
self._uuid = uuid
self._userid = userid
self._soledad = soledad
self._memstore = MemoryStore(
permanent_store=SoledadStore(soledad))
theAccount = SoledadBackedAccount(
uuid, soledad=soledad,
memstore=self._memstore)
self.theAccount = theAccount
def _create_account(self, uuid, soledad_session):
memstore = MemoryStore(
permanent_store=SoledadStore(soledad_session.soledad))
return SoledadBackedAccount(uuid, soledad_session.soledad, memstore)
class MBOXPlumber(object):
"""
An class that can fix things inside a soledadbacked account.
The idea is to gather in this helper different fixes for mailboxes
that can be invoked when data migration in the client is needed.
"""
def __init__(self, userid, passwd, mdir=None):
"""
Initialize the plumber with all that's needed to authenticate
against the provider.
:param userid: user identifier, foo@bar
:type userid: basestring
:param passwd: the soledad passphrase
:type passwd: basestring
:param mdir: a path to a maildir to import
:type mdir: str or None
"""
self.userid = userid
self.passwd = passwd
user, provider = userid.split('@')
self.user = user
self.mdir = mdir
self.sol = None
self._settings = Settings()
provider_config_path = os.path.join(get_path_prefix(),
get_provider_path(provider))
provider_config = ProviderConfig()
loaded = provider_config.load(provider_config_path)
if not loaded:
print "could not load provider config!"
return self.exit()
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(
self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
memstore = MemoryStore(
permanent_store=SoledadStore(self.sol),
write_period=5)
self.acct = SoledadBackedAccount(self.userid, self.sol,
memstore=memstore)
return True
#
# Account repairing
#
def repair_account(self, *args):
"""
Repair mbox uids for all mboxes in this account.
"""
init = self._init_local_soledad()
if not init:
return self.exit()
for mbox_name in self.acct.mailboxes:
self.repair_mbox_uids(mbox_name)
print "done."
self.exit()
def repair_mbox_uids(self, mbox_name):
"""
Repair indexes for a given mbox.
:param mbox_name: mailbox to repair
:type mbox_name: basestring
"""
print
print "REPAIRING INDEXES FOR MAILBOX %s" % (mbox_name,)
print "----------------------------------------------"
mbox = self.acct.getMailbox(mbox_name)
len_mbox = mbox.getMessageCount()
print "There are %s messages" % (len_mbox,)
last_ok = True if mbox.last_uid == len_mbox else False
uids_iter = mbox.messages.all_msg_iter()
dupes = self._has_dupes(uids_iter)
if last_ok and not dupes:
print "Mbox does not need repair."
return
# XXX CHANGE? ----
msgs = mbox.messages.get_all()
for zindex, doc in enumerate(msgs):
mindex = zindex + 1
old_uid = doc.content['uid']
doc.content['uid'] = mindex
self.sol.put_doc(doc)
if mindex != old_uid:
print "%s -> %s (%s)" % (mindex, doc.content['uid'], old_uid)
old_last_uid = mbox.last_uid
mbox.last_uid = len_mbox
print "LAST UID: %s (%s)" % (mbox.last_uid, old_last_uid)
def _has_dupes(self, sequence):
"""
Return True if the given sequence of ints has duplicates.
:param sequence: a sequence of ints
:type sequence: sequence
:rtype: bool
"""
d = defaultdict(lambda: 0)
for uid in sequence:
d[uid] += 1
if d[uid] != 1:
return True
return False
#
# Maildir import
#
def import_mail(self, mail_filename):
"""
Import a single mail into a mailbox.
:param mbox: the Mailbox instance to save in.
:type mbox: SoledadMailbox
:param mail_filename: the filename to the mail file to save
:type mail_filename: basestring
:return: a deferred
"""
def saved(_):
print "message added"
with open(mail_filename) as f:
mail_string = f.read()
# uid = self._mbox.getUIDNext()
# print "saving with UID: %s" % uid
d = self._mbox.messages.add_msg(
mail_string, notify_on_disk=True)
return d
def import_maildir(self, mbox_name="INBOX"):
"""
Import all mails in a maildir.
We will process all subfolders as beloging
to the same mailbox (cur, new, tmp).
"""
# TODO parse hierarchical subfolders into
# inferior mailboxes.
if not os.path.isdir(self.mdir):
print "ERROR: maildir path does not exist."
return
init = self._init_local_soledad()
if not init:
return self.exit()
mbox = self.acct.getMailbox(mbox_name)
self._mbox = mbox
len_mbox = mbox.getMessageCount()
mail_files_g = flatten(
map(partial(os.path.join, f), files)
for f, _, files in os.walk(self.mdir))
# we only coerce the generator to give the
# len, but we could skip than and inform at the end.
mail_files = list(mail_files_g)
print "Got %s mails to import into %s (%s)" % (
len(mail_files), mbox_name, len_mbox)
def all_saved(_):
print "all messages imported"
deferreds = []
for f_name in mail_files:
deferreds.append(self.import_mail(f_name))
print "deferreds: ", deferreds
d1 = defer.gatherResults(deferreds, consumeErrors=False)
d1.addCallback(all_saved)
d1.addCallback(self._cbExit)
def _cbExit(self, ignored):
return self.exit()
def exit(self):
from twisted.internet import reactor
try:
if self.sol:
self.sol.close()
reactor.stop()
except Exception:
pass
return
class MBOXPlumber(object):
"""
An class that can fix things inside a soledadbacked account.
The idea is to gather in this helper different fixes for mailboxes
that can be invoked when data migration in the client is needed.
"""
def __init__(self, userid, passwd, mdir=None):
"""
Initialize the plumber with all that's needed to authenticate
against the provider.
:param userid: user identifier, foo@bar
:type userid: basestring
:param passwd: the soledad passphrase
:type passwd: basestring
:param mdir: a path to a maildir to import
:type mdir: str or None
"""
self.userid = userid
self.passwd = passwd
user, provider = userid.split('@')
self.user = user
self.mdir = mdir
self.sol = None
self._settings = Settings()
provider_config_path = os.path.join(get_path_prefix(),
get_provider_path(provider))
provider_config = ProviderConfig()
loaded = provider_config.load(provider_config_path)
if not loaded:
print "could not load provider config!"
return self.exit()
def _init_local_soledad(self):
"""
Initialize local Soledad instance.
"""
self.uuid = self._settings.get_uuid(self.userid)
if not self.uuid:
print "Cannot get UUID from settings. Log in at least once."
return False
print "UUID: %s" % (self.uuid)
secrets, localdb = get_db_paths(self.uuid)
self.sol = initialize_soledad(self.uuid, self.userid, self.passwd,
secrets, localdb, "/tmp", "/tmp")
memstore = MemoryStore(permanent_store=SoledadStore(self.sol),
write_period=5)
self.acct = SoledadBackedAccount(self.userid,
self.sol,
memstore=memstore)
return True
#
# Account repairing
#
def repair_account(self, *args):
"""
Repair mbox uids for all mboxes in this account.
"""
init = self._init_local_soledad()
if not init:
return self.exit()
for mbox_name in self.acct.mailboxes:
self.repair_mbox_uids(mbox_name)
print "done."
self.exit()
def repair_mbox_uids(self, mbox_name):
"""
Repair indexes for a given mbox.
:param mbox_name: mailbox to repair
:type mbox_name: basestring
"""
print
print "REPAIRING INDEXES FOR MAILBOX %s" % (mbox_name, )
print "----------------------------------------------"
mbox = self.acct.getMailbox(mbox_name)
len_mbox = mbox.getMessageCount()
print "There are %s messages" % (len_mbox, )
last_ok = True if mbox.last_uid == len_mbox else False
uids_iter = mbox.messages.all_msg_iter()
dupes = self._has_dupes(uids_iter)
if last_ok and not dupes:
print "Mbox does not need repair."
return
# XXX CHANGE? ----
msgs = mbox.messages.get_all()
for zindex, doc in enumerate(msgs):
mindex = zindex + 1
old_uid = doc.content['uid']
doc.content['uid'] = mindex
self.sol.put_doc(doc)
if mindex != old_uid:
print "%s -> %s (%s)" % (mindex, doc.content['uid'], old_uid)
old_last_uid = mbox.last_uid
mbox.last_uid = len_mbox
print "LAST UID: %s (%s)" % (mbox.last_uid, old_last_uid)
def _has_dupes(self, sequence):
"""
Return True if the given sequence of ints has duplicates.
:param sequence: a sequence of ints
:type sequence: sequence
:rtype: bool
"""
d = defaultdict(lambda: 0)
for uid in sequence:
d[uid] += 1
if d[uid] != 1:
return True
return False
#
# Maildir import
#
def import_mail(self, mail_filename):
"""
Import a single mail into a mailbox.
:param mbox: the Mailbox instance to save in.
:type mbox: SoledadMailbox
:param mail_filename: the filename to the mail file to save
:type mail_filename: basestring
:return: a deferred
"""
def saved(_):
print "message added"
with open(mail_filename) as f:
mail_string = f.read()
# uid = self._mbox.getUIDNext()
# print "saving with UID: %s" % uid
d = self._mbox.messages.add_msg(mail_string, notify_on_disk=True)
return d
def import_maildir(self, mbox_name="INBOX"):
"""
Import all mails in a maildir.
We will process all subfolders as beloging
to the same mailbox (cur, new, tmp).
"""
# TODO parse hierarchical subfolders into
# inferior mailboxes.
if not os.path.isdir(self.mdir):
print "ERROR: maildir path does not exist."
return
init = self._init_local_soledad()
if not init:
return self.exit()
mbox = self.acct.getMailbox(mbox_name)
self._mbox = mbox
len_mbox = mbox.getMessageCount()
mail_files_g = flatten(
map(partial(os.path.join, f), files)
for f, _, files in os.walk(self.mdir))
# we only coerce the generator to give the
# len, but we could skip than and inform at the end.
mail_files = list(mail_files_g)
print "Got %s mails to import into %s (%s)" % (len(mail_files),
mbox_name, len_mbox)
def all_saved(_):
print "all messages imported"
deferreds = []
for f_name in mail_files:
deferreds.append(self.import_mail(f_name))
print "deferreds: ", deferreds
d1 = defer.gatherResults(deferreds, consumeErrors=False)
d1.addCallback(all_saved)
d1.addCallback(self._cbExit)
def _cbExit(self, ignored):
return self.exit()
def exit(self):
from twisted.internet import reactor
try:
if self.sol:
self.sol.close()
reactor.stop()
except Exception:
pass
return
