def prepare_sync(self):
"""Ensures that canonical tags are created for the account, and gets
and save Folder objects for folders on the IMAP backend. Returns a list
of tuples (folder_name, folder_id) for each folder we want to sync (in
order)."""
with mailsync_session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
Tag.create_canonical_tags(account.namespace, db_session)
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
sync_folder_names_ids = []
for folder_name in sync_folders:
try:
id_, = db_session.query(Folder.id). \
filter(Folder.name == folder_name,
Folder.account_id == self.account_id).one()
sync_folder_names_ids.append((folder_name, id_))
except NoResultFound:
log.error("Missing Folder object when starting sync",
folder_name=folder_name)
raise MailsyncError(
"Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
return sync_folder_names_ids
def create_canonical_tags(db):
"""Ensure that all canonical tags exist for the namespace we're testing
against. This is normally done when an account sync starts."""
from inbox.models import Namespace, Tag
namespace = db.session.query(Namespace).first()
Tag.create_canonical_tags(namespace, db.session)
db.session.commit()
def create_canonical_tags(db):
"""Ensure that all canonical tags exist for the namespace we're testing
against. This is normally done when an account sync starts."""
from inbox.models import Namespace, Tag
namespace = db.session.query(Namespace).first()
Tag.create_canonical_tags(namespace, db.session)
db.session.commit()
def prepare_sync(self):
"""Ensures that canonical tags are created for the account, and gets
and save Folder objects for folders on the IMAP backend. Returns a list
of tuples (folder_name, folder_id) for each folder we want to sync (in
order)."""
with mailsync_session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
Tag.create_canonical_tags(account.namespace, db_session)
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
sync_folder_names_ids = []
for folder_name in sync_folders:
try:
id_, = db_session.query(Folder.id). \
filter(Folder.name == folder_name,
Folder.account_id == self.account_id).one()
sync_folder_names_ids.append((folder_name, id_))
except NoResultFound:
log.error("Missing Folder object when starting sync",
folder_name=folder_name)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
return sync_folder_names_ids
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with mailsync_session_scope() as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(log, account, crispin_client.folder_names(),
db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {
name: id_
for id_, name in db_session.query(Folder.id, Folder.name).
filter_by(account_id=self.account_id)
}
saved_states = {
name: state
for name, state in db_session.query(
Folder.name, ImapFolderSyncStatus.state).join(
ImapFolderSyncStatus.folder).filter(
ImapFolderSyncStatus.account_id == self.account_id)
}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
log.info('initializing folder sync')
# STOPSHIP(emfree): replace by appropriate base class.
thread = self.sync_engine_class(
self.account_id, folder_name, folder_id_for[folder_name],
self.email_address, self.provider_name,
self.poll_frequency, self.syncmanager_lock,
self.refresh_flags_max, self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with mailsync_session_scope() as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {name: id_ for id_, name in db_session.query(
Folder.id, Folder.name).filter_by(account_id=self.account_id)}
saved_states = {name: state for name, state in
db_session.query(Folder.name,
ImapFolderSyncStatus.state)
.join(ImapFolderSyncStatus.folder)
.filter(ImapFolderSyncStatus.account_id ==
self.account_id)}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
log.error("Missing Folder object when starting sync",
folder_name=folder_name, folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
log.info('initializing folder sync')
# STOPSHIP(emfree): replace by appropriate base class.
thread = self.sync_engine_class(self.account_id, folder_name,
folder_id_for[folder_name],
self.email_address,
self.provider_name,
self.poll_frequency,
self.syncmanager_lock,
self.refresh_flags_max,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with session_scope(ignore_soft_deletes=False) as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(self.log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {
name: id_
for id_, name in db_session.query(Folder.id, Folder.name).
filter_by(account_id=self.account_id)
}
saved_states = {
name: state
for name, state in db_session.query(
Folder.name, ImapFolderSyncStatus.state).join(
ImapFolderSyncStatus.folder).filter(
ImapFolderSyncStatus.account_id == self.account_id)
}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
self.log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
self.log.info('initializing folder sync')
thread = ImapFolderSyncMonitor(
self.account_id, folder_name, folder_id_for[folder_name],
self.email_address, self.provider_name, self.shared_state,
self.folder_state_handlers, self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread):
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread):
self.log.info('folder sync finished')
# NOTE: Greenlet is automatically removed from the group
# after finishing.
self.folder_monitors.join()
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with session_scope(ignore_soft_deletes=False) as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(self.log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {name: id_ for id_, name in db_session.query(
Folder.id, Folder.name).filter_by(account_id=self.account_id)}
saved_states = {name: state for name, state in
db_session.query(Folder.name,
ImapFolderSyncStatus.state)
.join(ImapFolderSyncStatus.folder)
.filter(ImapFolderSyncStatus.account_id ==
self.account_id)}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
self.log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
self.log.info('initializing folder sync')
thread = ImapFolderSyncMonitor(self.account_id, folder_name,
folder_id_for[folder_name],
self.email_address,
self.provider_name,
self.shared_state,
self.folder_state_handlers,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread):
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread):
self.log.info('folder sync finished')
# NOTE: Greenlet is automatically removed from the group
# after finishing.
self.folder_monitors.join()
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with session_scope() as db_session:
saved_states = dict()
folder_id_for = dict()
for saved_state in db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id):
saved_states[saved_state.folder.name] = saved_state.state
folder_id_for[saved_state.folder.name] = saved_state.folder.id
# it's possible we've never started syncs for these folders before
for folder_id, folder_name, in \
db_session.query(Folder.id, Folder.name).filter_by(
account_id=self.account_id):
folder_id_for[folder_name] = folder_id
with connection_pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(self.log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
for folder_name in sync_folders:
if saved_states.get(folder_name) != 'finish':
self.log.info("Initializing folder sync for {0}"
.format(folder_name))
thread = ImapFolderSyncMonitor(self.account_id, folder_name,
folder_id_for[folder_name],
self.email_address,
self.provider,
self.shared_state,
self.folder_state_handlers,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread):
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread):
self.log.info("Folder sync for {} is done."
.format(folder_name))
# NOTE: Greenlet is automatically removed from the group
# after finishing.
self.folder_monitors.join()
def upgrade():
from inbox.models.session import session_scope
from inbox.models import Namespace, Tag
with session_scope() as db_session:
# Create the attachment tag
print "creating canonical tags..."
for ns in db_session.query(Namespace):
Tag.create_canonical_tags(ns, db_session)
db_session.commit()
conn = op.get_bind()
tag_id_for_namespace = dict([
(namespace_id, tag_id) for namespace_id, tag_id in conn.execute(
text("SELECT namespace_id, id FROM tag WHERE name = 'attachment'"))
])
print "have attachment tag for", len(tag_id_for_namespace), "namespaces"
existing_tagitems = set([
thread_id for thread_id, in
conn.execute(text(
"SELECT distinct(thread_id) FROM tagitem WHERE tag_id IN :tag_ids"
),
tag_ids=set(tag_id_for_namespace.values()))
])
q = """SELECT distinct(thread.id), namespace_id FROM thread
INNER JOIN message ON thread.id = message.thread_id
INNER JOIN part ON part.message_id = message.id
WHERE part.content_disposition IS NOT NULL
"""
if existing_tagitems:
print "skipping", len(existing_tagitems), \
"threads which already have the tag attachment"
q += " AND thread.id NOT IN :existing_tagitems"
q += " ORDER BY thread.id ASC"
for thread_id, namespace_id in \
conn.execute(text(q), existing_tagitems=existing_tagitems):
print thread_id
# We could bulk insert, but don't bother.
conn.execute(text("""
INSERT INTO tagitem (created_at, updated_at, thread_id, tag_id)
VALUES (UTC_TIMESTAMP(), UTC_TIMESTAMP(), :thread_id, :tag_id)
"""),
thread_id=thread_id,
tag_id=tag_id_for_namespace[namespace_id])
def upgrade():
from inbox.models.session import session_scope
from inbox.models import Namespace, Tag, Thread
from inbox.sqlalchemy_ext.util import safer_yield_per
from sqlalchemy import func
from sqlalchemy.orm import joinedload
with session_scope() as db_session:
# Create the attachment tag
for ns in db_session.query(Namespace):
Tag.create_canonical_tags(ns, db_session)
thread_count, = db_session.query(func.count(Thread.id)).one()
q = db_session.query(Thread).options(joinedload(Thread.messages))
processed_count = 0
for thr in safer_yield_per(q, Thread.id, 1, thread_count):
if any(m.attachments for m in thr.messages):
attachment_tag = thr.namespace.tags['attachment']
thr.apply_tag(attachment_tag)
processed_count += 1
print processed_count
def folder_sync_engine(db, monkeypatch):
# super ugly, but I don't want to have to mock tons of stuff
import inbox.mailsync.backends.imap.generic
from inbox.mailsync.backends.imap.generic import FolderSyncEngine
from inbox.models import Tag
monkeypatch.setattr(inbox.mailsync.backends.imap.generic,
"_pool", lambda(account): True)
# setup a dummy FolderSyncEngine - we only need to call a couple
# methods.
email = "*****@*****.**"
account = GenericAuthHandler('fastmail').create_account(
db.session, email, {"email": email, "password": "******"})
Tag.create_canonical_tags(account.namespace, db.session)
db.session.add(account)
db.session.commit()
engine = None
engine = FolderSyncEngine(account.id, "Inbox", 0,
email, "fastmail",
3200, None, 20, None)
return engine
def folder_sync_engine(db, monkeypatch):
# super ugly, but I don't want to have to mock tons of stuff
import inbox.mailsync.backends.imap.generic
from inbox.mailsync.backends.imap.generic import FolderSyncEngine
from inbox.models import Tag
monkeypatch.setattr(inbox.mailsync.backends.imap.generic,
"_pool", lambda(account): True)
# setup a dummy FolderSyncEngine - we only need to call a couple
# methods.
email = "*****@*****.**"
account = create_account(db.session, email,
{"email": email, "password": "******"})
Tag.create_canonical_tags(account.namespace, db.session)
db.session.add(account)
db.session.commit()
engine = None
engine = FolderSyncEngine(account.id, "Inbox", 0,
email, "fastmail",
3200, None, 20, None)
return engine
def create_canonical_tags(db, default_namespace):
"""Ensure that all canonical tags exist for the namespace we're testing
against. This is normally done when an account sync starts."""
Tag.create_canonical_tags(default_namespace, db.session)
db.session.commit()
def create_canonical_tags(db, default_namespace):
"""Ensure that all canonical tags exist for the namespace we're testing
against. This is normally done when an account sync starts."""
Tag.create_canonical_tags(default_namespace, db.session)
db.session.commit()
