def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
self.remove_deleted_uids(db_session, local_uids,
remote_uids)
new_uids = set(remote_uids) - local_uids
download_stack = UIDStack()
for uid in sorted(new_uids):
download_stack.put(uid, GenericUIDMetadata(self.throttled))
with mailsync_session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes, download_stack)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
self.download_uids(crispin_client, download_stack)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def __init__(self, account_id, folder_name, folder_id, email_address,
provider_name, poll_frequency, syncmanager_lock,
refresh_flags_max, retry_fail_classes):
bind_context(self, 'foldersyncengine', account_id, folder_id)
self.account_id = account_id
self.folder_name = folder_name
self.folder_id = folder_id
self.poll_frequency = poll_frequency
self.syncmanager_lock = syncmanager_lock
self.refresh_flags_max = refresh_flags_max
self.retry_fail_classes = retry_fail_classes
self.state = None
self.provider_name = provider_name
with mailsync_session_scope() as db_session:
account = db_session.query(Account).get(self.account_id)
self.throttled = account.throttled
self.namespace_id = account.namespace.id
assert self.namespace_id is not None, "namespace_id is None"
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
'finish': lambda self: 'finish',
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
self.remove_deleted_uids(db_session, local_uids,
remote_uids)
new_uids = set(remote_uids) - local_uids
download_stack = UIDStack()
for uid in sorted(new_uids):
download_stack.put(uid, GenericUIDMetadata(self.throttled))
with mailsync_session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes, download_stack)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
self.download_uids(crispin_client, download_stack)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
remote_uids = sorted(crispin_client.all_uids(), key=int)
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(
self.account_id, db_session, self.folder_id
)
common.remove_deleted_uids(
self.account_id, self.folder_id, set(local_uids) - set(remote_uids)
)
unknown_uids = set(remote_uids) - local_uids
with session_scope(self.namespace_id) as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
download_uid_count=len(unknown_uids),
)
change_poller = gevent.spawn(self.poll_for_changes)
bind_context(change_poller, "changepoller", self.account_id, self.folder_id)
if self.is_all_mail(crispin_client):
# Prioritize UIDs for messages in the inbox folder.
if len(remote_uids) < 1e6:
inbox_uids = set(
crispin_client.search_uids(["X-GM-LABELS", "inbox"])
)
else:
# The search above is really slow (times out) on really
# large mailboxes, so bound the search to messages within
# the past month in order to get anywhere.
since = datetime.utcnow() - timedelta(days=30)
inbox_uids = set(
crispin_client.search_uids(
["X-GM-LABELS", "inbox", "SINCE", since]
)
)
uids_to_download = sorted(unknown_uids - inbox_uids) + sorted(
unknown_uids & inbox_uids
)
else:
uids_to_download = sorted(unknown_uids)
for uids in chunk(reversed(uids_to_download), 1024):
g_metadata = crispin_client.g_metadata(uids)
# UIDs might have been expunged since sync started, in which
# case the g_metadata call above will return nothing.
# They may also have been preemptively downloaded by thread
# expansion. We can omit such UIDs.
uids = [u for u in uids if u in g_metadata and u not in self.saved_uids]
self.batch_download_uids(crispin_client, uids, g_metadata)
finally:
if change_poller is not None:
# schedule change_poller to die
gevent.kill(change_poller)
def __init__(self, account_id, folder_name, folder_id, email_address,
provider_name, poll_frequency, syncmanager_lock,
refresh_flags_max, retry_fail_classes):
bind_context(self, 'foldersyncengine', account_id, folder_id)
self.account_id = account_id
self.folder_name = folder_name
self.folder_id = folder_id
self.poll_frequency = poll_frequency
self.syncmanager_lock = syncmanager_lock
self.refresh_flags_max = refresh_flags_max
self.retry_fail_classes = retry_fail_classes
self.state = None
self.provider_name = provider_name
with mailsync_session_scope() as db_session:
account = db_session.query(Account).get(self.account_id)
self.throttled = account.throttled
self.namespace_id = account.namespace.id
assert self.namespace_id is not None, "namespace_id is None"
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
'finish': lambda self: 'finish',
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
def __init__(self, account_id, namespace_id, label_name, semaphore):
bind_context(self, 'renamehandler', account_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.label_name = label_name
self.log = log.new(account_id=account_id)
self.semaphore = semaphore
gevent.Greenlet.__init__(self)
def __init__(self, provider_name, account_id, namespace_id,
poll_frequency=300):
bind_context(self, 'contactsync', account_id)
self.log = logger.new(account_id=account_id, component='contact sync')
self.log.info('Begin syncing contacts...')
BaseSync.__init__(self, account_id, namespace_id, poll_frequency, -1,
'Contacts', provider_name)
def __init__(self, account_id, namespace_id, label_name,
message_ttl=DEFAULT_MESSAGE_TTL):
bind_context(self, 'renamehandler', account_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.label_name = label_name
self.log = log.new(account_id=account_id)
gevent.Greenlet.__init__(self)
def __init__(self, account_id, namespace_id, label_name, semaphore):
bind_context(self, 'renamehandler', account_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.label_name = label_name
self.log = log.new(account_id=account_id)
self.semaphore = semaphore
gevent.Greenlet.__init__(self)
def __init__(self, account_id, namespace_id, uid_accessor,
message_ttl=DEFAULT_MESSAGE_TTL):
bind_context(self, 'deletehandler', account_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.uids_for_message = uid_accessor
self.log = log.new(account_id=account_id)
self.message_ttl = datetime.timedelta(seconds=message_ttl)
gevent.Greenlet.__init__(self)
def __init__(self, account_id, namespace_id, uid_accessor,
message_ttl=DEFAULT_MESSAGE_TTL):
bind_context(self, 'deletehandler', account_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.uids_for_message = uid_accessor
self.log = log.new(account_id=account_id)
self.message_ttl = datetime.timedelta(seconds=message_ttl)
gevent.Greenlet.__init__(self)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
remote_uids = sorted(crispin_client.all_uids(), key=int)
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids) - set(remote_uids))
unknown_uids = set(remote_uids) - local_uids
with session_scope(self.namespace_id) as db_session:
self.update_uid_counts(
db_session, remote_uid_count=len(remote_uids),
download_uid_count=len(unknown_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
if self.is_all_mail(crispin_client):
# Prioritize UIDs for messages in the inbox folder.
if len(remote_uids) < 1e6:
inbox_uids = set(
crispin_client.search_uids(['X-GM-LABELS', 'inbox']))
else:
# The search above is really slow (times out) on really
# large mailboxes, so bound the search to messages within
# the past month in order to get anywhere.
since = datetime.utcnow() - timedelta(days=30)
inbox_uids = set(crispin_client.search_uids([
'X-GM-LABELS', 'inbox',
'SINCE', since]))
uids_to_download = (sorted(unknown_uids - inbox_uids) +
sorted(unknown_uids & inbox_uids))
else:
uids_to_download = sorted(unknown_uids)
for uids in chunk(reversed(uids_to_download), 1024):
g_metadata = crispin_client.g_metadata(uids)
# UIDs might have been expunged since sync started, in which
# case the g_metadata call above will return nothing.
# They may also have been preemptively downloaded by thread
# expansion. We can omit such UIDs.
uids = [u for u in uids if u in g_metadata and u not in
self.saved_uids]
self.batch_download_uids(crispin_client, uids, g_metadata)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def __init__(self, account_id, namespace_id, folder_name,
email_address, provider_name, syncmanager_lock):
with session_scope(namespace_id) as db_session:
try:
folder = db_session.query(Folder). \
filter(Folder.name == folder_name,
Folder.account_id == account_id).one()
except NoResultFound:
raise MailsyncError(u"Missing Folder '{}' on account {}"
.format(folder_name, account_id))
self.folder_id = folder.id
self.folder_role = folder.canonical_name
# Metric flags for sync performance
self.is_initial_sync = folder.initial_sync_end is None
self.is_first_sync = folder.initial_sync_start is None
self.is_first_message = self.is_first_sync
bind_context(self, 'foldersyncengine', account_id, self.folder_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.folder_name = folder_name
if self.folder_name.lower() == 'inbox':
self.poll_frequency = INBOX_POLL_FREQUENCY
else:
self.poll_frequency = DEFAULT_POLL_FREQUENCY
self.syncmanager_lock = syncmanager_lock
self.state = None
self.provider_name = provider_name
self.last_fast_refresh = None
self.flags_fetch_results = {}
self.conn_pool = connection_pool(self.account_id)
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
# Some generic IMAP servers are throwing UIDVALIDITY
# errors forever. Instead of resyncing those servers
# ad vitam, we keep track of the number of consecutive
# times we got such an error and bail out if it's higher than
# MAX_UIDINVALID_RESYNCS.
self.uidinvalid_count = 0
def __init__(self,
provider_name,
account_id,
namespace_id,
poll_frequency=300):
bind_context(self, 'contactsync', account_id)
self.log = logger.new(account_id=account_id, component='contact sync')
self.log.info('Begin syncing contacts...')
BaseSync.__init__(self, account_id, namespace_id, poll_frequency, -1,
'Contacts', provider_name)
def __init__(self, account_id, namespace_id, folder_name, email_address,
provider_name, syncmanager_lock, sync_signal):
with session_scope(namespace_id) as db_session:
try:
folder = db_session.query(Folder). \
filter(Folder.name == folder_name,
Folder.account_id == account_id).one()
except NoResultFound:
raise MailsyncError(
u"Missing Folder '{}' on account {}".format(
folder_name, account_id))
self.folder_id = folder.id
self.folder_role = folder.canonical_name
# Metric flags for sync performance
self.is_initial_sync = folder.initial_sync_end is None
self.is_first_sync = folder.initial_sync_start is None
self.is_first_message = self.is_first_sync
bind_context(self, 'foldersyncengine', account_id, self.folder_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.folder_name = folder_name
self.email_address = email_address
if self.folder_name.lower() == 'inbox':
self.poll_frequency = INBOX_POLL_FREQUENCY
else:
self.poll_frequency = DEFAULT_POLL_FREQUENCY
self.syncmanager_lock = syncmanager_lock
self.state = None
self.provider_name = provider_name
self.last_fast_refresh = None
self.flags_fetch_results = {}
self.conn_pool = connection_pool(self.account_id)
self.sync_signal = sync_signal
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
}
self.setup_heartbeats()
Greenlet.__init__(self)
# Some generic IMAP servers are throwing UIDVALIDITY
# errors forever. Instead of resyncing those servers
# ad vitam, we keep track of the number of consecutive
# times we got such an error and bail out if it's higher than
# MAX_UIDINVALID_RESYNCS.
self.uidinvalid_count = 0
def __init__(self, account, heartbeat=1):
bind_context(self, 'mailsyncmonitor', account.id)
self.shutdown = event.Event()
# how often to check inbox, in seconds
self.heartbeat = heartbeat
self.log = log.new(component='mail sync', account_id=account.id)
self.account_id = account.id
self.namespace_id = account.namespace.id
self.email_address = account.email_address
self.provider_name = account.provider
Greenlet.__init__(self)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
remote_uids = sorted(crispin_client.all_uids(), key=int)
remote_uid_count = len(remote_uids)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
self.remove_deleted_uids(db_session, local_uids,
remote_uids)
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session,
remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
remote_g_metadata = crispin_client.g_metadata(unknown_uids)
download_stack = UIDStack()
change_poller = spawn(self.poll_for_changes, download_stack)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
if self.is_all_mail(crispin_client):
# Put UIDs on the stack such that UIDs for messages in the
# inbox get downloaded first, and such that higher (i.e., more
# recent) UIDs get downloaded before lower ones.
inbox_uids = crispin_client.search_uids(['X-GM-LABELS inbox'])
inbox_uid_set = set(inbox_uids)
# Note that we have to be checking membership in a /set/ for
# performance.
ordered_uids_to_sync = [
u for u in sorted(remote_uids) if u not in inbox_uid_set
] + sorted(inbox_uids)
for uid in ordered_uids_to_sync:
if uid in remote_g_metadata:
metadata = GMetadata(remote_g_metadata[uid].msgid,
remote_g_metadata[uid].thrid,
self.throttled)
download_stack.put(uid, metadata)
self.__download_queued_threads(crispin_client, download_stack)
else:
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
self.download_uids(crispin_client, download_stack)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def __init__(self, account, heartbeat=1):
bind_context(self, 'mailsyncmonitor', account.id)
self.shutdown = event.Event()
# how often to check inbox, in seconds
self.heartbeat = heartbeat
self.log = log.new(component='mail sync', account_id=account.id)
self.account_id = account.id
self.namespace_id = account.namespace.id
self.email_address = account.email_address
self.provider_name = account.provider
Greenlet.__init__(self)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
remote_uids = sorted(crispin_client.all_uids(), key=int)
remote_uid_count = len(remote_uids)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
self.remove_deleted_uids(db_session, local_uids,
remote_uids)
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
remote_g_metadata = crispin_client.g_metadata(unknown_uids)
download_stack = UIDStack()
change_poller = spawn(self.poll_for_changes, download_stack)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
if self.is_all_mail(crispin_client):
# Put UIDs on the stack such that UIDs for messages in the
# inbox get downloaded first, and such that higher (i.e., more
# recent) UIDs get downloaded before lower ones.
inbox_uids = crispin_client.search_uids(['X-GM-LABELS inbox'])
inbox_uid_set = set(inbox_uids)
# Note that we have to be checking membership in a /set/ for
# performance.
ordered_uids_to_sync = [u for u in sorted(remote_uids) if u not
in inbox_uid_set] + sorted(inbox_uids)
for uid in ordered_uids_to_sync:
if uid in remote_g_metadata:
metadata = GMetadata(remote_g_metadata[uid].msgid,
remote_g_metadata[uid].thrid,
self.throttled)
download_stack.put(uid, metadata)
self.__download_queued_threads(crispin_client, download_stack)
else:
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
self.download_uids(crispin_client, download_stack)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id,
self.folder_id,
set(local_uids).difference(remote_uids),
)
new_uids = set(remote_uids).difference(local_uids)
with session_scope(self.namespace_id) as db_session:
account = db_session.query(Account).get(self.account_id)
throttled = account.throttled
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids),
)
change_poller = gevent.spawn(self.poll_for_changes)
bind_context(change_poller, "changepoller", self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
count = 0
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
count += 1
if throttled and count >= THROTTLE_COUNT:
# Throttled accounts' folders sync at a rate of
# 1 message/ minute, after the first approx. THROTTLE_COUNT
# messages per folder are synced.
# Note this is an approx. limit since we use the #(uids),
# not the #(messages).
gevent.sleep(THROTTLE_WAIT)
finally:
if change_poller is not None:
# schedule change_poller to die
gevent.kill(change_poller)
def __init__(self, email_address, provider_name, account_id, namespace_id,
poll_frequency=POLL_FREQUENCY):
bind_context(self, 'eventsync', account_id)
# Only Google for now, can easily parametrize by provider later.
self.provider = GoogleEventsProvider(account_id, namespace_id)
BaseSyncMonitor.__init__(self,
account_id,
namespace_id,
email_address,
EVENT_SYNC_FOLDER_ID,
EVENT_SYNC_FOLDER_NAME,
provider_name,
poll_frequency=poll_frequency)
def __init__(self, account_id, namespace_id, folder_name, email_address,
provider_name, syncmanager_lock):
with session_scope(namespace_id) as db_session:
try:
folder = db_session.query(Folder). \
filter(Folder.name == folder_name,
Folder.account_id == account_id).one()
except NoResultFound:
raise MailsyncError(
u"Missing Folder '{}' on account {}".format(
folder_name, account_id))
self.folder_id = folder.id
self.folder_role = folder.canonical_name
# Metric flags for sync performance
self.is_initial_sync = folder.initial_sync_end is None
self.is_first_sync = folder.initial_sync_start is None
self.is_first_message = self.is_first_sync
bind_context(self, 'foldersyncengine', account_id, self.folder_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.folder_name = folder_name
if self.folder_name.lower() == 'inbox':
self.poll_frequency = INBOX_POLL_FREQUENCY
else:
self.poll_frequency = DEFAULT_POLL_FREQUENCY
self.syncmanager_lock = syncmanager_lock
self.state = None
self.provider_name = provider_name
self.last_fast_refresh = None
self.flags_fetch_results = {}
self.conn_pool = connection_pool(self.account_id)
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids).difference(remote_uids),
db_session)
new_uids = set(remote_uids).difference(local_uids)
with session_scope(self.namespace_id) as db_session:
account = db_session.query(Account).get(self.account_id)
throttled = account.throttled
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
count = 0
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
count += 1
if throttled and count >= THROTTLE_COUNT:
# Throttled accounts' folders sync at a rate of
# 1 message/ minute, after the first approx. THROTTLE_COUNT
# messages per folder are synced.
# Note this is an approx. limit since we use the #(uids),
# not the #(messages).
sleep(THROTTLE_WAIT)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def __init__(self, email_address, provider_name, account_id, namespace_id,
poll_frequency=POLL_FREQUENCY):
bind_context(self, 'eventsync', account_id)
# Only Google for now, can easily parametrize by provider later.
self.provider = GoogleEventsProvider(account_id, namespace_id)
self.log = logger.new(account_id=account_id, component='calendar sync')
BaseSyncMonitor.__init__(self,
account_id,
namespace_id,
email_address,
EVENT_SYNC_FOLDER_ID,
EVENT_SYNC_FOLDER_NAME,
provider_name,
poll_frequency=poll_frequency,
scope='calendar')
def __init__(self, account_id, namespace_id, folder_name,
email_address, provider_name, syncmanager_lock):
with session_scope(namespace_id) as db_session:
try:
folder = db_session.query(Folder). \
filter(Folder.name == folder_name,
Folder.account_id == account_id).one()
except NoResultFound:
raise MailsyncError(u"Missing Folder '{}' on account {}"
.format(folder_name, account_id))
self.folder_id = folder.id
self.folder_role = folder.canonical_name
# Metric flags for sync performance
self.is_initial_sync = folder.initial_sync_end is None
self.is_first_sync = folder.initial_sync_start is None
self.is_first_message = self.is_first_sync
bind_context(self, 'foldersyncengine', account_id, self.folder_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.folder_name = folder_name
if self.folder_name.lower() == 'inbox':
self.poll_frequency = INBOX_POLL_FREQUENCY
else:
self.poll_frequency = DEFAULT_POLL_FREQUENCY
self.syncmanager_lock = syncmanager_lock
self.state = None
self.provider_name = provider_name
self.last_fast_refresh = None
self.conn_pool = connection_pool(self.account_id)
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
def __init__(self, account_id, folder_name, folder_id, email_address,
provider_name, syncmanager_lock):
bind_context(self, 'foldersyncengine', account_id, folder_id)
self.account_id = account_id
self.folder_name = folder_name
self.folder_id = folder_id
if self.folder_name.lower() == 'inbox':
self.poll_frequency = INBOX_POLL_FREQUENCY
else:
self.poll_frequency = DEFAULT_POLL_FREQUENCY
self.syncmanager_lock = syncmanager_lock
self.state = None
self.provider_name = provider_name
self.last_fast_refresh = None
self.conn_pool = connection_pool(self.account_id)
# Metric flags for sync performance
self.is_initial_sync = False
self.is_first_sync = False
self.is_first_message = False
with session_scope() as db_session:
account = Account.get(self.account_id, db_session)
self.namespace_id = account.namespace.id
assert self.namespace_id is not None, "namespace_id is None"
folder = Folder.get(self.folder_id, db_session)
if folder:
self.is_initial_sync = folder.initial_sync_end is None
self.is_first_sync = folder.initial_sync_start is None
self.is_first_message = self.is_first_sync
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
def __init__(self, email_address, provider_name, account_id, namespace_id,
poll_frequency=300):
bind_context(self, 'contactsync', account_id)
self.provider_name = provider_name
provider_cls = CONTACT_SYNC_PROVIDER_MAP[self.provider_name]
self.provider = provider_cls(account_id, namespace_id)
BaseSyncMonitor.__init__(self,
account_id,
namespace_id,
email_address,
CONTACT_SYNC_FOLDER_ID,
CONTACT_SYNC_FOLDER_NAME,
provider_name,
poll_frequency=poll_frequency,
scope='contacts')
def __init__(self, account_id, folder_name, folder_id, email_address,
provider_name, syncmanager_lock):
bind_context(self, 'foldersyncengine', account_id, folder_id)
self.account_id = account_id
self.folder_name = folder_name
self.folder_id = folder_id
if self.folder_name.lower() == 'inbox':
self.poll_frequency = INBOX_POLL_FREQUENCY
else:
self.poll_frequency = DEFAULT_POLL_FREQUENCY
self.syncmanager_lock = syncmanager_lock
self.state = None
self.provider_name = provider_name
self.last_fast_refresh = None
self.conn_pool = connection_pool(self.account_id)
# Metric flags for sync performance
self.is_initial_sync = False
self.is_first_sync = False
self.is_first_message = False
with session_scope() as db_session:
account = Account.get(self.account_id, db_session)
self.namespace_id = account.namespace.id
assert self.namespace_id is not None, "namespace_id is None"
folder = Folder.get(self.folder_id, db_session)
if folder:
self.is_initial_sync = folder.initial_sync_end is None
self.is_first_sync = folder.initial_sync_start is None
self.is_first_message = self.is_first_sync
self.state_handlers = {
'initial': self.initial_sync,
'initial uidinvalid': self.resync_uids,
'poll': self.poll,
'poll uidinvalid': self.resync_uids,
}
Greenlet.__init__(self)
self.heartbeat_status = HeartbeatStatusProxy(self.account_id,
self.folder_id,
self.folder_name,
email_address,
self.provider_name)
def __init__(self, email_address, provider_name, account_id, namespace_id,
poll_frequency=300):
bind_context(self, 'contactsync', account_id)
self.provider_name = provider_name
provider_cls = CONTACT_SYNC_PROVIDER_MAP[self.provider_name]
self.provider = provider_cls(account_id, namespace_id)
BaseSyncMonitor.__init__(self,
account_id,
namespace_id,
email_address,
CONTACT_SYNC_FOLDER_ID,
CONTACT_SYNC_FOLDER_NAME,
provider_name,
poll_frequency=poll_frequency,
retry_fail_classes=[ValidationError])
def __init__(self,
email_address,
provider_name,
account_id,
namespace_id,
poll_frequency=300):
bind_context(self, 'eventsync', account_id)
# Only Google for now, can easily parametrize by provider later.
self.provider = GoogleEventsProvider(account_id, namespace_id)
BaseSyncMonitor.__init__(self,
account_id,
namespace_id,
email_address,
EVENT_SYNC_FOLDER_ID,
EVENT_SYNC_FOLDER_NAME,
provider_name,
poll_frequency=poll_frequency)
def __init__(self, account, heartbeat=1, retry_fail_classes=[]):
bind_context(self, 'mailsyncmonitor', account.id)
self.shutdown = event.Event()
# how often to check inbox, in seconds
self.heartbeat = heartbeat
self.log = log.new(component='mail sync', account_id=account.id)
self.account_id = account.id
self.namespace_id = account.namespace.id
self.email_address = account.email_address
self.provider_name = account.provider
self.retry_fail_classes = retry_fail_classes
# Stuff that might be updated later and we want to keep a shared
# reference on child greenlets.
if not hasattr(self, 'shared_state'):
self.shared_state = dict()
Greenlet.__init__(self)
def __init__(self, account, heartbeat=1, retry_fail_classes=[]):
bind_context(self, "mailsyncmonitor", account.id)
self.shutdown = event.Event()
# how often to check inbox, in seconds
self.heartbeat = heartbeat
self.log = log.new(component="mail sync", account_id=account.id)
self.account_id = account.id
self.namespace_id = account.namespace.id
self.email_address = account.email_address
self.provider_name = account.provider
self.retry_fail_classes = retry_fail_classes
# Stuff that might be updated later and we want to keep a shared
# reference on child greenlets.
if not hasattr(self, "shared_state"):
self.shared_state = dict()
Greenlet.__init__(self)
def __init__(
self,
account_id,
namespace_id,
provider_name,
uid_accessor,
message_ttl=DEFAULT_MESSAGE_TTL,
thread_ttl=DEFAULT_THREAD_TTL,
):
bind_context(self, "deletehandler", account_id)
self.account_id = account_id
self.namespace_id = namespace_id
self.provider_name = provider_name
self.uids_for_message = uid_accessor
self.log = log.new(account_id=account_id)
self.message_ttl = datetime.timedelta(seconds=message_ttl)
self.thread_ttl = datetime.timedelta(seconds=thread_ttl)
gevent.Greenlet.__init__(self)
def __init__(
self, email_address, provider_name, account_id, namespace_id, poll_frequency=300
):
bind_context(self, "contactsync", account_id)
self.provider_name = provider_name
provider_cls = CONTACT_SYNC_PROVIDER_MAP[self.provider_name]
self.provider = provider_cls(account_id, namespace_id)
BaseSyncMonitor.__init__(
self,
account_id,
namespace_id,
email_address,
CONTACT_SYNC_FOLDER_ID,
CONTACT_SYNC_FOLDER_NAME,
provider_name,
poll_frequency=poll_frequency,
scope="contacts",
)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope() as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids).difference(remote_uids),
db_session)
new_uids = set(remote_uids).difference(local_uids)
with session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope() as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids).difference(remote_uids), db_session)
new_uids = set(remote_uids).difference(local_uids)
with session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)