def full_synchronize(self):
message_ids = self.connector.get_all_message_id_list()
# Check for message_ids that are saved as non email messages.
no_message_ids_in_db = set(
NoEmailMessageId.objects.filter(
account=self.email_account
).values_list('message_id', flat='true')
)
# Check for message_ids that are saved as email messages.
message_ids_in_db = set(
EmailMessage.objects.filter(
account=self.email_account
).values_list('message_id', flat='true')
)
# What do we need to do with every email message?
for i, message_dict in enumerate(message_ids):
logger.debug('Check for existing messages, %s/%s' % (i, len(message_ids)))
if message_dict['id'] in no_message_ids_in_db:
# Not an email, but chatmessage, skip.
pass
elif message_dict['id'] not in message_ids_in_db:
# Message is new.
app.send_task(
'download_email_message',
args=[self.email_account.id, message_dict['id']],
queue='email_first_sync'
)
else:
# We only need to update the labels for this message.
app.send_task(
'update_labels_for_message',
args=[self.email_account.id, message_dict['id']],
queue='email_first_sync'
)
# Finally, add a task to keep track when the sync queue is finished.
app.send_task(
'first_sync_finished',
args=[self.email_account.id],
queue='email_first_sync'
)
self.connector.save_history_id()
# Only if transaction was successful, we update the history ID.
logger.debug('Finished queuing up tasks for email sync, storing history id for %s' %
self.email_account.email_address)
def full_synchronize(self):
message_ids = self.connector.get_all_message_id_list()
# Check for message_ids that are saved as non email messages.
no_message_ids_in_db = set(
NoEmailMessageId.objects.filter(
account=self.email_account).values_list('message_id',
flat='true'))
# Check for message_ids that are saved as email messages.
message_ids_in_db = set(
EmailMessage.objects.filter(
account=self.email_account).values_list('message_id',
flat='true'))
# What do we need to do with every email message?
for i, message_dict in enumerate(message_ids):
logger.debug('Check for existing messages, %s/%s' %
(i, len(message_ids)))
if message_dict['id'] in no_message_ids_in_db:
# Not an email, but chatmessage, skip.
pass
elif message_dict['id'] not in message_ids_in_db:
# Message is new.
app.send_task('download_email_message',
args=[self.email_account.id, message_dict['id']],
queue='email_first_sync')
else:
# We only need to update the labels for this message.
app.send_task('update_labels_for_message',
args=[self.email_account.id, message_dict['id']],
queue='email_first_sync')
# Finally, add a task to keep track when the sync queue is finished.
app.send_task('full_sync_finished',
args=[self.email_account.id],
queue='email_first_sync')
# Only if transaction was successful, we update the history ID.
self.connector.save_history_id()
logger.debug(
'Finished queuing up tasks for email sync, storing history id for %s'
% self.email_account)
def sync_by_history(self):
"""
Synchronize EmailAccount by history.
Fetches the changes from the GMail api and creates tasks for the mutations.
"""
logger.info('updating history for %s with history_id %s' %
(self.email_account, self.email_account.history_id))
old_history_id = self.email_account.history_id
try:
history = self.connector.get_history()
except MailNotEnabledError:
self.email_account.is_authorized = False
self.email_account.is_syncing = False
self.email_account.save()
logger.error(
'Mail not enabled for this account. Mail sync stopped for account %s'
% self.email_account)
return
except NotFoundError:
# A NotFoundError (http error code 404) is given when the suplied historyId is invalid.
# Synchronization of email can be restored by initiating a full sync on the account. This is covered
# in the synchronize_email_account_scheduler task by looking at the sync_failure_count.
self.email_account.sync_failure_count += 1
self.email_account.save()
if self.email_account.sync_failure_count > settings.MAX_SYNC_FAILURES:
# Repeated sync failures indicate that a full sync did not restore the 404 error, so terminate
# syncing on this account altogether.
self.email_account.is_authorized = False
self.email_account.is_syncing = False
self.email_account.save()
logger.error(
'Repeated 404 error on incremental syncing. Authorization revoked for account %s'
% self.email_account)
return
self.connector.save_history_id()
if not len(history):
return
new_messages = set()
edit_labels = set()
# Collect message ids for new, removed email messages and email messages with label changes.
for history_item in history:
logger.debug('parsing history %s' % history_item)
# New email messages.
if 'messagesAdded' in history_item:
for item in history_item['messagesAdded']:
# Skip chat messages.
if 'labelIds' in item[
'message'] and settings.GMAIL_LABEL_CHAT not in item[
'message']['labelIds']:
logger.debug('Message added %s' %
item['message']['id'])
new_messages.add(item['message']['id'])
# Email messages with labels added.
for message in history_item.get('labelsAdded', []):
if settings.GMAIL_LABEL_CHAT in message['message']['labelIds']:
continue
if message['message']['id'] not in new_messages:
logger.debug('message updated %s',
message['message']['id'])
edit_labels.add(message['message']['id'])
# Email messages with labels removed.
for message in history_item.get('labelsRemoved', []):
if 'labelIds' in message[
'message'] and settings.GMAIL_LABEL_CHAT in message[
'message']['labelIds']:
continue
if message['message']['id'] not in new_messages:
logger.debug('message updated %s',
message['message']['id'])
edit_labels.add(message['message']['id'])
# Removed email messages.
for message in history_item.get('messagesDeleted', []):
logger.debug('deleting message %s' % message['message']['id'])
# When deleting the message, there is no need anymore to create a download it or update its labels.
new_messages.discard(message['message']['id'])
edit_labels.discard(message['message']['id'])
EmailMessage.objects.filter(
message_id=message['message']['id'],
account=self.email_account).order_by().delete()
# Create tasks to download email messages.
for message_id in new_messages:
logger.info('creating download_email_message for %s', message_id)
app.send_task('download_email_message',
args=[self.email_account.id, message_id])
# Creates tasks to update labeling for email messages.
for message_id in edit_labels:
logger.info('creating update_labels_for_message for %s',
message_id)
app.send_task('update_labels_for_message',
args=[self.email_account.id, message_id])
# Only update the unread count if the history id was updated.
if old_history_id != self.email_account.history_id:
# TODO: Find out if there is a better place to update unread count. download_email_message tasks aren't
# finished yet.
self.update_unread_count()
def sync_by_history(self):
"""
Synchronize EmailAccount by history.
Fetches the changes from the GMail api and creates tasks for the mutations.
"""
logger.info('updating history for %s with history_id %s' % (self.email_account, self.email_account.history_id))
old_history_id = self.email_account.history_id
history = self.connector.get_history()
if not len(history):
return
new_messages = set()
edit_labels = set()
# Collect message ids for new, removed email messages and email messages with label changes.
for history_item in history:
logger.debug('parsing history %s' % history_item)
# New email messages.
if 'messagesAdded' in history_item:
for item in history_item['messagesAdded']:
# Skip chat messages.
if 'labelIds' in item['message'] and settings.GMAIL_LABEL_CHAT not in item['message']['labelIds']:
logger.debug('Message added %s' % item['message']['id'])
new_messages.add(item['message']['id'])
# Email messages with labels added.
for message in history_item.get('labelsAdded', []):
if settings.GMAIL_LABEL_CHAT in message['message']['labelIds']:
continue
if message['message']['id'] not in new_messages:
logger.debug('message updated %s', message['message']['id'])
edit_labels.add(message['message']['id'])
# Email messages with labels removed.
for message in history_item.get('labelsRemoved', []):
if 'labelIds' in message['message'] and settings.GMAIL_LABEL_CHAT in message['message']['labelIds']:
continue
if message['message']['id'] not in new_messages:
logger.debug('message updated %s', message['message']['id'])
edit_labels.add(message['message']['id'])
# Removed email messages.
for message in history_item.get('messagesDeleted', []):
logger.debug('deleting message %s' % message['message']['id'])
# When deleting the message, there is no need anymore to create a download it or update its labels.
new_messages.discard(message['message']['id'])
edit_labels.discard(message['message']['id'])
EmailMessage.objects.filter(message_id=message['message']['id'], account=self.email_account).delete()
# Create tasks to download email messages.
for message_id in new_messages:
logger.info('creating download_email_message for %s', message_id)
app.send_task('download_email_message', args=[self.email_account.id, message_id])
# Creates tasks to update labeling for email messages.
for message_id in edit_labels:
logger.info('creating update_labels_for_message for %s', message_id)
app.send_task('update_labels_for_message', args=[self.email_account.id, message_id])
self.connector.save_history_id()
# Only update the unread count if the history id was updated.
if old_history_id != self.email_account.history_id:
self.update_unread_count()
def sync_by_history(self):
"""
Synchronize EmailAccount by history.
Fetches the changes from the GMail api and creates tasks for the mutations.
"""
logger.info('updating history for %s with history_id %s' % (self.email_account, self.email_account.history_id))
old_history_id = self.email_account.history_id
try:
history = self.connector.get_history()
except MailNotEnabledError:
self.email_account.is_authorized = False
self.email_account.is_syncing = False
self.email_account.save()
logger.error('Mail not enabled for this account. Mail sync stopped for account %s' % self.email_account)
return
except NotFoundError:
# A NotFoundError (http error code 404) is given when the suplied historyId is invalid.
# Synchronization of email can be restored by initiating a full sync on the account. This is covered
# in the synchronize_email_account_scheduler task by looking at the sync_failure_count.
self.email_account.sync_failure_count += 1
self.email_account.save()
if self.email_account.sync_failure_count > settings.MAX_SYNC_FAILURES:
# Repeated sync failures indicate that a full sync did not restore the 404 error, so terminate
# syncing on this account altogether.
self.email_account.is_authorized = False
self.email_account.is_syncing = False
self.email_account.save()
logger.error('Repeated 404 error on incremental syncing. Authorization revoked for account %s' %
self.email_account)
return
self.connector.save_history_id()
if not len(history):
return
new_messages = set()
edit_labels = set()
# Collect message ids for new, removed email messages and email messages with label changes.
for history_item in history:
logger.debug('parsing history %s' % history_item)
# New email messages.
if 'messagesAdded' in history_item:
for item in history_item['messagesAdded']:
# Skip chat messages.
if 'labelIds' in item['message'] and settings.GMAIL_LABEL_CHAT not in item['message']['labelIds']:
logger.debug('Message added %s' % item['message']['id'])
new_messages.add(item['message']['id'])
# Email messages with labels added.
for message in history_item.get('labelsAdded', []):
if settings.GMAIL_LABEL_CHAT in message['message']['labelIds']:
continue
if message['message']['id'] not in new_messages:
logger.debug('message updated %s', message['message']['id'])
edit_labels.add(message['message']['id'])
# Email messages with labels removed.
for message in history_item.get('labelsRemoved', []):
if 'labelIds' in message['message'] and settings.GMAIL_LABEL_CHAT in message['message']['labelIds']:
continue
if message['message']['id'] not in new_messages:
logger.debug('message updated %s', message['message']['id'])
edit_labels.add(message['message']['id'])
# Removed email messages.
for message in history_item.get('messagesDeleted', []):
logger.debug('deleting message %s' % message['message']['id'])
# When deleting the message, there is no need anymore to create a download it or update its labels.
new_messages.discard(message['message']['id'])
edit_labels.discard(message['message']['id'])
EmailMessage.objects.filter(
message_id=message['message']['id'], account=self.email_account
).order_by().delete()
# Create tasks to download email messages.
for message_id in new_messages:
logger.info('creating download_email_message for %s', message_id)
app.send_task('download_email_message', args=[self.email_account.id, message_id])
# Creates tasks to update labeling for email messages.
for message_id in edit_labels:
logger.info('creating update_labels_for_message for %s', message_id)
app.send_task('update_labels_for_message', args=[self.email_account.id, message_id])
# Only update the unread count if the history id was updated.
if old_history_id != self.email_account.history_id:
# TODO: Find out if there is a better place to update unread count. download_email_message tasks aren't
# finished yet.
self.update_unread_count()