def full_synchronize(self): message_ids = self.connector.get_all_message_id_list() # Check for message_ids that are saved as non email messages. no_message_ids_in_db = set( NoEmailMessageId.objects.filter( account=self.email_account ).values_list('message_id', flat='true') ) # Check for message_ids that are saved as email messages. message_ids_in_db = set( EmailMessage.objects.filter( account=self.email_account ).values_list('message_id', flat='true') ) # What do we need to do with every email message? for i, message_dict in enumerate(message_ids): logger.debug('Check for existing messages, %s/%s' % (i, len(message_ids))) if message_dict['id'] in no_message_ids_in_db: # Not an email, but chatmessage, skip. pass elif message_dict['id'] not in message_ids_in_db: # Message is new. app.send_task( 'download_email_message', args=[self.email_account.id, message_dict['id']], queue='email_first_sync' ) else: # We only need to update the labels for this message. app.send_task( 'update_labels_for_message', args=[self.email_account.id, message_dict['id']], queue='email_first_sync' ) # Finally, add a task to keep track when the sync queue is finished. app.send_task( 'first_sync_finished', args=[self.email_account.id], queue='email_first_sync' ) self.connector.save_history_id() # Only if transaction was successful, we update the history ID. logger.debug('Finished queuing up tasks for email sync, storing history id for %s' % self.email_account.email_address)
def full_synchronize(self): message_ids = self.connector.get_all_message_id_list() # Check for message_ids that are saved as non email messages. no_message_ids_in_db = set( NoEmailMessageId.objects.filter( account=self.email_account).values_list('message_id', flat='true')) # Check for message_ids that are saved as email messages. message_ids_in_db = set( EmailMessage.objects.filter( account=self.email_account).values_list('message_id', flat='true')) # What do we need to do with every email message? for i, message_dict in enumerate(message_ids): logger.debug('Check for existing messages, %s/%s' % (i, len(message_ids))) if message_dict['id'] in no_message_ids_in_db: # Not an email, but chatmessage, skip. pass elif message_dict['id'] not in message_ids_in_db: # Message is new. app.send_task('download_email_message', args=[self.email_account.id, message_dict['id']], queue='email_first_sync') else: # We only need to update the labels for this message. app.send_task('update_labels_for_message', args=[self.email_account.id, message_dict['id']], queue='email_first_sync') # Finally, add a task to keep track when the sync queue is finished. app.send_task('full_sync_finished', args=[self.email_account.id], queue='email_first_sync') # Only if transaction was successful, we update the history ID. self.connector.save_history_id() logger.debug( 'Finished queuing up tasks for email sync, storing history id for %s' % self.email_account)
def sync_by_history(self): """ Synchronize EmailAccount by history. Fetches the changes from the GMail api and creates tasks for the mutations. """ logger.info('updating history for %s with history_id %s' % (self.email_account, self.email_account.history_id)) old_history_id = self.email_account.history_id try: history = self.connector.get_history() except MailNotEnabledError: self.email_account.is_authorized = False self.email_account.is_syncing = False self.email_account.save() logger.error( 'Mail not enabled for this account. Mail sync stopped for account %s' % self.email_account) return except NotFoundError: # A NotFoundError (http error code 404) is given when the suplied historyId is invalid. # Synchronization of email can be restored by initiating a full sync on the account. This is covered # in the synchronize_email_account_scheduler task by looking at the sync_failure_count. self.email_account.sync_failure_count += 1 self.email_account.save() if self.email_account.sync_failure_count > settings.MAX_SYNC_FAILURES: # Repeated sync failures indicate that a full sync did not restore the 404 error, so terminate # syncing on this account altogether. self.email_account.is_authorized = False self.email_account.is_syncing = False self.email_account.save() logger.error( 'Repeated 404 error on incremental syncing. Authorization revoked for account %s' % self.email_account) return self.connector.save_history_id() if not len(history): return new_messages = set() edit_labels = set() # Collect message ids for new, removed email messages and email messages with label changes. for history_item in history: logger.debug('parsing history %s' % history_item) # New email messages. if 'messagesAdded' in history_item: for item in history_item['messagesAdded']: # Skip chat messages. if 'labelIds' in item[ 'message'] and settings.GMAIL_LABEL_CHAT not in item[ 'message']['labelIds']: logger.debug('Message added %s' % item['message']['id']) new_messages.add(item['message']['id']) # Email messages with labels added. for message in history_item.get('labelsAdded', []): if settings.GMAIL_LABEL_CHAT in message['message']['labelIds']: continue if message['message']['id'] not in new_messages: logger.debug('message updated %s', message['message']['id']) edit_labels.add(message['message']['id']) # Email messages with labels removed. for message in history_item.get('labelsRemoved', []): if 'labelIds' in message[ 'message'] and settings.GMAIL_LABEL_CHAT in message[ 'message']['labelIds']: continue if message['message']['id'] not in new_messages: logger.debug('message updated %s', message['message']['id']) edit_labels.add(message['message']['id']) # Removed email messages. for message in history_item.get('messagesDeleted', []): logger.debug('deleting message %s' % message['message']['id']) # When deleting the message, there is no need anymore to create a download it or update its labels. new_messages.discard(message['message']['id']) edit_labels.discard(message['message']['id']) EmailMessage.objects.filter( message_id=message['message']['id'], account=self.email_account).order_by().delete() # Create tasks to download email messages. for message_id in new_messages: logger.info('creating download_email_message for %s', message_id) app.send_task('download_email_message', args=[self.email_account.id, message_id]) # Creates tasks to update labeling for email messages. for message_id in edit_labels: logger.info('creating update_labels_for_message for %s', message_id) app.send_task('update_labels_for_message', args=[self.email_account.id, message_id]) # Only update the unread count if the history id was updated. if old_history_id != self.email_account.history_id: # TODO: Find out if there is a better place to update unread count. download_email_message tasks aren't # finished yet. self.update_unread_count()
def sync_by_history(self): """ Synchronize EmailAccount by history. Fetches the changes from the GMail api and creates tasks for the mutations. """ logger.info('updating history for %s with history_id %s' % (self.email_account, self.email_account.history_id)) old_history_id = self.email_account.history_id history = self.connector.get_history() if not len(history): return new_messages = set() edit_labels = set() # Collect message ids for new, removed email messages and email messages with label changes. for history_item in history: logger.debug('parsing history %s' % history_item) # New email messages. if 'messagesAdded' in history_item: for item in history_item['messagesAdded']: # Skip chat messages. if 'labelIds' in item['message'] and settings.GMAIL_LABEL_CHAT not in item['message']['labelIds']: logger.debug('Message added %s' % item['message']['id']) new_messages.add(item['message']['id']) # Email messages with labels added. for message in history_item.get('labelsAdded', []): if settings.GMAIL_LABEL_CHAT in message['message']['labelIds']: continue if message['message']['id'] not in new_messages: logger.debug('message updated %s', message['message']['id']) edit_labels.add(message['message']['id']) # Email messages with labels removed. for message in history_item.get('labelsRemoved', []): if 'labelIds' in message['message'] and settings.GMAIL_LABEL_CHAT in message['message']['labelIds']: continue if message['message']['id'] not in new_messages: logger.debug('message updated %s', message['message']['id']) edit_labels.add(message['message']['id']) # Removed email messages. for message in history_item.get('messagesDeleted', []): logger.debug('deleting message %s' % message['message']['id']) # When deleting the message, there is no need anymore to create a download it or update its labels. new_messages.discard(message['message']['id']) edit_labels.discard(message['message']['id']) EmailMessage.objects.filter(message_id=message['message']['id'], account=self.email_account).delete() # Create tasks to download email messages. for message_id in new_messages: logger.info('creating download_email_message for %s', message_id) app.send_task('download_email_message', args=[self.email_account.id, message_id]) # Creates tasks to update labeling for email messages. for message_id in edit_labels: logger.info('creating update_labels_for_message for %s', message_id) app.send_task('update_labels_for_message', args=[self.email_account.id, message_id]) self.connector.save_history_id() # Only update the unread count if the history id was updated. if old_history_id != self.email_account.history_id: self.update_unread_count()
def sync_by_history(self): """ Synchronize EmailAccount by history. Fetches the changes from the GMail api and creates tasks for the mutations. """ logger.info('updating history for %s with history_id %s' % (self.email_account, self.email_account.history_id)) old_history_id = self.email_account.history_id try: history = self.connector.get_history() except MailNotEnabledError: self.email_account.is_authorized = False self.email_account.is_syncing = False self.email_account.save() logger.error('Mail not enabled for this account. Mail sync stopped for account %s' % self.email_account) return except NotFoundError: # A NotFoundError (http error code 404) is given when the suplied historyId is invalid. # Synchronization of email can be restored by initiating a full sync on the account. This is covered # in the synchronize_email_account_scheduler task by looking at the sync_failure_count. self.email_account.sync_failure_count += 1 self.email_account.save() if self.email_account.sync_failure_count > settings.MAX_SYNC_FAILURES: # Repeated sync failures indicate that a full sync did not restore the 404 error, so terminate # syncing on this account altogether. self.email_account.is_authorized = False self.email_account.is_syncing = False self.email_account.save() logger.error('Repeated 404 error on incremental syncing. Authorization revoked for account %s' % self.email_account) return self.connector.save_history_id() if not len(history): return new_messages = set() edit_labels = set() # Collect message ids for new, removed email messages and email messages with label changes. for history_item in history: logger.debug('parsing history %s' % history_item) # New email messages. if 'messagesAdded' in history_item: for item in history_item['messagesAdded']: # Skip chat messages. if 'labelIds' in item['message'] and settings.GMAIL_LABEL_CHAT not in item['message']['labelIds']: logger.debug('Message added %s' % item['message']['id']) new_messages.add(item['message']['id']) # Email messages with labels added. for message in history_item.get('labelsAdded', []): if settings.GMAIL_LABEL_CHAT in message['message']['labelIds']: continue if message['message']['id'] not in new_messages: logger.debug('message updated %s', message['message']['id']) edit_labels.add(message['message']['id']) # Email messages with labels removed. for message in history_item.get('labelsRemoved', []): if 'labelIds' in message['message'] and settings.GMAIL_LABEL_CHAT in message['message']['labelIds']: continue if message['message']['id'] not in new_messages: logger.debug('message updated %s', message['message']['id']) edit_labels.add(message['message']['id']) # Removed email messages. for message in history_item.get('messagesDeleted', []): logger.debug('deleting message %s' % message['message']['id']) # When deleting the message, there is no need anymore to create a download it or update its labels. new_messages.discard(message['message']['id']) edit_labels.discard(message['message']['id']) EmailMessage.objects.filter( message_id=message['message']['id'], account=self.email_account ).order_by().delete() # Create tasks to download email messages. for message_id in new_messages: logger.info('creating download_email_message for %s', message_id) app.send_task('download_email_message', args=[self.email_account.id, message_id]) # Creates tasks to update labeling for email messages. for message_id in edit_labels: logger.info('creating update_labels_for_message for %s', message_id) app.send_task('update_labels_for_message', args=[self.email_account.id, message_id]) # Only update the unread count if the history id was updated. if old_history_id != self.email_account.history_id: # TODO: Find out if there is a better place to update unread count. download_email_message tasks aren't # finished yet. self.update_unread_count()