def run(self, batch=None, user_process_id=None): user_process = MoveUserProcess.get_by_id(user_process_id) move_process = user_process.move_process_key.get() failed_messages = [] #Create connection imap = IMAPHelper() imap.oauth1_2lo_login(user_email=user_process.user_email) for msg_id in batch: message_process = MoveMessageProcess( email=user_process.user_email, message_id=msg_id, user_process_key=user_process.key, status=constants.STARTED) message_process_key = message_process.put() try: move_message(user_process=user_process, message_process_id=message_process_key.id(), label=move_process.tag, imap=imap) except Exception as e: logging.exception( 'Failed while moving message [%s] for user [%s], ' 'try again...', msg_id, user_process.user_email) failed_messages.append( (yield MoveMessageProcessPipeline( message_process_id=message_process_key.id(), move_process_id=move_process.key.id())) ) imap.close()
def get_messages(user_email=None, tag=None, process_id=None): imap = None msg_ids = [] try: imap = IMAPHelper() imap.oauth1_2lo_login(user_email=user_email) try: if tag: logging.info('Creating label [%s]', tag) imap.create_label(tag) msg_ids = imap.list_messages(only_from_trash=True) except Exception as e: logging.exception('Error creating label or retrieving messages for ' 'user [%s]', user_email) processed_user = ProcessedUser.get_by_id(email) if not processed_user: processed_user = ProcessedUser(id=user_email, ok_count=0, error_count=0, total_count=list(), error_description=list()) processed_user.error_description.append(e.message) processed_user.put() return [] except Exception as e: logging.exception('Authentication or connection problem for user ' '[%s]', user_email) processed_user = ProcessedUser.get_by_id(user_email) if not processed_user: processed_user = ProcessedUser(id=user_email, ok_count=0, error_count=0, total_count=list(), error_description=list()) processed_user.error_description.append(e.message) processed_user.put() return [] finally: if imap: imap.close() # Assuming IMAP connection was OK if len(msg_ids) > 0: counter.load_and_increment_counter('%s_total_count' % user_email, delta=len(msg_ids), namespace=str(process_id)) return chunkify(msg_ids, num_chunks=constants.USER_CONNECTION_LIMIT) else: counter.load_and_increment_counter('%s_total_count' % user_email, delta=0, namespace=str(process_id)) return []
def move_messages(user_email=None, tag=None, chunk_ids=list(), process_id=None, retry_count=0, chunk_sizes=None): moved_successfully = [] imap = None number_moved_successfully = 0 number_moved_unsuccessfully = 0 if len(chunk_ids) <= 0: return True try: imap = IMAPHelper() imap.oauth1_2lo_login(user_email=user_email) imap.select(only_from_trash=True) for i, chunk in enumerate(chunk_ids): chunk_size = chunk_sizes[i] try: result, data = imap.copy_message( msg_id="%s:%s" % (chunk[0], chunk[-1]), destination_label=tag ) if result == 'OK': counter.load_and_increment_counter( '%s_ok_count' % (user_email), namespace=str(process_id), delta=chunk_size) moved_successfully.extend(chunk) number_moved_successfully += chunk_size else: counter.load_and_increment_counter( '%s_error_count' % user_email, namespace=str(process_id), delta=chunk_size) number_moved_unsuccessfully += chunk_size logging.error( 'Error moving message IDs [%s-%s] for user [%s]: ' 'Result [%s] data [%s]', chunk[0], chunk[-1], user_email, result, data) except Exception as e: logging.exception( 'Failed moving message range IDs [%s-%s] for user [%s]', chunk[0], chunk[-1], user_email) remaining = [] remaining_chunk_sizes = [] number_moved_unsuccessfully += chunk_size for j, original_chunk in enumerate(chunk_ids): if original_chunk not in moved_successfully: remaining.append(original_chunk) remaining_chunk_sizes.append(chunk_sizes[j]) # Keep retrying if messages are being moved if retry_count >= 3 and len(moved_successfully) == 0: logging.error('Giving up with remaining [%s] messages for ' 'user [%s]', number_moved_unsuccessfully, user_email) counter.load_and_increment_counter( '%s_error_count' % user_email, delta=number_moved_unsuccessfully, namespace=str(process_id)) else: logging.info( 'Scheduling [%s] remaining messages for user [%s]', number_moved_unsuccessfully, user_email) deferred.defer(move_messages, user_email=user_email, tag=tag, chunk_ids=remaining, process_id=process_id, retry_count=retry_count + 1, chunk_sizes=remaining_chunk_sizes) break except Exception as e: logging.exception('Authentication, connection or select problem for ' 'user [%s]', user_email) counter.load_and_increment_counter( '%s_error_count' % user_email, delta=len(chunk_ids), namespace=str(process_id)) finally: logging.info( 'Succesfully moved [%s] messages for user [%s] in this task', number_moved_successfully, user_email) if imap: imap.close()