def get_messages(user_email=None, tag=None, process_id=None): imap = None msg_ids = [] try: imap = IMAPHelper() imap.oauth1_2lo_login(user_email=user_email) try: if tag: logging.info('Creating label [%s]', tag) imap.create_label(tag) msg_ids = imap.list_messages(only_from_trash=True) except Exception as e: logging.exception('Error creating label or retrieving messages for ' 'user [%s]', user_email) processed_user = ProcessedUser.get_by_id(email) if not processed_user: processed_user = ProcessedUser(id=user_email, ok_count=0, error_count=0, total_count=list(), error_description=list()) processed_user.error_description.append(e.message) processed_user.put() return [] except Exception as e: logging.exception('Authentication or connection problem for user ' '[%s]', user_email) processed_user = ProcessedUser.get_by_id(user_email) if not processed_user: processed_user = ProcessedUser(id=user_email, ok_count=0, error_count=0, total_count=list(), error_description=list()) processed_user.error_description.append(e.message) processed_user.put() return [] finally: if imap: imap.close() # Assuming IMAP connection was OK if len(msg_ids) > 0: counter.load_and_increment_counter('%s_total_count' % user_email, delta=len(msg_ids), namespace=str(process_id)) return chunkify(msg_ids, num_chunks=constants.USER_CONNECTION_LIMIT) else: counter.load_and_increment_counter('%s_total_count' % user_email, delta=0, namespace=str(process_id)) return []
def generate_count_report(): # Process counters with the latest syntax futures = [] processes = MoveProcess.query().fetch() logging.info('Generating count report in [%s] processes', len(processes)) for process in processes: process_ok_count = 0 process_error_count = 0 process_total_count = 0 for email in process.emails: user = ProcessedUser.get_by_id(email) if not user: user = ProcessedUser(id=email, ok_count=0, error_count=0, total_count=list()) total_count = counter.load_and_get_count( '%s_total_count' % email, namespace=str(process.key.id())) if total_count: process_total_count += total_count else: total_count = 0 if total_count not in user.total_count: user.total_count.append(total_count) ok_count = counter.load_and_get_count( '%s_ok_count' % email, namespace=str(process.key.id())) if ok_count: process_ok_count += ok_count user.ok_count += ok_count error_count = counter.load_and_get_count( '%s_error_count' % email, namespace=str(process.key.id())) if error_count: process_error_count += error_count user.error_count += error_count futures.append(user.put_async()) logging.info('Updating process counters: total [%s] ok [%s] error [' '%s]', process_total_count, process_ok_count, process_error_count) process.ok_count = process_ok_count process.error_count = process_error_count process.total_count = process_total_count futures.append(process.put_async()) # Process futures [future.get_result() for future in futures]