def generate_count_report():
# Process counters with the latest syntax
futures = []
processes = MoveProcess.query().fetch()
logging.info('Generating count report in [%s] processes', len(processes))
for process in processes:
process_ok_count = 0
process_error_count = 0
process_total_count = 0
for email in process.emails:
user = ProcessedUser.get_by_id(email)
if not user:
user = ProcessedUser(id=email, ok_count=0, error_count=0,
total_count=list())
total_count = counter.load_and_get_count(
'%s_total_count' % email, namespace=str(process.key.id()))
if total_count:
process_total_count += total_count
else:
total_count = 0
if total_count not in user.total_count:
user.total_count.append(total_count)
ok_count = counter.load_and_get_count(
'%s_ok_count' % email, namespace=str(process.key.id()))
if ok_count:
process_ok_count += ok_count
user.ok_count += ok_count
error_count = counter.load_and_get_count(
'%s_error_count' % email, namespace=str(process.key.id()))
if error_count:
process_error_count += error_count
user.error_count += error_count
futures.append(user.put_async())
logging.info('Updating process counters: total [%s] ok [%s] error ['
'%s]', process_total_count, process_ok_count,
process_error_count)
process.ok_count = process_ok_count
process.error_count = process_error_count
process.total_count = process_total_count
futures.append(process.put_async())
# Process futures
[future.get_result() for future in futures]
def get_messages(user_email=None, tag=None, process_id=None):
imap = None
msg_ids = []
try:
imap = IMAPHelper()
imap.oauth1_2lo_login(user_email=user_email)
try:
if tag:
logging.info('Creating label [%s]', tag)
imap.create_label(tag)
msg_ids = imap.list_messages(only_from_trash=True)
except Exception as e:
logging.exception('Error creating label or retrieving messages for '
'user [%s]', user_email)
processed_user = ProcessedUser.get_by_id(email)
if not processed_user:
processed_user = ProcessedUser(id=user_email, ok_count=0,
error_count=0,
total_count=list(),
error_description=list())
processed_user.error_description.append(e.message)
processed_user.put()
return []
except Exception as e:
logging.exception('Authentication or connection problem for user '
'[%s]', user_email)
processed_user = ProcessedUser.get_by_id(user_email)
if not processed_user:
processed_user = ProcessedUser(id=user_email, ok_count=0,
error_count=0,
total_count=list(),
error_description=list())
processed_user.error_description.append(e.message)
processed_user.put()
return []
finally:
if imap:
imap.close()
# Assuming IMAP connection was OK
if len(msg_ids) > 0:
counter.load_and_increment_counter('%s_total_count' % user_email,
delta=len(msg_ids),
namespace=str(process_id))
return chunkify(msg_ids, num_chunks=constants.USER_CONNECTION_LIMIT)
else:
counter.load_and_increment_counter('%s_total_count' % user_email,
delta=0,
namespace=str(process_id))
return []
