def clean_messages(user_email=None, password=None, chunk_ids=list(),
retry_count=0, process_id=None):
cleaned_successfully = []
remaining = []
if len(chunk_ids) <= 0:
process = CleanUserProcess.get_by_id(process_id)
process.status = constants.FINISHED
process.put()
return True
try:
process = CleanUserProcess.get_by_id(process_id)
imap = IMAPHelper()
imap.login(email=user_email, password=process.source_password)
imap.select()
domain_name = user_email.split('@')[1]
primary_domain = PrimaryDomain.get_or_create(
domain_name)
try:
drive = DriveHelper(credentials_json=primary_domain.credentials,
admin_email=primary_domain.admin_email,
refresh_token=primary_domain.refresh_token)
folder = drive.get_folder(constants.ATTACHMENT_FOLDER)
if not folder:
folder = drive.create_folder(constants.ATTACHMENT_FOLDER)
sub_folder = drive.get_folder(user_email)
if not sub_folder:
sub_folder = drive.create_folder(user_email,
[{'id': folder['id']}])
except Exception as e:
logging.error(
"Couldn't authenticate drive for user %s" % user_email)
raise e
try:
migration = MigrationHelper(
credentials_json=primary_domain.credentials,
refresh_token=primary_domain.refresh_token)
except Exception as e:
logging.error(
"Couldn't authenticate migration api for user %s" % user_email)
raise e
for message_id in chunk_ids:
try:
result = clean_message(msg_id=message_id, imap=imap,
drive=drive,
migration=migration,
folder_id=sub_folder['id'],
user_email=user_email,
process_id=process_id)
if result:
counter.load_and_increment_counter(
'cleaning_%s_ok_count' % (user_email),
namespace=str(process_id))
cleaned_successfully.append(message_id)
else:
counter.load_and_increment_counter(
'cleaning_%s_error_count' % user_email,
namespace=str(process_id))
logging.error(
'Error cleaning message ID [%s] for user [%s]: [%s] ',
message_id, user_email, result)
except Exception as e:
logging.exception(
'Failed cleaning individual message ID [%s] for user [%s]',
message_id, user_email)
remaining = []
if retry_count < constants.MAX_CLEAN_RETRIES:
for chunk_msg in chunk_ids:
if chunk_msg not in cleaned_successfully:
remaining.append(chunk_msg)
logging.info(
'Scheduling [%s] remaining cleaning messages for user [%s]',
len(remaining), user_email)
deferred.defer(clean_messages, user_email=user_email,
chunk_ids=remaining,
process_id=process_id,
retry_count=retry_count + 1)
else:
for chunk_msg in chunk_ids:
if message_id == chunk_msg:
continue
if chunk_msg not in cleaned_successfully:
remaining.append(chunk_msg)
logging.info(
'Giving up cleaning message [%s] for '
'user [%s]', message_id,
user_email)
counter.load_and_increment_counter(
'cleaning_%s_error_count' % user_email,
delta=1,
namespace=str(process_id))
deferred.defer(clean_messages, user_email=user_email,
chunk_ids=remaining,
process_id=process_id)
break
except Exception as e:
logging.exception('Failed cleaning messages chunk')
raise e
finally:
if imap:
imap.close()
if len(chunk_ids) < 10 or (len(cleaned_successfully) + 10 > len(chunk_ids)):
process.status = constants.FINISHED
process.put()