def start_oauth2_dance(domain):
current_user = users.get_current_user()
user_email = current_user.email()
login_hint = user_email
user_domain = current_user.email().split('@')[1]
if user_domain != domain:
domain = user_domain
primary_domain = PrimaryDomain.get_or_create(domain)
approval_prompt = 'auto' if primary_domain.refresh_token else 'force'
scope = constants.OAUTH2_SCOPES
state = urllib.quote(domain)
redirect_uri = url_for('oauth_callback', _external=True)
oauth_helper = OAuthDanceHelper(scope=scope, redirect_uri=redirect_uri,
approval_prompt=approval_prompt)
url = oauth_helper.step1_get_authorize_url()
# TODO: Add a random token to avoid forgery
return redirect("%s&state=%s&login_hint=%s" % (url, state, login_hint))
def schedule_user_move(user_email=None, tag=None, move_process_key=None,
domain_name=None):
if domain_name:
try:
primary_domain = PrimaryDomain.get_or_create(
domain_name=domain_name)
if primary_domain.credentials:
email_settings_helper = EmailSettingsHelper(
credentials_json=primary_domain.credentials,
domain=domain_name,
refresh_token=primary_domain.refresh_token
)
email_settings_helper.enable_imap(user_email)
logging.info('IMAP enabled for [%s]',
user_email)
else:
logging.warn('Error trying to enable IMAP for user [%s]',
user_email)
except:
logging.exception('Domain [%s] is not authorized, IMAP not enabled',
domain_name)
for chunk_ids in get_messages(user_email=user_email, tag=tag,
process_id=move_process_key.id()):
if len(chunk_ids) > 0:
new_chunk_ids = []
chunk_sizes = []
for chunk in chunkify(chunk_ids,
chunk_size=constants.MESSAGE_BATCH_SIZE):
new_chunk_ids.append([chunk[0], chunk[-1]])
chunk_sizes.append(len(chunk))
logging.info('Scheduling user [%s] messages move', user_email)
deferred.defer(move_messages, user_email=user_email, tag=tag,
chunk_ids=new_chunk_ids,
process_id=move_process_key.id(),
chunk_sizes=chunk_sizes)
def oauth_callback():
code = request.args.get('code', None)
if not code:
logging.error('No code, no authorization')
abort(500)
state = request.args.get('state', None)
if not state:
logging.error('No state, no authorization')
abort(500)
domain_name = urllib.unquote(state)
redirect_uri = url_for('oauth_callback', _external=True)
oauth_helper = OAuthDanceHelper(redirect_uri=redirect_uri)
credentials = oauth_helper.step2_exchange(code)
primary_domain = PrimaryDomain.get_or_create(domain_name)
primary_domain.credentials = credentials.to_json()
if credentials.refresh_token:
primary_domain.refresh_token = credentials.refresh_token
user = users.get_current_user()
primary_domain.admin_email = user.email()
primary_domain.put()
return redirect(url_for('settings'))
def clean_messages(user_email=None, password=None, chunk_ids=list(),
retry_count=0, process_id=None):
cleaned_successfully = []
remaining = []
if len(chunk_ids) <= 0:
process = CleanUserProcess.get_by_id(process_id)
process.status = constants.FINISHED
process.put()
return True
try:
process = CleanUserProcess.get_by_id(process_id)
imap = IMAPHelper()
imap.login(email=user_email, password=process.source_password)
imap.select()
domain_name = user_email.split('@')[1]
primary_domain = PrimaryDomain.get_or_create(
domain_name)
try:
drive = DriveHelper(credentials_json=primary_domain.credentials,
admin_email=primary_domain.admin_email,
refresh_token=primary_domain.refresh_token)
folder = drive.get_folder(constants.ATTACHMENT_FOLDER)
if not folder:
folder = drive.create_folder(constants.ATTACHMENT_FOLDER)
sub_folder = drive.get_folder(user_email)
if not sub_folder:
sub_folder = drive.create_folder(user_email,
[{'id': folder['id']}])
except Exception as e:
logging.error(
"Couldn't authenticate drive for user %s" % user_email)
raise e
try:
migration = MigrationHelper(
credentials_json=primary_domain.credentials,
refresh_token=primary_domain.refresh_token)
except Exception as e:
logging.error(
"Couldn't authenticate migration api for user %s" % user_email)
raise e
for message_id in chunk_ids:
try:
result = clean_message(msg_id=message_id, imap=imap,
drive=drive,
migration=migration,
folder_id=sub_folder['id'],
user_email=user_email,
process_id=process_id)
if result:
counter.load_and_increment_counter(
'cleaning_%s_ok_count' % (user_email),
namespace=str(process_id))
cleaned_successfully.append(message_id)
else:
counter.load_and_increment_counter(
'cleaning_%s_error_count' % user_email,
namespace=str(process_id))
logging.error(
'Error cleaning message ID [%s] for user [%s]: [%s] ',
message_id, user_email, result)
except Exception as e:
logging.exception(
'Failed cleaning individual message ID [%s] for user [%s]',
message_id, user_email)
remaining = []
if retry_count < constants.MAX_CLEAN_RETRIES:
for chunk_msg in chunk_ids:
if chunk_msg not in cleaned_successfully:
remaining.append(chunk_msg)
logging.info(
'Scheduling [%s] remaining cleaning messages for user [%s]',
len(remaining), user_email)
deferred.defer(clean_messages, user_email=user_email,
chunk_ids=remaining,
process_id=process_id,
retry_count=retry_count + 1)
else:
for chunk_msg in chunk_ids:
if message_id == chunk_msg:
continue
if chunk_msg not in cleaned_successfully:
remaining.append(chunk_msg)
logging.info(
'Giving up cleaning message [%s] for '
'user [%s]', message_id,
user_email)
counter.load_and_increment_counter(
'cleaning_%s_error_count' % user_email,
delta=1,
namespace=str(process_id))
deferred.defer(clean_messages, user_email=user_email,
chunk_ids=remaining,
process_id=process_id)
break
except Exception as e:
logging.exception('Failed cleaning messages chunk')
raise e
finally:
if imap:
imap.close()
if len(chunk_ids) < 10 or (len(cleaned_successfully) + 10 > len(chunk_ids)):
process.status = constants.FINISHED
process.put()
def list_process():
form = CleanUserProcessForm()
user = users.get_current_user()
clean_process_saved = False
clean_processes = []
clean_process_query = CleanUserProcess.query(CleanUserProcess.owner_email == user.email()).order()
query_params = {}
if request.method == 'POST':
if form.validate_on_submit():
primary_domain = PrimaryDomain.get_or_create(
domain_name = user.email().split('@')[1])
logged_in = 'NO'
current_user = users.get_current_user()
if current_user.email() == primary_domain.admin_email:
imap = IMAPHelper()
logged_in, _ = imap.login(
form.data['source_email'], form.data['source_password'])
imap.close()
if logged_in != 'OK':
form.source_email.errors.append(
"Can't access the email with those credentials")
else:
clean_user_process = CleanUserProcess(
owner_email=user.email(),
destination_message_email=user.email(),
status=constants.STARTED
)
for key, value in form.data.iteritems():
setattr(clean_user_process, key, value)
clean_process_key = clean_user_process.put()
clean_process_saved = True
# TODO: process does not appears immediately after it's saved
# launch Pipeline
deferred.defer(schedule_user_cleaning, user_email=form.data['source_email'],
process_id=clean_process_key.id())
is_prev = request.args.get('prev', False)
url_cursor = request.args.get('cursor', None)
cursor = Cursor(urlsafe=url_cursor) if url_cursor else None
if is_prev:
clean_process_query = clean_process_query.order(
CleanUserProcess.created)
cursor = cursor.reversed()
else:
clean_process_query = clean_process_query.order(
-CleanUserProcess.created)
data, next_curs, more = clean_process_query.fetch_page(
constants.PAGE_SIZE, start_cursor=cursor)
clean_processes.extend(data)
if is_prev:
prev_curs = next_curs.reversed().urlsafe() if more else None
next_curs = url_cursor
else:
prev_curs = url_cursor
next_curs = next_curs.urlsafe() if more else None
return render_template('process.html', form=form, user=user.email(),
clean_process_saved=clean_process_saved,
clean_processes=clean_processes, next_curs=next_curs,
more=more, prev_curs=prev_curs)