Пример #1
0
def start_oauth2_dance(domain):
    current_user = users.get_current_user()
    user_email = current_user.email()
    login_hint = user_email
    user_domain = current_user.email().split('@')[1]

    if user_domain != domain:
        domain = user_domain

    primary_domain = PrimaryDomain.get_or_create(domain)
    approval_prompt = 'auto' if primary_domain.refresh_token else 'force'
    scope = constants.OAUTH2_SCOPES
    state = urllib.quote(domain)

    redirect_uri = url_for('oauth_callback', _external=True)
    oauth_helper = OAuthDanceHelper(scope=scope, redirect_uri=redirect_uri,
                                    approval_prompt=approval_prompt)
    url = oauth_helper.step1_get_authorize_url()
    # TODO: Add a random token to avoid forgery
    return redirect("%s&state=%s&login_hint=%s" % (url, state, login_hint))
Пример #2
0
def schedule_user_move(user_email=None, tag=None, move_process_key=None,
                       domain_name=None):
    if domain_name:
        try:
            primary_domain = PrimaryDomain.get_or_create(
                domain_name=domain_name)
            if primary_domain.credentials:
                email_settings_helper = EmailSettingsHelper(
                    credentials_json=primary_domain.credentials,
                    domain=domain_name,
                    refresh_token=primary_domain.refresh_token
                )
                email_settings_helper.enable_imap(user_email)
                logging.info('IMAP enabled for [%s]',
                             user_email)
            else:
                logging.warn('Error trying to enable IMAP for user [%s]',
                             user_email)
        except:
            logging.exception('Domain [%s] is not authorized, IMAP not enabled',
                              domain_name)

    for chunk_ids in get_messages(user_email=user_email, tag=tag,
                                  process_id=move_process_key.id()):
        if len(chunk_ids) > 0:
            new_chunk_ids = []
            chunk_sizes = []
            for chunk in chunkify(chunk_ids,
                              chunk_size=constants.MESSAGE_BATCH_SIZE):
                new_chunk_ids.append([chunk[0], chunk[-1]])
                chunk_sizes.append(len(chunk))
            logging.info('Scheduling user [%s] messages move', user_email)
            deferred.defer(move_messages, user_email=user_email, tag=tag,
                           chunk_ids=new_chunk_ids,
                           process_id=move_process_key.id(),
                           chunk_sizes=chunk_sizes)
Пример #3
0
def oauth_callback():
    code = request.args.get('code', None)
    if not code:
        logging.error('No code, no authorization')
        abort(500)
    state = request.args.get('state', None)
    if not state:
        logging.error('No state, no authorization')
        abort(500)
    domain_name = urllib.unquote(state)

    redirect_uri = url_for('oauth_callback', _external=True)
    oauth_helper = OAuthDanceHelper(redirect_uri=redirect_uri)
    credentials = oauth_helper.step2_exchange(code)

    primary_domain = PrimaryDomain.get_or_create(domain_name)
    primary_domain.credentials = credentials.to_json()
    if credentials.refresh_token:
        primary_domain.refresh_token = credentials.refresh_token
    user = users.get_current_user()
    primary_domain.admin_email = user.email()
    primary_domain.put()

    return redirect(url_for('settings'))
Пример #4
0
def clean_messages(user_email=None, password=None, chunk_ids=list(),
                   retry_count=0, process_id=None):
    cleaned_successfully = []
    remaining = []
    if len(chunk_ids) <= 0:
        process = CleanUserProcess.get_by_id(process_id)
        process.status = constants.FINISHED
        process.put()
        return True

    try:
        process = CleanUserProcess.get_by_id(process_id)
        imap = IMAPHelper()
        imap.login(email=user_email, password=process.source_password)
        imap.select()

        domain_name = user_email.split('@')[1]
        primary_domain = PrimaryDomain.get_or_create(
                domain_name)

        try:
            drive = DriveHelper(credentials_json=primary_domain.credentials,
                                admin_email=primary_domain.admin_email,
                                refresh_token=primary_domain.refresh_token)
            folder = drive.get_folder(constants.ATTACHMENT_FOLDER)
            if not folder:
                folder = drive.create_folder(constants.ATTACHMENT_FOLDER)
            sub_folder = drive.get_folder(user_email)
            if not sub_folder:
                sub_folder = drive.create_folder(user_email,
                                                 [{'id': folder['id']}])
        except Exception as e:
            logging.error(
                "Couldn't authenticate drive for user %s" % user_email)
            raise e

        try:
            migration = MigrationHelper(
                credentials_json=primary_domain.credentials,
                refresh_token=primary_domain.refresh_token)
        except Exception as e:
            logging.error(
                "Couldn't authenticate migration api for user %s" % user_email)
            raise e

        for message_id in chunk_ids:
            try:
                result = clean_message(msg_id=message_id, imap=imap,
                                       drive=drive,
                                       migration=migration,
                                       folder_id=sub_folder['id'],
                                       user_email=user_email,
                                       process_id=process_id)
                if result:
                    counter.load_and_increment_counter(
                        'cleaning_%s_ok_count' % (user_email),
                        namespace=str(process_id))
                    cleaned_successfully.append(message_id)
                else:
                    counter.load_and_increment_counter(
                        'cleaning_%s_error_count' % user_email,
                        namespace=str(process_id))
                    logging.error(
                        'Error cleaning message ID [%s] for user [%s]: [%s] ',
                        message_id, user_email, result)
            except Exception as e:
                logging.exception(
                    'Failed cleaning individual message ID [%s] for user [%s]',
                    message_id, user_email)
                remaining = []
                if retry_count < constants.MAX_CLEAN_RETRIES:
                    for chunk_msg in chunk_ids:
                        if chunk_msg not in cleaned_successfully:
                            remaining.append(chunk_msg)
                    logging.info(
                        'Scheduling [%s] remaining cleaning messages for user [%s]',
                        len(remaining), user_email)
                    deferred.defer(clean_messages, user_email=user_email,
                                   chunk_ids=remaining,
                                   process_id=process_id,
                                   retry_count=retry_count + 1)
                else:
                    for chunk_msg in chunk_ids:
                        if message_id == chunk_msg:
                            continue
                        if chunk_msg not in cleaned_successfully:
                            remaining.append(chunk_msg)
                    logging.info(
                        'Giving up cleaning message [%s] for '
                        'user [%s]', message_id,
                        user_email)
                    counter.load_and_increment_counter(
                        'cleaning_%s_error_count' % user_email,
                        delta=1,
                        namespace=str(process_id))
                    deferred.defer(clean_messages, user_email=user_email,
                                   chunk_ids=remaining,
                                   process_id=process_id)
                break

    except Exception as e:
        logging.exception('Failed cleaning messages chunk')
        raise e
    finally:
        if imap:
            imap.close()
        if len(chunk_ids) < 10 or (len(cleaned_successfully) + 10 > len(chunk_ids)):
            process.status = constants.FINISHED
            process.put()
Пример #5
0
def list_process():
    form = CleanUserProcessForm()
    user = users.get_current_user()
    clean_process_saved = False

    clean_processes = []
    clean_process_query = CleanUserProcess.query(CleanUserProcess.owner_email == user.email()).order()
    query_params = {}
    if request.method == 'POST':
        if form.validate_on_submit():
            primary_domain = PrimaryDomain.get_or_create(
                domain_name = user.email().split('@')[1])

            logged_in = 'NO'
            current_user = users.get_current_user()
            if current_user.email() == primary_domain.admin_email:
                imap = IMAPHelper()
                logged_in, _ = imap.login(
                    form.data['source_email'], form.data['source_password'])
                imap.close()

            if logged_in != 'OK':
                form.source_email.errors.append(
                    "Can't access the email with those credentials")
            else:
                clean_user_process = CleanUserProcess(
                    owner_email=user.email(),
                    destination_message_email=user.email(),
                    status=constants.STARTED
                )
                for key, value in form.data.iteritems():
                    setattr(clean_user_process, key, value)
                clean_process_key = clean_user_process.put()
                clean_process_saved = True
                # TODO: process does not appears immediately after it's saved
                # launch Pipeline
                deferred.defer(schedule_user_cleaning, user_email=form.data['source_email'],
                               process_id=clean_process_key.id())

    is_prev = request.args.get('prev', False)
    url_cursor = request.args.get('cursor', None)
    cursor = Cursor(urlsafe=url_cursor) if url_cursor else None

    if is_prev:
        clean_process_query = clean_process_query.order(
            CleanUserProcess.created)
        cursor = cursor.reversed()
    else:
        clean_process_query = clean_process_query.order(
            -CleanUserProcess.created)

    data, next_curs, more = clean_process_query.fetch_page(
        constants.PAGE_SIZE, start_cursor=cursor)
    clean_processes.extend(data)

    if is_prev:
        prev_curs = next_curs.reversed().urlsafe() if more else None
        next_curs = url_cursor
    else:
        prev_curs = url_cursor
        next_curs = next_curs.urlsafe() if more else None

    return render_template('process.html', form=form, user=user.email(),
                           clean_process_saved=clean_process_saved,
                           clean_processes=clean_processes, next_curs=next_curs,
                           more=more, prev_curs=prev_curs)