Exemplo n.º 1
0
def gmail_highestmodseq_update(crispin_client, db_session, log, folder_name,
                               uids, local_uids, status_cb, syncmanager_lock):
    g_metadata = crispin_client.g_metadata(uids)
    to_download = deduplicate_message_download(
        crispin_client, db_session, log, syncmanager_lock, g_metadata, uids)
    if folder_name != crispin_client.folder_names()['all']:
        flags = crispin_client.flags(to_download)
        message_download_stack = LifoQueue()
        for uid in to_download:
            if uid in flags and uid in g_metadata:
                # IMAP will just return no data for a UID if it's disappeared
                # from the folder in the meantime.
                message_download_stack.put(GMessage(
                    uid, g_metadata[uid], flags[uid].flags, flags[uid].labels))
        download_queued_threads(crispin_client, db_session, log, folder_name,
                                message_download_stack, status_cb,
                                syncmanager_lock)
    else:
        uid_download_stack = uid_list_to_stack(to_download)

        download_queued_uids(crispin_client, db_session, log, folder_name,
                             uid_download_stack, 0, uid_download_stack.qsize(),
                             status_cb, syncmanager_lock,
                             gmail_download_and_commit_uids,
                             create_gmail_message)
Exemplo n.º 2
0
Arquivo: gmail.py Projeto: caitp/inbox
def gmail_highestmodseq_update(crispin_client, db_session, log, folder_name,
                               uids, local_uids, status_cb, syncmanager_lock):
    g_metadata = crispin_client.g_metadata(uids)
    to_download = deduplicate_message_download(
        crispin_client, db_session, log, syncmanager_lock, g_metadata, uids)
    if folder_name == crispin_client.folder_names()['inbox']:
        flags = crispin_client.flags(to_download)
        message_download_stack = LifoQueue()
        for uid in to_download:
            if uid in flags and uid in g_metadata:
                # IMAP will just return no data for a UID if it's disappeared
                # from the folder in the meantime.
                message_download_stack.put(GMessage(
                    uid, g_metadata[uid], flags[uid].flags, flags[uid].labels))
        download_queued_threads(crispin_client, db_session, log, folder_name,
                                message_download_stack, status_cb,
                                syncmanager_lock)
    elif folder_name in uid_download_folders(crispin_client):
        uid_download_stack = uid_list_to_stack(to_download)

        download_queued_uids(crispin_client, db_session, log, folder_name,
                             uid_download_stack, 0, uid_download_stack.qsize(),
                             status_cb, syncmanager_lock,
                             gmail_download_and_commit_uids,
                             create_gmail_message)
    else:
        raise MailsyncError(
            "Unknown Gmail sync folder: {}".format(folder_name))
Exemplo n.º 3
0
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
                       shared_state, local_uids, uid_download_stack):
    remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
                                       folder_name, local_uids,
                                       shared_state['syncmanager_lock'])
    remote_uids = sorted(remote_g_metadata.keys(), key=int)
    log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
                                                    folder_name))
    if folder_name == crispin_client.folder_names()['all']:
        log.info("Already have {0} UIDs".format(len(local_uids)))

    with shared_state['syncmanager_lock']:
        log.debug("gmail_initial_sync grabbed syncmanager_lock")
        deleted_uids = remove_deleted_uids(
            crispin_client.account_id, db_session, log, folder_name,
            local_uids, remote_uids)
    local_uids = set(local_uids) - deleted_uids
    unknown_uids = set(remote_uids) - local_uids

    # folders that don't get thread expanded
    uid_download_folders = [crispin_client.folder_names()[tag] for tag in
                            ('trash', 'spam', 'all') if tag in
                            crispin_client.folder_names()]

    if folder_name == crispin_client.folder_names()['inbox']:
        # We don't do an initial dedupe for Inbox because we do thread
        # expansion, which means even if we have a given msgid downloaded, we
        # miiight not have the whole thread. This means that restarts cause
        # duplicate work, but hopefully these folders aren't too huge.
        message_download_stack = LifoQueue()
        flags = crispin_client.flags(unknown_uids)
        for uid in unknown_uids:
            if uid in flags:
                message_download_stack.put(
                    GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
                             flags[uid].labels))
        new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
                               crispin_client.PROVIDER, folder_name, log,
                               message_download_stack,
                               shared_state['poll_frequency'],
                               shared_state['syncmanager_lock'])
        download_queued_threads(crispin_client, db_session, log, folder_name,
                                message_download_stack,
                                shared_state['status_cb'],
                                shared_state['syncmanager_lock'])
    elif folder_name in uid_download_folders:
        full_download = deduplicate_message_download(
            crispin_client, db_session, log, shared_state['syncmanager_lock'],
            remote_g_metadata, unknown_uids)

        add_uids_to_stack(full_download, uid_download_stack)
        new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
                               crispin_client.PROVIDER, folder_name,
                               log, uid_download_stack,
                               shared_state['poll_frequency'],
                               shared_state['syncmanager_lock'])
        download_queued_uids(crispin_client, db_session, log, folder_name,
                             uid_download_stack, len(local_uids),
                             len(remote_uids), shared_state['status_cb'],
                             shared_state['syncmanager_lock'],
                             gmail_download_and_commit_uids,
                             create_gmail_message)
    else:
        raise MailsyncError(
            "Unknown Gmail sync folder: {}".format(folder_name))

    # Complete X-GM-MSGID mapping is no longer needed after initial sync.
    rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
                                          folder_name))

    new_uid_poller.kill()
Exemplo n.º 4
0
Arquivo: gmail.py Projeto: caitp/inbox
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
                       shared_state, local_uids, uid_download_stack):
    remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
                                       folder_name, local_uids,
                                       shared_state['syncmanager_lock'])
    remote_uids = sorted(remote_g_metadata.keys(), key=int)
    log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
                                                    folder_name))
    if folder_name == crispin_client.folder_names()['all']:
        log.info("Already have {0} UIDs".format(len(local_uids)))

    with shared_state['syncmanager_lock']:
        log.debug("gmail_initial_sync grabbed syncmanager_lock")
        deleted_uids = remove_deleted_uids(
            crispin_client.account_id, db_session, log, folder_name,
            local_uids, remote_uids)
    local_uids = set(local_uids) - deleted_uids
    unknown_uids = set(remote_uids) - local_uids

    if folder_name == crispin_client.folder_names()['inbox']:
        # We don't do an initial dedupe for Inbox because we do thread
        # expansion, which means even if we have a given msgid downloaded, we
        # miiight not have the whole thread. This means that restarts cause
        # duplicate work, but hopefully these folders aren't too huge.
        message_download_stack = LifoQueue()
        flags = crispin_client.flags(unknown_uids)
        for uid in unknown_uids:
            if uid in flags:
                message_download_stack.put(
                    GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
                             flags[uid].labels))
        new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
                               crispin_client.PROVIDER, folder_name, log,
                               message_download_stack,
                               shared_state['poll_frequency'],
                               shared_state['syncmanager_lock'])
        download_queued_threads(crispin_client, db_session, log, folder_name,
                                message_download_stack,
                                shared_state['status_cb'],
                                shared_state['syncmanager_lock'])
    elif folder_name in uid_download_folders(crispin_client):
        full_download = deduplicate_message_download(
            crispin_client, db_session, log, shared_state['syncmanager_lock'],
            remote_g_metadata, unknown_uids)

        add_uids_to_stack(full_download, uid_download_stack)
        new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
                               crispin_client.PROVIDER, folder_name,
                               log, uid_download_stack,
                               shared_state['poll_frequency'],
                               shared_state['syncmanager_lock'])
        download_queued_uids(crispin_client, db_session, log, folder_name,
                             uid_download_stack, len(local_uids),
                             len(remote_uids), shared_state['status_cb'],
                             shared_state['syncmanager_lock'],
                             gmail_download_and_commit_uids,
                             create_gmail_message)
    else:
        raise MailsyncError(
            "Unknown Gmail sync folder: {}".format(folder_name))

    # Complete X-GM-MSGID mapping is no longer needed after initial sync.
    rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
                                          folder_name))

    new_uid_poller.kill()
Exemplo n.º 5
0
Arquivo: gmail.py Projeto: jre21/inbox
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
                       shared_state, local_uids, uid_download_stack, c):
    remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
                                       folder_name, local_uids,
                                       shared_state['syncmanager_lock'], c)
    remote_uids = sorted(remote_g_metadata.keys(), key=int)
    log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
                                                    folder_name))
    if folder_name == crispin_client.folder_names(c)['all']:
        log.info("Already have {0} UIDs".format(len(local_uids)))

    deleted_uids = remove_deleted_uids(
        crispin_client.account_id, db_session, log, folder_name,
        local_uids, remote_uids, shared_state['syncmanager_lock'], c)
    local_uids = set(local_uids) - deleted_uids
    unknown_uids = set(remote_uids) - local_uids

    if folder_name != crispin_client.folder_names(c)['all']:
        # We don't do an initial dedupe for non-All Mail folders because
        # we do thread expansion, which means even if we have a given msgid
        # downloaded, we miiight not have the whole thread. This means that
        # restarts cause duplicate work, but hopefully these folders aren't
        # too huge.
        message_download_stack = LifoQueue()
        flags = crispin_client.flags(unknown_uids, c)
        for uid in unknown_uids:
            if uid in flags:
                message_download_stack.put(
                    GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
                             flags[uid].labels))
        new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
                               crispin_client.PROVIDER, folder_name, log,
                               message_download_stack,
                               shared_state['poll_frequency'],
                               shared_state['syncmanager_lock'])
        download_queued_threads(crispin_client, db_session, log, folder_name,
                                message_download_stack,
                                shared_state['status_cb'],
                                shared_state['syncmanager_lock'], c)
    else:
        full_download = deduplicate_message_download(
            crispin_client, db_session, log, remote_g_metadata, unknown_uids,
            c)

        add_uids_to_stack(full_download, uid_download_stack)
        new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
                               crispin_client.PROVIDER, folder_name,
                               log, uid_download_stack,
                               shared_state['poll_frequency'],
                               shared_state['syncmanager_lock'])
        download_queued_uids(crispin_client, db_session, log, folder_name,
                             uid_download_stack, len(local_uids),
                             len(remote_uids), shared_state['status_cb'],
                             shared_state['syncmanager_lock'],
                             gmail_download_and_commit_uids,
                             account.create_gmail_message, c)

    # Complete X-GM-MSGID mapping is no longer needed after initial sync.
    rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
                                          folder_name))

    new_uid_poller.kill()