예제 #1
0
파일: gmail.py 프로젝트: mdeng1024/inbox
    def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
                                       uids):
        """
        Deduplicate message download using X-GM-MSGID.

        Returns
        -------
        list
            Deduplicated UIDs.

        """
        with mailsync_session_scope() as db_session:
            local_g_msgids = g_msgids(self.account_id, db_session,
                                      in_={remote_g_metadata[uid].msgid
                                           for uid in uids if uid in
                                           remote_g_metadata})

        full_download, imapuid_only = partition(
            lambda uid: uid in remote_g_metadata and
            remote_g_metadata[uid].msgid in local_g_msgids,
            sorted(uids, key=int))
        if imapuid_only:
            log.info('skipping already downloaded uids',
                     count=len(imapuid_only))
            # Since we always download messages via All Mail and create the
            # relevant All Mail ImapUids too at that time, we don't need to
            # create them again here if we're deduping All Mail downloads.
            if crispin_client.selected_folder_name != \
                    crispin_client.folder_names()['all']:
                add_new_imapuids(crispin_client, remote_g_metadata,
                                 self.syncmanager_lock, imapuid_only)

        return full_download
예제 #2
0
파일: gmail.py 프로젝트: nvdnkpr/inbox
def deduplicate_message_download(crispin_client, db_session, log,
                                 syncmanager_lock, remote_g_metadata, uids):
    """ Deduplicate message download using X-GM-MSGID.

    Returns
    -------
    list
        Deduplicated UIDs.
    """
    local_g_msgids = set(account.g_msgids(crispin_client.account_id,
                                          db_session,
                                          in_=[remote_g_metadata[uid].msgid for
                                               uid in uids if uid in
                                               remote_g_metadata]))
    full_download, imapuid_only = partition(
        lambda uid: uid in remote_g_metadata and
        remote_g_metadata[uid].msgid in local_g_msgids,
        sorted(uids, key=int))
    if imapuid_only:
        log.info("Skipping {0} uids already downloaded"
                 .format(len(imapuid_only)))
        # Since we always download messages via All Mail and create the
        # relevant All Mail ImapUids too at that time, we don't need to create
        # them again here if we're deduping All Mail downloads.
        if crispin_client.selected_folder_name != \
                crispin_client.folder_names()['all']:
            add_new_imapuids(crispin_client, log, db_session,
                             remote_g_metadata, syncmanager_lock, imapuid_only)

    return full_download
예제 #3
0
    def index(self, transactions, db_session):
        """
        Translate database operations to CloudSearch index operations
        and perform them.

        """
        docs = []
        doc_service = get_doc_service()
        add_txns, delete_txns = partition(
            lambda trx: trx.command == 'delete', transactions)
        delete_docs = [{'type': 'delete', 'id': txn.record_id}
                       for txn in delete_txns]
        add_record_ids = [txn.record_id for txn in add_txns]
        add_records = db_session.query(Contact).options(
            joinedload("phone_numbers")).filter(
                Contact.id.in_(add_record_ids))
        add_docs = [{'type': 'add', 'id': obj.id,
                    'fields': cloudsearch_contact_repr(obj)}
                    for obj in add_records]
        docs = delete_docs + add_docs

        if docs:
            doc_service.upload_documents(
                documents=json.dumps(docs),
                contentType='application/json')

        self.log.info('docs indexed', adds=len(add_docs),
                      deletes=len(delete_docs))
예제 #4
0
    def index(self, transactions, db_session):
        """
        Translate database operations to CloudSearch index operations
        and perform them.

        """
        docs = []
        doc_service = get_doc_service()
        add_txns, delete_txns = partition(lambda trx: trx.command == 'delete',
                                          transactions)
        delete_docs = [{
            'type': 'delete',
            'id': txn.record_id
        } for txn in delete_txns]
        add_record_ids = [txn.record_id for txn in add_txns]
        add_records = db_session.query(Contact).options(
            joinedload("phone_numbers")).filter(Contact.id.in_(add_record_ids))
        add_docs = [{
            'type': 'add',
            'id': obj.id,
            'fields': cloudsearch_contact_repr(obj)
        } for obj in add_records]
        docs = delete_docs + add_docs

        if docs:
            doc_service.upload_documents(documents=json.dumps(docs),
                                         contentType='application/json')
            self._report_batch_upload()

        self.log.info('docs indexed',
                      adds=len(add_docs),
                      deletes=len(delete_docs))
예제 #5
0
    def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
                                       uids):
        """
        Deduplicate message download using X-GM-MSGID.

        Returns
        -------
        list
            Deduplicated UIDs.

        """
        with mailsync_session_scope() as db_session:
            local_g_msgids = g_msgids(self.namespace_id, db_session,
                                      in_={remote_g_metadata[uid].msgid
                                           for uid in uids if uid in
                                           remote_g_metadata})

        full_download, imapuid_only = partition(
            lambda uid: uid in remote_g_metadata and
            remote_g_metadata[uid].msgid in local_g_msgids,
            sorted(uids, key=int))
        if imapuid_only:
            log.info('downloading new uids for existing messages',
                     count=len(imapuid_only))
            add_new_imapuids(crispin_client, remote_g_metadata,
                             self.syncmanager_lock, imapuid_only)

        return full_download
예제 #6
0
    def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
                                       uids):
        """
        Deduplicate message download using X-GM-MSGID.

        Returns
        -------
        list
            Deduplicated UIDs.

        """
        with mailsync_session_scope() as db_session:
            local_g_msgids = g_msgids(self.namespace_id,
                                      db_session,
                                      in_={
                                          remote_g_metadata[uid].msgid
                                          for uid in uids
                                          if uid in remote_g_metadata
                                      })

        full_download, imapuid_only = partition(
            lambda uid: uid in remote_g_metadata and remote_g_metadata[uid].
            msgid in local_g_msgids, sorted(uids, key=int))
        if imapuid_only:
            log.info('downloading new uids for existing messages',
                     count=len(imapuid_only))
            add_new_imapuids(crispin_client, remote_g_metadata,
                             self.syncmanager_lock, imapuid_only)

        return full_download
예제 #7
0
파일: app.py 프로젝트: Inanny/inbox
def accounts():
    global last_calc_at, ACCOUNTS_INFO

    delta = (datetime.datetime.utcnow() - last_calc_at).seconds if \
        last_calc_at else None

    app.logger.error('In accounts, last_calc_at, delta: {0}, {1}'.format(
        str(last_calc_at), str(delta)))

    if not last_calc_at or delta >= recalc_after:
        app.logger.error('Recalc')

        ACCOUNTS_INFO = []
        accounts = g.db_session.query(Account).all()
        imap, eas = partition(lambda a: a.provider == 'eas', accounts)

        if imap:
            ACCOUNTS_INFO.extend(calculate_imap_status(g.db_session, imap))

        if eas and 'eas' in backend_module_registry:
            ACCOUNTS_INFO.extend(calculate_eas_status(g.db_session, eas))

        last_calc_at = datetime.datetime.utcnow()

    return json.dumps(ACCOUNTS_INFO, cls=DateTimeJSONEncoder)
예제 #8
0
파일: gmail.py 프로젝트: olofster/inbox
    def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
                                       uids):
        """
        Deduplicate message download using X-GM-MSGID.

        Returns
        -------
        list
            Deduplicated UIDs.

        """
        with mailsync_session_scope() as db_session:
            local_g_msgids = g_msgids(self.account_id, db_session,
                                      in_={remote_g_metadata[uid].msgid
                                           for uid in uids if uid in
                                           remote_g_metadata})

        full_download, imapuid_only = partition(
            lambda uid: uid in remote_g_metadata and
            remote_g_metadata[uid].msgid in local_g_msgids,
            sorted(uids, key=int))
        if imapuid_only:
            log.info('skipping already downloaded uids',
                     count=len(imapuid_only))
            # Since we always download messages via All Mail and create the
            # relevant All Mail ImapUids too at that time, we don't need to
            # create them again here if we're deduping All Mail downloads.
            if crispin_client.selected_folder_name != \
                    crispin_client.folder_names()['all']:
                add_new_imapuids(crispin_client, remote_g_metadata,
                                 self.syncmanager_lock, imapuid_only)

        return full_download
예제 #9
0
파일: app.py 프로젝트: 0xcd03/inbox
def accounts():
    global last_calc_at, ACCOUNTS_INFO

    delta = (datetime.datetime.utcnow() - last_calc_at).seconds if \
        last_calc_at else None

    app.logger.error('In accounts, last_calc_at, delta: {0}, {1}'.format(
        str(last_calc_at), str(delta)))

    if not last_calc_at or delta >= recalc_after:
        app.logger.error('Recalc')

        ACCOUNTS_INFO = []
        accounts = g.db_session.query(Account).all()
        imap, eas = partition(lambda a: a.provider == 'eas', accounts)

        if imap:
            ACCOUNTS_INFO.extend(calculate_imap_status(g.db_session, imap))

        if eas and 'eas' in backend_module_registry:
            ACCOUNTS_INFO.extend(calculate_eas_status(g.db_session, eas))

        last_calc_at = datetime.datetime.utcnow()

    return json.dumps(ACCOUNTS_INFO, cls=DateTimeJSONEncoder)
예제 #10
0
파일: gmail.py 프로젝트: caitp/inbox
def deduplicate_message_download(crispin_client, db_session, log,
                                 syncmanager_lock, remote_g_metadata, uids):
    """ Deduplicate message download using X-GM-MSGID.

    Returns
    -------
    list
        Deduplicated UIDs.
    """
    local_g_msgids = set(account.g_msgids(crispin_client.account_id,
                                          db_session,
                                          in_=[remote_g_metadata[uid].msgid for
                                               uid in uids if uid in
                                               remote_g_metadata]))
    full_download, imapuid_only = partition(
        lambda uid: uid in remote_g_metadata and
        remote_g_metadata[uid].msgid in local_g_msgids,
        sorted(uids, key=int))
    if imapuid_only:
        log.info("Skipping {0} uids already downloaded"
                 .format(len(imapuid_only)))
        # Since we always download messages via All Mail and create the
        # relevant All Mail ImapUids too at that time, we don't need to create
        # them again here if we're deduping All Mail downloads.
        if crispin_client.selected_folder_name != \
                crispin_client.folder_names()['all']:
            add_new_imapuids(crispin_client, log, db_session,
                             remote_g_metadata, syncmanager_lock, imapuid_only)

    return full_download
예제 #11
0
def new_or_updated(uids, local_uids):
    """
    HIGHESTMODSEQ queries return a list of messages that are *either*
    new *or* updated. We do different things with each, so we need to
    sort out which is which.

    """
    return partition(lambda x: x in local_uids, uids)
예제 #12
0
def new_or_updated(uids, local_uids):
    """
    HIGHESTMODSEQ queries return a list of messages that are *either*
    new *or* updated. We do different things with each, so we need to
    sort out which is which.

    """
    return partition(lambda x: x in local_uids, uids)
예제 #13
0
파일: gmail.py 프로젝트: nvasilakis/inbox
def deduplicate_message_download(crispin_client, db_session, log,
        remote_g_metadata, uids, c):
    """ Deduplicate message download using X-GM-MSGID. """
    local_g_msgids = set(account.g_msgids(crispin_client.account_id,
        db_session, in_=[remote_g_metadata[uid]['msgid'] for uid in uids]))
    full_download, imapuid_only = partition(
            lambda uid: remote_g_metadata[uid]['msgid'] in local_g_msgids,
            sorted(uids, key=int))
    log.info("Skipping {0} uids already downloaded".format(len(imapuid_only)))
    if len(imapuid_only) > 0:
        add_new_imapuid(crispin_client, db_session, remote_g_metadata,
                imapuid_only, c)

    return full_download
예제 #14
0
    def index(self, transactions, db_session):
        """
        Translate database operations to CloudSearch index operations
        and perform them.

        """
        docs = []
        doc_service = get_doc_service()
        add_txns, delete_txns = partition(lambda trx: trx.command == "delete", transactions)
        delete_docs = [{"type": "delete", "id": txn.record_id} for txn in delete_txns]
        add_record_ids = [txn.record_id for txn in add_txns]
        add_records = (
            db_session.query(Contact).options(joinedload("phone_numbers")).filter(Contact.id.in_(add_record_ids))
        )
        add_docs = [{"type": "add", "id": obj.id, "fields": cloudsearch_contact_repr(obj)} for obj in add_records]
        docs = delete_docs + add_docs

        if docs:
            doc_service.upload_documents(documents=json.dumps(docs), contentType="application/json")
            self._report_batch_upload()

        self.log.info("docs indexed", adds=len(add_docs), deletes=len(delete_docs))