def index(self, transactions, db_session): """ Translate database operations to CloudSearch index operations and perform them. """ docs = [] doc_service = get_doc_service() add_txns, delete_txns = partition(lambda trx: trx.command == 'delete', transactions) delete_docs = [{ 'type': 'delete', 'id': txn.record_id } for txn in delete_txns] add_record_ids = [txn.record_id for txn in add_txns] add_records = db_session.query(Contact).options( joinedload("phone_numbers")).filter(Contact.id.in_(add_record_ids)) add_docs = [{ 'type': 'add', 'id': obj.id, 'fields': cloudsearch_contact_repr(obj) } for obj in add_records] docs = delete_docs + add_docs if docs: doc_service.upload_documents(documents=json.dumps(docs), contentType='application/json') self._report_batch_upload() self.log.info('docs indexed', adds=len(add_docs), deletes=len(delete_docs))
def index(self, transactions, db_session): """ Translate database operations to CloudSearch index operations and perform them. """ docs = [] doc_service = get_doc_service() add_txns, delete_txns = partition( lambda trx: trx.command == 'delete', transactions) delete_docs = [{'type': 'delete', 'id': txn.record_id} for txn in delete_txns] add_record_ids = [txn.record_id for txn in add_txns] add_records = db_session.query(Contact).options( joinedload("phone_numbers")).filter( Contact.id.in_(add_record_ids)) add_docs = [{'type': 'add', 'id': obj.id, 'fields': cloudsearch_contact_repr(obj)} for obj in add_records] docs = delete_docs + add_docs if docs: doc_service.upload_documents( documents=json.dumps(docs), contentType='application/json') self.log.info('docs indexed', adds=len(add_docs), deletes=len(delete_docs))
def index(self, transactions, db_session): """ Translate database operations to CloudSearch index operations and perform them. """ docs = [] adds, deletes = 0, 0 doc_service = get_doc_service() for trx in transactions: if trx.command == 'delete': deletes += 1 doc = {'type': 'delete', 'id': trx.record_id} else: adds += 1 obj = db_session.query(Contact).get(trx.record_id) if obj is None: continue doc = { 'type': 'add', 'id': trx.record_id, 'fields': cloudsearch_contact_repr(obj) } docs.append(doc) if len(docs) > self.chunk_size: doc_service.upload_documents(documents=json.dumps(docs), contentType='application/json') docs = [] if docs: doc_service.upload_documents(documents=json.dumps(docs), contentType='application/json') self.log.info('docs indexed', adds=adds, deletes=deletes)
def index(self, transactions, db_session): """ Translate database operations to CloudSearch index operations and perform them. """ docs = [] adds, deletes = 0, 0 doc_service = get_doc_service() for trx in transactions: if trx.command == 'delete': deletes += 1 doc = {'type': 'delete', 'id': trx.record_id} else: adds += 1 obj = db_session.query(Contact).get(trx.record_id) if obj is None: continue doc = {'type': 'add', 'id': trx.record_id, 'fields': cloudsearch_contact_repr(obj)} docs.append(doc) if len(docs) > self.chunk_size: doc_service.upload_documents( documents=json.dumps(docs), contentType='application/json') docs = [] if docs: doc_service.upload_documents( documents=json.dumps(docs), contentType='application/json') self.log.info('docs indexed', adds=adds, deletes=deletes)
def index(self, transactions, db_session): """ Translate database operations to CloudSearch index operations and perform them. """ docs = [] doc_service = get_doc_service() add_txns, delete_txns = partition(lambda trx: trx.command == "delete", transactions) delete_docs = [{"type": "delete", "id": txn.record_id} for txn in delete_txns] add_record_ids = [txn.record_id for txn in add_txns] add_records = ( db_session.query(Contact).options(joinedload("phone_numbers")).filter(Contact.id.in_(add_record_ids)) ) add_docs = [{"type": "add", "id": obj.id, "fields": cloudsearch_contact_repr(obj)} for obj in add_records] docs = delete_docs + add_docs if docs: doc_service.upload_documents(documents=json.dumps(docs), contentType="application/json") self._report_batch_upload() self.log.info("docs indexed", adds=len(add_docs), deletes=len(delete_docs))