def _bulk_upsert(self, documents, namespace): with self.pgsql.cursor() as cursor: document_buffer = [] insert_accumulator = 0 for document in documents: document_buffer.append(document) insert_accumulator += 1 if insert_accumulator % self.chunk_size == 0: sql_bulk_insert(cursor, self.mappings, namespace, document_buffer, quiet=self.quiet) self.commit() document_buffer = [] LOG.info('%s %s copied...', insert_accumulator, namespace) sql_bulk_insert(cursor, self.mappings, namespace, document_buffer, quiet=self.quiet) self.commit()
def bulk_upsert(self, documents, namespace, timestamp): LOG.info('Inspecting %s...', namespace) if is_mapped(self.mappings, namespace): try: LOG.info('Mapping found for %s !...', namespace) LOG.info('Deleting all rows before update %s !...', namespace) db, collection = db_and_collection(namespace) for linked_table in self.get_linked_tables(db, collection): sql_delete_rows(self.pgsql.cursor(), linked_table) sql_delete_rows(self.pgsql.cursor(), collection) self.commit() self._bulk_upsert(documents, namespace) LOG.info('%s done.', namespace) except psycopg2.Error: LOG.error( "Impossible to bulk insert documents in namespace %s: %s", namespace, documents) if not self.quiet: LOG.error("Traceback:\n%s", traceback.format_exc())
def bulk_upsert(self, documents, namespace, timestamp): LOG.info('Inspecting %s...', namespace) if is_mapped(self.mappings, namespace): LOG.info('Mapping found for %s !...', namespace) LOG.info('Deleting all rows before update %s !...', namespace) db, collection = db_and_collection(namespace) for linked_table in self.get_linked_tables(db, collection): sql_delete_rows(self.pgsql.cursor(), linked_table) sql_delete_rows(self.pgsql.cursor(), collection) self.commit() self._bulk_upsert(documents, namespace) LOG.info('%s done.', namespace)