def _bulk_upsert(self, documents, namespace):
with self.pgsql.cursor() as cursor:
document_buffer = []
insert_accumulator = 0
for document in documents:
document_buffer.append(document)
insert_accumulator += 1
if insert_accumulator % self.chunk_size == 0:
sql_bulk_insert(cursor,
self.mappings,
namespace,
document_buffer,
quiet=self.quiet)
self.commit()
document_buffer = []
LOG.info('%s %s copied...', insert_accumulator, namespace)
sql_bulk_insert(cursor,
self.mappings,
namespace,
document_buffer,
quiet=self.quiet)
self.commit()
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
try:
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
except psycopg2.Error:
LOG.error(
"Impossible to bulk insert documents in namespace %s: %s",
namespace, documents)
if not self.quiet:
LOG.error("Traceback:\n%s", traceback.format_exc())
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)