def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
try:
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
except psycopg2.Error:
LOG.error(
"Impossible to bulk insert documents in namespace %s: %s",
namespace, documents)
if not self.quiet:
LOG.error("Traceback:\n%s", traceback.format_exc())
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
def test_sql_delete_rows(self):
cursor = MagicMock()
sql.sql_delete_rows(cursor, 'table')
cursor.execute.assert_called_with('DELETE FROM table')
def test_sql_delete_rows(self):
cursor = MagicMock()
sql.sql_delete_rows(cursor, 'table')
cursor.execute.assert_called_with('DELETE FROM table')