def test_is_mapped(self):
mapping = {'db': {'col': {'field': {'type': 'TEXT'}}}}
got = mappings.is_mapped(mapping, 'db.col', field_name=None)
self.assertTrue(got)
got = mappings.is_mapped(mapping, 'db.missing_col', field_name=None)
self.assertFalse(got)
got = mappings.is_mapped(mapping, 'missing_db.col', field_name=None)
self.assertFalse(got)
got = mappings.is_mapped(mapping, 'db.col', field_name='field')
self.assertTrue(got)
got = mappings.is_mapped(mapping, 'db.col', field_name='missing_field')
self.assertFalse(got)
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
try:
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
except psycopg2.Error:
LOG.error(
"Impossible to bulk insert documents in namespace %s: %s",
namespace, documents)
if not self.quiet:
LOG.error("Traceback:\n%s", traceback.format_exc())
def remove(self, document_id, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
with self.pgsql.cursor() as cursor:
db, collection = db_and_collection(namespace)
primary_key = self.mappings[db][collection]['pk']
cursor.execute("DELETE from {0} WHERE {1} = '{2}';".format(
collection.lower(), primary_key, str(document_id)))
self.commit()
def upsert(self, doc, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
try:
with self.pgsql.cursor() as cursor:
self._upsert(namespace, doc, cursor, timestamp)
self.commit()
except Exception as e:
LOG.error("Impossible to upsert %s to %s\n%s", doc, namespace, traceback.format_exc())
def upsert(self, doc, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
try:
with self.pgsql.cursor() as cursor:
self._upsert(namespace, doc, cursor, timestamp)
self.commit()
except Exception as e:
LOG.error("Impossible to upsert %s to %s\n%s", doc, namespace,
traceback.format_exc())
def remove(self, document_id, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
with self.pgsql.cursor() as cursor:
db, collection = db_and_collection(namespace)
primary_key = self.mappings[db][collection]['pk']
cursor.execute(
"DELETE from {0} WHERE {1} = '{2}';".format(collection.lower(), primary_key, str(document_id))
)
self.commit()
def remove(self, document_id, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
with self.pgsql.cursor() as cursor:
db, collection = db_and_collection(namespace)
primary_key = self.mappings[db][collection]['pk']
mapped_field = self.mappings[db][collection].get(primary_key, {})
field_type = mapped_field.get('type')
doc_id = to_sql_value(document_id, vtype=field_type)
cursor.execute("DELETE from {0} WHERE {1} = {2};".format(
collection.lower(), primary_key, doc_id))
self.commit()
def upsert(self, doc, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
try:
with self.pgsql.cursor() as cursor:
self._upsert(namespace, doc, cursor, timestamp)
self.commit()
except psycopg2.Error:
LOG.error(u"Impossible to upsert %s to %s", doc, namespace)
if not self.quiet:
LOG.error(u"Traceback:\n%s", traceback.format_exc())
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
def bulk_upsert(self, documents, namespace, timestamp):
LOG.info('Inspecting %s...', namespace)
if is_mapped(self.mappings, namespace):
LOG.info('Mapping found for %s !...', namespace)
LOG.info('Deleting all rows before update %s !...', namespace)
db, collection = db_and_collection(namespace)
for linked_table in self.get_linked_tables(db, collection):
sql_delete_rows(self.pgsql.cursor(), linked_table)
sql_delete_rows(self.pgsql.cursor(), collection)
self.commit()
self._bulk_upsert(documents, namespace)
LOG.info('%s done.', namespace)
def update(self, document_id, update_spec, namespace, timestamp):
if not is_mapped(self.mappings, namespace):
return
db, collection = db_and_collection(namespace)
updated_document = self.get_document_by_id(db, collection, document_id)
primary_key = self.mappings[db][collection]['pk']
mapped_field = self.mappings[db][collection].get(primary_key, {})
field_type = mapped_field.get('type')
doc_id = to_sql_value(document_id, vtype=field_type)
if updated_document is None:
return
for arrayField in get_any_array_fields(self.mappings, db, collection,
updated_document):
dest = self.mappings[db][collection][arrayField]['dest']
fk = self.mappings[db][collection][arrayField]['fk']
sql_delete_rows_where(self.pgsql.cursor(), dest,
"{0} = {1}".format(fk, doc_id))
self._upsert(namespace, updated_document, self.pgsql.cursor(),
timestamp)
self.commit()
