def test_is_mapped(self): mapping = {'db': {'col': {'field': {'type': 'TEXT'}}}} got = mappings.is_mapped(mapping, 'db.col', field_name=None) self.assertTrue(got) got = mappings.is_mapped(mapping, 'db.missing_col', field_name=None) self.assertFalse(got) got = mappings.is_mapped(mapping, 'missing_db.col', field_name=None) self.assertFalse(got) got = mappings.is_mapped(mapping, 'db.col', field_name='field') self.assertTrue(got) got = mappings.is_mapped(mapping, 'db.col', field_name='missing_field') self.assertFalse(got)
def bulk_upsert(self, documents, namespace, timestamp): LOG.info('Inspecting %s...', namespace) if is_mapped(self.mappings, namespace): try: LOG.info('Mapping found for %s !...', namespace) LOG.info('Deleting all rows before update %s !...', namespace) db, collection = db_and_collection(namespace) for linked_table in self.get_linked_tables(db, collection): sql_delete_rows(self.pgsql.cursor(), linked_table) sql_delete_rows(self.pgsql.cursor(), collection) self.commit() self._bulk_upsert(documents, namespace) LOG.info('%s done.', namespace) except psycopg2.Error: LOG.error( "Impossible to bulk insert documents in namespace %s: %s", namespace, documents) if not self.quiet: LOG.error("Traceback:\n%s", traceback.format_exc())
def remove(self, document_id, namespace, timestamp): if not is_mapped(self.mappings, namespace): return with self.pgsql.cursor() as cursor: db, collection = db_and_collection(namespace) primary_key = self.mappings[db][collection]['pk'] cursor.execute("DELETE from {0} WHERE {1} = '{2}';".format( collection.lower(), primary_key, str(document_id))) self.commit()
def upsert(self, doc, namespace, timestamp): if not is_mapped(self.mappings, namespace): return try: with self.pgsql.cursor() as cursor: self._upsert(namespace, doc, cursor, timestamp) self.commit() except Exception as e: LOG.error("Impossible to upsert %s to %s\n%s", doc, namespace, traceback.format_exc())
def remove(self, document_id, namespace, timestamp): if not is_mapped(self.mappings, namespace): return with self.pgsql.cursor() as cursor: db, collection = db_and_collection(namespace) primary_key = self.mappings[db][collection]['pk'] cursor.execute( "DELETE from {0} WHERE {1} = '{2}';".format(collection.lower(), primary_key, str(document_id)) ) self.commit()
def remove(self, document_id, namespace, timestamp): if not is_mapped(self.mappings, namespace): return with self.pgsql.cursor() as cursor: db, collection = db_and_collection(namespace) primary_key = self.mappings[db][collection]['pk'] mapped_field = self.mappings[db][collection].get(primary_key, {}) field_type = mapped_field.get('type') doc_id = to_sql_value(document_id, vtype=field_type) cursor.execute("DELETE from {0} WHERE {1} = {2};".format( collection.lower(), primary_key, doc_id)) self.commit()
def upsert(self, doc, namespace, timestamp): if not is_mapped(self.mappings, namespace): return try: with self.pgsql.cursor() as cursor: self._upsert(namespace, doc, cursor, timestamp) self.commit() except psycopg2.Error: LOG.error(u"Impossible to upsert %s to %s", doc, namespace) if not self.quiet: LOG.error(u"Traceback:\n%s", traceback.format_exc())
def bulk_upsert(self, documents, namespace, timestamp): LOG.info('Inspecting %s...', namespace) if is_mapped(self.mappings, namespace): LOG.info('Mapping found for %s !...', namespace) LOG.info('Deleting all rows before update %s !...', namespace) db, collection = db_and_collection(namespace) for linked_table in self.get_linked_tables(db, collection): sql_delete_rows(self.pgsql.cursor(), linked_table) sql_delete_rows(self.pgsql.cursor(), collection) self.commit() self._bulk_upsert(documents, namespace) LOG.info('%s done.', namespace)
def update(self, document_id, update_spec, namespace, timestamp): if not is_mapped(self.mappings, namespace): return db, collection = db_and_collection(namespace) updated_document = self.get_document_by_id(db, collection, document_id) primary_key = self.mappings[db][collection]['pk'] mapped_field = self.mappings[db][collection].get(primary_key, {}) field_type = mapped_field.get('type') doc_id = to_sql_value(document_id, vtype=field_type) if updated_document is None: return for arrayField in get_any_array_fields(self.mappings, db, collection, updated_document): dest = self.mappings[db][collection][arrayField]['dest'] fk = self.mappings[db][collection][arrayField]['fk'] sql_delete_rows_where(self.pgsql.cursor(), dest, "{0} = {1}".format(fk, doc_id)) self._upsert(namespace, updated_document, self.pgsql.cursor(), timestamp) self.commit()