def test_get_nested_field_from_document(self):
got = utils.get_nested_field_from_document(None, None)
self.assertIsNone(got)
doc = {}
got = utils.get_nested_field_from_document(doc, 'foo')
self.assertIsNone(got)
doc = {'foo': 'bar'}
got = utils.get_nested_field_from_document(doc, 'foo')
self.assertEqual(got, 'bar')
doc = {
'foo': {}
}
got = utils.get_nested_field_from_document(doc, 'foo.bar.baz')
self.assertIsNone(got)
doc = {
'foo': {
'bar': 'baz'
}
}
got = utils.get_nested_field_from_document(doc, 'foo.bar')
self.assertEqual(got, 'baz')
def test_get_nested_field_from_document(self):
got = utils.get_nested_field_from_document(None, None)
self.assertIsNone(got)
doc = {}
got = utils.get_nested_field_from_document(doc, 'foo')
self.assertIsNone(got)
doc = {'foo': 'bar'}
got = utils.get_nested_field_from_document(doc, 'foo')
self.assertEqual(got, 'bar')
doc = {
'foo': {}
}
got = utils.get_nested_field_from_document(doc, 'foo.bar.baz')
self.assertIsNone(got)
doc = {
'foo': {
'bar': 'baz'
}
}
got = utils.get_nested_field_from_document(doc, 'foo.bar')
self.assertEqual(got, 'baz')
def insert_document_arrays(collection, cursor, db, document, mapped_document, mappings, primary_key):
for arrayField in get_array_fields(mappings, db, collection, document):
dest = mappings[db][collection][arrayField]['dest']
fk = mappings[db][collection][arrayField]['fk']
linked_documents = get_nested_field_from_document(document, arrayField)
for linked_document in linked_documents:
linked_document[fk] = mapped_document[primary_key]
sql_bulk_insert(cursor, mappings, "{0}.{1}".format(db, dest), linked_documents)
def _upsert_array_fields(self, collection, cursor, db, document, mapped_document, namespace, timestamp):
for arrayField in get_array_fields(self.mappings, db, collection, document):
dest = self.mappings[db][collection][arrayField]['dest']
fk = self.mappings[db][collection][arrayField]['fk']
dest_namespace = u"{0}.{1}".format(db, dest)
linked_documents = get_nested_field_from_document(document, arrayField)
for linked_document in linked_documents:
linked_document[fk] = mapped_document[get_primary_key(self.mappings, namespace)]
self._upsert(dest_namespace, linked_document, cursor, timestamp)
def insert_scalar_arrays(collection, cursor, db, document, mapped_document, mappings, primary_key):
for arrayField in get_array_of_scalar_fields(mappings, db, collection, document):
dest = mappings[db][collection][arrayField]['dest']
fk = mappings[db][collection][arrayField]['fk']
value_field = mappings[db][collection][arrayField]['valueField']
scalar_values = get_nested_field_from_document(document, arrayField)
linked_documents = []
for value in scalar_values:
linked_documents.append({fk: mapped_document[primary_key], value_field: value})
sql_bulk_insert(cursor, mappings, "{0}.{1}".format(db, dest), linked_documents)
def upsert_scalar_array_fields(self, collection, cursor, db, document, mapped_document, namespace, timestamp):
for scalarArrayField in get_scalar_array_fields(self.mappings, db, collection):
dest = self.mappings[db][collection][scalarArrayField]['dest']
fk = self.mappings[db][collection][scalarArrayField]['fk']
value_field = self.mappings[db][collection][scalarArrayField]['valueField']
dest_namespace = u"{0}.{1}".format(db, dest)
values = get_nested_field_from_document(document, scalarArrayField)
if values is not None and isinstance(values, list):
for value in values:
updated_item = {fk: mapped_document[get_primary_key(self.mappings, namespace)], value_field: value}
self._upsert(dest_namespace, updated_item, cursor, timestamp)
def upsert_scalar_array_fields(self, collection, cursor, db, document, mapped_document, namespace, timestamp):
for scalarArrayField in get_scalar_array_fields(self.mappings, db, collection):
dest = self.mappings[db][collection][scalarArrayField]['dest']
fk = self.mappings[db][collection][scalarArrayField]['fk']
value_field = self.mappings[db][collection][scalarArrayField]['valueField']
dest_namespace = u"{0}.{1}".format(db, dest)
values = get_nested_field_from_document(document, scalarArrayField)
if values is not None and isinstance(values, list):
for value in values:
updated_item = {fk: mapped_document[get_primary_key(self.mappings, namespace)], value_field: value}
self._upsert(dest_namespace, updated_item, cursor, timestamp)
def insert_document_arrays(collection, query, db, document, mapped_document,
mappings, primary_key):
pk = mapped_document.get(
primary_key, ForeignKey('{0}.{1}'.format(collection, primary_key)))
for arrayField in get_array_fields(mappings, db, collection, document):
dest = mappings[db][collection][arrayField]['dest']
fk = mappings[db][collection][arrayField]['fk']
linked_documents = get_nested_field_from_document(document, arrayField)
for linked_document in linked_documents:
linked_document[fk] = pk
_sql_bulk_insert(query, mappings, "{0}.{1}".format(db, dest),
linked_documents)
def _upsert_array_fields(self, collection, cursor, db, document,
mapped_document, namespace, timestamp):
for arrayField in get_array_fields(self.mappings, db, collection,
document):
dest = self.mappings[db][collection][arrayField]['dest']
fk = self.mappings[db][collection][arrayField]['fk']
dest_namespace = u"{0}.{1}".format(db, dest)
linked_documents = get_nested_field_from_document(
document, arrayField)
for linked_document in linked_documents:
linked_document[fk] = mapped_document[get_primary_key(
self.mappings, namespace)]
self._upsert(dest_namespace, linked_document, cursor,
timestamp)
def insert_scalar_arrays(collection, query, db, document, mapped_document,
mappings, primary_key):
pk = mapped_document.get(
primary_key, ForeignKey('{0}.{1}'.format(collection, primary_key)))
for arrayField in get_array_of_scalar_fields(mappings, db, collection,
document):
dest = mappings[db][collection][arrayField]['dest']
fk = mappings[db][collection][arrayField]['fk']
value_field = mappings[db][collection][arrayField]['valueField']
scalar_values = get_nested_field_from_document(document, arrayField)
linked_documents = []
for value in scalar_values:
linked_documents.append({fk: pk, value_field: value})
_sql_bulk_insert(query, mappings, "{0}.{1}".format(db, dest),
linked_documents)