def _upsert(self, namespace, document, cursor, timestamp):
db, collection = db_and_collection(namespace)
mapped_document = get_mapped_document(self.mappings, document, namespace)
if mapped_document:
sql_insert(cursor, collection, mapped_document, self.mappings[db][collection]['pk'])
self._upsert_array_fields(collection, cursor, db, document, mapped_document, namespace, timestamp)
self.upsert_scalar_array_fields(collection, cursor, db, document, mapped_document, namespace, timestamp)
def _upsert(self, namespace, document, cursor, timestamp):
db, collection = db_and_collection(namespace)
mapped_document = get_mapped_document(self.mappings, document, namespace)
if mapped_document:
primary_key = self.mappings[db][collection]['pk']
sql_insert(cursor, collection, mapped_document, primary_key)
self._upsert_array_fields(collection, cursor, db, document, mapped_document, namespace, timestamp)
self.upsert_scalar_array_fields(collection, cursor, db, document, mapped_document, namespace, timestamp)
def _sql_bulk_insert(query, mappings, namespace, documents):
if not documents:
return
db, collection = db_and_collection(namespace)
primary_key = mappings[db][collection]['pk']
keys = [(k, v['dest']) for k, v in iteritems(mappings[db][collection])
if 'dest' in v
and v['type'] not in [ARRAY_TYPE, ARRAY_OF_SCALARS_TYPE]]
keys.sort(key=lambda x: x[1])
for document in documents:
mapped_document = get_mapped_document(mappings, document, namespace)
values = [
to_sql_value(extract_creation_date(mapped_document, primary_key),
vtype='TIMESTAMP')
]
for key, mapkey in keys:
field_mapping = mappings[db][collection][key]
if mapkey in mapped_document:
values.append(
to_sql_value(mapped_document[mapkey],
vtype=field_mapping['type']))
else:
values.append(to_sql_value(None, vtype=field_mapping['type']))
subquery = {
'collection': collection,
'document': {
'raw': document,
'mapped': mapped_document
},
'keys': ['_creationDate'] + [k[1] for k in keys],
'values': values,
'pk': primary_key,
'queries': []
}
query.append(subquery)
insert_document_arrays(collection, subquery['queries'], db, document,
mapped_document, mappings, primary_key)
insert_scalar_arrays(collection, subquery['queries'], db, document,
mapped_document, mappings, primary_key)
def sql_bulk_insert(cursor, mappings, namespace, documents):
if not documents:
return
db, collection = db_and_collection(namespace)
primary_key = mappings[db][collection]['pk']
keys = [
v['dest'] for k, v in iteritems(mappings[db][collection])
if 'dest' in v and v['type'] != ARRAY_TYPE
and v['type'] != ARRAY_OF_SCALARS_TYPE
]
keys.sort()
values = []
for document in documents:
mapped_document = get_mapped_document(mappings, document, namespace)
document_values = [
to_sql_value(
extract_creation_date(mapped_document,
mappings[db][collection]['pk']))
]
if not mapped_document:
break
for key in keys:
if key in mapped_document:
document_values.append(to_sql_value(mapped_document[key]))
else:
document_values.append(to_sql_value(None))
values.append(u"({0})".format(u','.join(document_values)))
insert_document_arrays(collection, cursor, db, document,
mapped_document, mappings, primary_key)
insert_scalar_arrays(collection, cursor, db, document, mapped_document,
mappings, primary_key)
if values:
sql = u"INSERT INTO {0} ({1}) VALUES {2}".format(
collection, u','.join(['_creationDate'] + keys), u",".join(values))
cursor.execute(sql)
def sql_bulk_insert(cursor, mappings, namespace, documents):
if not documents:
return
db, collection = db_and_collection(namespace)
primary_key = mappings[db][collection]['pk']
keys = unique([
v['dest'] for k, v in iteritems(mappings[db][collection])
if 'dest' in v and v['type'] != ARRAY_TYPE
and v['type'] != ARRAY_OF_SCALARS_TYPE
])
values = []
for document in documents:
mapped_document = get_mapped_document(mappings, document, namespace)
document_values = [to_sql_value(extract_creation_date(mapped_document, mappings[db][collection]['pk']))]
if not mapped_document:
break
for key in keys:
if key in mapped_document:
document_values.append(to_sql_value(mapped_document[key]))
else:
document_values.append(to_sql_value(None))
values.append(u"({0})".format(u','.join(document_values)))
insert_document_arrays(collection, cursor, db, document, mapped_document, mappings, primary_key)
insert_scalar_arrays(collection, cursor, db, document, mapped_document, mappings, primary_key)
if values:
sql = u"INSERT INTO {0} ({1}) VALUES {2}".format(
collection,
u','.join(['_creationDate'] + keys),
u",".join(values)
)
cursor.execute(sql)