def test_sql_bulk_insert_array(self):
cursor = MagicMock()
mapping = {
'db': {
'col1': {
'pk': '_id',
'_id': {
'type': 'INT'
},
'field1': {
'dest': 'col_array',
'type': '_ARRAY',
'fk': 'id_col1'
},
'field2': {
'dest': 'col_scalar',
'fk': 'id_col1',
'valueField': 'scalar',
'type': '_ARRAY_OF_SCALARS'
}
},
'col_array': {
'pk': '_id',
'field1': {
'dest': 'field1',
'type': 'TEXT'
},
'id_col1': {
'dest': 'id_col1',
'type': 'INT'
}
},
'col_scalar': {
'pk': '_id',
'scalar': {
'dest': 'scalar',
'type': 'INT'
},
'id_col1': {
'dest': 'id_col1',
'type': 'INT'
}
}
}
}
doc = {'_id': 1, 'field1': [{'field1': 'val'}], 'field2': [1, 2, 3]}
sql.sql_bulk_insert(cursor, mapping, 'db.col1', [doc, {}])
cursor.execute.assert_has_calls([
call(
'INSERT INTO col_array (_creationDate,field1,id_col1) VALUES (NULL,\'val\',1)'
),
call(
'INSERT INTO col_scalar (_creationDate,id_col1,scalar) VALUES (NULL,1,1),(NULL,1,2),(NULL,1,3)'
),
call('INSERT INTO col1 (_creationDate) VALUES (NULL)')
])
def _bulk_upsert(self, documents, namespace):
with self.pgsql.cursor() as cursor:
document_buffer = []
insert_accumulator = 0
for document in documents:
document_buffer.append(document)
insert_accumulator += 1
if insert_accumulator % self.chunk_size == 0:
sql_bulk_insert(cursor,
self.mappings,
namespace,
document_buffer,
quiet=self.quiet)
self.commit()
document_buffer = []
LOG.info('%s %s copied...', insert_accumulator, namespace)
sql_bulk_insert(cursor,
self.mappings,
namespace,
document_buffer,
quiet=self.quiet)
self.commit()
def test_sql_bulk_insert_array(self):
cursor = MagicMock()
mapping = {
'db': {
'col1': {
'pk': '_id',
'_id': {
'type': 'INT'
},
'field1': {
'dest': 'col_array',
'type': '_ARRAY',
'fk': 'id_col1'
},
'field2': {
'dest': 'col_scalar',
'fk': 'id_col1',
'valueField': 'scalar',
'type': '_ARRAY_OF_SCALARS'
}
},
'col_array': {
'pk': '_id',
'field1': {
'dest': 'field1',
'type': 'TEXT'
},
'id_col1': {
'dest': 'id_col1',
'type': 'INT'
}
},
'col_scalar': {
'pk': '_id',
'scalar': {
'dest': 'scalar',
'type': 'INT'
},
'id_col1': {
'dest': 'id_col1',
'type': 'INT'
}
}
}
}
doc = {'_id': 1, 'field1': [{'field1': 'val'}], 'field2': [1, 2, 3]}
sql.sql_bulk_insert(cursor, mapping, 'db.col1', [doc, {}])
cursor.execute.assert_has_calls([
call(TEST_SQL_BULK_INSERT_ARRAY_1),
call(TEST_SQL_BULK_INSERT_ARRAY_2)
])
def _upsert(self, namespace, document, cursor, timestamp):
db, collection = db_and_collection(namespace)
primary_key = self.mappings[db][collection]['pk']
sql_delete_rows_where(
cursor, collection,
'{0} = {1}'.format(primary_key,
to_sql_value(document[primary_key])))
sql_bulk_insert(cursor, self.mappings, namespace, [document])
self.commit()
def _bulk_upsert(self, documents, namespace):
with self.pgsql.cursor() as cursor:
document_buffer = []
insert_accumulator = 0
for document in documents:
document_buffer.append(document)
insert_accumulator += 1
if insert_accumulator % self.chunk_size == 0:
sql_bulk_insert(cursor, self.mappings, namespace, document_buffer)
self.commit()
document_buffer = []
LOG.info('%s %s copied...', insert_accumulator, namespace)
sql_bulk_insert(cursor, self.mappings, namespace, document_buffer)
self.commit()
def test_sql_bulk_insert(self):
cursor = MagicMock()
mapping = {
'db': {
'col': {
'pk': '_id',
'field1': {
'type': 'TEXT',
'dest': 'field1'
},
'field2.subfield': {
'type': 'TEXT',
'dest': 'field2_subfield'
}
}
}
}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [])
cursor.execute.assert_not_called()
doc = {'_id': 'foo', 'field1': 'val'}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [doc])
cursor.execute.assert_called_with(TEST_SQL_BULK_INSERT_1)
doc = {'_id': 'foo', 'field1': 'val1', 'field2': {'subfield': 'val2'}}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [doc])
cursor.execute.assert_called_with(TEST_SQL_BULK_INSERT_2)
def test_sql_bulk_insert(self):
cursor = MagicMock()
mapping = {
'db': {
'col': {
'pk': '_id',
'field1': {
'type': 'TEXT',
'dest': 'field1'
},
'field2.subfield': {
'type': 'TEXT',
'dest': 'field2_subfield'
}
}
}
}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [])
cursor.execute.assert_not_called()
doc = {
'_id': 'foo',
'field1': 'val'
}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [doc])
cursor.execute.assert_called_with(
"INSERT INTO col (_creationDate,field1,field2_subfield) VALUES (NULL,'val',NULL)"
)
doc = {
'_id': 'foo',
'field1': 'val1',
'field2': {
'subfield': 'val2'
}
}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [doc])
cursor.execute.assert_called_with(
"INSERT INTO col (_creationDate,field1,field2_subfield) VALUES (NULL,'val1','val2')"
)
def test_sql_bulk_insert(self):
cursor = MagicMock()
mapping = {
'db': {
'col': {
'pk': '_id',
'field1': {
'type': 'TEXT',
'dest': 'field1'
},
'field2.subfield': {
'type': 'TEXT',
'dest': 'field2_subfield'
}
}
}
}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [])
cursor.execute.assert_not_called()
doc = {'_id': 'foo', 'field1': 'val'}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [doc])
cursor.execute.assert_called_with(
"INSERT INTO col (_creationDate,field1,field2_subfield) VALUES (NULL,'val',NULL)"
)
doc = {'_id': 'foo', 'field1': 'val1', 'field2': {'subfield': 'val2'}}
sql.sql_bulk_insert(cursor, mapping, 'db.col', [doc])
cursor.execute.assert_called_with(
"INSERT INTO col (_creationDate,field1,field2_subfield) VALUES (NULL,'val1','val2')"
)
def test_sql_bulk_insert_array(self):
cursor = MagicMock()
mapping = {
'db': {
'col1': {
'pk': '_id',
'_id': {
'type': 'INT'
},
'field1': {
'dest': 'col_array',
'type': '_ARRAY',
'fk': 'id_col1'
},
'field2': {
'dest': 'col_scalar',
'fk': 'id_col1',
'valueField': 'scalar',
'type': '_ARRAY_OF_SCALARS'
}
},
'col_array': {
'pk': '_id',
'field1': {
'dest': 'field1',
'type': 'TEXT'
},
'id_col1': {
'dest': 'id_col1',
'type': 'INT'
}
},
'col_scalar': {
'pk': '_id',
'scalar': {
'dest': 'scalar',
'type': 'INT'
},
'id_col1': {
'dest': 'id_col1',
'type': 'INT'
}
}
}
}
doc = {
'_id': 1,
'field1': [
{'field1': 'val'}
],
'field2': [1, 2, 3]
}
sql.sql_bulk_insert(cursor, mapping, 'db.col1', [doc, {}])
cursor.execute.assert_has_calls([
call('INSERT INTO col_array (_creationDate,field1,id_col1) VALUES (NULL,\'val\',1)'),
call('INSERT INTO col_scalar (_creationDate,id_col1,scalar) VALUES (NULL,1,1),(NULL,1,2),(NULL,1,3)'),
call('INSERT INTO col1 (_creationDate) VALUES (NULL)')
])