def test_sync_existing_objs_some_deleted_some_updated():
"""
Tests when some existing objects will be deleted on a queryset. Run syncing
with some update fields.
"""
objs = [
ddf.G(models.TestModel, int_field=i, float_field=i) for i in range(5)
]
results = pgbulk.sync(
models.TestModel.objects.filter(int_field__lt=4),
[
models.TestModel(int_field=1, float_field=2),
models.TestModel(int_field=2, float_field=2),
models.TestModel(int_field=3, float_field=2),
],
['int_field'],
['float_field'],
returning=True,
ignore_duplicate_updates=True,
)
assert len(list(results)) == 4
assert len(list(results.deleted)) == 1
assert len(list(results.updated)) == 2
assert len(list(results.untouched)) == 1
assert list(results.deleted)[0].id == objs[0].id
def test_sync_existing_objs_some_deleted_wo_update():
"""
Tests when some existing objects will be deleted on a queryset. Run syncing
with no update fields and verify they are untouched in the sync
"""
objs = [
ddf.G(models.TestModel, int_field=i, float_field=i) for i in range(5)
]
results = pgbulk.sync(
models.TestModel.objects.filter(int_field__lt=4),
[
models.TestModel(int_field=1, float_field=2),
models.TestModel(int_field=2, float_field=2),
models.TestModel(int_field=3, float_field=2),
],
['int_field'],
[],
returning=True,
)
assert len(list(results)) == 4
assert len(list(results.deleted)) == 1
assert len(list(results.untouched)) == 3
assert list(results.deleted)[0].id == objs[0].id
def test_upsert_update_fields_returning():
"""
Tests the case when all updates were previously stored and the int
field is used as a uniqueness constraint. Assert returned values are
expected and that it updates all fields by default
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
test_models = [
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
for i in range(3)
]
# Update using the int field as a uniqueness constraint
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='0', float_field=0),
models.TestModel(int_field=1, char_field='1', float_field=1),
models.TestModel(int_field=2, char_field='2', float_field=2),
],
['int_field'],
returning=True,
)
assert list(results.created) == []
assert {u.id for u in results.updated} == {t.id for t in test_models}
assert {u.int_field for u in results.updated} == {0, 1, 2}
assert {u.float_field for u in results.updated} == {0, 1, 2}
assert {u.char_field for u in results.updated} == {'0', '1', '2'}
def test_upsert_some_updates_unique_int_char_field_queryset():
"""
Tests the case when some updates were previously stored and a queryset
is used on the bulk upsert.
"""
# Create previously stored test models with a unique int field and -1
# for all other fields
for i in range(3):
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
# Update using the int field as a uniqueness constraint on a queryset.
# Only one object should be updated.
pgbulk.upsert(
models.TestModel.objects.filter(int_field=0),
[
models.TestModel(int_field=0, char_field='0', float_field=0),
models.TestModel(int_field=4, char_field='1', float_field=1),
models.TestModel(int_field=5, char_field='2', float_field=2),
],
['int_field'],
['float_field'],
)
# Verify that two new objecs were created
assert models.TestModel.objects.count() == 5
assert models.TestModel.objects.filter(char_field='-1').count() == 3
for i, model_obj in enumerate(
models.TestModel.objects.filter(char_field='-1').order_by('int_field')
):
assert model_obj.int_field == i
assert model_obj.char_field == '-1'
def test_sync_existing_objs_some_deleted():
"""
Tests when some existing objects will be deleted.
"""
extant_obj1 = ddf.G(models.TestModel, int_field=1, float_field=1)
extant_obj2 = ddf.G(models.TestModel, int_field=2, float_field=1)
extant_obj3 = ddf.G(models.TestModel, int_field=3, float_field=1)
pgbulk.sync(
models.TestModel,
[
models.TestModel(int_field=3, float_field=2),
models.TestModel(int_field=4, float_field=2),
models.TestModel(int_field=5, float_field=2),
],
['int_field'],
['float_field'],
)
assert models.TestModel.objects.count() == 3
assert models.TestModel.objects.filter(int_field=3).exists()
assert models.TestModel.objects.filter(int_field=4).exists()
assert models.TestModel.objects.filter(int_field=5).exists()
with pytest.raises(models.TestModel.DoesNotExist):
models.TestModel.objects.get(id=extant_obj1.id)
with pytest.raises(models.TestModel.DoesNotExist):
models.TestModel.objects.get(id=extant_obj2.id)
test_model = models.TestModel.objects.get(id=extant_obj3.id)
assert test_model.int_field == 3
def test_upsert_some_updates_unique_int_char_field_update_float_field():
"""
Tests the case when some updates were previously stored and the int and
char fields are used as a uniqueness constraint. Only updates the float
field.
"""
# Create previously stored test models with a unique int and char field
for i in range(2):
ddf.G(models.TestModel, int_field=i, char_field=str(i), float_field=-1)
# Update using the int field as a uniqueness constraint. The first two
# are updated while the third is created
pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='0', float_field=0),
models.TestModel(int_field=1, char_field='1', float_field=1),
models.TestModel(int_field=2, char_field='2', float_field=2),
],
['int_field', 'char_field'],
['float_field'],
)
# Verify that the float field was updated for the first two models and
# the char field was not updated for the first two. The char field,
# however, should be '2' for the third model since it was created
assert models.TestModel.objects.count() == 3
for i, model_obj in enumerate(
models.TestModel.objects.order_by('int_field')
):
assert model_obj.int_field == i
assert model_obj.char_field == str(i)
assert model_obj.float_field == i # almostequals
def test_upsert_all_updates_unique_int_field_update_float_field():
"""
Tests the case when all updates were previously stored and the int
field is used as a uniqueness constraint. Only updates the float field
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
for i in range(3):
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
# Update using the int field as a uniqueness constraint
pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='0', float_field=0),
models.TestModel(int_field=1, char_field='1', float_field=1),
models.TestModel(int_field=2, char_field='2', float_field=2),
],
['int_field'],
update_fields=['float_field'],
)
# Verify that the float field was updated
assert models.TestModel.objects.count() == 3
for i, model_obj in enumerate(
models.TestModel.objects.order_by('int_field')
):
assert model_obj.int_field == i
assert model_obj.char_field == '-1'
assert model_obj.float_field == i # almostequals
def test_upsert_update_duplicate_fields_returning_some_updated_ignore_dups():
"""
Tests the case when all updates were previously stored and the upsert
tries to update the rows with duplicate values. Test when some aren't
duplicates and return untouched results. There will be no untouched
results in this test since we turn off ignoring duplicate updates
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
for i in range(3):
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
# Update using the int field as a uniqueness constraint
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='-1', float_field=-1),
models.TestModel(int_field=1, char_field='-1', float_field=-1),
models.TestModel(int_field=2, char_field='0', float_field=-1),
models.TestModel(int_field=3, char_field='3', float_field=3),
],
['int_field'],
['char_field', 'float_field'],
returning=['char_field'],
ignore_duplicate_updates=False,
return_untouched=True,
)
assert len(list(results.untouched)) == 0
assert len(list(results.updated)) == 3
assert len(list(results.created)) == 1
assert list(results.created)[0].char_field == '3'
def test_sync_existing_objs_all_deleted():
"""
Tests when there are existing objects that will all be deleted.
"""
extant_obj1 = ddf.G(models.TestModel, int_field=1)
extant_obj2 = ddf.G(models.TestModel, int_field=2)
extant_obj3 = ddf.G(models.TestModel, int_field=3)
pgbulk.sync(
models.TestModel,
[
models.TestModel(int_field=4),
models.TestModel(int_field=5),
models.TestModel(int_field=6),
],
['int_field'],
['float_field'],
)
assert models.TestModel.objects.count() == 3
assert models.TestModel.objects.filter(int_field=4).exists()
assert models.TestModel.objects.filter(int_field=5).exists()
assert models.TestModel.objects.filter(int_field=6).exists()
with pytest.raises(models.TestModel.DoesNotExist):
models.TestModel.objects.get(id=extant_obj1.id)
with pytest.raises(models.TestModel.DoesNotExist):
models.TestModel.objects.get(id=extant_obj2.id)
with pytest.raises(models.TestModel.DoesNotExist):
models.TestModel.objects.get(id=extant_obj3.id)
def test_upsert_no_update_fields_returning():
"""
Tests the case when all updates were previously stored and the int
field is used as a uniqueness constraint. This test does not update
any fields
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
for i in range(3):
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
# Update using the int field as a uniqueness constraint
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='0', float_field=0),
models.TestModel(int_field=1, char_field='1', float_field=1),
models.TestModel(int_field=2, char_field='2', float_field=2),
],
['int_field'],
[],
returning=True,
)
assert list(results) == []
def test_upsert_update_duplicate_fields_returning_none_updated():
"""
Tests the case when all updates were previously stored and the upsert
tries to update the rows with duplicate values.
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
for i in range(3):
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
# Update using the int field as a uniqueness constraint
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='-1', float_field=-1),
models.TestModel(int_field=1, char_field='-1', float_field=-1),
models.TestModel(int_field=2, char_field='-1', float_field=-1),
],
['int_field'],
['char_field', 'float_field'],
returning=True,
ignore_duplicate_updates=True,
)
assert list(results) == []
def test_upsert_wo_update_fields():
"""
Tests bulk_upsert with no update fields. This function in turn should
just do a bulk create for any models that do not already exist.
"""
# Create models that already exist
ddf.G(models.TestModel, int_field=1, float_field=1)
ddf.G(models.TestModel, int_field=2, float_field=2)
# Perform a bulk_upsert with one new model
pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=1, float_field=3),
models.TestModel(int_field=2, float_field=3),
models.TestModel(int_field=3, float_field=3),
],
['int_field'],
update_fields=[],
)
# Three objects should now exist, but no float fields should be updated
assert models.TestModel.objects.count() == 3
for test_model, expected_int_value in zip(
models.TestModel.objects.order_by('int_field'), [1, 2, 3]
):
assert test_model.int_field == expected_int_value
assert test_model.float_field == expected_int_value
def test_upsert_return_created_updated_values():
"""
Tests returning values when the items are either updated or created.
"""
# Create an item that will be updated
ddf.G(models.TestModel, int_field=2, float_field=1.0)
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=1, float_field=3.0),
models.TestModel(int_field=2.0, float_field=3.0),
models.TestModel(int_field=3, float_field=3.0),
models.TestModel(int_field=4, float_field=3.0),
],
['int_field'],
['float_field'],
returning=True,
)
created = list(results.created)
updated = list(results.updated)
assert len(created) == 3
assert len(updated) == 1
for test_model, expected_int in zip(
sorted(created, key=lambda k: k.int_field), [1, 3, 4]
):
assert test_model.int_field == expected_int
assert test_model.float_field == 3.0 # almostequals
assert test_model.id is not None
assert updated[0].int_field == 2
assert updated[0].float_field == 3.0 # almostequals
assert updated[0].id is not None
assert models.TestModel.objects.count() == 4
def test_upsert_some_updates_unique_timezone_field_update_float_field():
"""
Tests the case when some updates were previously stored and the timezone
field is used as a uniqueness constraint. Only updates the float field.
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
for i in ['US/Eastern', 'US/Central']:
ddf.G(
models.TestUniqueTzModel,
time_zone=i,
char_field='-1',
float_field=-1,
)
# Update using the int field as a uniqueness constraint. The first two
# are updated while the third is created
pgbulk.upsert(
models.TestUniqueTzModel,
[
models.TestModel(
time_zone=timezone('US/Eastern'), char_field='0', float_field=0
),
models.TestModel(
time_zone=timezone('US/Central'), char_field='1', float_field=1
),
models.TestModel(
time_zone=timezone('UTC'), char_field='2', float_field=2
),
],
['time_zone'],
['float_field'],
)
# Verify that the float field was updated for the first two models and
# the char field was not updated for the first two. The char field,
# however, should be '2' for the third model since it was created
m1 = models.TestUniqueTzModel.objects.get(time_zone=timezone('US/Eastern'))
assert m1.char_field == '-1'
assert m1.float_field == 0 # almostequals
m2 = models.TestUniqueTzModel.objects.get(time_zone=timezone('US/Central'))
assert m2.char_field == '-1'
assert m2.float_field == 1 # almostequals
m3 = models.TestUniqueTzModel.objects.get(time_zone=timezone('UTC'))
assert m3.char_field == '2'
assert m3.float_field == 2 # almostequals
def test_sync_no_existing_objs():
"""
Tests when there are no existing objects before the sync.
"""
pgbulk.sync(
models.TestModel,
[
models.TestModel(int_field=1),
models.TestModel(int_field=3),
models.TestModel(int_field=4),
],
['int_field'],
['float_field'],
)
assert models.TestModel.objects.count() == 3
assert models.TestModel.objects.filter(int_field=1).exists()
assert models.TestModel.objects.filter(int_field=3).exists()
assert models.TestModel.objects.filter(int_field=4).exists()
def test_upsert_return_list_of_values():
"""
Tests that values that are created are returned properly when returning
is True. Set returning to a list of fields
"""
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=1, float_field=2),
models.TestModel(int_field=3, float_field=4),
models.TestModel(int_field=4, float_field=5),
],
['int_field'],
['float_field'],
returning=['float_field'],
)
assert len(list(results.created)) == 3
with pytest.raises(AttributeError):
list(results.created)[0].int_field
assert {2, 4, 5} == {m.float_field for m in results.created}
def test_upsert_no_updates_unique_int_char_field():
"""
Tests the case when no updates were previously stored and the int and
char fields are used as a uniqueness constraint. In this case, there
is data previously stored, but the uniqueness constraints don't match.
"""
# Create previously stored test models with a unique int field and -1 for
# all other fields
for i in range(3):
ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1)
# Update using the int and char field as a uniqueness constraint. All
# three objects are created
pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=3, char_field='0', float_field=0),
models.TestModel(int_field=4, char_field='1', float_field=1),
models.TestModel(int_field=5, char_field='2', float_field=2),
],
['int_field', 'char_field'],
['float_field'],
)
# Verify that no updates occured
assert models.TestModel.objects.count() == 6
assert models.TestModel.objects.filter(char_field='-1').count() == 3
for i, model_obj in enumerate(
models.TestModel.objects.filter(char_field='-1').order_by('int_field')
):
assert model_obj.int_field == i
assert model_obj.char_field == '-1'
assert model_obj.float_field == -1 # almostequals
assert models.TestModel.objects.exclude(char_field='-1').count() == 3
for i, model_obj in enumerate(
models.TestModel.objects.exclude(char_field='-1').order_by('int_field')
):
assert model_obj.int_field == i + 3
assert model_obj.char_field == str(i)
assert model_obj.float_field == i # almostequals
def test_upsert_no_updates():
"""
Tests the case when no updates were previously stored (i.e objects are only
created)
"""
pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=0, char_field='0', float_field=0),
models.TestModel(int_field=1, char_field='1', float_field=1),
models.TestModel(int_field=2, char_field='2', float_field=2),
],
['int_field'],
['char_field', 'float_field'],
)
for i, model_obj in enumerate(
models.TestModel.objects.order_by('int_field')
):
assert model_obj.int_field == i
assert model_obj.char_field == str(i)
assert model_obj.float_field == i # almostequals
def test_sync_existing_objs_some_deleted_w_queryset():
"""
Tests when some existing objects will be deleted on a queryset
"""
extant_obj0 = ddf.G(models.TestModel, int_field=0, float_field=1)
extant_obj1 = ddf.G(models.TestModel, int_field=1, float_field=1)
extant_obj2 = ddf.G(models.TestModel, int_field=2, float_field=1)
extant_obj3 = ddf.G(models.TestModel, int_field=3, float_field=1)
extant_obj4 = ddf.G(models.TestModel, int_field=4, float_field=0)
pgbulk.sync(
models.TestModel.objects.filter(int_field__lt=4),
[
models.TestModel(int_field=1, float_field=2),
models.TestModel(int_field=2, float_field=2),
models.TestModel(int_field=3, float_field=2),
],
['int_field'],
['float_field'],
)
assert models.TestModel.objects.count() == 4
assert models.TestModel.objects.filter(int_field=1).exists()
assert models.TestModel.objects.filter(int_field=2).exists()
assert models.TestModel.objects.filter(int_field=3).exists()
with pytest.raises(models.TestModel.DoesNotExist):
models.TestModel.objects.get(id=extant_obj0.id)
test_model = models.TestModel.objects.get(id=extant_obj1.id)
assert test_model.float_field == 2
test_model = models.TestModel.objects.get(id=extant_obj2.id)
assert test_model.float_field == 2
test_model = models.TestModel.objects.get(id=extant_obj3.id)
assert test_model.float_field == 2
test_model = models.TestModel.objects.get(id=extant_obj4.id)
assert test_model.float_field == 0
def test_upsert_return_created_values():
"""
Tests that values that are created are returned properly when returning
is True.
"""
results = pgbulk.upsert(
models.TestModel,
[
models.TestModel(int_field=1),
models.TestModel(int_field=3),
models.TestModel(int_field=4),
],
['int_field'],
['float_field'],
returning=True,
)
assert len(list(results.created)) == 3
for test_model, expected_int in zip(
sorted(results.created, key=lambda k: k.int_field), [1, 3, 4]
):
assert test_model.int_field == expected_int
assert test_model.id is not None
assert models.TestModel.objects.count() == 3