def test_sync_existing_objs_some_deleted_some_updated(): """ Tests when some existing objects will be deleted on a queryset. Run syncing with some update fields. """ objs = [ ddf.G(models.TestModel, int_field=i, float_field=i) for i in range(5) ] results = pgbulk.sync( models.TestModel.objects.filter(int_field__lt=4), [ models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), models.TestModel(int_field=3, float_field=2), ], ['int_field'], ['float_field'], returning=True, ignore_duplicate_updates=True, ) assert len(list(results)) == 4 assert len(list(results.deleted)) == 1 assert len(list(results.updated)) == 2 assert len(list(results.untouched)) == 1 assert list(results.deleted)[0].id == objs[0].id
def test_sync_existing_objs_some_deleted_wo_update(): """ Tests when some existing objects will be deleted on a queryset. Run syncing with no update fields and verify they are untouched in the sync """ objs = [ ddf.G(models.TestModel, int_field=i, float_field=i) for i in range(5) ] results = pgbulk.sync( models.TestModel.objects.filter(int_field__lt=4), [ models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), models.TestModel(int_field=3, float_field=2), ], ['int_field'], [], returning=True, ) assert len(list(results)) == 4 assert len(list(results.deleted)) == 1 assert len(list(results.untouched)) == 3 assert list(results.deleted)[0].id == objs[0].id
def test_upsert_update_fields_returning(): """ Tests the case when all updates were previously stored and the int field is used as a uniqueness constraint. Assert returned values are expected and that it updates all fields by default """ # Create previously stored test models with a unique int field and -1 for # all other fields test_models = [ ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) for i in range(3) ] # Update using the int field as a uniqueness constraint results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='0', float_field=0), models.TestModel(int_field=1, char_field='1', float_field=1), models.TestModel(int_field=2, char_field='2', float_field=2), ], ['int_field'], returning=True, ) assert list(results.created) == [] assert {u.id for u in results.updated} == {t.id for t in test_models} assert {u.int_field for u in results.updated} == {0, 1, 2} assert {u.float_field for u in results.updated} == {0, 1, 2} assert {u.char_field for u in results.updated} == {'0', '1', '2'}
def test_upsert_some_updates_unique_int_char_field_queryset(): """ Tests the case when some updates were previously stored and a queryset is used on the bulk upsert. """ # Create previously stored test models with a unique int field and -1 # for all other fields for i in range(3): ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) # Update using the int field as a uniqueness constraint on a queryset. # Only one object should be updated. pgbulk.upsert( models.TestModel.objects.filter(int_field=0), [ models.TestModel(int_field=0, char_field='0', float_field=0), models.TestModel(int_field=4, char_field='1', float_field=1), models.TestModel(int_field=5, char_field='2', float_field=2), ], ['int_field'], ['float_field'], ) # Verify that two new objecs were created assert models.TestModel.objects.count() == 5 assert models.TestModel.objects.filter(char_field='-1').count() == 3 for i, model_obj in enumerate( models.TestModel.objects.filter(char_field='-1').order_by('int_field') ): assert model_obj.int_field == i assert model_obj.char_field == '-1'
def test_sync_existing_objs_some_deleted(): """ Tests when some existing objects will be deleted. """ extant_obj1 = ddf.G(models.TestModel, int_field=1, float_field=1) extant_obj2 = ddf.G(models.TestModel, int_field=2, float_field=1) extant_obj3 = ddf.G(models.TestModel, int_field=3, float_field=1) pgbulk.sync( models.TestModel, [ models.TestModel(int_field=3, float_field=2), models.TestModel(int_field=4, float_field=2), models.TestModel(int_field=5, float_field=2), ], ['int_field'], ['float_field'], ) assert models.TestModel.objects.count() == 3 assert models.TestModel.objects.filter(int_field=3).exists() assert models.TestModel.objects.filter(int_field=4).exists() assert models.TestModel.objects.filter(int_field=5).exists() with pytest.raises(models.TestModel.DoesNotExist): models.TestModel.objects.get(id=extant_obj1.id) with pytest.raises(models.TestModel.DoesNotExist): models.TestModel.objects.get(id=extant_obj2.id) test_model = models.TestModel.objects.get(id=extant_obj3.id) assert test_model.int_field == 3
def test_upsert_some_updates_unique_int_char_field_update_float_field(): """ Tests the case when some updates were previously stored and the int and char fields are used as a uniqueness constraint. Only updates the float field. """ # Create previously stored test models with a unique int and char field for i in range(2): ddf.G(models.TestModel, int_field=i, char_field=str(i), float_field=-1) # Update using the int field as a uniqueness constraint. The first two # are updated while the third is created pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='0', float_field=0), models.TestModel(int_field=1, char_field='1', float_field=1), models.TestModel(int_field=2, char_field='2', float_field=2), ], ['int_field', 'char_field'], ['float_field'], ) # Verify that the float field was updated for the first two models and # the char field was not updated for the first two. The char field, # however, should be '2' for the third model since it was created assert models.TestModel.objects.count() == 3 for i, model_obj in enumerate( models.TestModel.objects.order_by('int_field') ): assert model_obj.int_field == i assert model_obj.char_field == str(i) assert model_obj.float_field == i # almostequals
def test_upsert_all_updates_unique_int_field_update_float_field(): """ Tests the case when all updates were previously stored and the int field is used as a uniqueness constraint. Only updates the float field """ # Create previously stored test models with a unique int field and -1 for # all other fields for i in range(3): ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) # Update using the int field as a uniqueness constraint pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='0', float_field=0), models.TestModel(int_field=1, char_field='1', float_field=1), models.TestModel(int_field=2, char_field='2', float_field=2), ], ['int_field'], update_fields=['float_field'], ) # Verify that the float field was updated assert models.TestModel.objects.count() == 3 for i, model_obj in enumerate( models.TestModel.objects.order_by('int_field') ): assert model_obj.int_field == i assert model_obj.char_field == '-1' assert model_obj.float_field == i # almostequals
def test_upsert_update_duplicate_fields_returning_some_updated_ignore_dups(): """ Tests the case when all updates were previously stored and the upsert tries to update the rows with duplicate values. Test when some aren't duplicates and return untouched results. There will be no untouched results in this test since we turn off ignoring duplicate updates """ # Create previously stored test models with a unique int field and -1 for # all other fields for i in range(3): ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) # Update using the int field as a uniqueness constraint results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='-1', float_field=-1), models.TestModel(int_field=1, char_field='-1', float_field=-1), models.TestModel(int_field=2, char_field='0', float_field=-1), models.TestModel(int_field=3, char_field='3', float_field=3), ], ['int_field'], ['char_field', 'float_field'], returning=['char_field'], ignore_duplicate_updates=False, return_untouched=True, ) assert len(list(results.untouched)) == 0 assert len(list(results.updated)) == 3 assert len(list(results.created)) == 1 assert list(results.created)[0].char_field == '3'
def test_sync_existing_objs_all_deleted(): """ Tests when there are existing objects that will all be deleted. """ extant_obj1 = ddf.G(models.TestModel, int_field=1) extant_obj2 = ddf.G(models.TestModel, int_field=2) extant_obj3 = ddf.G(models.TestModel, int_field=3) pgbulk.sync( models.TestModel, [ models.TestModel(int_field=4), models.TestModel(int_field=5), models.TestModel(int_field=6), ], ['int_field'], ['float_field'], ) assert models.TestModel.objects.count() == 3 assert models.TestModel.objects.filter(int_field=4).exists() assert models.TestModel.objects.filter(int_field=5).exists() assert models.TestModel.objects.filter(int_field=6).exists() with pytest.raises(models.TestModel.DoesNotExist): models.TestModel.objects.get(id=extant_obj1.id) with pytest.raises(models.TestModel.DoesNotExist): models.TestModel.objects.get(id=extant_obj2.id) with pytest.raises(models.TestModel.DoesNotExist): models.TestModel.objects.get(id=extant_obj3.id)
def test_upsert_no_update_fields_returning(): """ Tests the case when all updates were previously stored and the int field is used as a uniqueness constraint. This test does not update any fields """ # Create previously stored test models with a unique int field and -1 for # all other fields for i in range(3): ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) # Update using the int field as a uniqueness constraint results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='0', float_field=0), models.TestModel(int_field=1, char_field='1', float_field=1), models.TestModel(int_field=2, char_field='2', float_field=2), ], ['int_field'], [], returning=True, ) assert list(results) == []
def test_upsert_update_duplicate_fields_returning_none_updated(): """ Tests the case when all updates were previously stored and the upsert tries to update the rows with duplicate values. """ # Create previously stored test models with a unique int field and -1 for # all other fields for i in range(3): ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) # Update using the int field as a uniqueness constraint results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='-1', float_field=-1), models.TestModel(int_field=1, char_field='-1', float_field=-1), models.TestModel(int_field=2, char_field='-1', float_field=-1), ], ['int_field'], ['char_field', 'float_field'], returning=True, ignore_duplicate_updates=True, ) assert list(results) == []
def test_upsert_wo_update_fields(): """ Tests bulk_upsert with no update fields. This function in turn should just do a bulk create for any models that do not already exist. """ # Create models that already exist ddf.G(models.TestModel, int_field=1, float_field=1) ddf.G(models.TestModel, int_field=2, float_field=2) # Perform a bulk_upsert with one new model pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=1, float_field=3), models.TestModel(int_field=2, float_field=3), models.TestModel(int_field=3, float_field=3), ], ['int_field'], update_fields=[], ) # Three objects should now exist, but no float fields should be updated assert models.TestModel.objects.count() == 3 for test_model, expected_int_value in zip( models.TestModel.objects.order_by('int_field'), [1, 2, 3] ): assert test_model.int_field == expected_int_value assert test_model.float_field == expected_int_value
def test_upsert_return_created_updated_values(): """ Tests returning values when the items are either updated or created. """ # Create an item that will be updated ddf.G(models.TestModel, int_field=2, float_field=1.0) results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=1, float_field=3.0), models.TestModel(int_field=2.0, float_field=3.0), models.TestModel(int_field=3, float_field=3.0), models.TestModel(int_field=4, float_field=3.0), ], ['int_field'], ['float_field'], returning=True, ) created = list(results.created) updated = list(results.updated) assert len(created) == 3 assert len(updated) == 1 for test_model, expected_int in zip( sorted(created, key=lambda k: k.int_field), [1, 3, 4] ): assert test_model.int_field == expected_int assert test_model.float_field == 3.0 # almostequals assert test_model.id is not None assert updated[0].int_field == 2 assert updated[0].float_field == 3.0 # almostequals assert updated[0].id is not None assert models.TestModel.objects.count() == 4
def test_upsert_some_updates_unique_timezone_field_update_float_field(): """ Tests the case when some updates were previously stored and the timezone field is used as a uniqueness constraint. Only updates the float field. """ # Create previously stored test models with a unique int field and -1 for # all other fields for i in ['US/Eastern', 'US/Central']: ddf.G( models.TestUniqueTzModel, time_zone=i, char_field='-1', float_field=-1, ) # Update using the int field as a uniqueness constraint. The first two # are updated while the third is created pgbulk.upsert( models.TestUniqueTzModel, [ models.TestModel( time_zone=timezone('US/Eastern'), char_field='0', float_field=0 ), models.TestModel( time_zone=timezone('US/Central'), char_field='1', float_field=1 ), models.TestModel( time_zone=timezone('UTC'), char_field='2', float_field=2 ), ], ['time_zone'], ['float_field'], ) # Verify that the float field was updated for the first two models and # the char field was not updated for the first two. The char field, # however, should be '2' for the third model since it was created m1 = models.TestUniqueTzModel.objects.get(time_zone=timezone('US/Eastern')) assert m1.char_field == '-1' assert m1.float_field == 0 # almostequals m2 = models.TestUniqueTzModel.objects.get(time_zone=timezone('US/Central')) assert m2.char_field == '-1' assert m2.float_field == 1 # almostequals m3 = models.TestUniqueTzModel.objects.get(time_zone=timezone('UTC')) assert m3.char_field == '2' assert m3.float_field == 2 # almostequals
def test_sync_no_existing_objs(): """ Tests when there are no existing objects before the sync. """ pgbulk.sync( models.TestModel, [ models.TestModel(int_field=1), models.TestModel(int_field=3), models.TestModel(int_field=4), ], ['int_field'], ['float_field'], ) assert models.TestModel.objects.count() == 3 assert models.TestModel.objects.filter(int_field=1).exists() assert models.TestModel.objects.filter(int_field=3).exists() assert models.TestModel.objects.filter(int_field=4).exists()
def test_upsert_return_list_of_values(): """ Tests that values that are created are returned properly when returning is True. Set returning to a list of fields """ results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=3, float_field=4), models.TestModel(int_field=4, float_field=5), ], ['int_field'], ['float_field'], returning=['float_field'], ) assert len(list(results.created)) == 3 with pytest.raises(AttributeError): list(results.created)[0].int_field assert {2, 4, 5} == {m.float_field for m in results.created}
def test_upsert_no_updates_unique_int_char_field(): """ Tests the case when no updates were previously stored and the int and char fields are used as a uniqueness constraint. In this case, there is data previously stored, but the uniqueness constraints don't match. """ # Create previously stored test models with a unique int field and -1 for # all other fields for i in range(3): ddf.G(models.TestModel, int_field=i, char_field='-1', float_field=-1) # Update using the int and char field as a uniqueness constraint. All # three objects are created pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=3, char_field='0', float_field=0), models.TestModel(int_field=4, char_field='1', float_field=1), models.TestModel(int_field=5, char_field='2', float_field=2), ], ['int_field', 'char_field'], ['float_field'], ) # Verify that no updates occured assert models.TestModel.objects.count() == 6 assert models.TestModel.objects.filter(char_field='-1').count() == 3 for i, model_obj in enumerate( models.TestModel.objects.filter(char_field='-1').order_by('int_field') ): assert model_obj.int_field == i assert model_obj.char_field == '-1' assert model_obj.float_field == -1 # almostequals assert models.TestModel.objects.exclude(char_field='-1').count() == 3 for i, model_obj in enumerate( models.TestModel.objects.exclude(char_field='-1').order_by('int_field') ): assert model_obj.int_field == i + 3 assert model_obj.char_field == str(i) assert model_obj.float_field == i # almostequals
def test_upsert_no_updates(): """ Tests the case when no updates were previously stored (i.e objects are only created) """ pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=0, char_field='0', float_field=0), models.TestModel(int_field=1, char_field='1', float_field=1), models.TestModel(int_field=2, char_field='2', float_field=2), ], ['int_field'], ['char_field', 'float_field'], ) for i, model_obj in enumerate( models.TestModel.objects.order_by('int_field') ): assert model_obj.int_field == i assert model_obj.char_field == str(i) assert model_obj.float_field == i # almostequals
def test_sync_existing_objs_some_deleted_w_queryset(): """ Tests when some existing objects will be deleted on a queryset """ extant_obj0 = ddf.G(models.TestModel, int_field=0, float_field=1) extant_obj1 = ddf.G(models.TestModel, int_field=1, float_field=1) extant_obj2 = ddf.G(models.TestModel, int_field=2, float_field=1) extant_obj3 = ddf.G(models.TestModel, int_field=3, float_field=1) extant_obj4 = ddf.G(models.TestModel, int_field=4, float_field=0) pgbulk.sync( models.TestModel.objects.filter(int_field__lt=4), [ models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), models.TestModel(int_field=3, float_field=2), ], ['int_field'], ['float_field'], ) assert models.TestModel.objects.count() == 4 assert models.TestModel.objects.filter(int_field=1).exists() assert models.TestModel.objects.filter(int_field=2).exists() assert models.TestModel.objects.filter(int_field=3).exists() with pytest.raises(models.TestModel.DoesNotExist): models.TestModel.objects.get(id=extant_obj0.id) test_model = models.TestModel.objects.get(id=extant_obj1.id) assert test_model.float_field == 2 test_model = models.TestModel.objects.get(id=extant_obj2.id) assert test_model.float_field == 2 test_model = models.TestModel.objects.get(id=extant_obj3.id) assert test_model.float_field == 2 test_model = models.TestModel.objects.get(id=extant_obj4.id) assert test_model.float_field == 0
def test_upsert_return_created_values(): """ Tests that values that are created are returned properly when returning is True. """ results = pgbulk.upsert( models.TestModel, [ models.TestModel(int_field=1), models.TestModel(int_field=3), models.TestModel(int_field=4), ], ['int_field'], ['float_field'], returning=True, ) assert len(list(results.created)) == 3 for test_model, expected_int in zip( sorted(results.created, key=lambda k: k.int_field), [1, 3, 4] ): assert test_model.int_field == expected_int assert test_model.id is not None assert models.TestModel.objects.count() == 3