def test_not_found(self):
"""
Test that get_search_app_by_model raises LookupError if it
can't find the right search app for the search model passed in.
"""
with pytest.raises(LookupError):
get_search_app_by_model(mock.Mock())
def test_not_found(self, mocked_load_search_apps):
"""
Test that get_search_app_by_model raises LookupError if it
can't find the right search app for the model passed in.
"""
mocked_load_search_apps.return_value = {
'app1': mock.Mock(),
}
with pytest.raises(LookupError):
get_search_app_by_model(mock.Mock())
def sync_related_objects_task(
self,
related_model_label,
related_obj_pk,
related_obj_field_name,
related_obj_filter=None,
):
"""
Syncs objects related to another object via a specified field.
For example, this task would sync the interactions of a company if given the following
arguments:
related_model_label='company.Company'
related_obj_pk=company.pk
related_obj_field_name='interactions'
Note that a lower priority (higher number) is used for syncing related objects, as syncing
them is less important than syncing the primary object that was modified.
If an error occurs, the task will be automatically retried with an exponential back-off.
The wait between attempts is approximately 2 ** attempt_num seconds (with some jitter
added).
"""
related_model = apps.get_model(related_model_label)
related_obj = related_model.objects.get(pk=related_obj_pk)
manager = getattr(related_obj, related_obj_field_name)
if related_obj_filter:
manager = manager.filter(**related_obj_filter)
queryset = manager.values_list('pk', flat=True)
search_app = get_search_app_by_model(manager.model)
for pk in queryset:
sync_object_task.apply_async(args=(search_app.name, pk),
priority=self.priority)
def test_collector(monkeypatch, setup_es):
"""
Test that the collector collects and deletes all the django objects deleted.
"""
obj = SimpleModel.objects.create()
sync_object_async(ESSimpleModel, SimpleModel, str(obj.pk))
setup_es.indices.refresh()
search_app = get_search_app_by_model(SimpleModel)
es_doc = ESSimpleModel.es_document(obj)
assert SimpleModel.objects.count() == 1
collector = Collector()
# check that the post/pre_delete callbacks of SimpleModel are in the collected
# signal receivers to disable
simplemodel_receivers = [
receiver for receiver in collector.signal_receivers_to_disable
if receiver.sender is SimpleModel
]
assert simplemodel_receivers
assert {receiver.signal
for receiver in simplemodel_receivers
} == {post_delete, pre_delete}
# mock the receiver methods so that we can check they are called
for receiver in collector.signal_receivers_to_disable:
monkeypatch.setattr(receiver, 'connect', mock.Mock())
monkeypatch.setattr(receiver, 'disconnect', mock.Mock())
collector.connect()
# check that the existing signal receivers are disconnected
for receiver in collector.signal_receivers_to_disable:
assert receiver.disconnect.called
assert not receiver.connect.called
obj.delete()
collector.disconnect()
# check that the existing signal receivers are connected back
for receiver in collector.signal_receivers_to_disable:
assert receiver.connect.called
assert collector.deletions == {
SimpleModel: [es_doc],
}
read_alias = search_app.es_model.get_read_alias()
assert SimpleModel.objects.count() == 0
assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 1
collector.delete_from_es()
setup_es.indices.refresh()
assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 0
def _collect(self, sender, instance, **kwargs):
"""Logic that gets run on post_delete."""
model = instance.__class__
es_model = get_search_app_by_model(model).es_model
es_doc = es_model.es_document(instance)
self.deletions[model].append(es_doc)
def _collect(self, instance):
"""Logic that gets run on post_delete."""
model = instance.__class__
es_model = get_search_app_by_model(model).es_model
es_doc = es_model.es_document(instance, include_index=False, include_source=False)
self.deletions[model].append(es_doc)
def test_simulate(
model_name,
config,
track_return_values,
es_with_signals,
es_collector_context_manager,
):
"""
Test that if --simulate is passed in, the command only simulates the action
without making any actual changes.
"""
# Set up the state before running the command
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
command = delete_old_records.Command()
mapping = MAPPING[model_name]
model_factory = mapping['factory']
has_search_app = not mapping.get('has_no_search_app')
with es_collector_context_manager as collector:
for _ in range(3):
_create_model_obj(model_factory,
**mapping['expired_objects_kwargs'][0])
collector.flush_and_refresh()
model = apps.get_model(model_name)
if has_search_app:
search_app = get_search_app_by_model(model)
read_alias = search_app.es_model.get_read_alias()
assert es_with_signals.count(read_alias,
doc_type=search_app.name)['count'] == 3
assert model.objects.count() == 3
# Run the command
management.call_command(command, model_name, simulate=True)
es_with_signals.indices.refresh()
# Check which models were deleted prior to the rollback
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert model._meta.label in {model._meta.label}
assert actual_deleted_models - {model._meta.label
} <= mapping['implicitly_deletable_models']
assert deletions_by_model[model._meta.label] == 3
# Check that nothing has actually been deleted
assert model.objects.count() == 3
if has_search_app:
assert es_with_signals.count(read_alias,
doc_type=search_app.name)['count'] == 3
def test_simulate(
cleanup_configs,
track_return_values,
es_with_signals,
es_collector_context_manager,
):
"""
Test that if --simulate is passed in, the command only simulates the action
without making any actual changes.
"""
# Set up the state before running the command
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
model_name, config = cleanup_configs
filter_config = config.filters[0]
command = delete_orphans.Command()
mapping = MAPPINGS[model_name]
model_factory = mapping['factory']
datetime_older_than_threshold = filter_config.cut_off_date - relativedelta(
days=1)
with es_collector_context_manager as collector:
for _ in range(3):
create_orphanable_model(model_factory, filter_config,
datetime_older_than_threshold)
collector.flush_and_refresh()
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
read_alias = search_app.es_model.get_read_alias()
assert model.objects.count() == 3
assert es_with_signals.count(read_alias,
doc_type=search_app.name)['count'] == 3
# Run the command
management.call_command(command, model_name, simulate=True)
es_with_signals.indices.refresh()
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
expected_deleted_models = {model._meta.label} | set(
mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
assert deletions_by_model[model._meta.label] == 3
# Check that nothing has actually been deleted
assert model.objects.count() == 3
assert es_with_signals.count(read_alias,
doc_type=search_app.name)['count'] == 3
def _collect(self, obj):
"""
Logic run on post_save for models of all search apps.
Note: This does not use transaction.on_commit(), because transactions in tests
are not committed. Be careful if reusing this logic in production code (as you would
usually want to delay syncing until the transaction is committed).
"""
model = obj.__class__
search_app = get_search_app_by_model(model)
self.collected_apps.add(search_app)
def test_found(self, mocked_load_search_apps):
"""
Test that get_search_app_by_model returns the right
search app for the model passed in.
"""
model = mock.Mock()
search_app = mock.Mock(queryset=mock.Mock(model=model))
mocked_load_search_apps.return_value = {
'app1': mock.Mock(),
'app2': search_app,
}
assert get_search_app_by_model(model) == search_app
def test_simulate(cleanup_commands_and_configs, track_return_values, setup_es,
caplog):
"""
Test that if --simulate is passed in, the command only simulates the action
without making any actual changes.
"""
caplog.set_level('INFO')
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
command, model_name, config = cleanup_commands_and_configs
mapping = MAPPINGS[model_name]
model_factory = mapping['factory']
datetime_older_than_threshold = FROZEN_TIME - config.age_threshold - relativedelta(
days=1)
for _ in range(3):
create_orphanable_model(model_factory, config,
datetime_older_than_threshold)
setup_es.indices.refresh()
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
read_alias = search_app.es_model.get_read_alias()
assert model.objects.count() == 3
assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 3
management.call_command(command, model_name, simulate=True)
setup_es.indices.refresh()
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
assert deletions_by_model[model._meta.label] == 3
expected_deleted_models = {model._meta.label} | set(
mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
# Check that nothing has actually been deleted
assert model.objects.count() == 3
assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 3
def test_update_es_after_deletions(setup_es):
"""
Test that the context manager update_es_after_deletions collects and deletes
all the django objects deleted.
"""
obj = SimpleModel.objects.create()
sync_object_async(ESSimpleModel, SimpleModel, str(obj.pk))
setup_es.indices.refresh()
search_app = get_search_app_by_model(SimpleModel)
read_alias = search_app.es_model.get_read_alias()
assert SimpleModel.objects.count() == 1
assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 1
with update_es_after_deletions():
obj.delete()
setup_es.indices.refresh()
assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 0
def test_run(
model_name,
config,
mapping,
dep_factory,
dep_field_name,
track_return_values,
opensearch_with_signals,
opensearch_collector_context_manager,
):
"""
Test that:
- a record without any objects referencing it but not old enough
doesn't get deleted
- a record without any objects referencing it and old gets deleted
- a record with another object referencing it doesn't get deleted
"""
# Set up the state before running the command
command = delete_orphans.Command()
model_factory = mapping['factory']
filter_config = config.filters[0]
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
datetime_within_threshold = filter_config.cut_off_date
datetime_older_than_threshold = filter_config.cut_off_date - relativedelta(days=1)
with opensearch_collector_context_manager as collector:
# this orphan should NOT get deleted because not old enough
create_orphanable_model(model_factory, filter_config, datetime_within_threshold)
# this orphan should get deleted because old
create_orphanable_model(model_factory, filter_config, datetime_older_than_threshold)
# this object should NOT get deleted because it has another object referencing it
non_orphan = create_orphanable_model(
model_factory,
filter_config,
datetime_older_than_threshold,
)
is_m2m = dep_factory._meta.model._meta.get_field(dep_field_name).many_to_many
dep_factory(
**{dep_field_name: [non_orphan] if is_m2m else non_orphan},
)
collector.flush_and_refresh()
# 3 + 1 in case of self-references
total_model_records = 3 + (1 if dep_factory == model_factory else 0)
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
read_alias = search_app.search_model.get_read_alias()
assert model.objects.count() == total_model_records
assert opensearch_with_signals.count(index=read_alias)['count'] == total_model_records
# Run the command
management.call_command(command, model_name)
opensearch_with_signals.indices.refresh()
# Check that the records have been deleted
assert model.objects.count() == total_model_records - 1
assert opensearch_with_signals.count(index=read_alias)['count'] == total_model_records - 1
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
assert deletions_by_model[model._meta.label] == 1
expected_deleted_models = {model._meta.label} | set(mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
def _delete_from_es(self):
for model, es_docs in self.deletions.items():
search_app = get_search_app_by_model(model)
delete_documents(search_app.es_model.get_write_alias(), es_docs)
def test_run(
model_label,
factory_kwargs,
relation_mapping,
relation_factory_kwargs,
is_expired,
track_return_values,
setup_es,
):
"""Tests the delete_old_records commands for various cases specified by MAPPING above."""
mapping = MAPPING[model_label]
model_factory = mapping['factory']
command = delete_old_records.Command()
model = apps.get_model(model_label)
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
obj = _create_model_obj(model_factory, **factory_kwargs)
total_model_records = 1
if relation_mapping:
relation_model = relation_mapping['factory']._meta.get_model_class()
relation_field = relation_model._meta.get_field(
relation_mapping['field'])
relation_factory_arg = [obj] if relation_field.many_to_many else obj
_create_model_obj(
relation_mapping['factory'],
**relation_factory_kwargs,
**{relation_mapping['field']: relation_factory_arg},
)
if relation_mapping['factory']._meta.get_model_class() is model:
total_model_records += 1
num_expired_records = 1 if is_expired else 0
search_app = get_search_app_by_model(model)
doc_type = search_app.name
read_alias = search_app.es_model.get_read_alias()
setup_es.indices.refresh()
assert model.objects.count() == total_model_records
assert setup_es.count(read_alias,
doc_type=doc_type)['count'] == total_model_records
management.call_command(command, model_label)
setup_es.indices.refresh()
# Check if the object has been deleted
assert model.objects.count() == total_model_records - num_expired_records
assert setup_es.count(read_alias,
doc_type=doc_type)['count'] == (total_model_records -
num_expired_records)
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
if is_expired:
assert deletions_by_model[model._meta.label] == num_expired_records
assert model._meta.label in {model._meta.label}
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models - {model._meta.label
} <= mapping['implicitly_deletable_models']
def test_run(cleanup_mapping, track_return_values, setup_es):
"""
Test that:
- a record without any objects referencing it but not old enough
doesn't get deleted
- a record without any objects referencing it and old gets deleted
- a record with another object referencing it doesn't get deleted
"""
command, model_name, config, mapping, dep_factory, dep_field_name = cleanup_mapping
model_factory = mapping['factory']
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
datetime_within_threshold = FROZEN_TIME - config.age_threshold
datetime_older_than_threshold = datetime_within_threshold - relativedelta(
days=1)
# this orphan should NOT get deleted because not old enough
create_orphanable_model(model_factory, config, datetime_within_threshold)
# this orphan should get deleted because old
create_orphanable_model(model_factory, config,
datetime_older_than_threshold)
# this object should NOT get deleted because it has another object referencing it
non_orphan = create_orphanable_model(model_factory, config,
datetime_older_than_threshold)
is_m2m = dep_factory._meta.model._meta.get_field(
dep_field_name).many_to_many
dep_factory(**{dep_field_name: [non_orphan] if is_m2m else non_orphan}, )
# 3 + 1 in case of self-references
total_model_records = 3 + (1 if dep_factory == model_factory else 0)
setup_es.indices.refresh()
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
doc_type = search_app.name
read_alias = search_app.es_model.get_read_alias()
assert model.objects.count() == total_model_records
assert setup_es.count(read_alias,
doc_type=doc_type)['count'] == total_model_records
management.call_command(delete_orphans.Command(), model_name)
setup_es.indices.refresh()
# Check that the records have been deleted
assert model.objects.count() == total_model_records - 1
assert setup_es.count(
read_alias, doc_type=doc_type)['count'] == total_model_records - 1
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
assert deletions_by_model[model._meta.label] == 1
expected_deleted_models = {model._meta.label} | set(
mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
def test_run(
model_label,
factory_kwargs,
relation_mapping,
relation_factory_kwargs,
is_expired,
track_return_values,
opensearch_with_signals,
opensearch_collector_context_manager,
):
"""Tests the delete_old_records commands for various cases specified by MAPPING above."""
# Set up the state before running the command
mapping = MAPPING[model_label]
model_factory = mapping['factory']
command = delete_old_records.Command()
model = apps.get_model(model_label)
has_search_app = not mapping.get('has_no_search_app')
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
with opensearch_collector_context_manager as collector:
obj = _create_model_obj(model_factory, **factory_kwargs)
total_model_records = 1
if relation_mapping:
relation_model = relation_mapping['factory']._meta.get_model_class(
)
relation_field = relation_model._meta.get_field(
relation_mapping['field'])
relation_factory_arg = [obj
] if relation_field.many_to_many else obj
_create_model_obj(
relation_mapping['factory'],
**relation_factory_kwargs,
**{relation_mapping['field']: relation_factory_arg},
)
if relation_mapping['factory']._meta.get_model_class() is model:
total_model_records += 1
collector.flush_and_refresh()
num_expired_records = 1 if is_expired else 0
if has_search_app:
search_app = get_search_app_by_model(model)
read_alias = search_app.search_model.get_read_alias()
assert opensearch_with_signals.count(
index=read_alias)['count'] == total_model_records
assert model.objects.count() == total_model_records
# Run the command
management.call_command(command, model_label)
opensearch_with_signals.indices.refresh()
# Check if the object has been deleted
assert model.objects.count() == total_model_records - num_expired_records
if has_search_app:
assert opensearch_with_signals.count(
index=read_alias)['count'] == (total_model_records -
num_expired_records)
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
if is_expired:
assert deletions_by_model[model._meta.label] == num_expired_records
assert model._meta.label in {model._meta.label}
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models - {model._meta.label
} <= mapping['implicitly_deletable_models']
