def test_simulate(
cleanup_configs,
track_return_values,
es_with_signals,
es_collector_context_manager,
):
"""
Test that if --simulate is passed in, the command only simulates the action
without making any actual changes.
"""
# Set up the state before running the command
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
model_name, config = cleanup_configs
filter_config = config.filters[0]
command = delete_orphans.Command()
mapping = MAPPINGS[model_name]
model_factory = mapping['factory']
datetime_older_than_threshold = filter_config.cut_off_date - relativedelta(
days=1)
with es_collector_context_manager as collector:
for _ in range(3):
create_orphanable_model(model_factory, filter_config,
datetime_older_than_threshold)
collector.flush_and_refresh()
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
read_alias = search_app.es_model.get_read_alias()
assert model.objects.count() == 3
assert es_with_signals.count(read_alias,
doc_type=search_app.name)['count'] == 3
# Run the command
management.call_command(command, model_name, simulate=True)
es_with_signals.indices.refresh()
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
expected_deleted_models = {model._meta.label} | set(
mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
assert deletions_by_model[model._meta.label] == 3
# Check that nothing has actually been deleted
assert model.objects.count() == 3
assert es_with_signals.count(read_alias,
doc_type=search_app.name)['count'] == 3
def test_only_print_queries(cleanup_configs, monkeypatch, caplog):
"""
Test that if --only-print-queries is passed, the SQL query is printed but no deletions or
simulation occurs.
"""
caplog.set_level('INFO')
delete_mock = mock.Mock()
monkeypatch.setattr(QuerySet, 'delete', delete_mock)
model_name, config = cleanup_configs
filter_config = config.filters[0]
command = delete_orphans.Command()
model = apps.get_model(model_name)
mapping = MAPPINGS[model_name]
model_factory = mapping['factory']
datetime_older_than_threshold = filter_config.cut_off_date - relativedelta(
days=1)
for _ in range(3):
create_orphanable_model(model_factory, filter_config,
datetime_older_than_threshold)
management.call_command(command, model_name, only_print_queries=True)
assert not delete_mock.called
log_text = caplog.text.lower()
assert f'{model._meta.verbose_name_plural} to delete:' in log_text
for relation in get_relations_to_delete(model):
related_meta = relation.related_model._meta
expected_related_log = (
f'{related_meta.verbose_name_plural} to delete '
f'(via {related_meta.model_name}.{relation.remote_field.name}): '
).lower()
assert expected_related_log in log_text
assert f'from "{related_meta.db_table}"' in log_text
def test_with_es_exception(mocked_bulk):
"""
Test that if ES returns a 5xx error, the command completes but it also
raises a DataHubException with details of the error.
"""
mocked_bulk.return_value = (None, [{'delete': {'status': 500}}])
command = delete_orphans.Command()
model_name = next(iter(delete_orphans.Command.CONFIGS))
model_factory = MAPPINGS[model_name]['factory']
filter_config = delete_orphans.Command.CONFIGS[model_name].filters[0]
datetime_older_than_threshold = (FROZEN_TIME -
filter_config.age_threshold -
relativedelta(days=1))
create_orphanable_model(model_factory, filter_config,
datetime_older_than_threshold)
with pytest.raises(DataHubException):
management.call_command(command, model_name)
model = apps.get_model(model_name)
assert model.objects.count() == 0
def test_run(cleanup_mapping, track_return_values, setup_es):
"""
Test that:
- a record without any objects referencing it but not old enough
doesn't get deleted
- a record without any objects referencing it and old gets deleted
- a record with another object referencing it doesn't get deleted
"""
command, model_name, config, mapping, dep_factory, dep_field_name = cleanup_mapping
model_factory = mapping['factory']
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
datetime_within_threshold = FROZEN_TIME - config.age_threshold
datetime_older_than_threshold = datetime_within_threshold - relativedelta(
days=1)
# this orphan should NOT get deleted because not old enough
create_orphanable_model(model_factory, config, datetime_within_threshold)
# this orphan should get deleted because old
create_orphanable_model(model_factory, config,
datetime_older_than_threshold)
# this object should NOT get deleted because it has another object referencing it
non_orphan = create_orphanable_model(model_factory, config,
datetime_older_than_threshold)
is_m2m = dep_factory._meta.model._meta.get_field(
dep_field_name).many_to_many
dep_factory(**{dep_field_name: [non_orphan] if is_m2m else non_orphan}, )
# 3 + 1 in case of self-references
total_model_records = 3 + (1 if dep_factory == model_factory else 0)
setup_es.indices.refresh()
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
doc_type = search_app.name
read_alias = search_app.es_model.get_read_alias()
assert model.objects.count() == total_model_records
assert setup_es.count(read_alias,
doc_type=doc_type)['count'] == total_model_records
management.call_command(delete_orphans.Command(), model_name)
setup_es.indices.refresh()
# Check that the records have been deleted
assert model.objects.count() == total_model_records - 1
assert setup_es.count(
read_alias, doc_type=doc_type)['count'] == total_model_records - 1
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
assert deletions_by_model[model._meta.label] == 1
expected_deleted_models = {model._meta.label} | set(
mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
def test_run(
model_name,
config,
mapping,
dep_factory,
dep_field_name,
track_return_values,
opensearch_with_signals,
opensearch_collector_context_manager,
):
"""
Test that:
- a record without any objects referencing it but not old enough
doesn't get deleted
- a record without any objects referencing it and old gets deleted
- a record with another object referencing it doesn't get deleted
"""
# Set up the state before running the command
command = delete_orphans.Command()
model_factory = mapping['factory']
filter_config = config.filters[0]
delete_return_value_tracker = track_return_values(QuerySet, 'delete')
datetime_within_threshold = filter_config.cut_off_date
datetime_older_than_threshold = filter_config.cut_off_date - relativedelta(days=1)
with opensearch_collector_context_manager as collector:
# this orphan should NOT get deleted because not old enough
create_orphanable_model(model_factory, filter_config, datetime_within_threshold)
# this orphan should get deleted because old
create_orphanable_model(model_factory, filter_config, datetime_older_than_threshold)
# this object should NOT get deleted because it has another object referencing it
non_orphan = create_orphanable_model(
model_factory,
filter_config,
datetime_older_than_threshold,
)
is_m2m = dep_factory._meta.model._meta.get_field(dep_field_name).many_to_many
dep_factory(
**{dep_field_name: [non_orphan] if is_m2m else non_orphan},
)
collector.flush_and_refresh()
# 3 + 1 in case of self-references
total_model_records = 3 + (1 if dep_factory == model_factory else 0)
model = apps.get_model(model_name)
search_app = get_search_app_by_model(model)
read_alias = search_app.search_model.get_read_alias()
assert model.objects.count() == total_model_records
assert opensearch_with_signals.count(index=read_alias)['count'] == total_model_records
# Run the command
management.call_command(command, model_name)
opensearch_with_signals.indices.refresh()
# Check that the records have been deleted
assert model.objects.count() == total_model_records - 1
assert opensearch_with_signals.count(index=read_alias)['count'] == total_model_records - 1
# Check which models were actually deleted
return_values = delete_return_value_tracker.return_values
assert len(return_values) == 1
_, deletions_by_model = return_values[0]
assert deletions_by_model[model._meta.label] == 1
expected_deleted_models = {model._meta.label} | set(mapping['implicit_related_models'])
actual_deleted_models = { # only include models actually deleted
deleted_model
for deleted_model, deleted_count in deletions_by_model.items()
if deleted_count
}
assert actual_deleted_models == expected_deleted_models
