def test_not_found(self): """ Test that get_search_app_by_model raises LookupError if it can't find the right search app for the search model passed in. """ with pytest.raises(LookupError): get_search_app_by_model(mock.Mock())
def test_not_found(self, mocked_load_search_apps): """ Test that get_search_app_by_model raises LookupError if it can't find the right search app for the model passed in. """ mocked_load_search_apps.return_value = { 'app1': mock.Mock(), } with pytest.raises(LookupError): get_search_app_by_model(mock.Mock())
def sync_related_objects_task( self, related_model_label, related_obj_pk, related_obj_field_name, related_obj_filter=None, ): """ Syncs objects related to another object via a specified field. For example, this task would sync the interactions of a company if given the following arguments: related_model_label='company.Company' related_obj_pk=company.pk related_obj_field_name='interactions' Note that a lower priority (higher number) is used for syncing related objects, as syncing them is less important than syncing the primary object that was modified. If an error occurs, the task will be automatically retried with an exponential back-off. The wait between attempts is approximately 2 ** attempt_num seconds (with some jitter added). """ related_model = apps.get_model(related_model_label) related_obj = related_model.objects.get(pk=related_obj_pk) manager = getattr(related_obj, related_obj_field_name) if related_obj_filter: manager = manager.filter(**related_obj_filter) queryset = manager.values_list('pk', flat=True) search_app = get_search_app_by_model(manager.model) for pk in queryset: sync_object_task.apply_async(args=(search_app.name, pk), priority=self.priority)
def test_collector(monkeypatch, setup_es): """ Test that the collector collects and deletes all the django objects deleted. """ obj = SimpleModel.objects.create() sync_object_async(ESSimpleModel, SimpleModel, str(obj.pk)) setup_es.indices.refresh() search_app = get_search_app_by_model(SimpleModel) es_doc = ESSimpleModel.es_document(obj) assert SimpleModel.objects.count() == 1 collector = Collector() # check that the post/pre_delete callbacks of SimpleModel are in the collected # signal receivers to disable simplemodel_receivers = [ receiver for receiver in collector.signal_receivers_to_disable if receiver.sender is SimpleModel ] assert simplemodel_receivers assert {receiver.signal for receiver in simplemodel_receivers } == {post_delete, pre_delete} # mock the receiver methods so that we can check they are called for receiver in collector.signal_receivers_to_disable: monkeypatch.setattr(receiver, 'connect', mock.Mock()) monkeypatch.setattr(receiver, 'disconnect', mock.Mock()) collector.connect() # check that the existing signal receivers are disconnected for receiver in collector.signal_receivers_to_disable: assert receiver.disconnect.called assert not receiver.connect.called obj.delete() collector.disconnect() # check that the existing signal receivers are connected back for receiver in collector.signal_receivers_to_disable: assert receiver.connect.called assert collector.deletions == { SimpleModel: [es_doc], } read_alias = search_app.es_model.get_read_alias() assert SimpleModel.objects.count() == 0 assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 1 collector.delete_from_es() setup_es.indices.refresh() assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 0
def _collect(self, sender, instance, **kwargs): """Logic that gets run on post_delete.""" model = instance.__class__ es_model = get_search_app_by_model(model).es_model es_doc = es_model.es_document(instance) self.deletions[model].append(es_doc)
def _collect(self, instance): """Logic that gets run on post_delete.""" model = instance.__class__ es_model = get_search_app_by_model(model).es_model es_doc = es_model.es_document(instance, include_index=False, include_source=False) self.deletions[model].append(es_doc)
def test_simulate( model_name, config, track_return_values, es_with_signals, es_collector_context_manager, ): """ Test that if --simulate is passed in, the command only simulates the action without making any actual changes. """ # Set up the state before running the command delete_return_value_tracker = track_return_values(QuerySet, 'delete') command = delete_old_records.Command() mapping = MAPPING[model_name] model_factory = mapping['factory'] has_search_app = not mapping.get('has_no_search_app') with es_collector_context_manager as collector: for _ in range(3): _create_model_obj(model_factory, **mapping['expired_objects_kwargs'][0]) collector.flush_and_refresh() model = apps.get_model(model_name) if has_search_app: search_app = get_search_app_by_model(model) read_alias = search_app.es_model.get_read_alias() assert es_with_signals.count(read_alias, doc_type=search_app.name)['count'] == 3 assert model.objects.count() == 3 # Run the command management.call_command(command, model_name, simulate=True) es_with_signals.indices.refresh() # Check which models were deleted prior to the rollback return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert model._meta.label in {model._meta.label} assert actual_deleted_models - {model._meta.label } <= mapping['implicitly_deletable_models'] assert deletions_by_model[model._meta.label] == 3 # Check that nothing has actually been deleted assert model.objects.count() == 3 if has_search_app: assert es_with_signals.count(read_alias, doc_type=search_app.name)['count'] == 3
def test_simulate( cleanup_configs, track_return_values, es_with_signals, es_collector_context_manager, ): """ Test that if --simulate is passed in, the command only simulates the action without making any actual changes. """ # Set up the state before running the command delete_return_value_tracker = track_return_values(QuerySet, 'delete') model_name, config = cleanup_configs filter_config = config.filters[0] command = delete_orphans.Command() mapping = MAPPINGS[model_name] model_factory = mapping['factory'] datetime_older_than_threshold = filter_config.cut_off_date - relativedelta( days=1) with es_collector_context_manager as collector: for _ in range(3): create_orphanable_model(model_factory, filter_config, datetime_older_than_threshold) collector.flush_and_refresh() model = apps.get_model(model_name) search_app = get_search_app_by_model(model) read_alias = search_app.es_model.get_read_alias() assert model.objects.count() == 3 assert es_with_signals.count(read_alias, doc_type=search_app.name)['count'] == 3 # Run the command management.call_command(command, model_name, simulate=True) es_with_signals.indices.refresh() # Check which models were actually deleted return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] expected_deleted_models = {model._meta.label} | set( mapping['implicit_related_models']) actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert actual_deleted_models == expected_deleted_models assert deletions_by_model[model._meta.label] == 3 # Check that nothing has actually been deleted assert model.objects.count() == 3 assert es_with_signals.count(read_alias, doc_type=search_app.name)['count'] == 3
def _collect(self, obj): """ Logic run on post_save for models of all search apps. Note: This does not use transaction.on_commit(), because transactions in tests are not committed. Be careful if reusing this logic in production code (as you would usually want to delay syncing until the transaction is committed). """ model = obj.__class__ search_app = get_search_app_by_model(model) self.collected_apps.add(search_app)
def test_found(self, mocked_load_search_apps): """ Test that get_search_app_by_model returns the right search app for the model passed in. """ model = mock.Mock() search_app = mock.Mock(queryset=mock.Mock(model=model)) mocked_load_search_apps.return_value = { 'app1': mock.Mock(), 'app2': search_app, } assert get_search_app_by_model(model) == search_app
def test_simulate(cleanup_commands_and_configs, track_return_values, setup_es, caplog): """ Test that if --simulate is passed in, the command only simulates the action without making any actual changes. """ caplog.set_level('INFO') delete_return_value_tracker = track_return_values(QuerySet, 'delete') command, model_name, config = cleanup_commands_and_configs mapping = MAPPINGS[model_name] model_factory = mapping['factory'] datetime_older_than_threshold = FROZEN_TIME - config.age_threshold - relativedelta( days=1) for _ in range(3): create_orphanable_model(model_factory, config, datetime_older_than_threshold) setup_es.indices.refresh() model = apps.get_model(model_name) search_app = get_search_app_by_model(model) read_alias = search_app.es_model.get_read_alias() assert model.objects.count() == 3 assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 3 management.call_command(command, model_name, simulate=True) setup_es.indices.refresh() # Check which models were actually deleted return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] assert deletions_by_model[model._meta.label] == 3 expected_deleted_models = {model._meta.label} | set( mapping['implicit_related_models']) actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert actual_deleted_models == expected_deleted_models # Check that nothing has actually been deleted assert model.objects.count() == 3 assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 3
def test_update_es_after_deletions(setup_es): """ Test that the context manager update_es_after_deletions collects and deletes all the django objects deleted. """ obj = SimpleModel.objects.create() sync_object_async(ESSimpleModel, SimpleModel, str(obj.pk)) setup_es.indices.refresh() search_app = get_search_app_by_model(SimpleModel) read_alias = search_app.es_model.get_read_alias() assert SimpleModel.objects.count() == 1 assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 1 with update_es_after_deletions(): obj.delete() setup_es.indices.refresh() assert setup_es.count(read_alias, doc_type=search_app.name)['count'] == 0
def test_run( model_name, config, mapping, dep_factory, dep_field_name, track_return_values, opensearch_with_signals, opensearch_collector_context_manager, ): """ Test that: - a record without any objects referencing it but not old enough doesn't get deleted - a record without any objects referencing it and old gets deleted - a record with another object referencing it doesn't get deleted """ # Set up the state before running the command command = delete_orphans.Command() model_factory = mapping['factory'] filter_config = config.filters[0] delete_return_value_tracker = track_return_values(QuerySet, 'delete') datetime_within_threshold = filter_config.cut_off_date datetime_older_than_threshold = filter_config.cut_off_date - relativedelta(days=1) with opensearch_collector_context_manager as collector: # this orphan should NOT get deleted because not old enough create_orphanable_model(model_factory, filter_config, datetime_within_threshold) # this orphan should get deleted because old create_orphanable_model(model_factory, filter_config, datetime_older_than_threshold) # this object should NOT get deleted because it has another object referencing it non_orphan = create_orphanable_model( model_factory, filter_config, datetime_older_than_threshold, ) is_m2m = dep_factory._meta.model._meta.get_field(dep_field_name).many_to_many dep_factory( **{dep_field_name: [non_orphan] if is_m2m else non_orphan}, ) collector.flush_and_refresh() # 3 + 1 in case of self-references total_model_records = 3 + (1 if dep_factory == model_factory else 0) model = apps.get_model(model_name) search_app = get_search_app_by_model(model) read_alias = search_app.search_model.get_read_alias() assert model.objects.count() == total_model_records assert opensearch_with_signals.count(index=read_alias)['count'] == total_model_records # Run the command management.call_command(command, model_name) opensearch_with_signals.indices.refresh() # Check that the records have been deleted assert model.objects.count() == total_model_records - 1 assert opensearch_with_signals.count(index=read_alias)['count'] == total_model_records - 1 # Check which models were actually deleted return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] assert deletions_by_model[model._meta.label] == 1 expected_deleted_models = {model._meta.label} | set(mapping['implicit_related_models']) actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert actual_deleted_models == expected_deleted_models
def _delete_from_es(self): for model, es_docs in self.deletions.items(): search_app = get_search_app_by_model(model) delete_documents(search_app.es_model.get_write_alias(), es_docs)
def test_run( model_label, factory_kwargs, relation_mapping, relation_factory_kwargs, is_expired, track_return_values, setup_es, ): """Tests the delete_old_records commands for various cases specified by MAPPING above.""" mapping = MAPPING[model_label] model_factory = mapping['factory'] command = delete_old_records.Command() model = apps.get_model(model_label) delete_return_value_tracker = track_return_values(QuerySet, 'delete') obj = _create_model_obj(model_factory, **factory_kwargs) total_model_records = 1 if relation_mapping: relation_model = relation_mapping['factory']._meta.get_model_class() relation_field = relation_model._meta.get_field( relation_mapping['field']) relation_factory_arg = [obj] if relation_field.many_to_many else obj _create_model_obj( relation_mapping['factory'], **relation_factory_kwargs, **{relation_mapping['field']: relation_factory_arg}, ) if relation_mapping['factory']._meta.get_model_class() is model: total_model_records += 1 num_expired_records = 1 if is_expired else 0 search_app = get_search_app_by_model(model) doc_type = search_app.name read_alias = search_app.es_model.get_read_alias() setup_es.indices.refresh() assert model.objects.count() == total_model_records assert setup_es.count(read_alias, doc_type=doc_type)['count'] == total_model_records management.call_command(command, model_label) setup_es.indices.refresh() # Check if the object has been deleted assert model.objects.count() == total_model_records - num_expired_records assert setup_es.count(read_alias, doc_type=doc_type)['count'] == (total_model_records - num_expired_records) # Check which models were actually deleted return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] if is_expired: assert deletions_by_model[model._meta.label] == num_expired_records assert model._meta.label in {model._meta.label} actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert actual_deleted_models - {model._meta.label } <= mapping['implicitly_deletable_models']
def test_run(cleanup_mapping, track_return_values, setup_es): """ Test that: - a record without any objects referencing it but not old enough doesn't get deleted - a record without any objects referencing it and old gets deleted - a record with another object referencing it doesn't get deleted """ command, model_name, config, mapping, dep_factory, dep_field_name = cleanup_mapping model_factory = mapping['factory'] delete_return_value_tracker = track_return_values(QuerySet, 'delete') datetime_within_threshold = FROZEN_TIME - config.age_threshold datetime_older_than_threshold = datetime_within_threshold - relativedelta( days=1) # this orphan should NOT get deleted because not old enough create_orphanable_model(model_factory, config, datetime_within_threshold) # this orphan should get deleted because old create_orphanable_model(model_factory, config, datetime_older_than_threshold) # this object should NOT get deleted because it has another object referencing it non_orphan = create_orphanable_model(model_factory, config, datetime_older_than_threshold) is_m2m = dep_factory._meta.model._meta.get_field( dep_field_name).many_to_many dep_factory(**{dep_field_name: [non_orphan] if is_m2m else non_orphan}, ) # 3 + 1 in case of self-references total_model_records = 3 + (1 if dep_factory == model_factory else 0) setup_es.indices.refresh() model = apps.get_model(model_name) search_app = get_search_app_by_model(model) doc_type = search_app.name read_alias = search_app.es_model.get_read_alias() assert model.objects.count() == total_model_records assert setup_es.count(read_alias, doc_type=doc_type)['count'] == total_model_records management.call_command(delete_orphans.Command(), model_name) setup_es.indices.refresh() # Check that the records have been deleted assert model.objects.count() == total_model_records - 1 assert setup_es.count( read_alias, doc_type=doc_type)['count'] == total_model_records - 1 # Check which models were actually deleted return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] assert deletions_by_model[model._meta.label] == 1 expected_deleted_models = {model._meta.label} | set( mapping['implicit_related_models']) actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert actual_deleted_models == expected_deleted_models
def test_run( model_label, factory_kwargs, relation_mapping, relation_factory_kwargs, is_expired, track_return_values, opensearch_with_signals, opensearch_collector_context_manager, ): """Tests the delete_old_records commands for various cases specified by MAPPING above.""" # Set up the state before running the command mapping = MAPPING[model_label] model_factory = mapping['factory'] command = delete_old_records.Command() model = apps.get_model(model_label) has_search_app = not mapping.get('has_no_search_app') delete_return_value_tracker = track_return_values(QuerySet, 'delete') with opensearch_collector_context_manager as collector: obj = _create_model_obj(model_factory, **factory_kwargs) total_model_records = 1 if relation_mapping: relation_model = relation_mapping['factory']._meta.get_model_class( ) relation_field = relation_model._meta.get_field( relation_mapping['field']) relation_factory_arg = [obj ] if relation_field.many_to_many else obj _create_model_obj( relation_mapping['factory'], **relation_factory_kwargs, **{relation_mapping['field']: relation_factory_arg}, ) if relation_mapping['factory']._meta.get_model_class() is model: total_model_records += 1 collector.flush_and_refresh() num_expired_records = 1 if is_expired else 0 if has_search_app: search_app = get_search_app_by_model(model) read_alias = search_app.search_model.get_read_alias() assert opensearch_with_signals.count( index=read_alias)['count'] == total_model_records assert model.objects.count() == total_model_records # Run the command management.call_command(command, model_label) opensearch_with_signals.indices.refresh() # Check if the object has been deleted assert model.objects.count() == total_model_records - num_expired_records if has_search_app: assert opensearch_with_signals.count( index=read_alias)['count'] == (total_model_records - num_expired_records) # Check which models were actually deleted return_values = delete_return_value_tracker.return_values assert len(return_values) == 1 _, deletions_by_model = return_values[0] if is_expired: assert deletions_by_model[model._meta.label] == num_expired_records assert model._meta.label in {model._meta.label} actual_deleted_models = { # only include models actually deleted deleted_model for deleted_model, deleted_count in deletions_by_model.items() if deleted_count } assert actual_deleted_models - {model._meta.label } <= mapping['implicitly_deletable_models']