def test_sync_app_uses_latest_data(monkeypatch, setup_es): """Test that sync_app() picks up updates made to records between batches.""" CompanyFactory.create_batch(2, name='old name') def sync_objects_side_effect(*args, **kwargs): nonlocal mock_sync_objects ret = sync_objects(*args, **kwargs) if mock_sync_objects.call_count == 1: Company.objects.update(name='new name') return ret mock_sync_objects = Mock(side_effect=sync_objects_side_effect) monkeypatch.setattr('datahub.search.bulk_sync.sync_objects', mock_sync_objects) sync_app(CompanySearchApp, batch_size=1) setup_es.indices.refresh() company = mock_sync_objects.call_args_list[1][0][1][0] fetched_company = setup_es.get( index=CompanySearchApp.es_model.get_read_alias(), doc_type=CompanySearchApp.name, id=company.pk, ) assert fetched_company['_source']['name'] == 'new name'
def test_sync_app_logic(monkeypatch): """Tests syncing an app to Elasticsearch during a mapping migration.""" bulk_mock = Mock() monkeypatch.setattr('datahub.search.bulk_sync.bulk', bulk_mock) search_app = create_mock_search_app( current_mapping_hash='mapping-hash', target_mapping_hash='mapping-hash', read_indices=('index1', 'index2'), write_index='index1', queryset=MockQuerySet([Mock(id=1), Mock(id=2)]), ) sync_app(search_app, batch_size=1000) assert bulk_mock.call_args_list[0][1]['actions'] == [ { '_index': 'index1', '_id': 1, '_type': 'test-type', }, { '_index': 'index1', '_id': 2, '_type': 'test-type', }, ] assert bulk_mock.call_count == 1
def sync_model(search_app_name): """ Task that syncs a single model to Elasticsearch. acks_late is set to True so that the task restarts if interrupted. priority is set to the lowest priority (for Redis, 0 is the highest priority). """ search_app = get_search_app(search_app_name) sync_app(search_app)
def test_sync_app_with_overridden_batch_size(monkeypatch): """Tests syncing an app to OpenSearch with an overridden batch size.""" bulk_mock = Mock() monkeypatch.setattr('datahub.search.bulk_sync.bulk', bulk_mock) search_app = create_mock_search_app(queryset=MockQuerySet( [Mock(id=1), Mock(id=2)]), ) sync_app(search_app, batch_size=1) assert bulk_mock.call_count == 2
def test_sync_app_with_default_batch_size(monkeypatch): """Tests syncing an app to Elasticsearch with the default batch size.""" bulk_mock = Mock() monkeypatch.setattr('datahub.search.bulk_sync.bulk', bulk_mock) search_app = create_mock_search_app(queryset=MockQuerySet( [Mock(id=1), Mock(id=2)]), ) sync_app(search_app) assert bulk_mock.call_count == 1
def resync_after_migrate(search_app): """ Completes a migration by performing a full resync, updating aliases and removing old indices. """ if not search_app.search_model.was_migration_started(): logger.warning( f'No pending migration detected for the {search_app.name} search app, aborting ' f'resync...', ) return sync_app(search_app, post_batch_callback=delete_from_secondary_indices_callback) _clean_up_aliases_and_indices(search_app)
def sync_es(batch_size, search_apps): """Sends data to Elasticsearch.""" for app in search_apps: sync_app(app, batch_size=batch_size) logger.info('Elasticsearch sync complete!')