async def test_calculate_mapping_diff(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) version, new_index_name = await migrator.create_next_index() migrator.work_index_name = new_index_name mappings = get_mappings() index_settings = DEFAULT_SETTINGS.copy() index_settings.update(app_settings.get('index', {})) # tweak mappings so we can get the diff... for key, value in mappings.items(): # need to modify on *all* or it won't work with ES.. if 'creators' in value['properties']: value['properties']['creators']['type'] = 'text' mappings['Item']['properties']['foobar'] = { 'type': 'keyword', 'index': True } await search.conn.indices.close(new_index_name) await search.conn.indices.put_settings(index_settings, new_index_name) for key, value in mappings.items(): await search.conn.indices.put_mapping(new_index_name, key, value) await search.conn.indices.open(new_index_name) diff = await migrator.calculate_mapping_diff() assert len(diff['Folder']) == 1 assert len(diff['Item']) == 2
async def _test_migrate_while_content_getting_added(es_requester): async with es_requester as requester: add_count = await add_content(requester) container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) await search.refresh(container) await asyncio.sleep(3) assert add_count == await search.get_doc_count(container) migrator = Migrator(search, container, force=True) add_content_task1 = asyncio.ensure_future( add_content(requester, base_id='foo1-')) add_content_task2 = asyncio.ensure_future( add_content(requester, base_id='foo2-')) reindex_task = asyncio.ensure_future(migrator.run_migration()) await asyncio.wait( [add_content_task1, reindex_task, add_content_task2]) await search.refresh(container) await asyncio.sleep(3) idx_count = await search.get_doc_count(container) # +1 here because container ob now indexed and it isn't by default in tests assert (add_count * 3) + 1 == idx_count await tm.abort(txn=txn)
async def migrate_all(self, arguments): search = get_utility(ICatalogUtility) change_transaction_strategy('none') await asyncio.sleep(1) # since something initialize custom types... async for _, tm, container in get_containers(self.request): try: self.migrator = Migrator( search, container, response=printer(), full=arguments.full, force=arguments.force, log_details=arguments.log_details, memory_tracking=arguments.memory_tracking, reindex_security=arguments.reindex_security, mapping_only=arguments.mapping_only, cache=False) await self.migrator.run_migration() seconds = int(time.time() - self.migrator.start_time) logger.warning(f'''Finished migration: Total Seconds: {seconds} Processed: {self.migrator.processed} Indexed: {self.migrator.indexed} Objects missing: {len(self.migrator.missing)} Objects orphaned: {len(self.migrator.orphaned)} Mapping Diff: {self.migrator.mapping_diff} ''') finally: await tm.commit(self.request)
async def test_calculate_mapping_diff(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) version, new_index_name = await migrator.create_next_index() migrator.work_index_name = new_index_name mappings = get_mappings() index_settings = DEFAULT_SETTINGS.copy() index_settings.update(app_settings.get('index', {})) # tweak mappings so we can get the diff... if 'creators' in mappings['properties']: mappings['properties']['creators']['type'] = 'text' mappings['properties']['foobar'] = {'type': 'keyword', 'index': True} await search.conn.indices.close(new_index_name) await search.conn.indices.put_settings(body=index_settings, index=new_index_name) await search.conn.indices.put_mapping(index=new_index_name, doc_type=DOC_TYPE, body=mappings) await search.conn.indices.open(new_index_name) diff = await migrator.calculate_mapping_diff() assert len(diff[DOC_TYPE]) == 2
async def test_removes_orphans(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) await search.index( container, {"foobar": { "title": "foobar", "type_name": "Item" }}) # foobar here is an orphaned object because it doesn't reference # an object im = get_adapter(container, IIndexManager) index_name = await im.get_index_name() # alias doc = await search.get_connection().get(index=index_name, doc_type=DOC_TYPE, id="foobar") assert doc["found"] migrator = Migrator(search, container, force=True) await migrator.run_migration() async def _test(): with pytest.raises(aioelasticsearch.exceptions.NotFoundError): await search.get_connection().get(index=index_name, doc_type=DOC_TYPE, id="foobar") assert len(migrator.orphaned) == 1 assert migrator.orphaned[0] == "foobar" await run_with_retries(_test, requester)
async def test_delete_in_both_during_migration(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) migrator = Migrator(search, container, force=True) im = get_adapter(container, IIndexManager) index_name = await im.get_index_name() next_index_name = await migrator.setup_next_index() resp, _ = await requester( "POST", "/db/guillotina", data=json.dumps({ "@type": "Folder", "title": "Foobar1", "id": "foobar" }), ) await requester("DELETE", "/db/guillotina/foobar") async def _test(): with pytest.raises(aioelasticsearch.exceptions.NotFoundError): await search.get_connection().get(index=next_index_name, doc_type="_all", id=resp["@uid"]) with pytest.raises(aioelasticsearch.exceptions.NotFoundError): await search.get_connection().get(index=index_name, doc_type="_all", id=resp["@uid"]) await run_with_retries(_test, requester)
async def test_removes_orphans(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) await search.index( container, {'foobar': { 'title': 'foobar', 'type_name': 'Item' }}) # foobar here is an orphaned object because it doesn't reference an object index_name = await search.get_index_name(container) # alias doc = await search.conn.get(index=index_name, doc_type=DOC_TYPE, id='foobar') assert doc['found'] migrator = Migrator(search, container, force=True) await migrator.run_migration() async def _test(): with pytest.raises(aioelasticsearch.exceptions.NotFoundError): await search.conn.get(index=index_name, doc_type=DOC_TYPE, id='foobar') assert len(migrator.orphaned) == 1 assert migrator.orphaned[0] == 'foobar' await run_with_retries(_test, requester)
async def _test_new_deletes_are_performed_during_migration(es_requester): async with es_requester as requester: await add_content(requester) container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) await migrator.setup_next_index() await migrator.copy_to_next_index() await search.refresh(container, migrator.work_index_name) await search.refresh(container) num_docs = await search.get_doc_count(container, migrator.work_index_name) current_docs = await search.get_doc_count(container) assert num_docs == current_docs keys = await container.async_keys() key = random.choice(keys) ob = await container.async_get(key) keys = await ob.async_keys() key = random.choice(keys) ob = await ob.async_get(key) await search.remove(container, [( ob._p_oid, ob.type_name, get_content_path(ob) )], request=request) await search.refresh(container, migrator.work_index_name) await search.refresh(container) num_docs = await search.get_doc_count(container, migrator.work_index_name) current_count = await search.get_doc_count(container) assert num_docs == current_count
async def test_removes_orphans(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) await search.index( container, {'foobar': { 'title': 'foobar', 'type_name': 'Item' }}) # foobar here is an orphaned object because it doesn't reference an object index_name = await search.get_index_name(container) # alias doc = await search.conn.get(index_name, 'foobar') assert doc['found'] migrator = Migrator(search, container, force=True) await migrator.run_migration() await asyncio.sleep(1) await search.refresh(container, index_name) await asyncio.sleep(1) with pytest.raises(aioes.exception.NotFoundError): doc = await search.conn.get(index_name, 'foobar') assert len(migrator.orphaned) == 1 assert migrator.orphaned[0] == 'foobar'
async def test_create_next_index(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) name = await migrator.create_next_index() assert name == 'guillotina-db-guillotina_2'
def __init__(self, txn, tm, request, container, last_tid=-2, index_scroll='15m', hits_scroll='5m', use_tid_query=True): self.txn = txn self.tm = tm self.request = request self.container = container self.orphaned = set() self.missing = set() self.out_of_date = set() self.utility = getUtility(ICatalogUtility) self.migrator = Migrator(self.utility, self.container, full=True, bulk_size=10) self.cache = LRU(200) self.last_tid = last_tid print(f'Last TID: {self.last_tid}') self.use_tid_query = use_tid_query self.last_zoid = None # for state tracking so we get boundries right self.last_result_set = [] self.index_scroll = index_scroll self.hits_scroll = hits_scroll
async def _test_new_indexes_are_performed_during_migration(es_requester): async with es_requester as requester: await add_content(requester) container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) await migrator.setup_next_index() await migrator.copy_to_next_index() await asyncio.sleep(1) await search.refresh(container, migrator.work_index_name) await search.refresh(container) await asyncio.sleep(1) num_docs = await search.get_doc_count(container, migrator.work_index_name) assert num_docs == await search.get_doc_count(container) await add_content(requester, base_id='foobar1-') await asyncio.sleep(1) await search.refresh(container, migrator.work_index_name) await search.refresh(container) await asyncio.sleep(1) num_docs = await search.get_doc_count(container, migrator.work_index_name) assert num_docs == await search.get_doc_count(container)
async def test_delete_in_both_during_migration(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) im = get_adapter(container, IIndexManager) index_name = await im.get_index_name() next_index_name = await migrator.setup_next_index() resp, _ = await requester('POST', '/db/guillotina', data=json.dumps({ '@type': 'Folder', 'title': 'Foobar1', 'id': 'foobar' })) await requester('DELETE', '/db/guillotina/foobar') async def _test(): with pytest.raises(aioelasticsearch.exceptions.NotFoundError): await search.conn.get(index=next_index_name, doc_type='_all', id=resp['@uid']) with pytest.raises(aioelasticsearch.exceptions.NotFoundError): await search.conn.get(index=index_name, doc_type='_all', id=resp['@uid']) await run_with_retries(_test, requester)
async def test_search_works_on_new_docs_during_migration(es_requester): async with es_requester as requester: await add_content(requester, 2) container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) im = get_adapter(container, IIndexManager) index_name = await im.get_index_name() next_index_name = await migrator.setup_next_index() resp, _ = await requester('POST', '/db/guillotina', data=json.dumps({'@type': 'Item'})) async def _test(): result1 = await search.conn.get(index=next_index_name, doc_type='_all', id=resp['@uid']) assert result1 is not None result2 = await search.conn.get(index=index_name, doc_type='_all', id=resp['@uid']) assert result2 is not None await run_with_retries(_test, requester)
async def es_migrate(path, root, request, reindex_security=False, mapping_only=False, full=False, force=False): try: ob, end_path = await traverse(request, root, path.lstrip('/').split('/')) if len(end_path) != 0: raise Exception('Could not found object') search = getUtility(ICatalogUtility) migrator = Migrator(search, ob, reindex_security=reindex_security, full=full, force=force, mapping_only=mapping_only, request=request, log_details=True) await migrator.run_migration() finally: txn = get_transaction(request) if txn is not None: tm = get_tm(request) await tm.abort(txn=txn)
class MigrateCommand(Command): description = 'Migrate indexes' migrator = None def get_parser(self): parser = super(MigrateCommand, self).get_parser() parser.add_argument('--full', help='Do a full reindex', action='store_true') parser.add_argument('--force', help='Override failing migration if existing ' 'migration index exists', action='store_true') parser.add_argument('--log-details', action='store_true') parser.add_argument('--memory-tracking', action='store_true') parser.add_argument('--reindex-security', action='store_true') parser.add_argument('--mapping-only', action='store_true') return parser async def migrate_all(self, arguments): search = get_utility(ICatalogUtility) change_transaction_strategy('none') await asyncio.sleep(1) # since something initialize custom types... async for _, tm, container in get_containers(self.request): try: self.migrator = Migrator( search, container, response=printer(), full=arguments.full, force=arguments.force, log_details=arguments.log_details, memory_tracking=arguments.memory_tracking, reindex_security=arguments.reindex_security, mapping_only=arguments.mapping_only, cache=False) await self.migrator.run_migration() seconds = int(time.time() - self.migrator.start_time) logger.warning(f'''Finished migration: Total Seconds: {seconds} Processed: {self.migrator.processed} Indexed: {self.migrator.indexed} Objects missing: {len(self.migrator.missing)} Objects orphaned: {len(self.migrator.orphaned)} Mapping Diff: {self.migrator.mapping_diff} ''') finally: await tm.commit(self.request) def run(self, arguments, settings, app): loop = self.get_loop() try: loop.run_until_complete(self.migrate_all(arguments)) except KeyboardInterrupt: # pragma: no cover pass finally: if self.migrator.status != 'done': loop = self.get_loop() loop.run_until_complete(self.migrator.cancel_migration())
async def test_updates_index_data(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) version, new_index_name = await migrator.create_next_index() migrator.work_index_name = new_index_name await search.install_mappings_on_index(new_index_name) ob = create_content() ob.title = 'foobar' await migrator.index_object(ob, full=True) assert len(migrator.batch) == 1 assert [v for v in migrator.batch.values()][0]['action'] == 'index' await migrator.flush() assert len(migrator.batch) == 0 await migrator.join_futures() await asyncio.sleep(1) await search.refresh(container, new_index_name) await asyncio.sleep(1) assert await search.get_doc_count(container, new_index_name) == 1 # test updating doc migrator.mapping_diff = {'Item': {'title': {}}} ob.title = 'foobar-new' await migrator.index_object(ob, full=False) assert len(migrator.batch) == 1 assert [v for v in migrator.batch.values()][0]['action'] == 'update' assert len([v for v in migrator.batch.values()][0]['data']) == 2 assert [v for v in migrator.batch.values() ][0]['data']['title'] == 'foobar-new' await migrator.flush() assert len(migrator.batch) == 0 await migrator.join_futures() await asyncio.sleep(1) await search.refresh(container, new_index_name) await asyncio.sleep(1) doc = await search.conn.get(index=new_index_name, doc_type=DOC_TYPE, id=ob._p_oid) assert doc['_source']['title'] == 'foobar-new'
async def test_updates_index_data(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) migrator = Migrator(search, container, force=True) new_index_name = await migrator.create_next_index() migrator.work_index_name = new_index_name ob = create_content() ob.title = "foobar" await migrator.index_object(ob, full=True) assert len(migrator.batch) == 1 assert [v for v in migrator.batch.values()][0]["action"] == "index" await migrator.flush() assert len(migrator.batch) == 0 await migrator.join_futures() await asyncio.sleep(1) await search.refresh(container, new_index_name) await asyncio.sleep(1) assert await search.get_doc_count(container, new_index_name) == 1 # test updating doc migrator.mapping_diff = {"title": {}} ob.title = "foobar-new" await migrator.index_object(ob, full=False) assert len(migrator.batch) == 1 assert [v for v in migrator.batch.values()][0]["action"] == "update" assert len([v for v in migrator.batch.values()][0]["data"]) == 2 assert [v for v in migrator.batch.values() ][0]["data"]["title"] == "foobar-new" # noqa await migrator.flush() assert len(migrator.batch) == 0 await migrator.join_futures() await asyncio.sleep(1) await search.refresh(container, new_index_name) await asyncio.sleep(1) doc = await search.get_connection().get(index=new_index_name, doc_type=DOC_TYPE, id=ob.__uuid__) assert doc["_source"]["title"] == "foobar-new"
async def test_updates_index_name(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) existing_index = await search.get_real_index_name(container) assert await search.conn.indices.exists(existing_index) migrator = Migrator(search, container, force=True, request=request) await migrator.run_migration() assert not await search.conn.indices.exists(existing_index) assert search.conn.indices.exists(migrator.work_index_name) assert await search.get_real_index_name(container ) == migrator.work_index_name
async def test_migrate_content_index_works(es_requester): async with es_requester as requester: add_count = await add_content(requester) cresp, _ = await requester( "POST", "/db/guillotina/", data=json.dumps({ "@type": "UniqueIndexContent", "id": "foobar" }), ) await requester( "POST", "/db/guillotina/foobar", data=json.dumps({"@type": "IndexItemContent"}), ) container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) await search.refresh(container) await asyncio.sleep(3) assert (add_count + 1) == await search.get_doc_count( container, "guillotina-db-guillotina_1") assert (await search.get_doc_count( container, "1_guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa get_short_uid(cresp["@uid"])), ) == 1) migrator = Migrator(search, container, force=True) await migrator.run_migration() assert await search.get_connection().indices.exists( "guillotina-db-guillotina_2") assert not await search.get_connection().indices.exists( "guillotina-db-guillotina_1") assert await search.get_connection().indices.exists( "2_guillotina-db-guillotina__uniqueindexcontent-{}".format( get_short_uid(cresp["@uid"]))) assert not await search.get_connection().indices.exists( "1_guillotina-db-guillotina__uniqueindexcontent-{}".format( get_short_uid(cresp["@uid"]))) assert (add_count + 1) == await search.get_doc_count( container, "guillotina-db-guillotina_2") assert (await search.get_doc_count( container, "2_guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa get_short_uid(cresp["@uid"])), ) == 1)
async def test_updates_index_name(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) im = get_adapter(container, IIndexManager) existing_index = await im.get_real_index_name() assert await search.get_connection().indices.exists(existing_index) migrator = Migrator(search, container, force=True) await migrator.run_migration() assert not await search.get_connection().indices.exists(existing_index) assert await search.get_connection().indices.exists( migrator.work_index_name) assert await im.get_real_index_name() == migrator.work_index_name
async def test_calculate_mapping_diff(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) index_manager = get_adapter(container, IIndexManager) migrator = Migrator(search, container, force=True) new_index_name = await index_manager.start_migration() migrator.work_index_name = new_index_name mappings = await index_manager.get_mappings() # tweak mappings so we can get the diff... if "creators" in mappings["properties"]: mappings["properties"]["creators"]["type"] = "text" mappings["properties"]["foobar"] = {"type": "keyword", "index": True} await search.create_index(new_index_name, index_manager, mappings=mappings) diff = await migrator.calculate_mapping_diff() assert len(diff) == 2
async def test_calculate_mapping_diff(es_requester): async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) index_manager = get_adapter(container, IIndexManager) migrator = Migrator(search, container, force=True, request=request) new_index_name = await index_manager.start_migration() migrator.work_index_name = new_index_name mappings = await index_manager.get_mappings() # tweak mappings so we can get the diff... if 'creators' in mappings['properties']: mappings['properties']['creators']['type'] = 'text' mappings['properties']['foobar'] = {'type': 'keyword', 'index': True} await search.create_index(new_index_name, index_manager, mappings=mappings) diff = await migrator.calculate_mapping_diff() assert len(diff) == 2
def __init__(self, txn, tm, request, container, last_tid=-2): self.txn = txn self.tm = tm self.request = request self.container = container self.orphaned = [] self.missing = [] self.utility = getUtility(ICatalogUtility) self.migrator = Migrator( self.utility, self.container, full=True, bulk_size=10) self.cache = LRU(200) self.last_tid = last_tid self.use_tid_query = True self.last_zoid = None # for state tracking so we get boundries right self.last_result_set = []
async def test_fixes_missing(es_requester): async with es_requester as requester: await add_content(requester, 2, 2) container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) await asyncio.sleep(1) await search.refresh(container) await asyncio.sleep(1) original_count = await search.get_doc_count(container) keys = await container.async_keys() key = random.choice(keys) ob = await container.async_get(key) await search.remove(container, [(ob)], request=request) await asyncio.sleep(1) await search.refresh(container) await asyncio.sleep(1) assert original_count != await search.get_doc_count(container) im = get_adapter(container, IIndexManager) old_index_name = await im.get_real_index_name() responses = [] class Writer: def write(self, item): responses.append(item) migrator = Migrator(search, container, force=True, request=request, response=Writer()) await migrator.run_migration() assert migrator.status == 'done' await asyncio.sleep(1) await search.refresh(container) await asyncio.sleep(1) # new index should fix missing one, old index still has it missing num_docs = await search.get_doc_count(container, migrator.work_index_name) assert num_docs == original_count assert old_index_name != await im.get_real_index_name()
async def test_migrate_content_index_works(es_requester): async with es_requester as requester: add_count = await add_content(requester) cresp, _ = await requester('POST', '/db/guillotina/', data=json.dumps({ '@type': 'UniqueIndexContent', 'id': 'foobar' })) await requester('POST', '/db/guillotina/foobar', data=json.dumps({'@type': 'IndexItemContent'})) container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) await search.refresh(container) await asyncio.sleep(3) assert (add_count + 1) == await search.get_doc_count( container, 'guillotina-db-guillotina_1') assert await search.get_doc_count( container, '1_guillotina-db-guillotina__uniqueindexcontent-{}'.format( # noqa get_short_oid(cresp['@uid']))) == 1 migrator = Migrator(search, container, force=True) await migrator.run_migration() assert await search.conn.indices.exists('guillotina-db-guillotina_2') assert not await search.conn.indices.exists( 'guillotina-db-guillotina_1') assert await search.conn.indices.exists( '2_guillotina-db-guillotina__uniqueindexcontent-{}'.format( get_short_oid(cresp['@uid']))) assert not await search.conn.indices.exists( '1_guillotina-db-guillotina__uniqueindexcontent-{}'.format( get_short_oid(cresp['@uid']))) assert (add_count + 1) == await search.get_doc_count( container, 'guillotina-db-guillotina_2') assert await search.get_doc_count( container, '2_guillotina-db-guillotina__uniqueindexcontent-{}'.format( # noqa get_short_oid(cresp['@uid']))) == 1
def __init__(self, txn, tm, request, container, last_tid=-2): self.txn = txn self.tm = tm self.request = request self.container = container self.orphaned = set() self.missing = set() self.out_of_date = set() self.utility = get_utility(ICatalogUtility) self.migrator = Migrator( self.utility, self.container, full=True, bulk_size=10, lookup_index=True) self.index_manager = get_adapter(self.container, IIndexManager) self.cache = LRU(200) self.last_tid = last_tid self.use_tid_query = True self.last_zoid = None # for state tracking so we get boundries right self.last_result_set = []
async def test_migrate_get_all_uids(es_requester): async with es_requester as requester: await add_content(requester) container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) await asyncio.sleep(1) await search.refresh(container) await asyncio.sleep(1) current_count = await search.get_doc_count(container) migrator = Migrator(search, container, force=True) uids = await migrator.get_all_uids() assert len(uids) == current_count await tm.abort(txn=txn)
async def test_search_works_on_updated_docs_during_migration_when_present( es_requester, ): # noqa """ - started migration - doc update - doc also updated in next index """ async with es_requester as requester: container, request, txn, tm = await setup_txn_on_container(requester) search = get_utility(ICatalogUtility) migrator = Migrator(search, container, force=True) im = get_adapter(container, IIndexManager) index_name = await im.get_index_name() next_index_name = await migrator.setup_next_index() resp, _ = await requester( "POST", "/db/guillotina", data=json.dumps({ "@type": "Item", "title": "Foobar1", "id": "foobar" }), ) await asyncio.sleep(1) await requester("PATCH", "/db/guillotina/foobar", data=json.dumps({"title": "Foobar2"})) async def _test(): result1 = await search.get_connection().get(index=next_index_name, doc_type="_all", id=resp["@uid"]) assert result1 is not None assert result1["_source"]["title"] == "Foobar2" result2 = await search.get_connection().get(index=index_name, doc_type="_all", id=resp["@uid"]) assert result2 is not None assert result2["_source"]["title"] == "Foobar2" await run_with_retries(_test, requester)
async def test_unindex_during_next_index(es_requester): async with es_requester as requester: await add_content(requester, 2) container, request, txn, tm = await setup_txn_on_container(requester) search = getUtility(ICatalogUtility) migrator = Migrator(search, container, force=True, request=request) next_index_version, work_index_name = await migrator.create_next_index( ) await search.install_mappings_on_index(work_index_name) await search.activate_next_index(container, next_index_version, request=request) await tm.commit(txn=txn) container, request, txn, tm = await setup_txn_on_container(requester) keys = await container.async_keys() item = await container.async_get(keys[0]) aiotask_context.set('request', request) await notify(ObjectRemovedEvent(item, container, item.id)) request.execute_futures() await asyncio.sleep(1)