async def test_updates_out_of_data_es_entries(es_requester):
async with es_requester as requester:
await add_content(requester)
await asyncio.sleep(1)
container, request, txn, tm = await setup_txn_on_container(requester)
aiotask_context.set('request', request)
search = get_utility(ICatalogUtility)
index_name = await search.get_container_index_name(container)
await search.update_by_query({
'script': {
'lang': 'painless',
'inline': "ctx._source.remove('tid')"
}
}, indexes=[index_name])
async def _test():
assert await search.get_doc_count(container) == 110
await run_with_retries(_test, requester)
await asyncio.sleep(1)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 0
assert len(vacuum.missing) == 0
assert len(vacuum.out_of_date) == 110
await tm.abort(txn=txn)
async def test_removes_orphaned_es_entry(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = get_utility(ICatalogUtility)
await search.index(container, {
'foobar': {
'title': 'foobar',
'type_name': 'Item'
}
})
async def _test():
assert await search.get_doc_count(container) == 1
await run_with_retries(_test, requester)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_orphans()
await vacuum.check_missing()
assert len(vacuum.orphaned) == 1
assert len(vacuum.missing) == 0
assert len(vacuum.out_of_date) == 0
async def __test():
assert await search.get_doc_count(container) == 0
await run_with_retries(__test, requester)
await tm.abort(txn=txn)
async def test_vacuum_with_multiple_containers(es_requester):
async with es_requester as requester:
# create another container, force to iterate differently
_, status = await requester('POST',
'/db',
data=json.dumps({
'@type': 'Container',
'id': 'foobar'
}))
assert status == 200
await add_content(requester, num_items=100)
search = get_utility(ICatalogUtility)
container, request, txn, tm = await setup_txn_on_container(requester)
aiotask_context.set('request', request)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
async def ___test():
assert await search.get_doc_count(container) == 1010
await run_with_retries(___test, requester)
await tm.abort(txn=txn)
async def test_updates_out_of_data_es_entries(es_requester):
async with es_requester as requester:
await add_content(requester)
await asyncio.sleep(1)
container, request, txn, tm = await setup_txn_on_container(requester)
task_vars.request.set(request)
search = get_utility(ICatalogUtility)
index_name = await search.get_container_index_name(container)
await search.update_by_query(
{"script": {
"lang": "painless",
"inline": "ctx._source.tid = 0"
}},
indexes=[index_name],
)
async def _test():
assert await search.get_doc_count(container) == 110
await run_with_retries(_test, requester, retry_wait=1)
await asyncio.sleep(1)
vacuum = Vacuum(txn, tm, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 0
assert len(vacuum.missing) == 0
assert len(vacuum.out_of_date) == 110
await tm.abort(txn=txn)
async def test_removes_orphaned_es_entry(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
index_name = await search.get_index_name(container)
await search.index(
container, {'foobar': {
'title': 'foobar',
'type_name': 'Item'
}})
await search.refresh(container, index_name)
await asyncio.sleep(1)
vacuum = Vacuum(txn, tm, request, container)
await vacuum()
assert len(vacuum.orphaned) == 1
await tm.abort(txn=txn)
async def test_adds_missing_elasticsearch_entry(es_requester):
async with es_requester as requester:
await add_content(requester)
await asyncio.sleep(1)
container, request, txn, tm = await setup_txn_on_container(requester)
keys = await container.async_keys()
key = random.choice(keys)
ob = await container.async_get(key)
search = getUtility(ICatalogUtility)
await search.remove(container,
[(ob._p_oid, ob.type_name, get_content_path(ob))],
request=request)
await asyncio.sleep(1)
vacuum = Vacuum(txn, tm, request, container)
await vacuum()
assert len(vacuum.missing) > 0
await tm.abort(txn=txn)
async def test_adds_missing_elasticsearch_entry(es_requester):
async with es_requester as requester:
await add_content(requester)
search = get_utility(ICatalogUtility)
container, request, txn, tm = await setup_txn_on_container(requester)
aiotask_context.set('request', request)
async def _test():
assert await search.get_doc_count(container) == 110
await run_with_retries(_test, requester)
for key in await container.async_keys():
ob = await container.async_get(key)
await search.remove(container, [ob], request=request)
async def __test():
assert await search.get_doc_count(container) == 0
await run_with_retries(__test, requester)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 0
assert len(vacuum.out_of_date) == 0
assert len(vacuum.missing) == 110
async def ___test():
assert await search.get_doc_count(container) == 110
await run_with_retries(___test, requester)
await tm.abort(txn=txn)
async def test_vacuum_with_sub_indexes(es_requester):
async with es_requester as requester:
await add_content(requester, num_folders=2, num_items=5, path='/db/guillotina/')
cresp, _ = await requester(
'POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'title': 'UniqueIndexContent',
'id': 'foobar'
})
)
await add_content(requester, num_folders=2, num_items=5, path='/db/guillotina/foobar')
search = get_utility(ICatalogUtility)
content_index_name = 'guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(cresp['@uid'])
)
container, request, txn, tm = await setup_txn_on_container(requester)
aiotask_context.set('request', request)
async def _test():
assert await search.get_doc_count(container) == 13
assert await search.get_doc_count(index_name=content_index_name) == 12
await run_with_retries(_test, requester)
for key in await container.async_keys():
if key == 'foobar':
continue
ob = await container.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
foobar = await container.async_get('foobar')
for key in await foobar.async_keys():
ob = await foobar.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
await search.index(container, {
'foobar1': {
'title': 'foobar',
'type_name': 'Item'
}
})
await search.index(container, {
'foobar2': {
'title': 'foobar',
'type_name': 'Item',
'__indexes__': [content_index_name]
}
})
async def __test():
assert await search.get_doc_count(container) == 2
assert await search.get_doc_count(index_name=content_index_name) == 1
await run_with_retries(__test, requester)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 2
assert len(vacuum.out_of_date) == 0
assert len(vacuum.missing) == 24
async def ___test():
assert await search.get_doc_count(container) == 13
assert await search.get_doc_count(index_name=content_index_name) == 12
await run_with_retries(___test, requester)
await tm.abort(txn=txn)
async def test_reindexes_moved_content(es_requester):
async with es_requester as requester:
resp1, _ = await requester('POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'Folder',
'id': 'foobar'
}))
resp2, _ = await requester('POST',
'/db/guillotina/foobar',
data=json.dumps({
'@type': 'Folder',
'id': 'foobar'
}))
resp3, _ = await requester('POST',
'/db/guillotina/foobar/foobar',
data=json.dumps({
'@type': 'Folder',
'id': 'foobar'
}))
container, request, txn, tm = await setup_txn_on_container(requester)
search = get_utility(ICatalogUtility)
index_name = await search.get_container_index_name(container)
async def _test():
assert await search.get_doc_count(container) == 3
result = await search.conn.get(index=index_name,
doc_type='_all',
id=resp3['@uid'])
assert result is not None
await run_with_retries(_test, requester)
# mess with index data to make it look like it was moved
await search.conn.update(index=index_name,
id=resp1['@uid'],
doc_type=DOC_TYPE,
body={
"doc": {
"path":
"/moved-foobar",
"parent_uuid":
"FOOOBBAR MOVED TO NEW PARENT"
}
})
await search.conn.update(
index=index_name,
id=resp2['@uid'],
doc_type=DOC_TYPE,
body={"doc": {
"path": "/moved-foobar/foobar"
}})
await search.conn.update(
index=index_name,
id=resp3['@uid'],
doc_type=DOC_TYPE,
body={"doc": {
"path": "/moved-foobar/foobar/foobar"
}})
async def _test():
result = await search.conn.get(index=index_name,
doc_type='_all',
id=resp3['@uid'],
stored_fields='path')
assert result['fields']['path'] == ["/moved-foobar/foobar/foobar"]
result = await search.conn.get(index=index_name,
doc_type='_all',
id=resp1['@uid'],
stored_fields='path,parent_uuid')
assert result['fields']['path'] == ["/moved-foobar"]
assert result['fields']['parent_uuid'] == [
"FOOOBBAR MOVED TO NEW PARENT"
]
await run_with_retries(_test, requester)
await asyncio.sleep(2)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_missing()
assert len(vacuum.orphaned) == 0
assert len(vacuum.missing) == 1
await asyncio.sleep(2)
async def __test():
result = await search.conn.get(index=index_name,
doc_type='_all',
id=resp3['@uid'],
stored_fields='path,parent_uuid')
assert result['fields']['path'] == ["/foobar/foobar/foobar"]
result = await search.conn.get(index=index_name,
doc_type='_all',
id=resp1['@uid'],
stored_fields='path,parent_uuid')
assert result['fields']['path'] == ["/foobar"]
assert result['fields'][
'parent_uuid'] != "FOOOBBAR MOVED TO NEW PARENT" # noqa
await run_with_retries(__test, requester)
await tm.abort(txn=txn)
async def test_reindexes_moved_content(es_requester):
async with es_requester as requester:
resp1, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "Folder",
"id": "foobar"
}),
)
resp2, _ = await requester(
"POST",
"/db/guillotina/foobar",
data=json.dumps({
"@type": "Folder",
"id": "foobar"
}),
)
resp3, _ = await requester(
"POST",
"/db/guillotina/foobar/foobar",
data=json.dumps({
"@type": "Folder",
"id": "foobar"
}),
)
container, request, txn, tm = await setup_txn_on_container(requester)
search = get_utility(ICatalogUtility)
index_name = await search.get_container_index_name(container)
async def _test():
assert await search.get_doc_count(container) == 3
result = await search.get_connection().get(index=index_name,
doc_type="_all",
id=resp3["@uid"])
assert result is not None
await run_with_retries(_test, requester)
# mess with index data to make it look like it was moved
await search.get_connection().update(
index=index_name,
id=resp1["@uid"],
doc_type=DOC_TYPE,
body={
"doc": {
"path": "/moved-foobar",
"parent_uuid": "FOOOBBAR MOVED TO NEW PARENT",
}
},
)
await search.get_connection().update(
index=index_name,
id=resp2["@uid"],
doc_type=DOC_TYPE,
body={"doc": {
"path": "/moved-foobar/foobar"
}},
)
await search.get_connection().update(
index=index_name,
id=resp3["@uid"],
doc_type=DOC_TYPE,
body={"doc": {
"path": "/moved-foobar/foobar/foobar"
}},
)
async def _test():
result = await search.get_connection().get(
index=index_name,
doc_type="_all",
id=resp3["@uid"],
stored_fields="path",
)
assert result["fields"]["path"] == ["/moved-foobar/foobar/foobar"]
result = await search.get_connection().get(
index=index_name,
doc_type="_all",
id=resp1["@uid"],
stored_fields="path,parent_uuid",
)
assert result["fields"]["path"] == ["/moved-foobar"]
assert result["fields"]["parent_uuid"] == [
"FOOOBBAR MOVED TO NEW PARENT"
]
await run_with_retries(_test, requester)
await asyncio.sleep(2)
vacuum = Vacuum(txn, tm, container)
await vacuum.setup()
await vacuum.check_missing()
assert len(vacuum.orphaned) == 0
assert len(vacuum.missing) == 1
await asyncio.sleep(2)
async def __test():
result = await search.get_connection().get(
index=index_name,
doc_type="_all",
id=resp3["@uid"],
stored_fields="path,parent_uuid",
)
assert result["fields"]["path"] == ["/foobar/foobar/foobar"]
result = await search.get_connection().get(
index=index_name,
doc_type="_all",
id=resp1["@uid"],
stored_fields="path,parent_uuid",
)
assert result["fields"]["path"] == ["/foobar"]
assert (result["fields"]["parent_uuid"] !=
"FOOOBBAR MOVED TO NEW PARENT") # noqa
await run_with_retries(__test, requester)
await tm.abort(txn=txn)
async def test_vacuum_with_sub_indexes(es_requester):
async with es_requester as requester:
await add_content(requester,
num_folders=2,
num_items=5,
path="/db/guillotina/")
cresp, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"title": "UniqueIndexContent",
"id": "foobar",
}),
)
await add_content(requester,
num_folders=2,
num_items=5,
path="/db/guillotina/foobar") # noqa
search = get_utility(ICatalogUtility)
content_index_name = (
"guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])))
container, request, txn, tm = await setup_txn_on_container(requester)
task_vars.request.set(request)
await asyncio.sleep(1)
async def _test():
assert await search.get_doc_count(container) == 13
assert (await search.get_doc_count(index_name=content_index_name
) == 12) # noqa
await run_with_retries(_test, requester)
for key in await container.async_keys():
if key == "foobar":
continue
ob = await container.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
foobar = await container.async_get("foobar")
for key in await foobar.async_keys():
ob = await foobar.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
await search.index(
container, {"foobar1": {
"title": "foobar",
"type_name": "Item"
}})
await search.index(
container,
{
"foobar2": {
"title": "foobar",
"type_name": "Item",
"__indexes__": [content_index_name],
}
},
)
async def __test():
assert await search.get_doc_count(container) == 2
assert (await search.get_doc_count(index_name=content_index_name
) == 1) # noqa
await run_with_retries(__test, requester)
vacuum = Vacuum(txn, tm, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 2
assert len(vacuum.out_of_date) == 0
assert len(vacuum.missing) == 24
async def ___test():
assert await search.get_doc_count(container) == 13
assert (await search.get_doc_count(index_name=content_index_name
) == 12) # noqa
await run_with_retries(___test, requester)
await tm.abort(txn=txn)