async def test_migrate_content_index_works(es_requester):
async with es_requester as requester:
add_count = await add_content(requester)
cresp, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"id": "foobar"
}),
)
await requester(
"POST",
"/db/guillotina/foobar",
data=json.dumps({"@type": "IndexItemContent"}),
)
container, request, txn, tm = await setup_txn_on_container(requester)
search = get_utility(ICatalogUtility)
await search.refresh(container)
await asyncio.sleep(3)
assert (add_count + 1) == await search.get_doc_count(
container, "guillotina-db-guillotina_1")
assert (await search.get_doc_count(
container,
"1_guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])),
) == 1)
migrator = Migrator(search, container, force=True)
await migrator.run_migration()
assert await search.get_connection().indices.exists(
"guillotina-db-guillotina_2")
assert not await search.get_connection().indices.exists(
"guillotina-db-guillotina_1")
assert await search.get_connection().indices.exists(
"2_guillotina-db-guillotina__uniqueindexcontent-{}".format(
get_short_uid(cresp["@uid"])))
assert not await search.get_connection().indices.exists(
"1_guillotina-db-guillotina__uniqueindexcontent-{}".format(
get_short_uid(cresp["@uid"])))
assert (add_count + 1) == await search.get_doc_count(
container, "guillotina-db-guillotina_2")
assert (await search.get_doc_count(
container,
"2_guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])),
) == 1)
async def test_delete_base_removes_index_from_elastic(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
cresp, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"title": "UniqueIndexContent",
"id": "foobar",
}),
)
resp, _ = await requester(
"POST",
"/db/guillotina/foobar",
data=json.dumps({
"@type": "IndexItemContent",
"title": "IndexItemContent",
"id": "foobar",
}),
)
catalog = get_utility(ICatalogUtility)
await requester("DELETE", "/db/guillotina/foobar")
content_index_name = (
"guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])))
async def _test():
# should find in content index but not main index
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await catalog.get_connection().get(index=content_index_name,
doc_type="_all",
id=resp["@uid"])
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await catalog.get_connection().get(
index="guillotina-guillotina",
doc_type="_all",
id=cresp["@uid"])
await run_with_retries(_test, requester)
assert not await catalog.get_connection().indices.exists_alias(
"guillotina-db-guillotina__uniqueindexcontent-{}".format(
get_short_uid(resp["@uid"])))
assert not await catalog.get_connection().indices.exists(
"1_guillotina-db-guillotina__uniqueindexcontent-{}".format(
get_short_uid(resp["@uid"])))
async def test_create_index(es_requester):
async with es_requester as requester:
resp, status = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"title": "UniqueIndexContent",
"id": "foobar",
}),
)
catalog = get_utility(ICatalogUtility)
assert status == 201
# assert indexes were created
assert await catalog.get_connection().indices.exists_alias(
"guillotina-db-guillotina__uniqueindexcontent-{}".format(
get_short_uid(resp["@uid"])))
assert await catalog.get_connection().indices.exists(
"1_guillotina-db-guillotina__uniqueindexcontent-{}".format(
get_short_uid(resp["@uid"])))
def _generate_new_index_name(self):
"""
index name structure is:
- {settings-prefix}{container id}__{type}-{short uid}
"""
container_name = super()._generate_new_index_name()
return "{}{}{}-{}".format(
container_name,
SUB_INDEX_SEPERATOR,
self.context.type_name.lower(),
get_short_uid(self.context.__uuid__),
)
async def test_delete_resource(es_requester):
async with es_requester as requester:
cresp, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"title": "UniqueIndexContent",
"id": "foobar",
}),
)
resp, status = await requester(
"POST",
"/db/guillotina/foobar",
data=json.dumps({
"@type": "IndexItemContent",
"title": "IndexItemContent",
"id": "foobar",
}),
)
assert status == 201
content_index_name = (
"guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])))
search = get_utility(ICatalogUtility)
async def _test():
# should find in content index but not main index
result = await search.get_connection().get(
index=content_index_name, doc_type="_all", id=resp["@uid"])
assert result is not None
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await search.get_connection().get(
index="guillotina-guillotina",
doc_type="_all",
id=resp["@uid"])
await run_with_retries(_test, requester)
# now, delete it
await requester("DELETE", "/db/guillotina/foobar/foobar")
async def _test():
# should find in content index but not main index
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await search.get_connection().get(index=content_index_name,
doc_type="_all",
id=resp["@uid"])
await run_with_retries(_test, requester)
def __call__(self, id, parent=None, *args, **kw):
obj = super(ResourceFactory, self).__call__(*args, **kw)
if parent is not None:
obj.__parent__ = parent
obj.type_name = self.type_name
now = datetime.now(tz=_zone)
obj.creation_date = now
obj.modification_date = now
if id is None:
if obj.__uuid__ is None:
obj.__uuid__ = app_settings["uid_generator"](obj)
obj.id = uid.get_short_uid(obj.__uuid__)
else:
obj.id = id
obj.__name__ = obj.id
apply_markers(obj)
return obj
async def test_elastic_index_field(es_requester):
async with es_requester as requester:
cresp, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"title": "UniqueIndexContent",
"id": "foobar",
}),
)
resp, _ = await requester(
"POST",
"/db/guillotina/foobar",
data=json.dumps({
"@type": "IndexItemContent",
"title": "IndexItemContent",
"id": "foobar",
}),
)
content_index_name = (
"guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])))
search = get_utility(ICatalogUtility)
async def _test():
result = await search.get_connection().get(
index="guillotina-db-guillotina",
doc_type="_all",
id=cresp["@uid"])
assert result["_source"]["elastic_index"] == content_index_name
result = await search.get_connection().get(
index=content_index_name, doc_type="_all", id=resp["@uid"])
assert result["_source"]["title"] == "IndexItemContent"
await run_with_retries(_test, requester)
async def test_vacuum_with_sub_indexes(es_requester):
async with es_requester as requester:
await add_content(requester,
num_folders=2,
num_items=5,
path="/db/guillotina/")
cresp, _ = await requester(
"POST",
"/db/guillotina/",
data=json.dumps({
"@type": "UniqueIndexContent",
"title": "UniqueIndexContent",
"id": "foobar",
}),
)
await add_content(requester,
num_folders=2,
num_items=5,
path="/db/guillotina/foobar") # noqa
search = get_utility(ICatalogUtility)
content_index_name = (
"guillotina-db-guillotina__uniqueindexcontent-{}".format( # noqa
get_short_uid(cresp["@uid"])))
container, request, txn, tm = await setup_txn_on_container(requester)
task_vars.request.set(request)
await asyncio.sleep(1)
async def _test():
assert await search.get_doc_count(container) == 13
assert (await search.get_doc_count(index_name=content_index_name
) == 12) # noqa
await run_with_retries(_test, requester)
for key in await container.async_keys():
if key == "foobar":
continue
ob = await container.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
foobar = await container.async_get("foobar")
for key in await foobar.async_keys():
ob = await foobar.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
await search.index(
container, {"foobar1": {
"title": "foobar",
"type_name": "Item"
}})
await search.index(
container,
{
"foobar2": {
"title": "foobar",
"type_name": "Item",
"__indexes__": [content_index_name],
}
},
)
async def __test():
assert await search.get_doc_count(container) == 2
assert (await search.get_doc_count(index_name=content_index_name
) == 1) # noqa
await run_with_retries(__test, requester)
vacuum = Vacuum(txn, tm, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 2
assert len(vacuum.out_of_date) == 0
assert len(vacuum.missing) == 24
async def ___test():
assert await search.get_doc_count(container) == 13
assert (await search.get_doc_count(index_name=content_index_name
) == 12) # noqa
await run_with_retries(___test, requester)
await tm.abort(txn=txn)