async def test_elastic_index_field(es_requester):
async with es_requester as requester:
cresp, _ = await requester(
'POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'title': 'UniqueIndexContent',
'id': 'foobar'
})
)
resp, _ = await requester(
'POST',
'/db/guillotina/foobar',
data=json.dumps({
'@type': 'IndexItemContent',
'title': 'IndexItemContent',
'id': 'foobar'
})
)
content_index_name = 'guillotina-db-guillotina__uniqueindexcontent-{}'.format( # noqa
get_short_oid(cresp['@uid'])
)
search = get_utility(ICatalogUtility)
async def _test():
result = await search.conn.get(
index='guillotina-db-guillotina',
doc_type='_all', id=cresp['@uid'])
assert result['_source']['elastic_index'] == content_index_name
result = await search.conn.get(
index=content_index_name, doc_type='_all', id=resp['@uid'])
assert result['_source']['title'] == 'IndexItemContent'
await run_with_retries(_test, requester)
async def test_indexes_data_in_correct_indexes(es_requester):
async with es_requester as requester:
cresp, _ = await requester('POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'title': 'UniqueIndexContent',
'id': 'foobar'
}))
resp, status = await requester('POST',
'/db/guillotina/foobar',
data=json.dumps({
'@type': 'IndexItemContent',
'title': 'IndexItemContent',
'id': 'foobar'
}))
assert status == 201
content_index_name = 'guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(cresp['@uid']))
search = get_utility(ICatalogUtility)
async def _test():
# should find in content index but not main index
result = await search.conn.get(index=content_index_name,
doc_type='_all',
id=resp['@uid'])
assert result is not None
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await search.conn.get(index='guillotina-guillotina',
doc_type='_all',
id=resp['@uid'])
await run_with_retries(_test, requester)
async def test_delete_base_removes_index_from_elastic(es_requester):
async with es_requester as requester:
cresp, _ = await requester(
'POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'title': 'UniqueIndexContent',
'id': 'foobar'
})
)
resp, _ = await requester(
'POST',
'/db/guillotina/foobar',
data=json.dumps({
'@type': 'IndexItemContent',
'title': 'IndexItemContent',
'id': 'foobar'
})
)
catalog = get_utility(ICatalogUtility)
await requester('DELETE', '/db/guillotina/foobar')
content_index_name = 'guillotina-db-guillotina__uniqueindexcontent-{}'.format( # noqa
get_short_oid(cresp['@uid'])
)
async def _test():
# should find in content index but not main index
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await catalog.conn.get(
index=content_index_name, doc_type='_all', id=resp['@uid'])
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await catalog.conn.get(
index='guillotina-guillotina',
doc_type='_all', id=cresp['@uid'])
await run_with_retries(_test, requester)
assert not await catalog.conn.indices.exists_alias(
'guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(resp['@uid'])
))
assert not await catalog.conn.indices.exists(
'1_guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(resp['@uid'])
))
async def test_migrate_content_index_works(es_requester):
async with es_requester as requester:
add_count = await add_content(requester)
cresp, _ = await requester('POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'id': 'foobar'
}))
await requester('POST',
'/db/guillotina/foobar',
data=json.dumps({'@type': 'IndexItemContent'}))
container, request, txn, tm = await setup_txn_on_container(requester)
search = get_utility(ICatalogUtility)
await search.refresh(container)
await asyncio.sleep(3)
assert (add_count + 1) == await search.get_doc_count(
container, 'guillotina-db-guillotina_1')
assert await search.get_doc_count(
container,
'1_guillotina-db-guillotina__uniqueindexcontent-{}'.format( # noqa
get_short_oid(cresp['@uid']))) == 1
migrator = Migrator(search, container, force=True)
await migrator.run_migration()
assert await search.conn.indices.exists('guillotina-db-guillotina_2')
assert not await search.conn.indices.exists(
'guillotina-db-guillotina_1')
assert await search.conn.indices.exists(
'2_guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(cresp['@uid'])))
assert not await search.conn.indices.exists(
'1_guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(cresp['@uid'])))
assert (add_count + 1) == await search.get_doc_count(
container, 'guillotina-db-guillotina_2')
assert await search.get_doc_count(
container,
'2_guillotina-db-guillotina__uniqueindexcontent-{}'.format( # noqa
get_short_oid(cresp['@uid']))) == 1
def _generate_new_index_name(self):
'''
index name structure is:
- {settings-prefix}{container id}__{type}-{short uid}
'''
container_name = super()._generate_new_index_name()
return '{}{}{}-{}'.format(container_name, SUB_INDEX_SEPERATOR,
self.context.type_name.lower(),
get_short_oid(self.context._p_oid))
async def test_create_index(es_requester):
async with es_requester as requester:
resp, status = await requester('POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'title': 'UniqueIndexContent',
'id': 'foobar'
}))
catalog = get_utility(ICatalogUtility)
assert status == 201
# assert indexes were created
assert await catalog.conn.indices.exists_alias(
'guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(resp['@uid'])))
assert await catalog.conn.indices.exists(
'1_guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(resp['@uid'])))
def __call__(self, id, parent=None, *args, **kw):
obj = super(ResourceFactory, self).__call__(*args, **kw)
if parent is not None:
obj.__parent__ = parent
obj.type_name = self.type_name
now = datetime.now(tz=_zone)
obj.creation_date = now
obj.modification_date = now
if id is None:
if obj._p_oid is None:
obj._p_oid = app_settings['oid_generator'](obj)
obj.id = oid.get_short_oid(obj._p_oid)
else:
obj.id = id
obj.__name__ = obj.id
apply_markers(obj)
return obj
async def test_vacuum_with_sub_indexes(es_requester):
async with es_requester as requester:
await add_content(requester, num_folders=2, num_items=5, path='/db/guillotina/')
cresp, _ = await requester(
'POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'UniqueIndexContent',
'title': 'UniqueIndexContent',
'id': 'foobar'
})
)
await add_content(requester, num_folders=2, num_items=5, path='/db/guillotina/foobar')
search = get_utility(ICatalogUtility)
content_index_name = 'guillotina-db-guillotina__uniqueindexcontent-{}'.format(
get_short_oid(cresp['@uid'])
)
container, request, txn, tm = await setup_txn_on_container(requester)
aiotask_context.set('request', request)
async def _test():
assert await search.get_doc_count(container) == 13
assert await search.get_doc_count(index_name=content_index_name) == 12
await run_with_retries(_test, requester)
for key in await container.async_keys():
if key == 'foobar':
continue
ob = await container.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
foobar = await container.async_get('foobar')
for key in await foobar.async_keys():
ob = await foobar.async_get(key)
await search.remove(container, [ob], request=request)
await asyncio.sleep(1)
await search.index(container, {
'foobar1': {
'title': 'foobar',
'type_name': 'Item'
}
})
await search.index(container, {
'foobar2': {
'title': 'foobar',
'type_name': 'Item',
'__indexes__': [content_index_name]
}
})
async def __test():
assert await search.get_doc_count(container) == 2
assert await search.get_doc_count(index_name=content_index_name) == 1
await run_with_retries(__test, requester)
vacuum = Vacuum(txn, tm, request, container)
await vacuum.setup()
await vacuum.check_missing()
await vacuum.check_orphans()
assert len(vacuum.orphaned) == 2
assert len(vacuum.out_of_date) == 0
assert len(vacuum.missing) == 24
async def ___test():
assert await search.get_doc_count(container) == 13
assert await search.get_doc_count(index_name=content_index_name) == 12
await run_with_retries(___test, requester)
await tm.abort(txn=txn)