def __call__(self):
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
for obj in self.objects(
{'object_provides': IDexterityContent.__identifier__},
'Reindex path_depth in Solr for all DX objs.'):
processor.index(obj, ['path_depth'])
def reindex_object_security_without_children(obj):
"""Reindex only the security indices of the passed in object.
Sometimes we know we do not need to also reindex the security of all the
child objects, so we can directly go to what the upstream implementation
does per object.
As we're bypassing the normal machinery, likewise we need to do Solr
indexing by hand.
"""
catalog = api.portal.get_tool('portal_catalog')
security_idxs = CatalogAware._cmf_security_indexes
catalog.reindexObject(obj, idxs=security_idxs, update_metadata=False)
if is_solr_feature_enabled():
# Using catalogs reindexObject does not trigger corresponding solr
# reindexing, so we do it manually.
# Register collective.indexing hook, to make sure solr changes
# are realy send to solr. See
# collective.indexing.queue.IndexQueue.hook.
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
processor.index(obj, security_idxs)
def reindex_object_security_without_children(obj):
"""Reindex only the security indices of the passed in object.
Sometimes we know we do not need to also reindex the security of all the
child objects, so we can directly go to what the upstream implementation
does per object.
As we're bypassing the normal machinery, likewise we need to do Solr
indexing by hand.
"""
catalog = api.portal.get_tool('portal_catalog')
security_idxs = CatalogAware._cmf_security_indexes
catalog.reindexObject(obj, idxs=security_idxs, update_metadata=False)
if is_solr_feature_enabled():
# Using catalogs reindexObject does not trigger corresponding solr
# reindexing, so we do it manually.
# Register collective.indexing hook, to make sure solr changes
# are realy send to solr. See
# collective.indexing.queue.IndexQueue.hook.
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
processor.index(obj, security_idxs)
def test_reindex_object_security_honors_skip_self(self):
self.subfolder.manage_permission('View', roles=['Reader'], acquire=False)
self.subfolder.reindexObjectSecurity(skip_self=True)
getQueue().process()
self.assertEqual(0, len(self.connection.add.mock_calls))
def test_reindex_object_with_idxs_causes_atomic_update(self):
self.subfolder.reindexObject(idxs=['Title'])
getQueue().process()
self.connection.add.assert_called_once_with({
u'UID': IUUID(self.subfolder),
u'Title': {'set': u'My Subfolder'},
})
def __call__(self):
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
catalog = api.portal.get_tool('portal_catalog')
for obj in self.objects({'object_provides': IDexterityContent.__identifier__},
"Reindex sortable title."):
catalog.reindexObject(obj, idxs=['sortable_title'],
update_metadata=False)
processor.index(obj, ['sortable_title'])
def setUp(self):
super(TestCollectiveIndexingIntegration, self).setUp()
self.portal = self.layer['portal']
login(self.portal, TEST_USER_NAME)
setRoles(self.portal, TEST_USER_ID, ['Manager'])
# Prepare nested folders so 'View' isn't mapped to the 'Reader' role.
# First subtree has a leaf node folder with 'View' acquired, whereas
# the second subtree has a leaf node folder with AQ disabled for 'View'
self.folder = api.content.create(
type='Folder', title='My Folder',
id='folder', container=self.portal)
self.folder.manage_permission('View', roles=['Other'], acquire=False)
self.subfolder = api.content.create(
type='Folder', title='My Subfolder',
id='subfolder', container=self.folder)
self.subfolder.manage_permission('View', roles=['Other'], acquire=True)
self.folder2 = api.content.create(
type='Folder', title='My Folder 2',
id='folder2', container=self.portal)
self.folder2.manage_permission('View', roles=['Other'], acquire=False)
self.subfolder2_without_aq = api.content.create(
type='Folder', title='My Subfolder without acquired permission',
id='subfolder2_without_aq', container=self.folder2)
self.subfolder2_without_aq.manage_permission('View', roles=['Other'], acquire=False)
self.folder.reindexObjectSecurity()
self.folder2.reindexObjectSecurity()
self.subfolder.reindexObjectSecurity()
self.subfolder2_without_aq.reindexObjectSecurity()
# Flush queue to avoid having the above objects getting indexed at
# the end of the transaction, after we already installed the mocks
getQueue().process()
self.manager = MagicMock(name='SolrConnectionManager')
alsoProvides(self.manager, ISolrConnectionManager)
conn = MagicMock(name='SolrConnection')
conn.get = MagicMock(name='get', return_value=SolrResponse(
body=get_data('schema.json'), status=200))
type(self.manager).connection = PropertyMock(return_value=conn)
type(self.manager).schema = PropertyMock(
return_value=SolrSchema(self.manager))
sm = self.portal.getSiteManager()
sm.registerUtility(self.manager, ISolrConnectionManager)
self.connection = self.manager.connection
# Manager is memoized on the ISolrIndexQueueProcessor - reset it
queue_processor = getUtility(ISolrIndexQueueProcessor, name='ftw.solr')
queue_processor._manager = None
def test_reindex_object_security_stops_recursion_early_if_possible(self):
# Both folder2 and subfolder_without_aq haven't previously had 'View'
# mapped to any roles. subfolder_without_aq has Acquisition diabled
# for 'View', and should therefore be skipped during reindex because
# it's effective security index contents don't change.
self.folder2.manage_permission('View', roles=['Reader'], acquire=False)
self.folder2.reindexObjectSecurity()
getQueue().process()
self.connection.add.assert_called_once_with({
u'allowedRolesAndUsers': {'set': [u'Reader']},
u'UID': IUUID(self.folder2)})
def test_reindex_object_security_causes_atomic_update(self):
# Subfolder previously hasn't had 'View' mapped to any roles.
# Explicitly give it to 'Reader' role.
self.subfolder.manage_permission(
'View', roles=['Reader'], acquire=False)
self.subfolder.reindexObjectSecurity()
getQueue().process()
self.connection.add.assert_called_once_with({
u'UID': IUUID(self.subfolder),
u'allowedRolesAndUsers': {'set': [u'Reader']},
})
def test_reindex_object_causes_full_reindex_in_solr(self):
with freeze(datetime(2018, 8, 31, 13, 45)):
self.subfolder.reindexObject()
getQueue().process()
self.connection.add.assert_called_once_with({
u'UID': IUUID(self.subfolder),
u'Title': u'My Subfolder',
u'modified': u'2018-08-31T11:45:00.000Z',
u'SearchableText': 'subfolder My Subfolder ',
u'allowedRolesAndUsers': ['Other'],
u'path': u'/plone/folder/subfolder',
})
def __call__(self):
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
query = {'object_provides': ITask.__identifier__}
catalog = api.portal.get_tool('portal_catalog')
index = self.portal.portal_catalog.Indexes.get(INDEX_NAME)
if index.__class__.__name__ != 'BooleanIndex':
self.catalog_remove_index(INDEX_NAME)
self.catalog_add_index(INDEX_NAME, 'BooleanIndex')
for obj in self.objects(query, 'Reindex is_subtask'):
catalog.reindexObject(obj,
idxs=[INDEX_NAME],
update_metadata=False)
processor.index(obj, [INDEX_NAME])
def recursive_index_security(catalog, obj):
"""This function reindexes the security indexes for an object in a specific catalog
recursively.
It does this by walking down the tree and checking on every object whether a
the security indexes are already up to date.
If the security indexes are already up to date, it stops walking down the tree.
The aim of stopping to walk down is to improve the performance drastically on
large trees.
The expectation is that if a children is up to date but the parent wasnt, the
reason is usally that the children does not inherit the relevant values (for
example it does not inherit the View permission) and thus the grand-children will
also not change, so we can abort wakling down the path.
"""
indexes_to_update = []
for index_name in obj._cmf_security_indexes:
if not is_index_up_to_date(catalog, obj, index_name):
indexes_to_update.append(index_name)
if len(indexes_to_update) > 0:
indexer = getQueue()
indexer.reindex(obj, indexes_to_update)
# We assume that if the parent is up to date, all children are too.
# This basically only walks into the tree untill an object is up to date -
# usually because it does not inherit the security relevant thigns - and then
# stops.
for subobj in obj.objectValues():
recursive_index_security(catalog, subobj)
def __call__(self):
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
catalog = api.portal.get_tool('portal_catalog')
if not self.catalog_has_index('document_type'):
self.catalog_add_index('document_type', 'FieldIndex')
query = {
'object_provides': IBaseDocument.__identifier__,
}
for obj in self.objects(query, "Index document type."):
catalog.reindexObject(obj, idxs=['document_type'],
update_metadata=False)
processor.index(obj, ['document_type'])
def recursive_index_security(catalog, obj):
"""This function reindexes the security indexes for an object in a specific catalog
recursively.
It does this by walking down the tree and checking on every object whether a
the security indexes are already up to date.
If the security indexes are already up to date, it stops walking down the tree.
The aim of stopping to walk down is to improve the performance drastically on
large trees.
The expectation is that if a children is up to date but the parent wasnt, the
reason is usally that the children does not inherit the relevant values (for
example it does not inherit the View permission) and thus the grand-children will
also not change, so we can abort wakling down the path.
"""
indexes_to_update = []
# Since objectValues returns all objects, including placefulworkflow policy
# objects, we have to check if the object is Catalog aware.
if not ICatalogAware.providedBy(obj):
return
for index_name in obj._cmf_security_indexes:
if not is_index_up_to_date(catalog, obj, index_name):
indexes_to_update.append(index_name)
if len(indexes_to_update) > 0:
indexer = getQueue()
indexer.reindex(obj, indexes_to_update)
# We assume that if the parent is up to date, all children are too.
# This basically only walks into the tree untill an object is up to date -
# usually because it does not inherit the security relevant thigns - and then
# stops.
for subobj in obj.objectValues():
recursive_index_security(catalog, subobj)
def __call__(self):
getQueue().hook()
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
catalog = api.portal.get_tool('portal_catalog')
if not self.catalog_has_index('document_type'):
self.catalog_add_index('document_type', 'FieldIndex')
query = {
'object_provides': IBaseDocument.__identifier__,
}
for obj in self.objects(query, "Index document type."):
catalog.reindexObject(obj,
idxs=['document_type'],
update_metadata=False)
processor.index(obj, ['document_type'])
def reindexObject(self, idxs=None):
# `CMFCatalogAware.reindexObject` also updates the modification date
# of the object for the "reindex all" case. unfortunately, some other
# packages like `CMFEditions` check that date to see if the object was
# modified during the request, which fails when it's only set on commit
if idxs in (None, []) and hasattr(aq_base(self), 'notifyModified'):
self.notifyModified()
obj = filterTemporaryItems(self)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.reindex(obj, idxs)
def test_reindex_object_security_is_recursive(self):
# Both folder and subfolder haven't previously had 'View' mapped to
# any roles. Subfolder has 'View' set to acquire though. Giving it
# to 'Reader' on Folder should therefore propagate down to Subfolder.
self.folder.manage_permission('View', roles=['Reader'], acquire=False)
self.folder.reindexObjectSecurity()
getQueue().process()
expected_calls = [
call({
'allowedRolesAndUsers': {'set': [u'Reader']},
u'UID': IUUID(self.folder)}),
call({
'allowedRolesAndUsers': {'set': [u'Other', u'Reader']},
u'UID': IUUID(self.subfolder)})
]
self.assertEqual(2, len(self.connection.add.mock_calls))
# XXX: Why isn't call order stable here?
self.connection.add.assert_has_calls(expected_calls, any_order=True)
def reindexObject(self, idxs=None):
# `CMFCatalogAware.reindexObject` also updates the modification date
# of the object for the "reindex all" case. unfortunately, some other
# packages like `CMFEditions` check that date to see if the object was
# modified during the request, which fails when it's only set on commit
if idxs in (None, []) and hasattr(aq_base(self), 'notifyModified'):
self.notifyModified()
obj = filterTemporaryItems(self)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.reindex(obj, idxs)
def objectMoved(ev):
if ev.newParent is None or ev.oldParent is None:
# it's an IObjectRemovedEvent or IObjectAddedEvent
return
if ev.newParent is ev.oldParent:
# it's a renaming operation
dispatchToSublocations(ev.object, ev)
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.index(obj)
def objectMoved(ev):
if ev.newParent is None or ev.oldParent is None:
# it's an IObjectRemovedEvent or IObjectAddedEvent
return
if ev.newParent is ev.oldParent:
# it's a renaming operation
dispatchToSublocations(ev.object, ev)
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.index(obj)
def __iter__(self):
for item in self.previous:
pathkey = self.pathkey(*item.keys())[0]
if not pathkey: # not enough info
yield item
continue
path = item[pathkey]
ob = traverse(self.context, str(path).lstrip('/'), None)
if ob is None:
yield item
continue # object not found
if not isinstance(ob, CatalogAware):
yield item
continue # can't notify portal_catalog
if self.verbose: # add a log to display reindexation progess
self.counter += 1
logger.info('Reindex object %s (%s)', path, self.counter)
# update catalog
if self.indexes:
self.portal_catalog.reindexObject(ob, idxs=self.indexes)
else:
self.portal_catalog.reindexObject(ob)
# solr reindexing
if self.solr_enabled:
# Register collective.indexing hook, to make sure solr changes
# are realy send to solr. See
# collective.indexing.queue.IndexQueue.hook.
getQueue().hook()
# Using catalogs reindexObject does not trigger corresponding solr
# reindexing, so we do it manually.
processor = getUtility(IIndexQueueProcessor, name='ftw.solr')
processor.index(ob)
yield item
def objectModified(ev):
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is None or indexer is None:
return
if getattr(ev, 'descriptions', None): # not used by archetypes/plone atm
# build the list of to be updated attributes
attrs = []
for desc in ev.descriptions:
if isinstance(desc, Attributes):
attrs.extend(desc.attributes)
indexer.reindex(obj, attrs)
if 'allow' in attrs: # dispatch to sublocations on security changes
dispatchToSublocations(ev.object, ev)
else:
# with no descriptions (of changed attributes) just reindex all
indexer.reindex(obj)
def objectModified(ev):
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is None or indexer is None:
return
if getattr(ev, 'descriptions', None): # not used by archetypes/plone atm
# build the list of to be updated attributes
attrs = []
for desc in ev.descriptions:
if isinstance(desc, Attributes):
attrs.extend(desc.attributes)
indexer.reindex(obj, attrs)
if 'allow' in attrs: # dispatch to sublocations on security changes
dispatchToSublocations(ev.object, ev)
else:
# with no descriptions (of changed attributes) just reindex all
indexer.reindex(obj)
def unindexObject(self):
obj = filterTemporaryItems(self, checkId=False)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.unindex(obj)
def objectRemoved(ev):
obj = filterTemporaryItems(ev.object, checkId=False)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.unindex(obj)
def objectAdded(ev):
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.index(obj)
def indexObject(self):
obj = filterTemporaryItems(self)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.index(obj)
def unindexObject(self):
obj = filterTemporaryItems(self, checkId=False)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.unindex(obj)
def indexObject(self):
obj = filterTemporaryItems(self)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.index(obj)
def test_system_roles_dont_mistakenly_terminate_recursion(self):
"""This test makes sure that the special shortcut treatment of the
'Authenticated' and 'Anonymous' roles in the allowedRolesAndUsers
indexer doesn't mistakenly stop down propagation of security
reindexing in our reindexObjectSecurity patch.
"""
# Prepare a situation as follows:
#
# - folder2 has the View permission assigned to Authenticated and
# Reader roles
# - Its child, subfolder2_without_aq, only has View assigned to Reader,
# and it doesn't acquire the View permission
# - Give the user the local role Reader on the container folder2
# - Because of local role inheritance, the user also has Reader on
# subfolder2_without_aq. This is the *only* setting by which he
# gets the 'View' permission on subfolder2_without_aq
self.folder2.manage_permission(
'View', roles=['Authenticated', 'Reader'], acquire=False)
self.subfolder2_without_aq.manage_permission(
'View', roles=['Reader'], acquire=False)
self.folder2.manage_setLocalRoles(TEST_USER_ID, ['Reader'])
# Guard assertion: Make sure the user doesn't have any local roles
# other than Owner on subfolder2_without_aq
self.assertEqual(
(('test_user_1_', ('Owner',)),),
self.subfolder2_without_aq.get_local_roles())
self.folder2.reindexObjectSecurity()
self.subfolder2_without_aq.reindexObjectSecurity()
getQueue().process()
expected_calls = [
# folder2 only has the Authenticated role indexed (but not Reader)
# because of the shortcut in the allowedRolesAndUsers indexer
call({
'allowedRolesAndUsers': {'set': [u'Authenticated']},
u'UID': IUUID(self.folder2)}),
# subfolder2_without_aq has TEST_USER_ID in its
# allowedRolesAndUsers because he gets View via the inherited
# Reader local role
call({
'allowedRolesAndUsers': {'set': [u'user:test_user_1_', u'Reader']},
u'UID': IUUID(self.subfolder2_without_aq)})
]
self.assertEqual(2, len(self.connection.add.mock_calls))
self.connection.add.assert_has_calls(expected_calls, any_order=True)
self.connection.add.reset_mock()
# Now remove the Reader local role for TEST_USER_ID
self.folder2.manage_setLocalRoles(TEST_USER_ID, ['Authenticated'])
self.folder2.reindexObjectSecurity()
getQueue().process()
expected_calls = [
call({
'allowedRolesAndUsers': {'set': [u'Authenticated']},
u'UID': IUUID(self.folder2)}),
# TEST_USER_ID should be gone from allowedRolesAndUsers, because
# he doesn't inherit the Reader local role anymore
call({
'allowedRolesAndUsers': {'set': [u'Reader']},
u'UID': IUUID(self.subfolder2_without_aq)})
]
self.assertEqual(2, len(self.connection.add.mock_calls))
self.connection.add.assert_has_calls(expected_calls, any_order=True)
def objectRemoved(ev):
obj = filterTemporaryItems(ev.object, checkId=False)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.unindex(obj)
def setUp(self):
self.me = getQueue()
self.failUnless(IIndexQueue.providedBy(self.me),
'non-queued indexer found')
def objectAdded(ev):
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.index(obj)
def objectTransitioned(ev):
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.reindex(obj)
def get_indexing_queue_length(self):
return getQueue().length()
def objectTransitioned(ev):
obj = filterTemporaryItems(ev.object)
indexer = getQueue()
if obj is not None and indexer is not None:
indexer.reindex(obj)
def setUp(self):
self.me = getQueue()
self.failUnless(IIndexQueue.providedBy(self.me), "non-queued indexer found")
def get_indexing_queue_length(self):
return getQueue().length()
