def searchResults(self, query=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
# Make sure any pending index tasks have been processed
processQueue()
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(query, dict) and not show_inactive:
show_inactive = 'show_inactive' in query
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not self.allow_inactive(kw):
kw['effectiveRange'] = DateTime()
sort_on = kw.get('sort_on')
if sort_on and sort_on not in self.indexes():
# I get crazy sort_ons like '194' or 'null'.
kw.pop('sort_on')
return ZCatalog.searchResults(self, query, **kw)
def searchResults(self, query=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
# Make sure any pending index tasks have been processed
processQueue()
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(query, dict) and not show_inactive:
show_inactive = 'show_inactive' in query
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not self.allow_inactive(kw):
kw['effectiveRange'] = DateTime()
sort_on = kw.get('sort_on')
if sort_on and sort_on not in self.indexes():
# I get crazy sort_ons like '194' or 'null'.
kw.pop('sort_on')
return ZCatalog.searchResults(self, query, **kw)
def getKeywords(self, context=None, indexName="Subject"):
processQueue()
if indexName not in self.getKeywordIndexes():
raise ValueError("%s is not a valid field" % indexName)
catalog = getToolByName(self, "portal_catalog")
keywords = catalog.uniqueValuesFor(indexName)
return list(sorted(keywords, key=lambda x: x.lower()))
def afterSetUp(self):
self.catalog = self.portal.portal_catalog
self.folder.invokeFactory('Document',
id='doc',
title='Foo',
description='Bar')
self.catalog.unindexObject(self.folder.doc)
processQueue()
def getKeywords(self, context=None, indexName="Subject"):
processQueue()
if indexName not in self.getKeywordIndexes():
raise ValueError("%s is not a valid field" % indexName)
catalog = getToolByName(self, "portal_catalog")
keywords = catalog.uniqueValuesFor(indexName)
return list(sorted(keywords, key=lambda x: x.lower()))
def test_delete_object(self):
obj = createObject(self.portal,
'Event',
'event_to_delete',
title='Some Event')
obj_uid = getUID(obj)
self.portal.manage_delObjects(['event_to_delete'])
processQueue()
self.assertEqual(self.get_hook().index, {})
self.assertEqual(self.get_hook().remove, {obj_uid})
def searchResults(self, REQUEST=None, **kw):
"""
Calls ZCatalog.searchResults with extra arguments that
limit the results to what the user is allowed to see.
"""
processQueue()
user = getSecurityManager().getUser()
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not _checkPermission(AccessInactivePortalContent, self):
now = DateTime()
self._convertQuery(kw)
# Intersect query restrictions with those implicit to the tool
for k in 'effective', 'expires':
if k in kw:
range = kw[k]['range'] or ''
query = kw[k]['query']
if not isinstance(query, (tuple, list)):
query = (query, )
else:
range = ''
query = None
if range.find('min') > -1:
lo = min(query)
else:
lo = None
if range.find('max') > -1:
hi = max(query)
else:
hi = None
if k == 'effective':
if hi is None or hi > now:
hi = now
if lo is not None and hi < lo:
return ()
else: # 'expires':
if lo is None or lo < now:
lo = now
if hi is not None and hi < lo:
return ()
# Rebuild a query
if lo is None:
query = hi
range = 'max'
elif hi is None:
query = lo
range = 'min'
else:
query = (lo, hi)
range = 'min:max'
kw[k] = {'query': query, 'range': range}
return ZCatalog.searchResults(self, REQUEST, **kw)
def test_has_right_brain_data(self):
current_length = len(self.catalog._catalog.uids)
obj = createObject(self.portal, 'Event', 'event', title='Some Event')
self.assertEqual(current_length + 1, len(self.catalog._catalog.uids))
self.assertEqual(self.get_hook().index, {getUID(obj): obj})
self.portal.manage_delObjects(['event'])
# uid not actually removed until this if catalog optimized
processQueue()
self.assertEqual(current_length, len(self.catalog._catalog.uids))
self.assertEqual(self.get_hook().remove, {getUID(obj)})
self.assertEqual(self.get_hook().index, {})
def ensure_plone_catalog_queue_processed():
"""In order for marking added objects as safe in plone.protect,
we must ensure that the catalog is processed and the data structures
actually are created at this point.
"""
try:
from Products.CMFCore.indexing import processQueue
except ImportError:
pass
else:
processQueue()
def testReindexTitleOnly(self):
# Reindexing should only index the Title
self.catalog.indexObject(self.folder.doc)
processQueue()
self.folder.doc.setTitle('Fred')
self.folder.doc.setDescription('BamBam')
self.catalog.reindexObject(self.folder.doc, idxs=['Title'])
self.assertEqual(len(self.catalog(getId='doc')), 1)
self.assertEqual(len(self.catalog(Title='Fred')), 1)
# Description index did not change
self.assertEqual(len(self.catalog(Description='Bar')), 1)
self.assertEqual(len(self.catalog(Description='BamBam')), 0)
def testReindexObjectSkipsMetadata(self):
# Reindexing should not update metadata when update_metadata=0
self.catalog.indexObject(self.folder.doc)
processQueue()
self.folder.doc.setTitle('Fred')
self.folder.doc.setDescription('BamBam')
self.catalog.reindexObject(self.folder.doc, update_metadata=0)
brain = self.catalog(getId='doc')[0]
# Metadata did not change
self.assertEqual(brain.getId, 'doc')
self.assertEqual(brain.Title, 'Foo')
self.assertEqual(brain.Description, 'Bar')
def setUpPloneSite(self, portal):
setRoles(portal, TEST_USER_ID, ['Manager'])
wftool = getToolByName(portal, 'portal_workflow')
wftool.setDefaultChain('simple_publication_workflow')
portal.invokeFactory('Folder', 'test-folder')
portal.invokeFactory('Document', 'front-page')
portal.invokeFactory('Folder', 'news')
wftool.doActionFor(portal.news, 'publish')
portal.news.invokeFactory('News Item', 'news1')
setRoles(portal, TEST_USER_ID, ['Member'])
from Products.CMFCore.indexing import processQueue
processQueue()
def search(self, query, sort_index=None, reverse=0, limit=None, merge=1):
# Wrap search() the same way that searchResults() is
# Make sure any pending index tasks have been processed
processQueue()
user = _getAuthenticatedUser(self)
query['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not self.allow_inactive(query):
query['effectiveRange'] = DateTime()
return super().search(query, sort_index, reverse, limit, merge)
def unrestrictedSearchResults(self, REQUEST=None, **kw):
"""Calls ZCatalog.searchResults directly without restrictions.
This method returns every also not yet effective and already expired
objects regardless of the roles the caller has.
CAUTION: Care must be taken not to open security holes by
exposing the results of this method to non authorized callers!
If you're in doubt if you should use this method or
'searchResults' use the latter.
"""
processQueue()
return ZCatalog.searchResults(self, REQUEST, **kw)
def search(self, query,
sort_index=None, reverse=0, limit=None, merge=1):
# Wrap search() the same way that searchResults() is
# Make sure any pending index tasks have been processed
processQueue()
user = _getAuthenticatedUser(self)
query['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not self.allow_inactive(query):
query['effectiveRange'] = DateTime()
return super(CatalogTool, self).search(
query, sort_index, reverse, limit, merge)
def testIndexIdOnly(self):
# Indexing should only index the id
#
# TODO: Demonstrate that the behavior is independent
# of index type.
#
self.catalog.indexObject(self.folder.doc, idxs=['getId'])
processQueue()
# The document is cataloged
path = self.catalog._CatalogTool__url(self.folder.doc)
self.assertTrue(path in self.catalog._catalog.paths.values())
# But it is not returned when searching...
self.assertEqual(len(self.catalog(getId='doc')), 0) # <-- Should be 1
self.assertEqual(len(self.catalog(Title='Foo')), 0)
self.assertEqual(len(self.catalog(Description='Bar')), 0)
def testIndexTitleOnly(self):
# Indexing should only index the Title
#
# TODO: This does not work as expected. The object
# appears to be in the catalog but is not returned
# by searchResults()!?
#
self.catalog.indexObject(self.folder.doc, idxs=['Title'])
processQueue()
# The document is cataloged
path = self.catalog._CatalogTool__url(self.folder.doc)
self.assertTrue(path in self.catalog._catalog.paths.values())
# But it is not returned when searching...
self.assertEqual(len(self.catalog(getId='doc')), 0)
self.assertEqual(len(self.catalog(Title='Foo')), 0) # <-- Should be 1
self.assertEqual(len(self.catalog(Description='Bar')), 0)
def setUp(self):
super(IntegrationTestCase, self).setUp()
self.portal = self.layer['portal']
self.request = self.layer
setRoles(self.portal, TEST_USER_ID, ['Manager'])
self.view = self.portal.restrictedTraverse('@@mass-edit')
self.portal.invokeFactory('Document', 'page1')
self.portal.invokeFactory('Document', 'page2')
self.page1 = self.portal['page1']
self.page2 = self.portal['page2']
directlyProvides(self.page1, IMassEditTest)
directlyProvides(self.page2, IMassEditTest)
self.page1.reindexObject()
self.page2.reindexObject()
processQueue()
def testUpdateTypeIndex(self):
# conjure up incorrect catalog data...
info = self.portal.portal_types.getTypeInfo('File')
info.title = 'Foo'
foo = self.folder[self.folder.invokeFactory('File', id='foo')]
processQueue()
info.title = 'File'
# make sure it's actually wrong...
catalog = self.portal.portal_catalog
self.assertFalse(catalog(Type='File'))
self.assertTrue(catalog(Type='Foo'))
# fix using the maintenance view & check again...
maintenance = self.portal.unrestrictedTraverse('blob-maintenance')
maintenance.updateTypeIndex()
self.assertEqual([b.getObject() for b in catalog(Type='File')], [foo])
self.assertFalse(catalog(Type='Foo'))
def test_reference_vocab(self):
self.dummy.Schema()['field'].set(self.dummy, 'getRawRelatedItems')
self.folder.invokeFactory('Document', 'doc1')
self.folder.invokeFactory('Document', 'doc2')
uid1 = self.folder.doc1.UID()
uid2 = self.folder.doc2.UID()
self.folder.doc1.setRelatedItems([uid2])
self.folder.doc2.setRelatedItems([uid1])
self.folder.doc1.reindexObject()
self.folder.doc2.reindexObject()
processQueue()
vocab = self.dummy.getCurrentValues()
self.assertEqual(len(vocab), 2)
self.assertTrue(uid1 in vocab.keys())
self.assertTrue(uid2 in vocab.keys())
self.assertTrue('doc1' in vocab.values())
self.assertTrue('doc2' in vocab.values())
def test_reference_vocab(self):
self.dummy.Schema()['field'].set(self.dummy, 'getRawRelatedItems')
self.folder.invokeFactory('Document', 'doc1')
self.folder.invokeFactory('Document', 'doc2')
uid1 = self.folder.doc1.UID()
uid2 = self.folder.doc2.UID()
self.folder.doc1.setRelatedItems([uid2])
self.folder.doc2.setRelatedItems([uid1])
self.folder.doc1.reindexObject()
self.folder.doc2.reindexObject()
processQueue()
vocab = self.dummy.getCurrentValues()
self.assertEqual(len(vocab), 2)
self.assertTrue(uid1 in vocab.keys())
self.assertTrue(uid2 in vocab.keys())
self.assertTrue('doc1' in vocab.values())
self.assertTrue('doc2' in vocab.values())
def testReindexNumericalTitle(self):
from Products.CMFCore.utils import getToolByName
# Create 2 pages, one with a numerical title
portal = self.portal
self.setRoles(['Manager'])
catalog = getToolByName(portal, 'portal_catalog')
portal.invokeFactory(
id='num-title',
type_name='Document',
title='10 green bottles, hanging on the wall',
)
portal.invokeFactory(
id='accidentally-fall',
type_name='Document',
title='And if one green bottle should accidentally fall',
)
processQueue()
# Change title of both, shouldn't be reindexed yet
portal['accidentally-fall'].title = 'fell'
portal['num-title'].title = '9 green bottles, hanging on the wall'
self.assertEqual(
catalog(id=portal['num-title'].id)[0].Title,
'10 green bottles, hanging on the wall',
)
self.assertEqual(
catalog(id=portal['accidentally-fall'].id)[0].Title,
'And if one green bottle should accidentally fall',
)
# Only the numerical title got reindexed
portal.portal_setup.runAllImportStepsFromProfile(
'profile-plone.app.theming:default')
alphas.reindex_sortable_title(portal.portal_setup)
self.assertEqual(
catalog(id=portal['num-title'].id)[0].Title,
'9 green bottles, hanging on the wall',
)
self.assertEqual(
catalog(id=portal['accidentally-fall'].id)[0].Title,
'And if one green bottle should accidentally fall',
)
def afterSetUp(self):
""" After setup
"""
setSite(self.portal)
provideAdapter(ThemeTaggableMerged)
provideAdapter(AttributeAnnotations,
provides=IAnnotations,
adapts=[IThemeTaggable])
classImplements(ATNewsItem, IThemeTaggable)
self.setRoles('Manager')
# create and theme tag a news item
self.portal.invokeFactory('News Item', id='news1')
obj = self.portal.news1
themes = IThemeTagging(obj)
themes.tags = ['agriculture']
self.portal.portal_catalog.reindexObject(self.portal.news1)
self.portal.portal_catalog.reindexObject(self.portal)
processQueue()
def testReindexNumericalTitle(self):
from Products.CMFCore.utils import getToolByName
# Create 2 pages, one with a numerical title
portal = self.portal
self.setRoles(['Manager'])
catalog = getToolByName(portal, 'portal_catalog')
portal.invokeFactory(
id='num-title', type_name='Document',
title='10 green bottles, hanging on the wall',
)
portal.invokeFactory(
id='accidentally-fall', type_name='Document',
title='And if one green bottle should accidentally fall',
)
processQueue()
# Change title of both, shouldn't be reindexed yet
portal['accidentally-fall'].title = 'fell'
portal['num-title'].title = '9 green bottles, hanging on the wall'
self.assertEqual(
catalog(id=portal['num-title'].id)[0].Title,
'10 green bottles, hanging on the wall',
)
self.assertEqual(
catalog(id=portal['accidentally-fall'].id)[0].Title,
'And if one green bottle should accidentally fall',
)
# Only the numerical title got reindexed
portal.portal_setup.runAllImportStepsFromProfile(
'profile-plone.app.theming:default')
alphas.reindex_sortable_title(portal.portal_setup)
self.assertEqual(
catalog(id=portal['num-title'].id)[0].Title,
'9 green bottles, hanging on the wall',
)
self.assertEqual(
catalog(id=portal['accidentally-fall'].id)[0].Title,
'And if one green bottle should accidentally fall',
)
def testVocabularyEncoding(self):
"""The vocabulary should not return the binary encoded token
("N=C3=A5=C3=B8=C3=AF"), but instead the value as the id in the result
set. Fixes an encoding problem. See:
https://github.com/plone/Products.CMFPlone/issues/650
"""
test_val = 'Nåøï'
self.portal.invokeFactory('Document', id="page", title="page")
self.portal.page.subject = (test_val, )
self.portal.page.reindexObject(idxs=['Subject'])
processQueue()
self.request.form['name'] = 'plone.app.vocabularies.Keywords'
results = getMultiAdapter((self.portal, self.request),
name='getVocabulary')()
results = json.loads(results)
result = results['results'][0]
self.assertEqual(result['text'], test_val)
self.assertEqual(result['id'], test_val)
def searchResults(self, query=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
# Make sure any pending index tasks have been processed
processQueue()
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(query, dict) and not show_inactive:
show_inactive = 'show_inactive' in query
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not self.allow_inactive(kw):
kw['effectiveRange'] = DateTime()
# filter out invalid sort_on indexes
sort_on = kw.get('sort_on') or []
if isinstance(sort_on, six.string_types):
sort_on = [sort_on]
valid_indexes = self.indexes()
try:
sort_on = [idx for idx in sort_on if idx in valid_indexes]
except TypeError:
# sort_on is not iterable
sort_on = []
if not sort_on:
kw.pop('sort_on', None)
else:
kw['sort_on'] = sort_on
return ZCatalog.searchResults(self, query, **kw)
def searchResults(self, query=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
# Make sure any pending index tasks have been processed
processQueue()
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(query, dict) and not show_inactive:
show_inactive = 'show_inactive' in query
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not self.allow_inactive(kw):
kw['effectiveRange'] = DateTime()
# filter out invalid sort_on indexes
sort_on = kw.get('sort_on') or []
if isinstance(sort_on, six.string_types):
sort_on = [sort_on]
valid_indexes = self.indexes()
try:
sort_on = [idx for idx in sort_on if idx in valid_indexes]
except TypeError:
# sort_on is not iterable
sort_on = []
if not sort_on:
kw.pop('sort_on', None)
else:
kw['sort_on'] = sort_on
return ZCatalog.searchResults(self, query, **kw)
def search(self, *args, **kw):
# Wrap search() the same way that searchResults() is
# Make sure any pending index tasks have been processed
processQueue()
query = {}
if args:
query = args[0]
elif 'query_request' in kw:
query = kw.get('query_request')
kw['query_request'] = query.copy()
user = _getAuthenticatedUser(self)
query['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not self.allow_inactive(kw):
query['effectiveRange'] = DateTime()
kw['query_request'] = query
return super(CatalogTool, self).search(**kw)
def testVocabularyEncoding(self):
"""The vocabulary should not return the binary encoded token
("N=C3=A5=C3=B8=C3=AF"), but instead the value as the id in the result
set. Fixes an encoding problem. See:
https://github.com/plone/Products.CMFPlone/issues/650
"""
test_val = u'Nåøï'
self.portal.invokeFactory('Document', id="page", title="page")
self.portal.page.subject = (test_val,)
self.portal.page.reindexObject(idxs=['Subject'])
processQueue()
self.request.form['name'] = 'plone.app.vocabularies.Keywords'
results = getMultiAdapter(
(self.portal, self.request),
name='getVocabulary'
)()
results = json.loads(results)
result = results['results'][0]
self.assertEqual(result['text'], test_val)
self.assertEqual(result['id'], test_val)
def testMergedThemes(self):
""" air_quality is merged with air
create and theme tag a news item
"""
self.portal.invokeFactory('News Item', id='news2')
obj = self.portal.news2
themes = IThemeTagging(obj)
themes.tags = ['air']
self.portal.portal_catalog.reindexObject(self.portal.news2)
self.portal.invokeFactory('News Item', id='news3')
obj = self.portal.news3
themes = IThemeTagging(obj)
themes.tags = ['air_quality']
self.portal.portal_catalog.reindexObject(self.portal.news3)
processQueue()
res = self.portal.portal_catalog.searchResults(
getThemes=['air_quality'], portal_type='News Item')
self.assertEqual(len(res), 1)
# only air is mapped to air_quality not the other way around
res = self.portal.portal_catalog.searchResults(getThemes=['air'],
portal_type='News Item')
self.assertEqual(len(res), 1)
def test25_versioningRestoresInsideRefsFolderOrder(self):
# Enable OMInsideChildrensModifier
portal_modifier = self.portal.portal_modifier
portal_modifier.edit("OMOutsideChildrensModifier", enabled=False,
condition="python: False")
portal_modifier.edit("OMInsideChildrensModifier", enabled=True,
condition="python: portal_type=='Folder'")
portal_repo = self.portal.portal_repository
fol = self.portal.fol
# save change no 1
fol.setTitle('v1 of fol')
fol.invokeFactory('Document', 'doc3')
fol.invokeFactory('Document', 'doc4')
portal_repo.applyVersionControl(fol, comment='first save')
orig_order = fol.objectIds()
fol.setTitle('v2 of fol')
fol.moveObjectsToTop(['doc2'])
self.assertEqual(fol.objectIds()[0], 'doc2')
# Test that a retrieve uses the order from the repo copy
repo_fol1 = portal_repo.retrieve(fol, 0).object
self.assertEqual(fol.objectIds()[0], 'doc2')
self.assertNotEqual(fol.objectIds(), orig_order)
self.assertEqual(repo_fol1.objectIds()[0], 'doc1')
# Test that a revert restores the order and objects from the
# repo copy
portal_repo.revert(fol)
self.assertEqual(fol.objectIds()[0], 'doc1')
self.assertEqual(fol.objectIds(), orig_order)
# See how we interact with some adds deletes and reorders
fol.invokeFactory('Document', 'doc5')
fol.moveObjectsToTop(['doc3'])
fol.moveObjectsToTop(['doc4'])
fol.manage_delObjects(['doc2'])
processQueue()
transaction.savepoint(optimistic=True)
doc3 = fol['doc3']
doc4 = fol['doc4']
self.assertNotEqual(fol.objectIds(), orig_order)
repo_fol1 = portal_repo.retrieve(fol, 0).object
# Test that a retrieve uses the repository order and items,
# but does not affect the working copy
repo_fol1 = portal_repo.retrieve(fol, 0).object
self.assertEqual(repo_fol1.objectIds(), orig_order)
self.assertNotEqual(getattr(repo_fol1, 'doc2', None), None)
self.assertEqual(getattr(repo_fol1, 'doc5', None), None)
self.assertNotEqual(fol.objectIds(), orig_order)
self.assertEqual(fol.objectIds()[0], 'doc4')
self.assertEqual(fol.objectIds()[1], 'doc3')
self.assertEqual(fol['doc3'], doc3)
self.assertEqual(fol['doc4'], doc4)
self.assertEqual(getattr(fol, 'doc2', None), None)
self.assertNotEqual(getattr(fol, 'doc5', None), None)
# Test that a revert restores the missing child from the repo
# copy, removed the newly created child and restored the order
portal_repo.revert(fol)
self.assertEqual(list(fol.objectIds()), orig_order)
self.assertEqual(getattr(fol, 'doc5', None), None)
# Test the BTreeFolder internals
self.assertEqual(fol._tree.get('doc5', None), None)
self.assertEqual(fol._count(), 4)
self.assertEqual(fol._mt_index[doc3.meta_type].get('doc5', None),
None)
self.assertEqual(fol._tree['doc3'], fol['doc3'].aq_base)
self.assertEqual(fol._mt_index[doc3.meta_type]['doc3'], 1)
def getCounter(self):
processQueue()
return self._counter is not None and self._counter() or 0
def maybe_process_indexing_queue(self):
if not IS_PLONE_5:
return
from Products.CMFCore.indexing import processQueue
processQueue()
def test15_retrieveInsideRefsFolderWithAddedOrDeletedObjects(self):
portal_repo = self.portal.portal_repository
fol = self.portal.fol
doc1 = fol.doc1
doc2 = fol.doc2
# just configure the standard folder to treat the children as
# inside references. For this we reconfigure the standard modifiers.
portal_modifier = self.portal.portal_modifier
portal_modifier.edit("OMOutsideChildrensModifier", enabled=False,
condition="python: False")
portal_modifier.edit("OMInsideChildrensModifier", enabled=True,
condition="python: portal_type=='Folder'")
# save change no 1
fol.setTitle('v1 of fol')
doc1.setTitle("v1 of doc1")
doc2.setTitle("v1 of doc2")
orig_ids = fol.objectIds()
orig_values = fol.objectValues()
portal_repo.applyVersionControl(fol, comment='first save')
retrieved_data = portal_repo.retrieve(fol, 0)
ret_folder = retrieved_data.object
self.assertEqual(ret_folder.objectIds(), orig_ids)
ret_values = ret_folder.objectValues()
# The values are not identical to the stored values because they are
# retrieved from the repository.
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
# remove an item
fol.manage_delObjects('doc2')
processQueue()
cur_ids = fol.objectIds()
self.assertEqual(len(cur_ids), 1)
# retrieve should retrieve missing sub-objects
retrieved_data = portal_repo.retrieve(fol, 0)
ret_folder = retrieved_data.object
self.assertEqual(ret_folder.objectIds(), orig_ids)
ret_values = ret_folder.objectValues()
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
# We should not have altered the folder itself on retrieve
self.assertEqual(fol.objectIds(), cur_ids)
# add new item
fol.invokeFactory('Document', 'doc3')
doc3 = fol.doc3
doc3.setTitle('v1 of doc3')
cur_ids = fol.objectIds()
self.assertEqual(len(cur_ids), 2)
# retrieve should not add new sub-objects
retrieved_data = portal_repo.retrieve(fol, 0)
ret_folder = retrieved_data.object
self.assertEqual(ret_folder.objectIds(), orig_ids)
ret_values = ret_folder.objectValues()
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
# We should not have altered the folder itself on retrieve
self.assertEqual(fol.objectIds(), cur_ids)
# revert to original state, ensure that subobject changes are
# reverted
portal_repo.revert(fol, 0)
fol = self.portal.fol
# check if reversion worked correctly
self.assertEqual(fol.objectIds(), orig_ids)
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
def getCounter(self):
processQueue()
return self._counter is not None and self._counter() or 0
def test25_versioningRestoresInsideRefsFolderOrder(self):
# Enable OMInsideChildrensModifier
portal_modifier = self.portal.portal_modifier
portal_modifier.edit("OMOutsideChildrensModifier", enabled=False,
condition="python: False")
portal_modifier.edit("OMInsideChildrensModifier", enabled=True,
condition="python: portal_type=='Folder'")
portal_repo = self.portal.portal_repository
fol = self.portal.fol
# save change no 1
fol.setTitle('v1 of fol')
fol.invokeFactory('Document', 'doc3')
fol.invokeFactory('Document', 'doc4')
portal_repo.applyVersionControl(fol, comment='first save')
orig_order = fol.objectIds()
fol.setTitle('v2 of fol')
fol.moveObjectsToTop(['doc2'])
self.failUnlessEqual(fol.objectIds()[0], 'doc2')
# Test that a retrieve uses the order from the repo copy
repo_fol1 = portal_repo.retrieve(fol, 0).object
self.failUnlessEqual(fol.objectIds()[0], 'doc2')
self.failIfEqual(fol.objectIds(), orig_order)
self.failUnlessEqual(repo_fol1.objectIds()[0], 'doc1')
# Test that a revert restores the order and objects from the
# repo copy
portal_repo.revert(fol)
self.failUnlessEqual(fol.objectIds()[0], 'doc1')
self.failUnlessEqual(fol.objectIds(), orig_order)
# See how we interact with some adds deletes and reorders
fol.invokeFactory('Document', 'doc5')
fol.moveObjectsToTop(['doc3'])
fol.moveObjectsToTop(['doc4'])
fol.manage_delObjects(['doc2'])
processQueue()
transaction.savepoint(optimistic=True)
doc3 = fol['doc3']
doc4 = fol['doc4']
self.failIfEqual(fol.objectIds(), orig_order)
repo_fol1 = portal_repo.retrieve(fol, 0).object
# Test that a retrieve uses the repository order and items,
# but does not affect the working copy
repo_fol1 = portal_repo.retrieve(fol, 0).object
self.failUnlessEqual(repo_fol1.objectIds(), orig_order)
self.failIfEqual(getattr(repo_fol1, 'doc2', None), None)
self.failUnlessEqual(getattr(repo_fol1, 'doc5', None), None)
self.failIfEqual(fol.objectIds(), orig_order)
self.failUnlessEqual(fol.objectIds()[0], 'doc4')
self.failUnlessEqual(fol.objectIds()[1], 'doc3')
self.failUnlessEqual(fol['doc3'], doc3)
self.failUnlessEqual(fol['doc4'], doc4)
self.failUnlessEqual(getattr(fol, 'doc2', None), None)
self.failIfEqual(getattr(fol, 'doc5', None), None)
# Test that a revert restores the missing child from the repo
# copy, removed the newly created child and restored the order
portal_repo.revert(fol)
self.failUnlessEqual(list(fol.objectIds()), orig_order)
self.failUnlessEqual(getattr(fol, 'doc5', None), None)
# Test the BTreeFolder internals
self.failUnlessEqual(fol._tree.get('doc5', None), None)
self.failUnlessEqual(fol._count(), 4)
self.failUnlessEqual(fol._mt_index[doc3.meta_type].get('doc5', None),
None)
self.failUnlessEqual(fol._tree['doc3'], fol['doc3'].aq_base)
self.failUnlessEqual(fol._mt_index[doc3.meta_type]['doc3'], 1)
def test15_storageStatistics(self):
self.maxDiff = None
portal_storage = self.portal.portal_historiesstorage
cmf_uid = 1
obj1 = CMFDummy('obj', cmf_uid)
obj1.text = 'v1 of text'
portal_storage.register(cmf_uid, ObjectData(obj1), metadata=self.buildMetadata('saved v1'))
obj2 = CMFDummy('obj', cmf_uid)
obj2.text = 'v2 of text'
portal_storage.save(cmf_uid, ObjectData(obj2), metadata=self.buildMetadata('saved v2'))
obj3 = CMFDummy('obj', cmf_uid)
obj3.text = 'v3 of text'
portal_storage.save(cmf_uid, ObjectData(obj3), metadata=self.buildMetadata('saved v3'))
obj4 = CMFDummy('obj', cmf_uid)
obj4.text = 'v4 of text'
self.portal._setObject('obj', obj4)
self.portal.portal_catalog.indexObject(self.portal.obj)
portal_storage.save(cmf_uid, ObjectData(obj4), metadata=self.buildMetadata('saved v4'))
cmf_uid = 2
tomorrow = DateTime() + 1
obj5 = CMFDummy('tomorrow', cmf_uid, effective=tomorrow)
obj5.allowedRolesAndUsers = ['Anonymous']
self.portal._setObject('tomorrow', obj5)
self.portal.portal_catalog.indexObject(self.portal.tomorrow)
portal_storage.register(cmf_uid, ObjectData(obj5), metadata=self.buildMetadata('effective tomorrow'))
cmf_uid = 3
yesterday = DateTime() - 1
obj6 = CMFDummy('yesterday', cmf_uid, expires=yesterday)
obj6.allowedRolesAndUsers = ['Anonymous']
self.portal._setObject('yesterday', obj6)
self.portal.portal_catalog.indexObject(self.portal.yesterday)
portal_storage.register(cmf_uid, ObjectData(obj6), metadata=self.buildMetadata('expired yesterday'))
cmf_uid = 4
obj7 = CMFDummy('public', cmf_uid)
obj7.text = 'visible for everyone'
obj7.allowedRolesAndUsers = ['Anonymous']
self.portal._setObject('public', obj7)
self.portal.portal_catalog.indexObject(self.portal.public)
portal_storage.register(cmf_uid, ObjectData(obj7), metadata=self.buildMetadata('saved public'))
processQueue()
got = portal_storage.zmi_getStorageStatistics()
expected = {'deleted': [],
'summaries': {
'totalHistories': 4,
'deletedVersions': 0,
'existingVersions': 7,
'deletedHistories': 0,
# time may easily be different
# 'time': '0.00',
'totalVersions': 7,
'existingAverage': '1.8',
'existingHistories': 4,
'deletedAverage': 'n/a',
'totalAverage': '1.8'},
'existing': [
{
'url': 'http://nohost/plone/obj',
'history_id': 1,
'length': 4,
'path': '/obj',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}, {
'url': 'http://nohost/plone/tomorrow',
'history_id': 2,
'length': 1,
'path': '/tomorrow',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}, {
'url': 'http://nohost/plone/yesterday',
'history_id': 3,
'length': 1,
'path': '/yesterday',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}, {
'url': 'http://nohost/plone/public',
'history_id': 4,
'length': 1,
'path': '/public',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}]}
self.assertEqual(expected['deleted'], got['deleted'])
self.assertTrue('summaries' in got)
self.assertTrue('time' in got['summaries'])
for key, value in expected['summaries'].items():
self.assertEqual(value, got['summaries'][key])
self.assertEqual(len(expected['existing']), len(got['existing']))
for idx in range(len(expected['existing'])):
exp = expected['existing'][idx]
actual = got['existing'][idx]
for key, value in exp.items():
self.assertEqual(actual[key], value)
# The actual size is not important and we want robust tests,
# s. https://github.com/plone/Products.CMFEditions/issues/31
self.failUnless(actual['size'] > 0)
def test15_storageStatistics(self):
self.maxDiff = None
portal_storage = self.portal.portal_historiesstorage
cmf_uid = 1
obj1 = CMFDummy('obj', cmf_uid)
obj1.text = 'v1 of text'
portal_storage.register(cmf_uid, ObjectData(obj1), metadata=self.buildMetadata('saved v1'))
obj2 = CMFDummy('obj', cmf_uid)
obj2.text = 'v2 of text'
portal_storage.save(cmf_uid, ObjectData(obj2), metadata=self.buildMetadata('saved v2'))
obj3 = CMFDummy('obj', cmf_uid)
obj3.text = 'v3 of text'
portal_storage.save(cmf_uid, ObjectData(obj3), metadata=self.buildMetadata('saved v3'))
obj4 = CMFDummy('obj', cmf_uid)
obj4.text = 'v4 of text'
self.portal._setObject('obj', obj4)
self.portal.portal_catalog.indexObject(self.portal.obj)
portal_storage.save(cmf_uid, ObjectData(obj4), metadata=self.buildMetadata('saved v4'))
cmf_uid = 2
tomorrow = DateTime() + 1
obj5 = CMFDummy('tomorrow', cmf_uid, effective=tomorrow)
obj5.allowedRolesAndUsers = ['Anonymous']
self.portal._setObject('tomorrow', obj5)
self.portal.portal_catalog.indexObject(self.portal.tomorrow)
portal_storage.register(cmf_uid, ObjectData(obj5), metadata=self.buildMetadata('effective tomorrow'))
cmf_uid = 3
yesterday = DateTime() - 1
obj6 = CMFDummy('yesterday', cmf_uid, expires=yesterday)
obj6.allowedRolesAndUsers = ['Anonymous']
self.portal._setObject('yesterday', obj6)
self.portal.portal_catalog.indexObject(self.portal.yesterday)
portal_storage.register(cmf_uid, ObjectData(obj6), metadata=self.buildMetadata('expired yesterday'))
cmf_uid = 4
obj7 = CMFDummy('public', cmf_uid)
obj7.text = 'visible for everyone'
obj7.allowedRolesAndUsers = ['Anonymous']
self.portal._setObject('public', obj7)
self.portal.portal_catalog.indexObject(self.portal.public)
portal_storage.register(cmf_uid, ObjectData(obj7), metadata=self.buildMetadata('saved public'))
processQueue()
got = portal_storage.zmi_getStorageStatistics()
expected = {'deleted': [],
'summaries': {
'totalHistories': 4,
'deletedVersions': 0,
'existingVersions': 7,
'deletedHistories': 0,
# time may easily be different
# 'time': '0.00',
'totalVersions': 7,
'existingAverage': '1.8',
'existingHistories': 4,
'deletedAverage': 'n/a',
'totalAverage': '1.8'},
'existing': [
{
'url': 'http://nohost/plone/obj',
'history_id': 1,
'length': 4,
'path': '/obj',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}, {
'url': 'http://nohost/plone/tomorrow',
'history_id': 2,
'length': 1,
'path': '/tomorrow',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}, {
'url': 'http://nohost/plone/yesterday',
'history_id': 3,
'length': 1,
'path': '/yesterday',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}, {
'url': 'http://nohost/plone/public',
'history_id': 4,
'length': 1,
'path': '/public',
'sizeState': 'approximate',
'portal_type': 'Dummy',
}]}
self.assertEqual(expected['deleted'], got['deleted'])
self.assertTrue('summaries' in got)
self.assertTrue('time' in got['summaries'])
for key, value in expected['summaries'].items():
self.assertEqual(value, got['summaries'][key])
self.assertEqual(len(expected['existing']), len(got['existing']))
for idx in range(len(expected['existing'])):
exp = expected['existing'][idx]
actual = got['existing'][idx]
for key, value in exp.items():
self.assertEqual(actual[key], value)
# The actual size is not important and we want robust tests,
# s. https://github.com/plone/Products.CMFEditions/issues/31
self.assertTrue(actual['size'] > 0)
def test15_retrieveInsideRefsFolderWithAddedOrDeletedObjects(self):
portal_repo = self.portal.portal_repository
fol = self.portal.fol
doc1 = fol.doc1
doc2 = fol.doc2
# just configure the standard folder to treat the children as
# inside references. For this we reconfigure the standard modifiers.
portal_modifier = self.portal.portal_modifier
portal_modifier.edit("OMOutsideChildrensModifier", enabled=False,
condition="python: False")
portal_modifier.edit("OMInsideChildrensModifier", enabled=True,
condition="python: portal_type=='Folder'")
# save change no 1
fol.setTitle('v1 of fol')
doc1.setTitle("v1 of doc1")
doc2.setTitle("v1 of doc2")
orig_ids = fol.objectIds()
orig_values = fol.objectValues()
portal_repo.applyVersionControl(fol, comment='first save')
retrieved_data = portal_repo.retrieve(fol, 0)
ret_folder = retrieved_data.object
self.assertEqual(ret_folder.objectIds(), orig_ids)
ret_values = ret_folder.objectValues()
# The values are not identical to the stored values because they are
# retrieved from the repository.
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
# remove an item
fol.manage_delObjects('doc2')
processQueue()
cur_ids = fol.objectIds()
self.assertEqual(len(cur_ids), 1)
# retrieve should retrieve missing sub-objects
retrieved_data = portal_repo.retrieve(fol, 0)
ret_folder = retrieved_data.object
self.assertEqual(ret_folder.objectIds(), orig_ids)
ret_values = ret_folder.objectValues()
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
# We should not have altered the folder itself on retrieve
self.assertEqual(fol.objectIds(), cur_ids)
# add new item
fol.invokeFactory('Document', 'doc3')
doc3 = fol.doc3
doc3.setTitle('v1 of doc3')
cur_ids = fol.objectIds()
self.assertEqual(len(cur_ids), 2)
# retrieve should not add new sub-objects
retrieved_data = portal_repo.retrieve(fol, 0)
ret_folder = retrieved_data.object
self.assertEqual(ret_folder.objectIds(), orig_ids)
ret_values = ret_folder.objectValues()
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())
# We should not have altered the folder itself on retrieve
self.assertEqual(fol.objectIds(), cur_ids)
# revert to original state, ensure that subobject changes are
# reverted
portal_repo.revert(fol, 0)
fol = self.portal.fol
# check if reversion worked correctly
self.assertEqual(fol.objectIds(), orig_ids)
for i in range(len(ret_values)):
self.assertEqual(ret_values[i].getId(), orig_values[i].getId())
self.assertEqual(ret_values[i].Title(), orig_values[i].Title())