def __call__(self, **kwargs):
""" Evolve
"""
start = datetime.now(bucharest)
logger.info('Reindex stiam.ro.source started: %s',
start.strftime('%Y-%m-%d %H:%M:%S'))
end = datetime.now(bucharest) + timedelta(3)
leQuery = Le('stiam.ro.effective', end)
query = SearchQuery(leQuery)
brains = query.searchResults()
logger.info('Reindex %s newsitems', len(brains))
intids = getUtility(IIntIds)
sourceIndex = getUtility(interfaces.IIndex, 'stiam.ro.source')
for index, brain in enumerate(brains):
server = brain.__parent__.__parent__.__name__
oid = intids.getId(brain)
sourceIndex.doIndex(oid, server)
if index % 500 == 0:
logger.info('Indexed %s items', index)
logger.info('Reindex stiam.ro.source done: %s',
datetime.now(bucharest) - start)
def items(self):
source = self.context.__parent__.__name__
source = Eq('stiam.ro.source', source)
tags = [self.context.__name__]
tags = AnyOf('stiam.ro.tags', tags)
query = SearchQuery(source).And(tags)
brains = query.searchResults(sort_index='stiam.ro.effective',
reverse=True, limit=30)
duplicate = ""
index = 0
for brain in brains:
if index >= 15:
raise StopIteration
title = getattr(brain, "title", "")
s = SequenceMatcher(lambda x: x == "", title, duplicate)
if s.ratio() > 0.6:
continue
duplicate = title
index += 1
yield brain
def items(self):
tags = self.context.tags
max_items = self.context.max_items
anyOfQuery = AnyOf("jokes.tags", tags)
query = SearchQuery(anyOfQuery)
brains = query.searchResults(sort_index="jokes.effective", reverse=True, limit=max_items)
for brain in brains:
yield brain
def findWorkItemsForItemAndProcessId(item, process_id):
""" Returns workitems for 'item' 'process_id'."""
intids = zope.component.getUtility(IIntIds, context=getSite())
iid = intids.queryId(item, None)
if not iid: return [] # maybe but we are not able to know
oidsQuery = AnyOf('workflow-relevant-oids', [iid])
pidQuery = Eq('workitem-processid', process_id)
query = SearchQuery(oidsQuery).And(pidQuery)
return query.searchResults()
def values(self):
intids = zope.component.getUtility(IIntIds, context=self.context)
oid = intids.queryId(self.context)
if oid:
query = SearchQuery(AnyOf('workflow-relevant-oids', [oid]))
res = query.apply()
#raise Exception(res)
for intid in res:
yield intids.getObject(intid)
def test_IfIndexed(self):
from zope.intid.interfaces import IIntIds
intids = zope.component.getUtility(IIntIds, context=self.root)
iid = intids.register(self.item)
self.startWorkflow()
from z3c.indexer.query import AnyOf
from z3c.indexer.search import SearchQuery
query = SearchQuery(AnyOf('workflow-relevant-oids', iid))
res = query.apply()
self.assertTrue(len(res) == 1)
def removeAttributionSubscriber(category, event):
""" Remove attributions if a category is removed."""
intids = zope.component.getUtility(IIntIds, context=category)
container = zope.component.getUtility(interfaces.ICategoriesContainer,
context=category)
query = SearchQuery(AnyOf(ATTRIBUTION_INDEX, (category.__name__, )))
result = query.apply()
for intid in result:
categorizable = intids.getObject(intid)
attribution = interfaces.IAttribution(categorizable)
attribution.unattribute(category.__name__)
def test_IndexedUpToDateWhenCategoryRemoved(self):
""" Test if an attribution is indexed at all."""
self.root['c'] = catable = Categorizable()
attribution = interfaces.IAttribution(catable)
attribution.attribute('cat22')
del self.categories['set2']['cat22']
from z3c.indexer.query import AnyOf
from z3c.indexer.search import SearchQuery
query = SearchQuery(AnyOf('attribution-set', ('cat22', )))
result = query.apply()
self.assertTrue(len(result) == 0)
def moveAttributionSubscriber(category, event):
""" Keep an attribution up to date if a category is renamed."""
if event.oldName and event.newName:
intids = zope.component.getUtility(IIntIds, context=category)
container = zope.component.getUtility(interfaces.ICategoriesContainer,
context=category)
query = SearchQuery(AnyOf(ATTRIBUTION_INDEX, (event.oldName, )))
result = query.apply()
for intid in result:
categorizable = intids.getObject(intid)
attribution = interfaces.IAttribution(categorizable)
attribution.unattribute(event.oldName)
attribution.attribute(event.newName)
def items(self):
tags = self.context.tags or [self.context.__name__]
now = datetime.now(bucharest)
max_items = 500
logger.info('Query %s start %s', tags, now.strftime('%H:%M:%S'))
anyOfQuery = AnyOf('stiam.ro.tags', tags)
query = SearchQuery(anyOfQuery)
brains = query.searchResults(sort_index='stiam.ro.effective',
reverse=True, limit=max_items)
logger.info('Query %s end %s', tags, datetime.now(bucharest).strftime('%H:%M:%S'))
for brain in brains:
yield brain
def getSimilarWorkItems(self):
""" See ISimilarWorkItems"""
ids = []
intids = zope.component.getUtility(IIntIds, context=self.context)
for obj in self.objects:
intid = intids.queryId(obj, None)
if intid is not None:
ids.append(intid)
#raise Exception(ids)
query = SearchQuery(AnyOf('workflow-relevant-oids', ids))
res = query.apply()
#raise Exception(res)
for intid in res:
if intid != intids.getId(self.context):
#raise Exception(intid)
yield intids.getObject(intid)
def _cleanup(self, **kwargs):
""" Cleanup thumbs
"""
index = 0
now = datetime.now(bucharest)
logger.info('cleanup old images started ...')
geQuery = Le('stiam.ro.effective', now)
query = SearchQuery(geQuery)
brains = query.apply()
intids = queryUtility(IIntIds)
if not intids:
logger.warn('No intids utility. Cleanup aborted.')
return index
for docid in brains:
try:
doc = intids.getObject(docid)
except KeyError, err:
logger.exception(err)
continue
for image in doc.values():
if not IImage.providedBy(image):
continue
if IBlobImage.providedBy(image):
continue
try:
del doc[image.__name__]
except Exception, err:
logger.exception(err)
continue
index += 1
# Transaction commit
if index % 20 == 0:
logger.info('Commit transaction %s', index)
transaction.commit()
def items(self):
tags = self.context.tags or [self.context.__name__]
max_items = self.context.max_items * 2
anyOfQuery = AnyOf('stiam.ro.tags', tags)
query = SearchQuery(anyOfQuery)
brains = query.searchResults(sort_index='stiam.ro.effective',
reverse=True, limit=max_items)
index = 0
duplicate = ""
for brain in brains:
if index >= max_items / 2:
raise StopIteration
title = getattr(brain, "title", "")
s = SequenceMatcher(lambda x: x == "", title, duplicate)
if s.ratio() > 0.6:
continue
duplicate = title
index += 1
yield brain
def test_IfIndexed(self):
""" Test if an attribution is indexed at all."""
self.root['c'] = catable = Categorizable()
attribution = interfaces.IAttribution(catable)
attribution.attribute('cat21')
zope.event.notify(ObjectModifiedEvent(catable))
from z3c.indexer.query import AnyOf
from z3c.indexer.search import SearchQuery
query = SearchQuery(AnyOf('attribution-set', ('cat21', )))
result = query.apply()
self.assertTrue(len(result) == 1)
attribution.unattribute('cat21')
zope.event.notify(ObjectModifiedEvent(catable))
query = SearchQuery(AnyOf('attribution-set', ('cat21', )))
result = query.apply()
self.assertTrue(len(result) == 0)
def test_IndexedUpToDateWhenCategoryMovedAndRenamed(self):
""" Test if an attribution is indexed at all."""
self.root['c'] = catable = Categorizable()
attribution = interfaces.IAttribution(catable)
attribution.attribute('cat11')
zope.event.notify(ObjectModifiedEvent(catable))
from zope.copypastemove.interfaces import IObjectMover
mover = IObjectMover(self.categories['set1']['cat11'])
mover.moveTo(self.categories['set2'], new_name='moved')
zope.event.notify(ObjectModifiedEvent(catable))
from z3c.indexer.query import AnyOf
from z3c.indexer.search import SearchQuery
query = SearchQuery(AnyOf('attribution-set', ('cat22', )))
result = query.apply()
self.assertTrue(len(result) == 0)
query = SearchQuery(AnyOf('attribution-set', ('moved', )))
result = query.apply()
self.assertTrue(len(result) == 1)
def values(self):
principal_id = self.request.principal.id
contributorQuery = AnyOf('workitem-contributors', (principal_id, ))
worklistQuery = Eq('worklist-value', 'contributor')
query = SearchQuery(contributorQuery).And(worklistQuery)
return query.searchResults()
