def __call__(self, **kwargs): """ Evolve """ start = datetime.now(bucharest) logger.info('Reindex stiam.ro.source started: %s', start.strftime('%Y-%m-%d %H:%M:%S')) end = datetime.now(bucharest) + timedelta(3) leQuery = Le('stiam.ro.effective', end) query = SearchQuery(leQuery) brains = query.searchResults() logger.info('Reindex %s newsitems', len(brains)) intids = getUtility(IIntIds) sourceIndex = getUtility(interfaces.IIndex, 'stiam.ro.source') for index, brain in enumerate(brains): server = brain.__parent__.__parent__.__name__ oid = intids.getId(brain) sourceIndex.doIndex(oid, server) if index % 500 == 0: logger.info('Indexed %s items', index) logger.info('Reindex stiam.ro.source done: %s', datetime.now(bucharest) - start)
def items(self): source = self.context.__parent__.__name__ source = Eq('stiam.ro.source', source) tags = [self.context.__name__] tags = AnyOf('stiam.ro.tags', tags) query = SearchQuery(source).And(tags) brains = query.searchResults(sort_index='stiam.ro.effective', reverse=True, limit=30) duplicate = "" index = 0 for brain in brains: if index >= 15: raise StopIteration title = getattr(brain, "title", "") s = SequenceMatcher(lambda x: x == "", title, duplicate) if s.ratio() > 0.6: continue duplicate = title index += 1 yield brain
def items(self): tags = self.context.tags max_items = self.context.max_items anyOfQuery = AnyOf("jokes.tags", tags) query = SearchQuery(anyOfQuery) brains = query.searchResults(sort_index="jokes.effective", reverse=True, limit=max_items) for brain in brains: yield brain
def findWorkItemsForItemAndProcessId(item, process_id): """ Returns workitems for 'item' 'process_id'.""" intids = zope.component.getUtility(IIntIds, context=getSite()) iid = intids.queryId(item, None) if not iid: return [] # maybe but we are not able to know oidsQuery = AnyOf('workflow-relevant-oids', [iid]) pidQuery = Eq('workitem-processid', process_id) query = SearchQuery(oidsQuery).And(pidQuery) return query.searchResults()
def values(self): intids = zope.component.getUtility(IIntIds, context=self.context) oid = intids.queryId(self.context) if oid: query = SearchQuery(AnyOf('workflow-relevant-oids', [oid])) res = query.apply() #raise Exception(res) for intid in res: yield intids.getObject(intid)
def test_IfIndexed(self): from zope.intid.interfaces import IIntIds intids = zope.component.getUtility(IIntIds, context=self.root) iid = intids.register(self.item) self.startWorkflow() from z3c.indexer.query import AnyOf from z3c.indexer.search import SearchQuery query = SearchQuery(AnyOf('workflow-relevant-oids', iid)) res = query.apply() self.assertTrue(len(res) == 1)
def removeAttributionSubscriber(category, event): """ Remove attributions if a category is removed.""" intids = zope.component.getUtility(IIntIds, context=category) container = zope.component.getUtility(interfaces.ICategoriesContainer, context=category) query = SearchQuery(AnyOf(ATTRIBUTION_INDEX, (category.__name__, ))) result = query.apply() for intid in result: categorizable = intids.getObject(intid) attribution = interfaces.IAttribution(categorizable) attribution.unattribute(category.__name__)
def test_IndexedUpToDateWhenCategoryRemoved(self): """ Test if an attribution is indexed at all.""" self.root['c'] = catable = Categorizable() attribution = interfaces.IAttribution(catable) attribution.attribute('cat22') del self.categories['set2']['cat22'] from z3c.indexer.query import AnyOf from z3c.indexer.search import SearchQuery query = SearchQuery(AnyOf('attribution-set', ('cat22', ))) result = query.apply() self.assertTrue(len(result) == 0)
def moveAttributionSubscriber(category, event): """ Keep an attribution up to date if a category is renamed.""" if event.oldName and event.newName: intids = zope.component.getUtility(IIntIds, context=category) container = zope.component.getUtility(interfaces.ICategoriesContainer, context=category) query = SearchQuery(AnyOf(ATTRIBUTION_INDEX, (event.oldName, ))) result = query.apply() for intid in result: categorizable = intids.getObject(intid) attribution = interfaces.IAttribution(categorizable) attribution.unattribute(event.oldName) attribution.attribute(event.newName)
def items(self): tags = self.context.tags or [self.context.__name__] now = datetime.now(bucharest) max_items = 500 logger.info('Query %s start %s', tags, now.strftime('%H:%M:%S')) anyOfQuery = AnyOf('stiam.ro.tags', tags) query = SearchQuery(anyOfQuery) brains = query.searchResults(sort_index='stiam.ro.effective', reverse=True, limit=max_items) logger.info('Query %s end %s', tags, datetime.now(bucharest).strftime('%H:%M:%S')) for brain in brains: yield brain
def getSimilarWorkItems(self): """ See ISimilarWorkItems""" ids = [] intids = zope.component.getUtility(IIntIds, context=self.context) for obj in self.objects: intid = intids.queryId(obj, None) if intid is not None: ids.append(intid) #raise Exception(ids) query = SearchQuery(AnyOf('workflow-relevant-oids', ids)) res = query.apply() #raise Exception(res) for intid in res: if intid != intids.getId(self.context): #raise Exception(intid) yield intids.getObject(intid)
def _cleanup(self, **kwargs): """ Cleanup thumbs """ index = 0 now = datetime.now(bucharest) logger.info('cleanup old images started ...') geQuery = Le('stiam.ro.effective', now) query = SearchQuery(geQuery) brains = query.apply() intids = queryUtility(IIntIds) if not intids: logger.warn('No intids utility. Cleanup aborted.') return index for docid in brains: try: doc = intids.getObject(docid) except KeyError, err: logger.exception(err) continue for image in doc.values(): if not IImage.providedBy(image): continue if IBlobImage.providedBy(image): continue try: del doc[image.__name__] except Exception, err: logger.exception(err) continue index += 1 # Transaction commit if index % 20 == 0: logger.info('Commit transaction %s', index) transaction.commit()
def items(self): tags = self.context.tags or [self.context.__name__] max_items = self.context.max_items * 2 anyOfQuery = AnyOf('stiam.ro.tags', tags) query = SearchQuery(anyOfQuery) brains = query.searchResults(sort_index='stiam.ro.effective', reverse=True, limit=max_items) index = 0 duplicate = "" for brain in brains: if index >= max_items / 2: raise StopIteration title = getattr(brain, "title", "") s = SequenceMatcher(lambda x: x == "", title, duplicate) if s.ratio() > 0.6: continue duplicate = title index += 1 yield brain
def test_IfIndexed(self): """ Test if an attribution is indexed at all.""" self.root['c'] = catable = Categorizable() attribution = interfaces.IAttribution(catable) attribution.attribute('cat21') zope.event.notify(ObjectModifiedEvent(catable)) from z3c.indexer.query import AnyOf from z3c.indexer.search import SearchQuery query = SearchQuery(AnyOf('attribution-set', ('cat21', ))) result = query.apply() self.assertTrue(len(result) == 1) attribution.unattribute('cat21') zope.event.notify(ObjectModifiedEvent(catable)) query = SearchQuery(AnyOf('attribution-set', ('cat21', ))) result = query.apply() self.assertTrue(len(result) == 0)
def test_IndexedUpToDateWhenCategoryMovedAndRenamed(self): """ Test if an attribution is indexed at all.""" self.root['c'] = catable = Categorizable() attribution = interfaces.IAttribution(catable) attribution.attribute('cat11') zope.event.notify(ObjectModifiedEvent(catable)) from zope.copypastemove.interfaces import IObjectMover mover = IObjectMover(self.categories['set1']['cat11']) mover.moveTo(self.categories['set2'], new_name='moved') zope.event.notify(ObjectModifiedEvent(catable)) from z3c.indexer.query import AnyOf from z3c.indexer.search import SearchQuery query = SearchQuery(AnyOf('attribution-set', ('cat22', ))) result = query.apply() self.assertTrue(len(result) == 0) query = SearchQuery(AnyOf('attribution-set', ('moved', ))) result = query.apply() self.assertTrue(len(result) == 1)
def values(self): principal_id = self.request.principal.id contributorQuery = AnyOf('workitem-contributors', (principal_id, )) worklistQuery = Eq('worklist-value', 'contributor') query = SearchQuery(contributorQuery).And(worklistQuery) return query.searchResults()