class GroupingStorageValues(Implicit, Owned, persistent.Persistent):
""" A datastructure to store the UIDs of objects appearing under a specific
grouping.
It conforms to the requirements imposed by OrderedBTreeFolderBase on
its sub-objects (acquisition aware, ownable, persistent).
"""
def __init__(self, uids):
self.archived = False
self.uids = OOTreeSet()
self.uids.update(uids)
def __iter__(self):
return self.uids.__iter__()
def __contains__(self, item):
return item in self.uids
def __len__(self):
return len(self.uids)
def add(self, item):
self.uids.insert(item)
def discard(self, item):
self.uids.remove(item)
def remove(self, item):
self.uids.remove(item)
def pop(self):
return self.uids.pop()
class GroupingStorageValues(
Implicit, Owned, persistent.Persistent):
""" A datastructure to store the UIDs of objects appearing under a specific
grouping.
It conforms to the requirements imposed by OrderedBTreeFolderBase on
its sub-objects (acquisition aware, ownable, persistent).
"""
def __init__(self, uids):
self.archived = False
self.uids = OOTreeSet()
self.uids.update(uids)
def __iter__(self):
return self.uids.__iter__()
def __contains__(self, item):
return item in self.uids
def __len__(self):
return len(self.uids)
def add(self, item):
self.uids.insert(item)
def discard(self, item):
self.uids.remove(item)
def remove(self, item):
self.uids.remove(item)
def pop(self):
return self.uids.pop()
def convertListToOOTreeSet(self):
"conver the list object to an OOTreeSet"
if len(self.entry) and isinstance(self.entry, types.ListType):
temp = OOTreeSet()
for i in self.entry:
temp.insert(i)
self.setObject('entry', temp)
def convertListToOOTreeSet(self):
"conver the list object to an OOTreeSet"
if len(self.entry) and isinstance(self.entry, types.ListType):
temp = OOTreeSet()
for i in self.entry:
temp.insert(i)
self.setObject('entry', temp)
def _put_into_indexes(self, obj):
"""
Put publication into all indexes.
Attr:
obj (obj): Indexable object.
Raises:
UnindexableObject: When there is no index (property) which can be
used to index `obj` in database.
"""
no_of_used_indexes = 0
for field_name, db_index in list(self._get_db_fields(obj)):
attr_value = getattr(obj, field_name)
if attr_value is None: # index only by set attributes
continue
container = db_index.get(attr_value, None)
if container is None:
container = OOTreeSet()
db_index[attr_value] = container
container.insert(obj)
no_of_used_indexes += 1
# make sure that atleast one `attr_value` was used
if no_of_used_indexes <= 0:
raise UnindexableObject(
"You have to use atleast one of the identificators!"
)
def __init__(self, sessionId):
self._sessionId = sessionId
self._questionId = OOBTree()
self._answer = OOBTree()
self._nav = OOBTree()
self._context = OOBTree()
self._loginStatus = OOBTree()
self._page = OOBTree()
self._query = OOBTree()
self._qstPage = OOTreeSet()
class Branding(grok.Annotation):
def __init__(self):
self._brands = OOTreeSet()
def addBrand(self, brand):
self._brands.insert(brand)
def getBrands(self):
return list(self._brands)
class Branding(grok.Annotation):
grok.implements(IBranding)
def __init__(self):
self._brands = OOTreeSet()
def addBrand(self, brand):
self._brands.insert(brand)
def getBrands(self):
return list(self._brands)
def clear(self):
'''clear the index.'''
l = self.__len__
if isinstance(l, Length): l.set(0)
else: self.__len__ = Length()
try:
self.numObjects.set(0)
except AttributeError:
self.numObjects = Length()
if self.ReverseOrder: self._reverseOrder = OOTreeSet()
self._setup()
class Session(Implicit, Persistent, RoleManager, Item):
def __init__(self, sessionId):
self._sessionId = sessionId
self._questionId = OOBTree()
self._answer = OOBTree()
self._nav = OOBTree()
self._context = OOBTree()
self._loginStatus = OOBTree()
self._page = OOBTree()
self._query = OOBTree()
self._qstPage = OOTreeSet()
def addAnswer(self, questionId, answer, nav, time, context, loginStatus,
page, query):
if not self._questionId.has_key(time):
self._questionId[time] = questionId
self._answer[time] = answer
self._nav[time] = nav
self._context[time] = context
self._loginStatus[time] = loginStatus
self._page[time] = page
self._query[time] = query
self._qstPage.insert(str(questionId) + str(page))
return True
return False
def getTotFeedback(self):
return len(self._questionId)
#note: isAnswered(questionId, sessionId=sessionId, page=url)
def isAnswered(self, questionId, **kwargs):
if 'page' in kwargs:
qstPage = str(questionId) + str(kwargs['page'])
return True if self._qstPage.has_key(qstPage) else False
else:
return True if questionId in self._questionId.values() else False
def getAnswerList(self):
answers = []
for time in self._questionId.keys():
tmp = [
self._sessionId, time,
self._questionId.get(time),
self._answer.get(time),
self._nav.get(time),
self._context.get(time),
self._loginStatus.get(time),
self._page.get(time),
self._query.get(time)
]
answers.append(tmp)
return answers
class Comments(grok.Annotation):
grok.context(Article) # this is actually the default
grok.implements(IComments)
grok.name('annotations.Comments') # this is actually the default
def __init__(self):
self.comments = OOTreeSet()
def addComment(self, text):
self.comments.insert(text)
def getComments(self):
return list(self.comments)
class Comments(grok.Annotation):
grok.context(Article) # this is actually the default
grok.implements(IComments)
grok.name('annotations.Comments') # this is actually the default
def __init__(self):
self.comments = OOTreeSet()
def addComment(self, text):
self.comments.insert(text)
def getComments(self):
return list(self.comments)
def relink(self, page, links):
for link in self.links.get(page, ()):
oldlinks = self.backlinks[link]
oldlinks.remove(page)
if not oldlinks:
del self.backlinks[link]
self.links[page] = OOTreeSet(links)
for link in links:
data = self.backlinks.get(link)
if data is None:
data = OOTreeSet()
self.backlinks[link] = data
data.insert(page)
def logstorage(self):
""" This is an annotation OOTreeSet where we can store log entries
"""
annotations = IAnnotations(self.context)
if self.logstorage_key not in annotations:
annotations[self.logstorage_key] = OOTreeSet()
return annotations[self.logstorage_key]
def _insert(self, entry):
if entry.main not in self._index:
self._index[entry.main] = OOBTree()
if entry.sub not in self._index[entry.main]:
self._index[entry.main][entry.sub] = OOTreeSet()
self._index[entry.main][entry.sub].insert(entry)
def __init__(self, name, searcher, parent, other_tags, tag_path):
self.name = name
self.__name__ = name.encode('utf-8')
self.__parent__ = parent
self.searcher = searcher
self.other_tags = difference(other_tags, OOTreeSet([self.__name__]))
self.tag_path = tag_path + [name]
class Subscription(Persistent):
implements(ISubscription)
adapts(ISubscribable)
def __init__(self):
self.subscribers = OOTreeSet()
def subscribe(self, principal_id):
self.subscribers.insert(principal_id)
def unsubscribe(self, principal_id):
self.subscribers.remove(principal_id)
def send(self, message, subject):
for pid in self.subscribers.keys():
profile = get_profile(pid, "basic")
send_mail(message, subject, profile.email)
def index(self, conf, element):
self._indexCheck()
# self.get is equivalent to root['indexByConf']
if not self.get().has_key(conf):
self.get()[conf] = OOTreeSet()
self.get()[conf].insert(element)
class Subscription(Persistent):
implements(ISubscription)
adapts(ISubscribable)
def __init__(self):
self.subscribers = OOTreeSet()
def subscribe(self, principal_id):
self.subscribers.insert(principal_id)
def unsubscribe(self, principal_id):
self.subscribers.remove(principal_id)
def send(self, message, subject):
for pid in self.subscribers.keys():
profile = get_profile(pid, "basic")
send_mail(message, subject, profile.email)
def categoryConfDictToTreeSet(dbi, withRBDB, prevVersion):
"""
Replacing the conference dictionary in the Category objects by a OOTreeSet.
"""
for categ in CategoryManager()._getIdx().itervalues():
categ.conferencesBackup = categ.conferences.values()
categ.conferences = OOTreeSet(categ.conferences.itervalues())
if len(categ.conferences) != len(categ.conferencesBackup):
print "Problem migrating conf dict to tree set: %s" % categ.getId()
def set_exported_content_tags(self, item, language, tags):
entry = (item.id, language)
prev_tags = self._entry_tags.get(entry)
self._entry_tags[entry] = tags
if prev_tags:
for tag in prev_tags:
tag_entries = self._entries_by_tag.get(tag)
if tag_entries:
tag_entries.remove(entry)
for tag in tags:
tag_entries = self._entries_by_tag.get(tag)
if tag_entries is None:
tag_entries = OOTreeSet()
self._entries_by_tag[tag] = tag_entries
tag_entries.insert(entry)
def clear(self): '''clear the index.''' l = self.__len__ if isinstance(l, Length): l.set(0) else: self.__len__ = Length() try: self.numObjects.set(0) except AttributeError: self.numObjects= Length() if self.ReverseOrder: self._reverseOrder = OOTreeSet() self._setup()
class PhotoAlbum(ContentContainer):
interface.implements(IPhotoAlbum)
pageCount = FieldProperty(IPhotoAlbum['pageCount'])
def __init__(self, **kw):
super(PhotoAlbum, self).__init__(**kw)
self.albums = OOTreeSet()
self.photos = OOTreeSet()
self.__dict__['totalPhotos'] = Length(0)
def __setitem__(self, name, item):
super(PhotoAlbum, self).__setitem__(name, item)
if IPhoto.providedBy(item):
self.photos.insert(name)
elif IPhotoAlbum.providedBy(item):
self.albums.insert(name)
def __delitem__(self, name):
if name in self.photos:
self.photos.remove(name)
elif name in self.albums:
self.albums.remove(name)
super(PhotoAlbum, self).__delitem__(name)
@property
def total(self):
return len(self.photos)
@property
def totalAlbums(self):
return len(self.albums)
@property
def totalPhotos(self):
return self.__dict__['totalPhotos']()
def listPhotos(self):
photos = self.photos
for name in self:
if name in photos:
yield name
def listPhotoAlbums(self):
albums = self.albums
for name in self:
if name in albums:
yield name
class Assignments(Persistent, Location):
interface.implements(IAssignments)
def __init__(self, doc_id):
self.doc_id = doc_id
self.assignees = OOTreeSet()
def assign(self, principals):
modified = False
context = getUtility(IIntIds).getObject(self.doc_id)
for pid in tuple(self.assignees):
if pid not in principals:
self.assignees.remove(pid)
event.notify(PrincipalUnassignedEvent(context, pid))
for pid in principals:
if pid not in self.assignees:
self.assignees.insert(pid)
event.notify(PrincipalAssignedEvent(context, pid))
def reassign(self, principals):
modified = False
context = getUtility(IIntIds).getObject(self.doc_id)
for pid in principals:
if pid not in self.assignees:
self.assignees.insert(pid)
event.notify(PrincipalAssignedEvent(context, pid))
def unassign(self, principals):
context = getUtility(IIntIds).getObject(self.doc_id)
for principal in principals:
if principal in self.assignees:
self.assignees.remove(principal)
event.notify(PrincipalUnassignedEvent(context, principal))
def getAssignees(self):
getPrincipal = getUtility(IAuthentication).getPrincipal
assignees = []
for pid in self.assignees:
try:
assignees.append(getPrincipal(pid))
except PrincipalLookupError:
pass
return assignees
def isAssigned(self, pid):
return pid in self.assignees
def insert(self, parent, child):
# Establish a new path to child_id for each path that leads to
# parent_id.
child_id = child._p_oid
new_paths = []
for parent_path in self.get_paths(parent):
new_paths.append(parent_path + (child_id, ))
if not self.paths.has_key(child_id):
self.paths[child_id] = OOTreeSet()
for path in new_paths:
self.paths[child_id].insert(path)
return new_paths
def enqueue(self, t, obj):
"""
Add an element to the queue
"""
if t not in self._container:
self._container[t] = OOTreeSet()
if obj in self._container[t]:
raise DuplicateElementException(obj)
self._container[t].add(obj)
self._elem_counter.change(1)
def index(self, element):
self._indexCheck()
# self.get is equivalent to root['IndexByCRName']
if self.get().has_key(element.getTitle()):
for cr in self.get()[element.getTitle()]:
#if there is a chat room repeated in the server (either ours or an external one) we have a problem
if cr.getTitle() == element.getTitle() and cr.getHost() == element.getHost():
raise NoReportError( _('There is already a chat room in that server with that name, please choose another one'))
else:
self.get()[element.getTitle()] = OOTreeSet()
self.get()[element.getTitle()].insert(element)
def _get_persistent_class_keys(cls):
index_key = cls.full_name + "-keys"
keys = datastore.root.get(index_key)
if keys is None:
if isinstance(cls.primary_member, schema.Integer):
keys = IOTreeSet()
else:
keys = OOTreeSet()
datastore.root[index_key] = keys
return keys
class ManagableIndex(OFolder,Normalize, Ignore):
'''Abstract base class for 'ManagableIndex'.'''
security= ClassSecurityInfo()
security.declareProtected(
ManageManagableIndexes,
'addAttributeLookupForm',
'addExpressionEvaluatorForm',
'addValueProvider',
'manage_propertiesForm', 'manage_changeProperties', 'manage_editProperties',
'indexSize',
)
security.declarePrivate(
'getReverseOrder',
)
icon= 'misc_/PluginIndexes/index.gif'
manage_options= (
OFolder.manage_options[:1]
+ OFolder.manage_options[2:]
)
operators= ('or', 'and',)
useOperator= 'or'
query_options= ('operator', 'range', 'usage', 'match', 'isearch', 'isearch_filter')
Combiners= ('useFirst',)
NormalizerProperty= 'NormalizeTerm'
IgnoreProperty= 'StopTermPredicate'
_properties= (
(
{ 'id' : 'CombineType', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listCombiners',
'label':'Combine type: determines how values from value providers are combined'},
{'id':'PrenormalizeTerm', 'type':'string', 'mode':'w',
'label':'Term prenormalizer: applied to terms before term expansion in queries and (always) before stop term elimination; used e.g. for case normalization, stemming, phonetic normalization, ...',},
{'id' : IgnoreProperty, 'type' : 'string', 'mode' : 'w',
'label':'Stop term predicate: used to recognized and eliminate stop terms; used always (except in range queries) after prenormalization',},
{'id' : NormalizerProperty, 'type' : 'string', 'mode' : 'w',
'label':'Term normalizer: applied to terms before type checking; used e.g. for encoding the term into a efficient form',},
{ 'id' : 'TermType', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listTermTypes',
'label':'Term type: used to convert and check the terms type; may allows to choose specially optimized index structures (e.g. for integer types) or provide additional features (e.g. term expansions for string types) -- clear+reindex after change!',
},
{ 'id' : 'TermTypeExtra', 'type' : 'string', 'mode' : 'w',
'label':'Term type argument: required by some term types (see the documentation)',},
{ 'id' : 'TermCopy', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listCopyTypes',
'label':'Control term copying: may be necessary for mutable terms to prevent index corruption',},
)
)
TermType= _TermTypeList[0]
TermTypeExtra= ''
TermCopy= _TermCopyList[0]
CombineType= Combiners[0]
NormalizeTerm= ''
PrenormalizeTerm= ''
StopTermPredicate= ''
__implements__= PluggableIndexInterface
def __init__(self, name, extra=None, caller=None):
self.id= name
def setProperties(obj, values, special):
__traceback_info__ = obj
allowed = dict.fromkeys(special)
allowed.update(dict.fromkeys([p['id'] for p in obj._properties]))
for k in values.keys():
if k not in allowed: raise ValueError('not a known property: %s' % k)
obj.manage_changeProperties(**values)
if extra: setProperties(self, extra, ('ValueProviders',))
providers = extra and extra.get('ValueProviders')
if providers is None: self._createDefaultValueProvider()
else:
for p in providers:
vp = self.addValueProvider(p['id'], p['type'])
setProperties(vp, p, ('id', 'type',))
self.clear()
def clear(self):
'''clear the index.'''
l = self.__len__
if isinstance(l, Length): l.set(0)
else: self.__len__ = Length()
try: self.numObjects.set(0)
except AttributeError: self.numObjects= Length()
if self.ReverseOrder: self._reverseOrder = OOTreeSet()
self._setup()
def __len__(self):
'''Python 2.4 requires this to be defined inside the class.'''
l = self.__len__
if not isinstance(l, Length): l = self.__len__ = Length()
return l()
def indexSize(self):
return self.__len__()
def _setup(self):
self._unindex= IOBTree()
treeType = self.TermType in _IntegerTypes and IOBTree or OOBTree
self._index= treeType()
def _createDefaultValueProvider(self):
self.addValueProvider(self.id,'AttributeLookup')
## term expansion -- provided for indexes with declared "string" and "ustring"
## term types
def matchGlob(self, t):
'''match '*' (match any sequence) and '?' (match any character) in *t* returning a list of terms.'''
# leads to wrong result -- would need to check against index
# if not ('*' in t or '?' in t): return [t]
regexp = glob2regexp(t)
return self.matchRegexp(regexp+'$')
_matchType = None
def matchRegexp(self, regexp):
'''match *regexp* into a list of matching terms.
Note that for efficiency reasons, the regular expression
should have an easily recognizable plain text prefix -- at
least for large indexes.
'''
prefix, regexp = _splitPrefixRegexp(regexp)
termType = self._matchType or self.TermType
if termType == 'string': prefix = str(prefix); regexp = str(regexp)
elif termType == 'ustring': prefix = unicode(prefix); regexp = unicode(regexp)
elif termType == 'asis': pass
else: raise ValueError(
"Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
% (self.id, termType)
)
index = self._getMatchIndex(); pn = len(prefix)
l = []; match = compile(regexp).match
for t in index.keys(prefix): # terms starting prefix
if not t.startswith(prefix): break
if match(t[pn:]): l.append(t)
return l
def _getMatchIndex(self):
'''the index used for expansion'''
return self._index
## match filtering
def matchFilterGlob(self, t):
'''see 'matchGlob' but for filtering.'''
regexp = glob2regexp(t)
return self.matchFilterRegexp(regexp+'$')
def matchFilterRegexp(self, regexp):
'''see 'matchRegexp' but for filtering.'''
termType = self._matchType or self.TermType
if termType == 'string': regexp = str(regexp)
elif termType == 'ustring': regexp = unicode(regexp)
elif termType == 'asis': pass
else: raise ValueError(
"Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
% (self.id, termType)
)
return compile(regexp).match
## Responsibilities from 'PluggableIndexInterface'
# 'getId' -- inherited from 'SimpleItem'
def getEntryForObject(self,documentId, default= None):
'''Information for *documentId*.'''
info= self._unindex.get(documentId)
if info is None: return default
return repr(info)
def index_object(self,documentId,obj,threshold=None):
'''index *obj* as *documentId*.
Commit subtransaction when *threshold* index terms have been indexed.
Return the number of index terms indexed.
'''
# Note: objPath should be provided by the catalog -- but it is not
try: objPath = obj.getPhysicalPath()
except: objPath = None
__traceback_info__ = self.id, objPath
val= self._evaluate(obj)
# see whether something changed - do nothing, if it did not
oldval= self._unindex.get(documentId)
if val == oldval: return 0
# some data types, e.g. "OOSet"s do not define a senseful "==".
# provide a hook to handle this case
customEqual= self._equalValues
if customEqual is not None and customEqual(val,oldval): return 0
# remove state info
update= self._update
if update is None or oldval is None or val is None:
# no optimization
if oldval is not None: self._unindex_object(documentId,oldval,val is None)
if val is None: return 0
rv= self._indexValue(documentId,val,threshold)
if oldval is None: self.numObjects.change(1)
else:
# optimization
rv= update(documentId,val,oldval,threshold)
if isinstance(rv, tuple): return rv[0]
# remember indexed value
self._unindex[documentId]= val
return rv
def unindex_object(self,documentId):
'''unindex *documentId*.
ATT: why do we not have a *threshold*????
'''
# Note: objPath/documentId should be provided by the catalog -- but it is not
__traceback_info__ = self.id, documentId
val= self._unindex.get(documentId)
if val is None: return # not indexed
self._unindex_object(documentId,val,1)
def _unindex_object(self,documentId,val,remove):
self._unindexValue(documentId,val)
if remove:
del self._unindex[documentId]
self.numObjects.change(-1)
def uniqueValues(self, name=None, withLengths=0):
'''unique values for *name* (???).
If *withLength*, returns sequence of tuples *(value,length)*.
'''
if name is None: name= self.id
if name != self.id: return ()
values= self._index.keys()
if not withLengths: return tuple(values)
return tuple([(value,self._len(value)) for value in values])
def _apply_index(self,request, cid= ''):
'''see 'PluggableIndex'.
What is *cid* for???
'''
__traceback_info__ = self.id
record= parseIndexRequest(request, self.id, self.query_options)
terms= record.keys
if terms is None: return
__traceback_info__ = self.id, record.keys
op= record.get('operator', self.useOperator)
if op not in self.operators:
raise ValueError("operator not permitted: %s" % op)
combine= op == 'or' and union or intersection
filteredSearch = None
if record.get('isearch') and record.get('isearch_filter') \
and self.supportFiltering and IFilter is not None:
filteredSearch = self._getFilteredISearch(record)
if filteredSearch is None:
match = record.get('match')
if match is not None:
l = []; match = getattr(self, 'match' + match.capitalize())
prenorm = self._prenormalizeTerm
for t in terms:
t = prenorm(t, None)
if t is not None: l.extend(match(t))
terms = l
range= record.get('range')
if range is not None:
terms= [self._standardizeTerm(t,elimStopTerm=0, prenormalize=not match) for t in terms]
range= range.split(':'); lo= hi= None
if 'min' in range: lo= min(terms)
if 'max' in range: hi= max(terms)
terms= self._enumerateRange(lo,hi)
else:
terms= [self._standardizeTerm(t, prenormalize=not match) for t in terms]
if filteredSearch is None: r = self._search(terms,combine,record)
else: r = filteredSearch
if r is None: return
return r, self.id
#################################################################
# search
def _search(self,terms,combine,record):
return setOperation(
combine is union and 'or' or 'and',
[self._searchTerm(t,record) for t in terms],
record.get('isearch'),
)
def _searchTerm(self,term,record):
return self._load(term)
def _enumerateRange(self,min,max):
'''enumerate terms between *min* and *max*.'''
return self._index.keys(min,max)
#################################################################
# filtering
supportFiltering = False
def _getFilteredISearch(self, query):
'''return a filtered search for *query*, if this seems promissing, or 'None'.
'''
preds = []
enumerator = self._getFilterEnumerator(); makeFilter = self._makeFilter
terms = query.keys
match = query.get('match'); range = query.get('range')
op = query.get('operator', self.useOperator)
if match is not None:
# we do not want to filter combined 'match' and 'range' queries
if range is not None: return
# can only filter 'or' matches
if op != 'or': return
# we can filter 'match' queries only if there is no 'normalizer'
# maybe, we should not filter, if there is an 'ignorer'?
if self._hasNormalizer(): return
l = []; match = getattr(self, 'matchFilter' + match.capitalize())
prenorm = self._prenormalizeTerm
for t in terms:
t = prenorm(t, None)
if t is not None: preds.append(match(t))
else:
range= query.get('range')
if range is not None:
# can only filter 'or' ranges
if op != 'or': return
terms= [self._standardizeTerm(t,elimStopTerm=0, prenormalize=not match) for t in terms]
range= range.split(':'); lo= hi= None
if 'min' in range: lo= min(terms)
if 'max' in range: hi= max(terms)
preds.append(_rangeChecker(lo,hi))
else:
makePred = self._makeTermPredicate; standardize = self._standardizeTerm
preds = [
makePred(standardize(t, prenormalize=not match))
for t in terms
]
subsearches = [IFilter(makeFilter(pred), enumerator) for pred in preds]
return self._combineSubsearches(subsearches, op)
def _combineSubsearches(self, subsearches, op):
if len(subsearches) == 1: return subsearches[0]
combine = op == 'or' and IOr or IAnd
search = combine(*subsearches); search.complete()
return search
def _getFilterEnumerator(self):
return Enumerator(self._unindex)
def _makeTermPredicate(self, term):
'''this is adequate for field and keyword indexes.'''
return lambda x, t=term: x == t
def _makeFilter(self, pred):
'''a document filter 'did -> True/False' checking term predicate *pred*.
This definition is adequate, when the predicate can be directly
applied to the 'unindex' value.
'''
def check(did):
dv = self._unindex.get(did)
return dv is not None and pred(dv)
return check
#################################################################
# required for sorting
# no longer needed for Zope 2.7 -- keep for compatibility
def keyForDocument(self, docId): return self._unindex[docId]
def items(self):
return [(k,self._load(k)) for k in self._index.keys()]
#################################################################
# Reverse ordering support
def getReverseOrder(self):
'''return the keys in reverse order.'''
if self.ReverseOrder:
return _LazyMap(lambda x: x.getValue(), self._reverseOrder.keys())
#################################################################
# Storage API
# we implement a small optimization
# a single document is stored as integer; more are stored as an IITreeSet
ReverseOrder = 0
def _insert(self,term,docId, _isInstance= isinstance, _IntType= IntType):
'''index *docId* under *term*.'''
index= self._index
dl= index.get(term)
if dl is None:
index[term]= docId; self.__len__.change(1)
if self.ReverseOrder: self._reverseOrder.insert(reverseOrder(term))
return
if _isInstance(dl,_IntType): dl= index[term]= IITreeSet((dl,))
dl.insert(docId)
def _remove(self,term,docId, _isInstance= isinstance, _IntType= IntType):
'''unindex *docId* under *term*.'''
index= self._index
dl= index.get(term); isInt= _isInstance(dl,_IntType)
if dl is None or isInt and dl != docId:
raise ValueError('Attempt to remove nonexisting document %s from %s'
% (docId, self.id)
)
if isInt: dl = None
else: dl.remove(docId)
if not dl:
del index[term]; self.__len__.change(-1)
if self.ReverseOrder: self._reverseOrder.remove(reverseOrder(term))
def _load(self,term, _isInstance= isinstance, _IntType= IntType):
'''the docId list for *term*.'''
index= self._index
dl= index.get(term)
if dl is None: return IISet()
if _isInstance(dl,_IntType): return IISet((dl,))
return dl
def _len(self,term):
'''the number of documents indexed under *term*.'''
return len(self._load(term))
###########################################################################
## methods to maintain auxiliary indexes
## we implement the same optimization as for the main index
def _insertAux(self, index, term, docId):
'''index *docId* under *term*.'''
dl= index.get(term)
if dl is None: index[term]= docId; return
if isinstance(dl,int): dl= index[term]= IITreeSet((dl,))
dl.insert(docId)
def _removeAux(self, index, term, docId):
'''unindex *docId* under *term*.'''
dl= index.get(term); isInt= isinstance(dl,int)
if dl is None or isInt and dl != docId:
raise ValueError('Attempt to remove nonexisting document %s from %s'
% (docId, self.id)
)
if isInt: dl = None
else: dl.remove(docId)
if not dl: del index[term]
def _loadAux(self,index, term):
'''the docId list for *term*.'''
dl= index.get(term)
if dl is None: return IISet()
if isinstance(dl,int): return IISet((dl,))
return dl
#################################################################
# Term standardization and checking
def listTermTypes(self):
'''the sequence of supported term types.'''
return _TermTypeList
def listCopyTypes(self):
'''the sequence of term copy types.'''
return _TermCopyList
def listCombiners(self):
'''the sequence of combine types.'''
return self.Combiners
def _standardizeTerm(self, value, object=None, copy=False, elimStopTerm=True, prenormalize=True):
if prenormalize:
value = self._prenormalizeTerm(value, object)
if value is None: return
if elimStopTerm:
value= self._ignore(value,object)
if value is None: return
value= self._normalize(value,object)
if value is None: return
tt= self.TermType
checker= _TermTypeMap[tt]
if checker: value= checker(self,value,object)
if copy and tt in ('not checked', 'instance', 'expression checked',):
copy= _CopyTypeMap[self.TermCopy]
if copy: value= copy(value)
return value
_prenormalizer = None
def _prenormalizeTerm(self, value, object):
PT = self.PrenormalizeTerm
if not PT: return value
normalizer = self._prenormalizer
if normalizer is None:
normalizer = self._prenormalizer = Normalize()
normalizer.NormalizerProperty = 'PrenormalizeTerm'
return normalizer._normalize(value, object)
#################################################################
# Evaluation
def _evaluate(self,object):
'''evaluate *object* with respect to this index.'''
l= []; v= None
combiner= self.CombineType; useFirst= combiner == 'useFirst'
for vp in self.objectValues():
v= vp.evaluate(object)
if v is not None:
if useFirst: break
l.append(v)
if useFirst:
if v is None: return
return self._standardizeValue(v,object)
return getattr(self,'_combine_' + combiner)(l,object)
def _standardizeValue(self,value,object):
return self._standardizeTerm(value,object,1)
#################################################################
# to be defined by concrete deriving classes
# _indexValue(self,documentId,val,threshold)
# _unindexValue(self,documentId,val)
#################################################################
# optionally defined by concrete deriving classes
# _update(self,documentId,val,oldval,threshold)
# returns number of entries added; if tuple, _unindex already updated
# _equalValues(self,val1,val2) -- true, if standardized values are equal
_update= None
_equalValues= None
#################################################################
# Value provider management
def all_meta_types(self):
return (
{ 'name' : AttributeLookup.meta_type,
'action' : 'addAttributeLookupForm',
'permission' : ManageManagableIndexes,
},
{ 'name' : ExpressionEvaluator.meta_type,
'action' : 'addExpressionEvaluatorForm',
'permission' : ManageManagableIndexes,
},
)
def addAttributeLookupForm(self):
'''addForm for 'AttributeLookup' value provider.'''
return addForm.__of__(self)(
type= 'AttributeLookup',
description= '''An AttributeLookup is a value provider which evaluates an object by looking up an attribute of the object.''',
action= 'addValueProvider',
)
def addExpressionEvaluatorForm(self):
'''addForm for 'ExpressionEvaluator' value provider.'''
return addForm.__of__(self)(
type= 'ExpressionEvaluator',
description= '''An ExpressionEvaluator is a value provider which evaluates an expression.''',
action= 'addValueProvider',
)
def addValueProvider(self,id,type, RESPONSE=None):
'''add a value provider with *id* of *type*.'''
if type not in ('AttributeLookup', 'ExpressionEvaluator'):
raise ValueError('unknown type')
cl= _mdict[type]
# try to avaid a name conflict
eid= id
if not id.endswith('_') and hasattr(aq_base(self),id): eid= id + '_'
vp= cl(); vp.id= eid
if id != eid and type == 'AttributeLookup': vp.Name= id
self._setObject(eid, vp)
vp= self._getOb(eid)
if RESPONSE is None: return vp
RESPONSE.redirect('%s/manage_workspace' % vp.absolute_url())
def __init__(self, **kw):
super(PhotoAlbum, self).__init__(**kw)
self.albums = OOTreeSet()
self.photos = OOTreeSet()
self.__dict__['totalPhotos'] = Length(0)
def test_comparison(sheraf_connection, m0, m1, m2):
assert QuerySet() is not None
assert QuerySet([m0]) == QuerySet([m0])
assert QuerySet([m0]) == QuerySet(m for m in [m0])
assert QuerySet([m0]) == QuerySet(OOTreeSet([m0]))
def __init__(self, doc_id):
self.doc_id = doc_id
self.assignees = OOTreeSet()
def __init__(self):
self.comments = OOTreeSet()
def replaceEntries(self,items):
"replace all the entries with these ones"
self.entry = OOTreeSet()
for item in items:
self.entry.insert(item)
def setUp(self):
self.t = OOTreeSet()
def main():
m = Mock()
oset = OOTreeSet()
oset.add(m)
class MultipleValuesIndex(Index):
"""An `index <Index>` where keys may be defined multiple times."""
accepts_multiple_values = True
@overrides(Index.__init__)
def __init__(self, pairs = None):
self.__items = OOTreeSet()
self.__descending_items = OOTreeSet()
Index.__init__(self, pairs)
@overrides(Index.add)
def add(self, key, value):
self.__items.insert(Entry(key, value))
self.__descending_items.insert(DescEntry(key, value))
@overrides(Index.remove)
def remove(self, key, value = undefined):
entry = Entry(key, value)
desc_entry = DescEntry(key, value)
# Remove all values for the given key
if value is undefined:
while True:
try:
self.__items.remove(entry)
self.__descending_items.remove(desc_entry)
except KeyError:
break
# Remove a specific value for the given key
else:
try:
self.__items.remove(entry)
self.__descending_items.remove(desc_entry)
except KeyError:
pass
@overrides(Index.items)
def items(
self,
min = undefined,
max = undefined,
exclude_min = False,
exclude_max = False,
descending = False
):
if descending:
return self.__descending_items.keys(
min = None if max is undefined
else DescEntry(
max,
lower_bound if exclude_max else upper_bound
),
max = None if min is undefined
else DescEntry(
min,
upper_bound if exclude_min else lower_bound
)
)
else:
return self.__items.keys(
min = None if min is undefined
else Entry(
min,
upper_bound if exclude_min else lower_bound
),
max = None if max is undefined
else Entry(
max,
lower_bound if exclude_max else upper_bound
)
)
@overrides(Index.min_key)
def min_key(self, exclude_none = False):
if exclude_none:
for key in self.keys(min = None, include_min = False):
return key
else:
return self.__items.minKey().key
@overrides(Index.max_key)
def max_key(self):
return self.__items.maxKey().key
@overrides(Index.__len__)
def __len__(self):
return len(self.__items)
@overrides(Index.__bool__)
def __bool__(self):
return bool(self.__items)
@overrides(Index.__contains__)
def __contains__(self, key):
for value in self.values(key = key):
return True
return False
class Members(BaseContentContainer):
interface.implements(IWorkspace, IMembers, IMemberRoleManagement)
title = _(u'Members')
description = u''
approving = True
def __init__(self, **kw):
super(Members, self).__init__(**kw)
self.joined = OOBTree()
self.notapproved = OOTreeSet()
@getproperty
def joining(self):
roles = []
for rid, setting in IRolePermissionMap(
self.__parent__).getRolesForPermission('zojax.JoinGroup'):
if setting is Allow:
role = queryUtility(IRole, rid)
if IPublicRole.providedBy(role):
roles.append(rid)
return roles
@setproperty
def joining(self, value):
roles = IRolePermissionManager(self.__parent__)
for rid in value:
role = queryUtility(IRole, rid)
if IPublicRole.providedBy(role):
roles.grantPermissionToRole('zojax.JoinGroup', rid)
else:
roles.denyPermissionToRole('zojax.JoinGroup', rid)
@getproperty
def invites(self):
value = []
roles = [
role for role, setting in IRolePermissionMap(
self.__parent__).getRolesForPermission('zojax.InviteGroupMember')
if setting is Allow]
if 'group.Member' in roles:
value.append('member')
if 'group.Manager' in roles:
value.append('manager')
return value
@setproperty
def invites(self, value):
roles = IRolePermissionManager(self.__parent__)
for val, role in [('member', 'group.Member'),
('manager', 'group.Manager')]:
if val in value:
roles.grantPermissionToRole('zojax.InviteGroupMember', role)
else:
roles.unsetPermissionFromRole('zojax.InviteGroupMember', role)
def join(self):
request = getRequest()
if request is not None:
self.joinPrincipal(request.principal.id, self.approving)
def remove(self):
request = getRequest()
if request is not None:
self.removePrincipal(request.principal.id)
def joinPrincipal(self, principalId, approved=True,
_td = timedelta(milliseconds=1)):
if principalId not in self:
member = Member()
event.notify(ObjectCreatedEvent(member))
self[principalId] = member
joined = parseDatetimetz(str(datetime.now()))
while joined in self.joined:
joined = joined + _td
member.joined = joined
self.joined[joined] = principalId
if not approved:
member.approved = False
self.notapproved.insert(principalId)
else:
event.notify(MemberApprovedEvent(member))
event.notify(ObjectModifiedEvent(self.__parent__))
updatePermissions(self.__parent__)
def isMember(self, principalId):
return principalId in self
def isManager(self, principalId):
return principalId in self.managers
@Lazy
def managers(self):
self.managers = ()
self._p_changed = True
return self.managers
@property
def principals(self):
return self.keys()
def toMember(self, id):
if id in self:
if id in self.managers:
managers = list(self.managers)
managers.remove(id)
self.managers = tuple(managers)
updatePermissions(self.__parent__)
event.notify(ObjectModifiedEvent(self.__parent__))
def toManager(self, id):
if id in self:
if id not in self.managers:
managers = list(self.managers)
managers.append(id)
self.managers = tuple(managers)
updatePermissions(self.__parent__)
event.notify(ObjectModifiedEvent(self.__parent__))
def approve(self, id):
if id in self:
member = self[id]
if not member.approved:
del member.approved
self.notapproved.remove(id)
event.notify(MemberApprovedEvent(member))
event.notify(ObjectModifiedEvent(self.__parent__))
updatePermissions(self.__parent__)
def __delitem__(self, key):
member = self[key]
self.toMember(key)
del self.joined[member.joined]
if key in self.notapproved:
self.notapproved.remove(key)
super(Members, self).__delitem__(key)
updatePermissions(self.__parent__)
event.notify(ObjectModifiedEvent(self.__parent__))
def invite(self, principal, message):
invitation = MemberInvitation(
principal, getUtility(IIntIds).getId(self.__parent__), message)
IOwnership(invitation).ownerId = getRequest().principal.id
event.notify(ObjectCreatedEvent(invitation))
getUtility(IInvitations).storeInvitation(invitation)
return invitation
def removeInvitation(self, id):
configlet = removeAllProxies(getUtility(IInvitations))
invitation = configlet.get(id)
if invitation is not None and \
invitation.oid == getUtility(IIntIds).getId(self.__parent__):
del configlet[id]
def __or__(self, other):
return QuerySet(union(OOTreeSet(self), OOTreeSet(other)))
def __init__(self):
self._brands = OOTreeSet()
def addEntry(self, item):
"Add an entry to the list"
if self.entry is None:
self.entry = OOTreeSet()
self.entry.insert(item)
def __and__(self, other):
return QuerySet(intersection(OOTreeSet(self), OOTreeSet(other)))
def __init__(self, **kw):
super(Members, self).__init__(**kw)
self.joined = OOBTree()
self.notapproved = OOTreeSet()
def __xor__(self, other):
return QuerySet(difference(OOTreeSet(self), OOTreeSet(other)))
def __init__(self, uids):
self.archived = False
self.uids = OOTreeSet()
self.uids.update(uids)
def __init__(self):
self.subscribers = OOTreeSet()
def insert(self, key, value):
"Insert value under key."
if not self._index.has_key(key):
self._index[key] = OOTreeSet()
self._index[key].insert(value)
def __init__(self):
self._brands = OOTreeSet()
class SimpleList(UserObject):
"SimpleList object"
security = ClassSecurityInfo()
meta_type = "SimpleList"
radioOptions = ( ('append', 'Add'), ('remove', 'Remove') , ('replace', 'Replace') )
entry = None
security.declareProtected('CompoundDoc: Modify SimpleList', 'addEntries')
def addEntries(self,items):
"add these items to our current list of items"
if self.entry is None:
self.replaceEntries(items)
for item in items:
if not self.entry.has_key(item):
self.entry.insert(item)
security.declareProtected('CompoundDoc: Modify SimpleList', 'removeEntries')
def removeEntries(self,items):
"removes these items from the current set of entries"
if self.entry is not None:
for item in items:
if self.entry.has_key(item):
self.entry.remove(item)
security.declareProtected('CompoundDoc: Modify SimpleList', 'replaceEntries')
def replaceEntries(self,items):
"replace all the entries with these ones"
self.entry = OOTreeSet()
for item in items:
self.entry.insert(item)
security.declareProtected('CompoundDoc: View SimpleList', 'drawViewWindows')
def drawViewWindows(self):
"draw the list of addresses in windows format"
return '\r\n'.join(self.getEntries())
security.declareProtected('CompoundDoc: View SimpleList', 'drawViewMac')
def drawViewMac(self):
"draw the list of addresses in mac format"
return '\r'.join(self.getEntries())
security.declareProtected('CompoundDoc: View SimpleList', 'drawViewUnix')
def drawViewUnix(self):
"draw the list of addresses in unix format"
return '\n'.join(self.getEntries())
radioLookup = {'append' : addEntries, 'remove' : removeEntries , 'replace' : replaceEntries }
security.declareProtected('CompoundDoc: Add List Item', 'addEntry')
def addEntry(self, item):
"Add an entry to the list"
if self.entry is None:
self.entry = OOTreeSet()
self.entry.insert(item)
security.declareProtected('CompoundDoc: Del List Item', 'delEntry')
def delEntry(self, item):
"Remove an entry form the list"
if self.entry.has_key(item):
self.entry.remove(item)
if not len(self.entry):
self.entry = None
security.declareProtected('CompoundDoc: Has List Item', 'hasEntry')
def hasEntry(self,item):
"Do we have this item"
if self.entry is not None:
return self.entry.has_key(item)
security.declareProtected('CompoundDoc: Get List Items', 'getEntries')
def getEntries(self):
"Return all the entries as a list"
if self.entry is not None:
return self.entry.keys()
return OOTreeSet()
security.declareProtected('CompoundDoc: Get List Items', 'getTree')
def getTree(self):
"Return all the entries as the native format"
if self.entry is not None:
return self.entry
return OOTreeSet()
security.declareProtected('CompoundDoc: Clear List Items', 'clearEntries')
def clearEntries(self):
"Remove all the entries from the object"
self.entry = None
security.declarePrivate('before_manage_edit')
def before_manage_edit(self, dict):
"process the edits"
if 'editAdd' in dict and 'editAddName' in dict:
self.addEntries(dict['editAddName'])
if 'editDel' in dict and 'editDelName' in dict:
self.removeEntries(dict['editDelName'])
if 'editClear' in dict:
self.clearEntries()
data = dict.pop('data', None)
if data is not None:
temp = data.read().split()
self.radioLookup[dict['fileSettings']](self,temp)
security.declareProtected('View management screens', 'edit')
def edit(self, *args, **kw):
"Inline edit short object"
temp = []
append = temp.append
append(self.text_area('editAddName', '', formType="tokens"))
append('<p>%s</p>' % self.create_button("editAdd", "Add Entries"))
append(self.text_area('editDelName', '', formType="tokens"))
append('<p>%s</p>' % self.create_button("editDel", "Delete Entries"))
append('<p>Upload File:')
append(self.input_file('data'))
temp.extend(self.radio_list('fileSettings', self.radioOptions, selected='append'))
append('</p>')
append(self.create_button("editClear", "Clear All Entries"))
append('<p>View Email Addresses:')
path = self.absolute_url_path()
format = ' <a href="%s">%s</a> '
append(format % (os.path.join(path, 'drawViewWindows'), 'Windows'))
append(format % (os.path.join(path, 'drawViewUnix'), 'Unix'))
append(format % (os.path.join(path, 'drawViewMac'), 'Mac'))
append('</p>')
return ''.join(temp)
security.declareProtected('View', 'view')
def view(self):
"Inline draw view"
return self.unorderedList(self.getEntries())
security.declarePrivate('PrincipiaSearchSource')
def PrincipiaSearchSource(self):
"This is the basic search function"
return ' '.join(self.getEntries())
security.declarePrivate('classUpgrader')
def classUpgrader(self):
"upgrade this class"
self.convertListToOOTreeSet()
security.declarePrivate('convertListToOOTreeSet')
def convertListToOOTreeSet(self):
"conver the list object to an OOTreeSet"
if len(self.entry) and isinstance(self.entry, types.ListType):
temp = OOTreeSet()
for i in self.entry:
temp.insert(i)
self.setObject('entry', temp)
convertListToOOTreeSet = utility.upgradeLimit(convertListToOOTreeSet, 141)
def __init__(self, pairs = None):
self.__items = OOTreeSet()
self.__descending_items = OOTreeSet()
Index.__init__(self, pairs)
def _items(self):
res = {}
tag_set = OOTreeSet(self.tags)
for i in self.tags:
res[i] = Tag(i, self.__parent__, self, tag_set, [])
return res
def __init__(self, uids):
self.archived = False
self.uids = OOTreeSet()
self.uids.update(uids)
class ManagableIndex(OFolder, Normalize, Ignore):
'''Abstract base class for 'ManagableIndex'.'''
security = ClassSecurityInfo()
security.declareProtected(
ManageManagableIndexes,
'addAttributeLookupForm',
'addExpressionEvaluatorForm',
'addValueProvider',
'manage_propertiesForm',
'manage_changeProperties',
'manage_editProperties',
'indexSize',
)
security.declarePrivate('getReverseOrder', )
icon = 'misc_/PluginIndexes/index.gif'
manage_options = (OFolder.manage_options[:1] + OFolder.manage_options[2:])
operators = (
'or',
'and',
)
useOperator = 'or'
query_options = ('operator', 'range', 'usage', 'match', 'isearch',
'isearch_filter')
Combiners = ('useFirst', )
NormalizerProperty = 'NormalizeTerm'
IgnoreProperty = 'StopTermPredicate'
_properties = ((
{
'id':
'CombineType',
'type':
'selection',
'mode':
'w',
'select_variable':
'listCombiners',
'label':
'Combine type: determines how values from value providers are combined'
},
{
'id':
'PrenormalizeTerm',
'type':
'string',
'mode':
'w',
'label':
'Term prenormalizer: applied to terms before term expansion in queries and (always) before stop term elimination; used e.g. for case normalization, stemming, phonetic normalization, ...',
},
{
'id':
IgnoreProperty,
'type':
'string',
'mode':
'w',
'label':
'Stop term predicate: used to recognized and eliminate stop terms; used always (except in range queries) after prenormalization',
},
{
'id':
NormalizerProperty,
'type':
'string',
'mode':
'w',
'label':
'Term normalizer: applied to terms before type checking; used e.g. for encoding the term into a efficient form',
},
{
'id':
'TermType',
'type':
'selection',
'mode':
'w',
'select_variable':
'listTermTypes',
'label':
'Term type: used to convert and check the terms type; may allows to choose specially optimized index structures (e.g. for integer types) or provide additional features (e.g. term expansions for string types) -- clear+reindex after change!',
},
{
'id':
'TermTypeExtra',
'type':
'string',
'mode':
'w',
'label':
'Term type argument: required by some term types (see the documentation)',
},
{
'id':
'TermCopy',
'type':
'selection',
'mode':
'w',
'select_variable':
'listCopyTypes',
'label':
'Control term copying: may be necessary for mutable terms to prevent index corruption',
},
))
TermType = _TermTypeList[0]
TermTypeExtra = ''
TermCopy = _TermCopyList[0]
CombineType = Combiners[0]
NormalizeTerm = ''
PrenormalizeTerm = ''
StopTermPredicate = ''
implements(IManagableIndex)
def __init__(self, name, extra=None, caller=None):
self.id = name
def setProperties(obj, values, special):
__traceback_info__ = obj
allowed = dict.fromkeys(special)
allowed.update(dict.fromkeys([p['id'] for p in obj._properties]))
for k in values.keys():
if k not in allowed:
raise ValueError('not a known property: %s' % k)
obj.manage_changeProperties(**values)
if extra:
# GenericSetup passes something stupid -- try to normalize
if not hasattr(extra, 'keys'): extra = extra.__dict__
setProperties(self, extra, ('ValueProviders', ))
providers = extra.get('ValueProviders')
else:
providers = None
if providers is None: self._createDefaultValueProvider()
else:
for p in providers:
vp = self.addValueProvider(p['id'], p['type'])
setProperties(vp, p, (
'id',
'type',
))
self.clear()
def clear(self):
'''clear the index.'''
l = self.__len__
if isinstance(l, Length): l.set(0)
else: self.__len__ = Length()
try:
self.numObjects.set(0)
except AttributeError:
self.numObjects = Length()
if self.ReverseOrder: self._reverseOrder = OOTreeSet()
self._setup()
def __len__(self):
'''Python 2.4 requires this to be defined inside the class.'''
l = self.__len__
if not isinstance(l, Length): l = self.__len__ = Length()
return l()
def indexSize(self):
return self.__len__()
def _setup(self):
self._unindex = IOBTree()
treeType = self.TermType in _IntegerTypes and IOBTree or OOBTree
self._index = treeType()
def _createDefaultValueProvider(self):
self.addValueProvider(self.id, 'AttributeLookup')
## term expansion -- provided for indexes with declared "string" and "ustring"
## term types
def matchGlob(self, t):
'''match '*' (match any sequence) and '?' (match any character) in *t* returning a list of terms.'''
# leads to wrong result -- would need to check against index
# if not ('*' in t or '?' in t): return [t]
regexp = glob2regexp(t)
return self.matchRegexp(regexp + '$')
_matchType = None
def matchRegexp(self, regexp):
'''match *regexp* into a list of matching terms.
Note that for efficiency reasons, the regular expression
should have an easily recognizable plain text prefix -- at
least for large indexes.
'''
prefix, regexp = _splitPrefixRegexp(regexp)
termType = self._matchType or self.TermType
if termType == 'string':
prefix = str(prefix)
regexp = str(regexp)
elif termType == 'ustring':
prefix = unicode(prefix)
regexp = unicode(regexp)
elif termType == 'asis':
pass
else:
raise ValueError(
"Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
% (self.id, termType))
index = self._getMatchIndex()
pn = len(prefix)
l = []
match = compile(regexp).match
for t in index.keys(prefix): # terms starting prefix
if not t.startswith(prefix): break
if match(t[pn:]): l.append(t)
return l
def _getMatchIndex(self):
'''the index used for expansion'''
return self._index
## match filtering
def matchFilterGlob(self, t):
'''see 'matchGlob' but for filtering.'''
regexp = glob2regexp(t)
return self.matchFilterRegexp(regexp + '$')
def matchFilterRegexp(self, regexp):
'''see 'matchRegexp' but for filtering.'''
termType = self._matchType or self.TermType
if termType == 'string': regexp = str(regexp)
elif termType == 'ustring': regexp = unicode(regexp)
elif termType == 'asis': pass
else:
raise ValueError(
"Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
% (self.id, termType))
return compile(regexp).match
## Responsibilities from 'IPluggableIndex'
# 'getId' -- inherited from 'SimpleItem'
def getEntryForObject(self, documentId, default=None):
'''Information for *documentId*.'''
info = self._unindex.get(documentId)
if info is None: return default
return repr(info)
def getIndexSourceNames(self):
"""we are more flexible than the framework expects,
therefore some value providers have no (single) attribute name.
We return the value provider's id in this case.
"""
return [p.getName() for p in self.objectValues()]
def index_object(self, documentId, obj, threshold=None):
'''index *obj* as *documentId*.
Commit subtransaction when *threshold* index terms have been indexed.
Return the number of index terms indexed.
'''
# Note: objPath should be provided by the catalog -- but it is not
try:
objPath = obj.getPhysicalPath()
except:
objPath = None
__traceback_info__ = self.id, objPath
val = self._evaluate(obj)
cuv = self._val2UnindexVal
if val is not None and cuv is not None: unindexVal = cuv(val)
else: unindexVal = val
# see whether something changed - do nothing, if it did not
oldval = self._unindex.get(documentId)
if unindexVal == oldval: return 0
# some data types, e.g. "OOSet"s do not define a senseful "==".
# provide a hook to handle this case
customEqual = self._equalValues
if customEqual is not None:
if customEqual(val, oldval): return 0
# remove state info
update = self._update
if update is None or oldval is None or val is None:
# no optimization
if oldval is not None:
self._unindex_object(documentId, oldval, val is None)
if val is None: return 0
rv = self._indexValue(documentId, val, threshold)
if oldval is None: self.numObjects.change(1)
else:
# optimization
rv = update(documentId, val, oldval, threshold)
if isinstance(rv, tuple): return rv[0]
# remember indexed value
self._unindex[documentId] = unindexVal
return rv
def unindex_object(self, documentId):
'''unindex *documentId*.
ATT: why do we not have a *threshold*????
'''
# Note: objPath/documentId should be provided by the catalog -- but it is not
__traceback_info__ = self.id, documentId
val = self._unindex.get(documentId)
if val is None: return # not indexed
cuv = self._unindexVal2Val
if cuv is not None: val = cuv(val)
self._unindex_object(documentId, val, 1)
def _unindex_object(self, documentId, val, remove):
self._unindexValue(documentId, val)
if remove:
del self._unindex[documentId]
self.numObjects.change(-1)
def hasUniqueValuesFor(self, name):
return name is None or name == self.id
def uniqueValues(self, name=None, withLengths=0):
'''unique values for *name* (???).
If *withLength*, returns sequence of tuples *(value,length)*.
'''
if not self.hasUniqueValuesFor(name): return ()
values = self._index.keys()
if not withLengths: return tuple(values)
return tuple([(value, self._len(value)) for value in values])
def _apply_index(self, request, cid=''):
'''see 'PluggableIndex'.
What is *cid* for???
'''
__traceback_info__ = self.id
record = parseIndexRequest(request, self.id, self.query_options)
terms = record.keys
if terms is None: return
__traceback_info__ = self.id, terms
op = record.get('operator', self.useOperator)
if op not in self.operators:
raise ValueError("operator not permitted: %s" % op)
combine = op == 'or' and union or intersection
filteredSearch = None
if record.get('isearch') and record.get('isearch_filter') \
and self.supportFiltering and IFilter is not None:
filteredSearch = self._getFilteredISearch(record)
if filteredSearch is None:
match = record.get('match')
if match is not None:
l = []
match = getattr(self, 'match' + match.capitalize())
prenorm = self._prenormalizeTerm
for t in terms:
t = prenorm(t, None)
if t is not None: l.extend(match(t))
terms = l
range = record.get('range')
if range is not None:
terms = [
self._standardizeTerm(t,
elimStopTerm=0,
prenormalize=not match)
for t in terms
]
range = range.split(':')
lo = hi = None
if 'min' in range: lo = min(terms)
if 'max' in range: hi = max(terms)
terms = self._enumerateRange(lo, hi)
else:
terms = [
self._standardizeTerm(t, prenormalize=not match)
for t in terms
]
if filteredSearch is None: r = self._search(terms, combine, record)
else: r = filteredSearch
if r is None: return
return r, self.id
#################################################################
# search
def _search(self, terms, combine, record):
return setOperation(
combine is union and 'or' or 'and',
[self._searchTerm(t, record) for t in terms],
record.get('isearch'),
)
def _searchTerm(self, term, record):
return self._load(term)
def _enumerateRange(self, min, max):
'''enumerate terms between *min* and *max*.'''
return self._index.keys(min, max)
#################################################################
# filtering
supportFiltering = False
def _getFilteredISearch(self, query):
'''return a filtered search for *query*, if this seems promissing, or 'None'.
'''
preds = []
enumerator = self._getFilterEnumerator()
makeFilter = self._makeFilter
terms = query.keys
match = query.get('match')
range = query.get('range')
op = query.get('operator', self.useOperator)
if match is not None:
# we do not want to filter combined 'match' and 'range' queries
if range is not None: return
# can only filter 'or' matches
if op != 'or': return
# we can filter 'match' queries only if there is no 'normalizer'
# maybe, we should not filter, if there is an 'ignorer'?
if self._hasNormalizer(): return
l = []
match = getattr(self, 'matchFilter' + match.capitalize())
prenorm = self._prenormalizeTerm
for t in terms:
t = prenorm(t, None)
if t is not None: preds.append(match(t))
else:
range = query.get('range')
if range is not None:
# can only filter 'or' ranges
if op != 'or': return
terms = [
self._standardizeTerm(t,
elimStopTerm=0,
prenormalize=not match)
for t in terms
]
range = range.split(':')
lo = hi = None
if 'min' in range: lo = min(terms)
if 'max' in range: hi = max(terms)
preds.append(_rangeChecker(lo, hi))
else:
makePred = self._makeTermPredicate
standardize = self._standardizeTerm
preds = [
makePred(standardize(t, prenormalize=not match))
for t in terms
]
subsearches = [IFilter(makeFilter(pred), enumerator) for pred in preds]
return self._combineSubsearches(subsearches, op)
def _combineSubsearches(self, subsearches, op):
if len(subsearches) == 1: return subsearches[0]
combine = op == 'or' and IOr or IAnd
search = combine(*subsearches)
search.complete()
return search
def _getFilterEnumerator(self):
return Enumerator(self._unindex)
def _makeTermPredicate(self, term):
'''this is adequate for field and keyword indexes.'''
return lambda x, t=term: x == t
def _makeFilter(self, pred):
'''a document filter 'did -> True/False' checking term predicate *pred*.
This definition is adequate, when the predicate can be directly
applied to the 'unindex' value.
'''
def check(did):
dv = self._unindex.get(did)
return dv is not None and pred(dv)
return check
#################################################################
# required for sorting
# no longer needed for Zope 2.7 -- keep for compatibility
def keyForDocument(self, docId):
return self._unindex[docId]
def items(self):
return [(k, self._load(k)) for k in self._index.keys()]
#################################################################
# Reverse ordering support
def getReverseOrder(self):
'''return the keys in reverse order.'''
if self.ReverseOrder:
return _LazyMap(lambda x: x.getValue(), self._reverseOrder.keys())
#################################################################
# Storage API
# we implement a small optimization
# a single document is stored as integer; more are stored as an IITreeSet
ReverseOrder = 0
def _insert(self, term, docId, _isInstance=isinstance, _IntType=IntType):
'''index *docId* under *term*.'''
index = self._index
dl = index.get(term)
if dl is None:
index[term] = docId
self.__len__.change(1)
if self.ReverseOrder: self._reverseOrder.insert(reverseOrder(term))
return
if _isInstance(dl, _IntType): dl = index[term] = IITreeSet((dl, ))
dl.insert(docId)
def _remove(self, term, docId, _isInstance=isinstance, _IntType=IntType):
'''unindex *docId* under *term*.'''
index = self._index
dl = index.get(term)
isInt = _isInstance(dl, _IntType)
if dl is None or isInt and dl != docId:
raise ValueError(
'Attempt to remove nonexisting document %s from %s' %
(docId, self.id))
if isInt: dl = None
else: dl.remove(docId)
if not dl:
del index[term]
self.__len__.change(-1)
if self.ReverseOrder: self._reverseOrder.remove(reverseOrder(term))
def _load(self, term, _isInstance=isinstance, _IntType=IntType):
'''the docId list for *term*.'''
index = self._index
dl = index.get(term)
if dl is None: return IISet()
if _isInstance(dl, _IntType): return IISet((dl, ))
return dl
def _len(self, term):
'''the number of documents indexed under *term*.'''
return len(self._load(term))
###########################################################################
## methods to maintain auxiliary indexes
## we implement the same optimization as for the main index
def _insertAux(self, index, term, docId):
'''index *docId* under *term*.'''
dl = index.get(term)
if dl is None:
index[term] = docId
return
if isinstance(dl, int): dl = index[term] = IITreeSet((dl, ))
dl.insert(docId)
def _removeAux(self, index, term, docId):
'''unindex *docId* under *term*.'''
dl = index.get(term)
isInt = isinstance(dl, int)
if dl is None or isInt and dl != docId:
raise ValueError(
'Attempt to remove nonexisting document %s from %s' %
(docId, self.id))
if isInt: dl = None
else: dl.remove(docId)
if not dl: del index[term]
def _loadAux(self, index, term):
'''the docId list for *term*.'''
dl = index.get(term)
if dl is None: return IISet()
if isinstance(dl, int): return IISet((dl, ))
return dl
#################################################################
# Term standardization and checking
def listTermTypes(self):
'''the sequence of supported term types.'''
return _TermTypeList
def listCopyTypes(self):
'''the sequence of term copy types.'''
return _TermCopyList
def listCombiners(self):
'''the sequence of combine types.'''
return self.Combiners
def _standardizeTerm(self,
value,
object=None,
copy=False,
elimStopTerm=True,
prenormalize=True):
if prenormalize:
value = self._prenormalizeTerm(value, object)
if value is None: return
if elimStopTerm:
value = self._ignore(value, object)
if value is None: return
value = self._normalize(value, object)
if value is None: return
tt = self.TermType
checker = _TermTypeMap[tt]
if checker: value = checker(self, value, object)
if copy and tt in (
'not checked',
'instance',
'expression checked',
):
copy = _CopyTypeMap[self.TermCopy]
if copy: value = copy(value)
return value
_prenormalizer = None
def _prenormalizeTerm(self, value, object):
PT = self.PrenormalizeTerm
if not PT: return value
normalizer = self._prenormalizer
if normalizer is None:
normalizer = self._prenormalizer = Normalize()
normalizer.NormalizerProperty = 'PrenormalizeTerm'
return normalizer._normalize(value, object)
#################################################################
# Evaluation
def _evaluate(self, object):
'''evaluate *object* with respect to this index.'''
l = []
v = None
combiner = self.CombineType
useFirst = combiner == 'useFirst'
for vp in self.objectValues():
v = vp.evaluate(object)
if v is not None:
if useFirst: break
l.append(v)
if useFirst:
if v is None: return
return self._standardizeValue(v, object)
return getattr(self, '_combine_' + combiner)(l, object)
def _standardizeValue(self, value, object):
return self._standardizeTerm(value, object, 1)
#################################################################
# to be defined by concrete deriving classes
# _indexValue(self,documentId,val,threshold)
# _unindexValue(self,documentId,val)
#################################################################
# optionally defined by concrete deriving classes
# _update(self,documentId,val,oldval,threshold)
# returns number of entries added; if tuple, _unindex already updated
# _equalValues(self,val1,val2) -- true, if standardized values are equal
# _val2UnindexVal(val)
# convert *val* to the unindex value (stored in '_unindex')
# _unindexVal2Val(unindexVal)
# its inverse
_update = None
_equalValues = None
_val2UnindexVal = _unindexVal2Val = None
#################################################################
# Value provider management
def all_meta_types(self):
return (
{
'name': AttributeLookup.meta_type,
'action': 'addAttributeLookupForm',
'permission': ManageManagableIndexes,
'instance': lambda id: self._make_child(id, 'AttributeLookup'),
},
{
'name': ExpressionEvaluator.meta_type,
'action': 'addExpressionEvaluatorForm',
'permission': ManageManagableIndexes,
'instance':
lambda id: self._make_child(id, 'ExpressionEvaluator'),
},
)
def addAttributeLookupForm(self):
'''addForm for 'AttributeLookup' value provider.'''
return addForm.__of__(self)(
type='AttributeLookup',
description=
'''An AttributeLookup is a value provider which evaluates an object by looking up an attribute of the object.''',
action='addValueProvider',
)
def addExpressionEvaluatorForm(self):
'''addForm for 'ExpressionEvaluator' value provider.'''
return addForm.__of__(self)(
type='ExpressionEvaluator',
description=
'''An ExpressionEvaluator is a value provider which evaluates an expression.''',
action='addValueProvider',
)
@staticmethod
def _make_child(id, type):
if type not in ('AttributeLookup', 'ExpressionEvaluator'):
raise ValueError('unknown type')
cl = _mdict[type]
vp = cl()
vp.id = id
return vp
def addValueProvider(self, id, type, RESPONSE=None):
'''add a value provider with *id* of *type*.'''
# try to avaid a name conflict
eid = id
if not id.endswith('_') and hasattr(aq_base(self), id): eid = id + '_'
vp = self._make_child(eid, type)
if id != eid and type == 'AttributeLookup': vp.Name = id
self._setObject(eid, vp)
vp = self._getOb(eid)
if RESPONSE is None: return vp
RESPONSE.redirect('%s/manage_workspace' % vp.absolute_url())
