def testFixed1843(self): from BTrees.IIBTree import IISet t = IISet() t.insert(1) # This one used to fail to raise the TypeError when it occurred. self.assertRaises(TypeError, t.keys, "") # This one used to segfault. self.assertRaises(TypeError, t.keys, 0, "")
def get(self, pattern): """ Query the lexicon for words matching a pattern.""" # single word pattern produce a slicing problem below. # Because the splitter throws away single characters we can # return an empty tuple here. if len(pattern)==1: return () wc_set = [self.multi_wc, self.single_wc] digrams = [] globbing = 0 for i in range(len(pattern)): if pattern[i] in wc_set: globbing = 1 continue if i == 0: digrams.insert(i, (self.eow + pattern[i]) ) digrams.append((pattern[i] + pattern[i+1])) else: try: if pattern[i+1] not in wc_set: digrams.append( pattern[i] + pattern[i+1] ) except IndexError: digrams.append( (pattern[i] + self.eow) ) if not globbing: result = self._lexicon.get(pattern, None) if result is None: return () return (result, ) ## now get all of the intsets that contain the result digrams result = None for digram in digrams: result=union(result, self._digrams.get(digram, None)) if not result: return () else: ## now we have narrowed the list of possible candidates ## down to those words which contain digrams. However, ## some words may have been returned that match digrams, ## but do not match 'pattern'. This is because some words ## may contain all matching digrams, but in the wrong ## order. expr = re.compile(self.createRegex(pattern)) words = [] hits = IISet() for x in result: if expr.match(self._inverseLex[x]): hits.insert(x) return hits
def get(self, pattern): """ Query the lexicon for words matching a pattern.""" # single word pattern produce a slicing problem below. # Because the splitter throws away single characters we can # return an empty tuple here. if len(pattern) == 1: return () wc_set = [self.multi_wc, self.single_wc] digrams = [] globbing = 0 for i in range(len(pattern)): if pattern[i] in wc_set: globbing = 1 continue if i == 0: digrams.insert(i, (self.eow + pattern[i])) digrams.append((pattern[i] + pattern[i + 1])) else: try: if pattern[i + 1] not in wc_set: digrams.append(pattern[i] + pattern[i + 1]) except IndexError: digrams.append((pattern[i] + self.eow)) if not globbing: result = self._lexicon.get(pattern, None) if result is None: return () return (result, ) ## now get all of the intsets that contain the result digrams result = None for digram in digrams: result = union(result, self._digrams.get(digram, None)) if not result: return () else: ## now we have narrowed the list of possible candidates ## down to those words which contain digrams. However, ## some words may have been returned that match digrams, ## but do not match 'pattern'. This is because some words ## may contain all matching digrams, but in the wrong ## order. expr = re.compile(self.createRegex(pattern)) words = [] hits = IISet() for x in result: if expr.match(self._inverseLex[x]): hits.insert(x) return hits
class AccountingFolder(BaseFolder, BrowserDefaultMixin): """ """ security = ClassSecurityInfo() implements(interfaces.IAccountingFolder) meta_type = 'AccountingFolder' _at_rename_after_creation = True schema = AccountingFolder_schema ##code-section class-header #fill in your manual code here ##/code-section class-header # Methods # Manually created methods def __init__(self, oid, **kwargs): BaseFolder.__init__(self, oid, **kwargs) self._closing_transfers = IISet() security.declareProtected(permissions.View, 'getAccountingRoot') def getAccountingRoot(self): ''' Return 'self' as accounting root ''' return self def displayContentsTab(self): """ Hide contents tab """ return False def registerClosingDate(self, date): """ register closing transfer date """ # strip time before insert date = int(DateTime(date.Date())) self._closing_transfers.insert(date) def getClosingDates(self): """ return all registered closing dates """ return self._closing_transfers
def nearResultSets(sets, index, distance=5, bidirectional=1): """ perform near search on results sets """ # One resultset consists of an IISet() or documentIds and # tuple whose first element is the word (from LexiconLookup()) # First we perform an intersection to get the documentIds of # those documents that contain all the words docids = intersectResultSets(sets).docIds() # Now we determine for every document the positions of all # the words inside the document. Then we compare all the positions # to determine neighbourship words = [] for set in sets: for word in set.words().keys(): words.append(word) res_docids = IISet() for docId in docids: # the posMap is a list of tuples(word,IISet[positions]) posMap = index.positionsFromDocumentLookup(docId, words) if bidirectional: if len(posMap.checkPositionMapBidirectional(distance)) > 0: res_docids.insert(docId) else: if len(posMap.checkPositionMapUnidirectional(distance)) > 0: res_docids.insert(docId) d = {} for w in words: d[w] = 1.0 return ResultSet(res_docids, d)
def checkCatalog(path, indexes): """ perform some consistency checks on a ZCatalog instance""" root = Zope2.app() try: catalog = root.unrestrictedTraverse(path) except AttributeError: print 'Error: catalog object not found' sys.exit(1) # get Catalog instance _cat = catalog._catalog # check Catalog internal BTrees l_data = list(_cat.data.keys()) l_data.sort() l_uids = list(_cat.uids.values()) l_uids.sort() l_paths = list(_cat.data.keys()) l_paths.sort() print "Checking catalog internal BTrees" print "\tINFO: Mapping data: %d entries" % len(l_data) print "\tINFO: Mapping uids: %d entries" % len(l_uids) print "\tINFO: Mapping paths: %d entries" % len(l_paths) if l_data == l_uids: print "\tOK: Mapping data equals Mapping uids" else: print "\tERR: Mapping data does not equal Mapping uids" if l_data == l_paths: print "\tOK: Mapping data equals Maaping paths" else: print "\tERR: Mapping data does not equal Maaping paths" # check BTrees of indexes for id, idx in _cat.indexes.items(): if indexes and not idx.meta_type in indexes: continue print "Checking index '%s' (type: %s)" % (id, idx.meta_type) if idx.meta_type in ['FieldIndex', 'KeywordIndex']: # check forward entries RIDS = IISet() for key, rids in idx._index.items(): if isinstance(rids, IntType): RIDS.insert(rids) else: map(RIDS.insert, rids.keys()) diff = difference(RIDS, IISet(_cat.data.keys())) if len(diff) != 0: print '\tERR: Problem with forward entries' print '\tERR: too much forward entries:', diff else: print '\tOK: Forward entries (%d entries)' % (len(RIDS)) elif idx.meta_type in ['PathIndex']: RIDS = IISet() for rids in map(None, idx._index.values()): map(RIDS.insert, rids.values()[0]) diff = difference(RIDS, IISet(_cat.data.keys())) if len(diff) != 0: print '\tERR: Problem with forward entries' print '\tERR: too much forward entries:', diff else: print '\tOK: Forward entries (%d entries)' % (len(RIDS)) if idx.meta_type in ['FieldIndex', 'KeywordIndex', 'PathIndex']: # check backward entries RIDS = IISet(idx._unindex.keys()) diff = difference(RIDS, IISet(_cat.data.keys())) if len(diff) != 0: print '\tERR: Problem with backward entries' print '\tERR: too much backward entries:', diff else: print '\tOK: Backward entries (%d entries)' % (len(RIDS))
def search(self, path, default_level=0, depth=-1, navtree=0, navtree_start=0): """ path is either a string representing a relative URL or a part of a relative URL or a tuple (path,level). level >= 0 starts searching at the given level level < 0 not implemented yet """ if isinstance(path, basestring): startlevel = default_level else: startlevel = int(path[1]) path = path[0] absolute_path = isinstance(path, basestring) and path.startswith('/') comps = filter(None, path.split('/')) orig_comps = [''] + comps[:] # Optimization - avoid using the root set # as it is common for all objects anyway and add overhead # There is an assumption about catalog/index having # the same container as content if default_level == 0: indexpath = list(filter(None, self.getPhysicalPath())) while min(len(indexpath), len(comps)): if indexpath[0] == comps[0]: del indexpath[0] del comps[0] startlevel += 1 else: break if len(comps) == 0: if depth == -1 and not navtree: return IISet(self._unindex.keys()) # Make sure that we get depth = 1 if in navtree mode # unless specified otherwise orig_depth = depth if depth == -1: depth = 0 or navtree # Optimized navtree starting with absolute path if absolute_path and navtree and depth == 1 and default_level == 0: set_list = [] # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: set_list.append(self._index_parents[parent_path]) except KeyError: pass return multiunion(set_list) # Optimized breadcrumbs elif absolute_path and navtree and depth == 0 and default_level == 0: item_list = IISet() # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: item_list.insert(self._index_items[parent_path]) except KeyError: pass return item_list # Specific object search elif absolute_path and orig_depth == 0 and default_level == 0: try: return IISet([self._index_items[path]]) except KeyError: return IISet() # Single depth search elif absolute_path and orig_depth == 1 and default_level == 0: # only get objects contained in requested folder try: return self._index_parents[path] except KeyError: return IISet() # Sitemaps, relative paths, and depth queries elif startlevel >= 0: pathset = None # Same as pathindex navset = None # For collecting siblings along the way depthset = None # For limiting depth if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(startlevel): navset = self._index[None][startlevel] for level in range(startlevel, startlevel + len(comps) + depth): if level - startlevel < len(comps): comp = comps[level - startlevel] if not self._index.has_key( comp) or not self._index[comp].has_key(level): # Navtree is inverse, keep going even for # nonexisting paths if navtree: pathset = IISet() else: return IISet() else: pathset = intersection(pathset, self._index[comp][level]) if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(level+depth): navset = union( navset, intersection(pathset, self._index[None][level + depth])) if level - startlevel >= len(comps) or navtree: if self._index.has_key(None) and self._index[None].has_key( level): depthset = union( depthset, intersection(pathset, self._index[None][level])) if navtree: return union(depthset, navset) or IISet() elif depth: return depthset or IISet() else: return pathset or IISet() else: results = IISet() for level in range(0, self._depth + 1): ids = None error = 0 for cn in range(0, len(comps)): comp = comps[cn] try: ids = intersection(ids, self._index[comp][level + cn]) except KeyError: error = 1 if error == 0: results = union(results, ids) return results
class Article(Posting): """ """ security = ClassSecurityInfo() meta_type ='Article' icon ='misc_/Zch/posting_img' security.declarePrivate('__init__') def __init__(self, id): Posting.__init__(self, id) self.ids =IISet() #Article has sub ids. security.declareProtected(View, 'relative_path') def relative_path(self): return self.id security.declareProtected(View, 'index_html') def index_html(self,REQUEST): """ Zch article main page (the read more page) """ return self.article_html(self,REQUEST) security.declareProtected(ManageZch, 'postingValues') def postingValues(self): """ return all replies """ return self.data_map(self.ids) security.declareProtected(View, 'comment_list_size') def comment_list_size(self, start=0, size=0): """ returns comment items """ if start: start = int(start) else: start = 0 if size: size = int(size) else: size = 0 # Adjust start to tnum if start == 1: start = 2 if size: size = size-1 # Convert to ids[] index number if start: start = start -2 if size == 0: ids = [id for id in self.ids][start:] else: ids = [id for id in self.ids][start:start+size] else: if size == 0: ids = [id for id in self.ids][:] else: ids = [id for id in self.ids][size*-1:] return self.data_map(ids) security.declareProtected(View, 'comment_list_from_to') def comment_list_from_to(self, from_tnum=0, to_tnum=0): """ returns comment items """ from_tnum = int(from_tnum) to_tnum = int(to_tnum) ids = [id for id in self.ids if (from_tnum == 0 or int(self.data[id].tnum) >= from_tnum) and (to_tnum == 0 or int(self.data[id].tnum) <= to_tnum)] return self.data_map(ids) security.declareProtected(AddCommentZch, 'addPosting') def addPosting(self, file='', REQUEST=None,RESPONSE=None): """ add a Comment """ index=1 id=self.createId() msg=Comment(id, self.id) err, sage = msg.__of__(self)._validation(REQUEST,RESPONSE,'delete attachment',file) if err: return err # Set thread number. msg.tnum = str(len(self.ids) + 2) if sage==0: self.modified=id self.ids.insert(id) self.data[id]=msg if index: msg.__of__(self).index() if RESPONSE: return self.showMessage(self, REQUEST=REQUEST, title='Comment Posted', message ='Your reply has been posted', action=self.absolute_url() ) return id security.declareProtected(View, 'recent_entry') def recent_entry(self): if len (self.ids) != 0: return self.data[self.ids[-1]].body else: return self.body security.declareProtected(View, 'recent_creator') def recent_creator(self): if len (self.ids) != 0: return self.data[self.ids[-1]].author else: return self.author security.declarePublic('__len__') def __len__(self): return len(self.ids) + 1 security.declareProtected(View, '__getitem__') def __getitem__(self,id): """ Get a posting from the ZchSite data store """ # make sure id is an integer try: if not isinstance(id,IntType): id=atoi(id) except ValueError: raise KeyError, id try: return Posting.__getitem__(self,id) except KeyError: try: return self.data[self.ids[id-2]].__of__(self) except: raise KeyError, id
def search(self, path, default_level=0, depth=-1, navtree=0, navtree_start=0): """ path is either a string representing a relative URL or a part of a relative URL or a tuple (path,level). level >= 0 starts searching at the given level level < 0 not implemented yet """ if isinstance(path, basestring): startlevel = default_level else: startlevel = int(path[1]) path = path[0] absolute_path = isinstance(path, basestring) and path.startswith('/') comps = filter(None, path.split('/')) orig_comps = [''] + comps[:] if depth > 0: raise ValueError, "Can't do depth searches anymore" if not comps: comps = ['dmd'] startlevel = 1 elif comps[0] == 'zport': comps = comps[1:] elif comps[0] != 'dmd': raise ValueError, "Depth searches must start with 'dmd'" startlevel = len(comps) #startlevel = len(comps)-1 if len(comps) > 1 else 1 if len(comps) == 0: if depth == -1 and not navtree: return IISet(self._unindex.keys()) # Make sure that we get depth = 1 if in navtree mode # unless specified otherwise orig_depth = depth if depth == -1: depth = 0 or navtree # Optimized navtree starting with absolute path if absolute_path and navtree and depth == 1 and default_level==0: set_list = [] # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: set_list.append(self._index_parents[parent_path]) except KeyError: pass return multiunion(set_list) # Optimized breadcrumbs elif absolute_path and navtree and depth == 0 and default_level==0: item_list = IISet() # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: item_list.insert(self._index_items[parent_path]) except KeyError: pass return item_list # Specific object search elif absolute_path and orig_depth == 0 and default_level == 0: try: return IISet([self._index_items[path]]) except KeyError: return IISet() # Single depth search elif absolute_path and orig_depth == 1 and default_level == 0: # only get objects contained in requested folder try: return self._index_parents[path] except KeyError: return IISet() # Sitemaps, relative paths, and depth queries elif startlevel >= 0: pathset = None # Same as pathindex navset = None # For collecting siblings along the way depthset = None # For limiting depth if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(startlevel): navset = self._index[None][startlevel] for level in range(startlevel, startlevel+len(comps)): if level <= len(comps): comp = "/".join(comps[:level]) if (not self._index.has_key(comp) or not self._index[comp].has_key(level)): # Navtree is inverse, keep going even for # nonexisting paths if navtree: pathset = IISet() else: return IISet() else: return self._index[comp][level] if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(level+depth): navset = union(navset, intersection(pathset, self._index[None][level+depth])) if level-startlevel >= len(comps) or navtree: if (self._index.has_key(None) and self._index[None].has_key(level)): depthset = union(depthset, intersection(pathset, self._index[None][level])) if navtree: return union(depthset, navset) or IISet() elif depth: return depthset or IISet() else: return pathset or IISet() else: results = IISet() for level in range(0,self._depth + 1): ids = None error = 0 for cn in range(0,len(comps)): comp = comps[cn] try: ids = intersection(ids,self._index[comp][level+cn]) except KeyError: error = 1 if error==0: results = union(results,ids) return results
def get(self, key, default=None): """Return the matched word against the key.""" r=IISet() wid=self._lexicon.get(key, default) if wid is not None: r.insert(wid) return r
def search(self, path, default_level=0, depth=-1, navtree=0, navtree_start=0): """ path is either a string representing a relative URL or a part of a relative URL or a tuple (path,level). level >= 0 starts searching at the given level level < 0 not implemented yet """ if isinstance(path, basestring): startlevel = default_level else: startlevel = int(path[1]) path = path[0] absolute_path = isinstance(path, basestring) and path.startswith('/') comps = filter(None, path.split('/')) orig_comps = [''] + comps[:] # Optimization - avoid using the root set # as it is common for all objects anyway and add overhead # There is an assumption about catalog/index having # the same container as content if default_level == 0: indexpath = list(filter(None, self.getPhysicalPath())) while min(len(indexpath), len(comps)): if indexpath[0] == comps[0]: del indexpath[0] del comps[0] startlevel += 1 else: break if len(comps) == 0: if depth == -1 and not navtree: return IISet(self._unindex.keys()) # Make sure that we get depth = 1 if in navtree mode # unless specified otherwise orig_depth = depth if depth == -1: depth = 0 or navtree # Optimized navtree starting with absolute path if absolute_path and navtree and depth == 1 and default_level==0: set_list = [] # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: set_list.append(self._index_parents[parent_path]) except KeyError: pass return multiunion(set_list) # Optimized breadcrumbs elif absolute_path and navtree and depth == 0 and default_level==0: item_list = IISet() # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: item_list.insert(self._index_items[parent_path]) except KeyError: pass return item_list # Specific object search elif absolute_path and orig_depth == 0 and default_level == 0: try: return IISet([self._index_items[path]]) except KeyError: return IISet() # Single depth search elif absolute_path and orig_depth == 1 and default_level == 0: # only get objects contained in requested folder try: return self._index_parents[path] except KeyError: return IISet() # Sitemaps, relative paths, and depth queries elif startlevel >= 0: pathset = None # Same as pathindex navset = None # For collecting siblings along the way depthset = None # For limiting depth if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(startlevel): navset = self._index[None][startlevel] for level in range(startlevel, startlevel+len(comps) + depth): if level-startlevel < len(comps): comp = comps[level-startlevel] if not self._index.has_key(comp) or not self._index[comp].has_key(level): # Navtree is inverse, keep going even for # nonexisting paths if navtree: pathset = IISet() else: return IISet() else: pathset = intersection(pathset, self._index[comp][level]) if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(level+depth): navset = union(navset, intersection(pathset, self._index[None][level+depth])) if level-startlevel >= len(comps) or navtree: if self._index.has_key(None) and self._index[None].has_key(level): depthset = union(depthset, intersection(pathset, self._index[None][level])) if navtree: return union(depthset, navset) or IISet() elif depth: return depthset or IISet() else: return pathset or IISet() else: results = IISet() for level in range(0,self._depth + 1): ids = None error = 0 for cn in range(0,len(comps)): comp = comps[cn] try: ids = intersection(ids,self._index[comp][level+cn]) except KeyError: error = 1 if error==0: results = union(results,ids) return results
def checkCatalog(path,indexes): """ perform some consistency checks on a ZCatalog instance""" root = Zope2.app() try: catalog = root.unrestrictedTraverse(path) except AttributeError: print 'Error: catalog object not found' sys.exit(1) # get Catalog instance _cat = catalog._catalog # check Catalog internal BTrees l_data = list(_cat.data.keys()) l_data.sort() l_uids = list(_cat.uids.values()) l_uids.sort() l_paths = list(_cat.data.keys()) l_paths.sort() print "Checking catalog internal BTrees" print "\tINFO: Mapping data: %d entries" % len(l_data) print "\tINFO: Mapping uids: %d entries" % len(l_uids) print "\tINFO: Mapping paths: %d entries" % len(l_paths) if l_data == l_uids: print "\tOK: Mapping data equals Mapping uids" else: print "\tERR: Mapping data does not equal Mapping uids" if l_data == l_paths: print "\tOK: Mapping data equals Maaping paths" else: print "\tERR: Mapping data does not equal Maaping paths" # check BTrees of indexes for id,idx in _cat.indexes.items(): if indexes and not idx.meta_type in indexes: continue print "Checking index '%s' (type: %s)" % (id, idx.meta_type) if idx.meta_type in ['FieldIndex','KeywordIndex']: # check forward entries RIDS = IISet() for key, rids in idx._index.items(): if isinstance(rids,IntType): RIDS.insert( rids ) else: map(RIDS.insert , rids.keys()) diff = difference(RIDS, IISet(_cat.data.keys())) if len(diff)!=0: print '\tERR: Problem with forward entries' print '\tERR: too much forward entries:', diff else: print '\tOK: Forward entries (%d entries)' % (len(RIDS)) elif idx.meta_type in ['PathIndex']: RIDS = IISet() for rids in map(None,idx._index.values()): map(RIDS.insert , rids.values()[0]) diff = difference(RIDS, IISet(_cat.data.keys())) if len(diff)!=0: print '\tERR: Problem with forward entries' print '\tERR: too much forward entries:', diff else: print '\tOK: Forward entries (%d entries)' % (len(RIDS)) if idx.meta_type in ['FieldIndex','KeywordIndex','PathIndex']: # check backward entries RIDS = IISet(idx._unindex.keys()) diff = difference(RIDS, IISet(_cat.data.keys())) if len(diff)!=0: print '\tERR: Problem with backward entries' print '\tERR: too much backward entries:', diff else: print '\tOK: Backward entries (%d entries)' % (len(RIDS))
def languageindex_search(self, language, fallback=True, res=None): main, sub = splitLanguage(language) if main not in self._index: return None if fallback: # Search in sorted order, specific sub tag first, None second subs = list(self._index[main].keys()) subs.sort() if sub in subs: subs.remove(sub) subs.insert(0, sub) else: subs = [sub] if not fallback and res is not None: # We do not support any optimization when fallback is enabled. # # TODO: The core loop is not in C here. Casual benchmarks suggest this # is still more effecient than trying to move it to C. The problem is # that we only have an IISet of docids as an input. We need to filter # this per language. The available index structures we have are: # # IndexEntry objects used as entries. Complex objects storing docid, # main and sub languages and UID of the canonical. Their hash and # compare function uses the canonical UID. # # self._index # An OOBTreeSet structure per language. In the outermost nodes we have # OOBTree's per language. Useful to get all items in a language. # Otherwise useless, as we would have to compare the docid attribute # of the object in the tree against our wanted set, requiring a full # loop over all items. # # self._unindex # An IOBTree of docid to entry. Better to match our docid wanted set, # but we would still have to compare the language code to the entry # object itself. # # self._sortindex # An IOBTree of docid to language tag. Looks like the best candidate # for us, as we can compare the language directly as a simple string # comparision. # # One thing to keep in mind, is that once we get a wanted set, this # will usually have gone through a path query already. This means # we will almost always already have matching set and won't filter # out any item at all. So the edge-case of a 100% match is actually # the most common one for us. # # Casual benchmarks show that trying to construct an IOBTree from the # wanted set and intersecting it with the sortindex is still slower # than having the core loop in Python code. tag = lang_tag(main, sub) result = IISet() for r in res: lang = self._sortindex.get(r) if lang == tag: result.insert(r) return result result = OOSet() for sublanguage in subs: result = oo_union(result, self._index[main][sublanguage]) return IISet(entry.docid for entry in result)
class ZchSite(ZCatalog.ZCatalog): """A Zch Site is a self contained web-based news publishing and discussion system""" meta_type ='Zch Site' description='Zch Site' security = ClassSecurityInfo() security.setPermissionDefault(ManageZch,('Manager',)) security.setPermissionDefault(AddArticleZch,('Manager',)) security.setPermissionDefault(AddCommentZch,('Anonymous','Manager',)) security.setPermissionDefault(View,('Anonymous','Manager',)) icon ='misc_/Zch/Zch_img' _properties=({'id':'title', 'type':'string','mode':'w'},) fileattache=0 sage=0 manage_options=({'label':'Contents', 'icon':icon, 'action':'manage_main', 'target':'manage_main'}, {'label':'View', 'icon':'', 'action':'index_html', 'target':'manage_main'}, {'label':'Postings', 'icon':'', 'action':'manage_postings', 'target':'manage_main'}, {'label':'Options', 'icon':'', 'action':'manage_editForm', 'target':'manage_main'}, {'label':'Properties', 'icon':'', 'action':'manage_propertiesForm', 'target':'manage_main'}, {'label':'Catalog', 'icon':'', 'action':'manage_catalogView', 'target':'manage_main'}, {'label':'Indexes', 'icon':'', 'action':'manage_catalogIndexes', 'target':'manage_main'}, {'label':'Security', 'icon':'', 'action':'manage_access', 'target':'manage_main'}, {'label':'Undo', 'icon':'', 'action':'manage_UndoForm', 'target':'manage_main'} ) security.declareProtected(ManageZch, 'manage_postings') manage_postings = HTMLFile('dtml/manage_postings', globals()) security.declareProtected(ManageZch, 'manage_editForm') manage_editForm = HTMLFile('dtml/editForm', globals()) security.declarePrivate('_buildIndexing') def _buildIndexing(self, id, title): # Initialise ZCatalog if not hasattr(self,'_catalog'): ZCatalog.ZCatalog.__init__(self, id, title) # delete any existing indexes for name in self.indexes(): self.delIndex(name) # add the default indexes for (name,index_type) in [('meta_type', 'FieldIndex'), ('author', 'FieldIndex'), ('body', 'ZCTextIndex'), ('title', 'ZCTextIndex'), ('date', 'FieldIndex')]: if index_type == 'ZCTextIndex': extras = EmptyClass() extras.doc_attr = name extras.index_type = 'Okapi BM25 Rank' extras.lexicon_id = 'lexicon' self.addIndex(name, index_type, extra=extras) else: self.addIndex(name,index_type) # delete the default metadata columns for name in self.schema(): self.delColumn(name) # Add the meta data columns for search results for name in ['id','title','absolute_url','author','date_posted','date','body', 'tnum']: self.addColumn(name,'') security.declareProtected(ManageZch, 'recatalogPostings') def recatalogPostings(self,REQUEST=None): """ Clear the Catalog and then Index all the postings. """ self._catalog.clear() for article_id in self.ids: article = self.data[article_id].__of__(self) if type(article.body)==type([]): article.body = join(article.body, '\n') for comment_id in article.ids: comment = self.data[comment_id].__of__(article) if type(comment.body)==type([]): comment.body = join(comment.body, '\n') self.catalog_object(comment, join(comment.getPhysicalPath(), '/')) self.catalog_object(article, join(article.getPhysicalPath(), '/')) if REQUEST is not None: return REQUEST.RESPONSE.redirect(REQUEST['HTTP_REFERER']) security.declareProtected(ManageZch, 'loadSkelton') def loadSkelton(self, REQUEST, skelton='zch'): "Add Page Template PythonScript, DTMLMethod and Image read from skelton directory." for entry in os.listdir(os.path.join(package_home(globals()), 'skelton', skelton)): if entry[-3:] == '.pt' or entry[-4:]=='.pys' or entry[-5:]=='.dtml' or entry[-4:]=='.gif': f=open(os.path.join(package_home(globals()), 'skelton', skelton, entry), 'rb') file=f.read() f.close() try: if entry[-3:] == '.pt': id = entry[:-3] manage_addPageTemplate(self, id, '', file, encoding='utf-8') elif entry[-4:] == '.pys': id = entry[:-4] manage_addPythonScript(self,id) self._getOb(id).write(file) elif entry[-5:] == '.dtml': id = entry[:-5] self.manage_addDTMLMethod(id,'',file) elif entry[-4:] == '.gif': id = entry[:-4] self.manage_addImage(id,file,content_type='image/gif') except: pass if REQUEST is not None: return REQUEST.RESPONSE.redirect(REQUEST['HTTP_REFERER']) security.declarePrivate('loadProperties') def loadProperties(self, skelton): "Add properties from 'properties' file." p = re.compile(r'(\w+?):(\w+?)=\s*(.*)\s*') newprop = list(self._properties) f = open(os.path.join(package_home(globals()), 'skelton', skelton, 'properties'), 'r') for s in f: if s[0] == '#': continue m = p.match(s) if m: newprop.append({'id':m.group(1), 'type':m.group(2), 'mode': 'wd'}) f.close() self._properties = tuple(newprop) f = open(os.path.join(package_home(globals()), 'skelton', skelton, 'properties'), 'r') for s in f: if s[0] == '#': continue m = p.match(s) if m: self._updateProperty(m.group(1), m.group(3)) f.close() security.declarePrivate('__init__') def __init__(self, id, title, skelton, fileattache, parent, elements): if elements: from Products.ZCTextIndex.ZCTextIndex import manage_addLexicon manage_addLexicon(self,id='lexicon',elements = elements) self.__of__(parent)._buildIndexing(id,title) t=time() self.created = t self.modified = t self.fileattache = fileattache self.data =IOBTree() # id -> Message self.ids =IISet() # ids of children self.loadSkelton(None, skelton) self.loadProperties(skelton) self.skelton = skelton security.declarePublic('__len__') def __len__(self): return len(self.ids) + 1 security.declareProtected(View, '__getitem__') def __getitem__(self,id): """ Get a posting from the ZchSite data store """ # make sure id is an integer try: if not isinstance(id,IntType): id=atoi(id) except ValueError: raise KeyError, id # make sure it's in our list of children if not self.ids.has_key(id): raise KeyError, id # return the posting return self.data[id].__of__(self) security.declareProtected(View, 'zchcrypt') def zchcrypt(self,word,key): import hmac, base64 h = hmac.new(key) h.update(word) return base64.encodestring(h.digest())[:-3] security.declareProtected(View, 'zchfqdn') def zchfqdn(self,n): return getfqdn(n) security.declarePrivate('delItem') def delItem(self,id): if not self.data.has_key(id): return if self.ids.has_key(id): # article article = self.data[id].__of__(self) for comment_id in article.ids: obj = self.data[comment_id].__of__(article) self.uncatalog_object(obj.getPhysicalPath()) del self.data[comment_id] self.uncatalog_object(article.getPhysicalPath()) del self.data[id] self.ids.remove(id) else: # comment parent = self.data[self.data[id].parent_id].__of__(self) # remove it from it's parents list of ids obj = self.data[id].__of__(parent) self.uncatalog_object(obj.getPhysicalPath()) del self.data[id] parent.ids.remove(id) security.declarePrivate('createId') def createId(self): id=int(time()) while self.data.has_key(id): id=id+1 return id security.declarePrivate('data_map') def data_map(self,ids): result=[] for id in ids: result.append(self.data[id].__of__(self)) return result security.declareProtected(View, 'article_list') def article_list(self, size=None): """ returns article items """ def cmp_by_modified(x, y): return cmp(y.modified, x.modified) items = self.data_map(self.ids) items.sort(cmp_by_modified) if size: items = items[:size] for i in range(len(items)): items[i].sequence_number = i + 1 return items security.declareProtected(ManageZch, 'postingValues') postingValues = article_list security.declareProtected(View, 'tpId') def tpId(self): return self.id security.declareProtected(View, 'tpURL') def tpURL(self): return self.id security.declareProtected(View, 'this') def this(self): return self security.declareProtected(View, 'site_url') def site_url(self): # """ url of the Zch main page """ return self.absolute_url() security.declareProtected(View, 'has_items') def has_items(self): return len(self.ids) security.declareProtected(View, 'item_count') def item_count(self): return len(self.data) security.declareProtected(AddArticleZch, 'addPosting') def addPosting(self,file='',REQUEST=None,RESPONSE=None, index=1): """ add an article """ id=self.createId() msg=Article(id) err, sage = msg.__of__(self)._validation(REQUEST,RESPONSE,'delete attachment',file) if err: return err # Set thread number. msg.tnum = '1' self.ids.insert(id) self.data[id]=msg if index: msg.__of__(self).index() if RESPONSE: return self.showMessage(self, REQUEST=REQUEST, title='Article Posted', message ='Your article has been posted', action=self.absolute_url() ) return id security.declareProtected(View, 'search') def search(self,REQUEST): """ fulfill a search request """ if REQUEST.has_key('op') and REQUEST['op']=='articles': REQUEST.set('meta_type','Article') sr=self.__call__(REQUEST) rc=len(sr) return self.showSearchResults(self,REQUEST,search_results=sr, result_count=rc) security.declareProtected(ManageZch, 'manage_edit') def manage_edit(self, REQUEST=None, fileattache=0): """ edit Zch options """ self.fileattache = fileattache if REQUEST is not None: return REQUEST.RESPONSE.redirect(REQUEST['HTTP_REFERER']) security.declareProtected(ManageZch, 'manage_delete') def manage_delete(self,ids=[],REQUEST=None): """ delete selected articles from a Zch site """ ids=map(atoi, ids) for id in ids: self.delItem(id) if REQUEST is not None: return REQUEST.RESPONSE.redirect(REQUEST['HTTP_REFERER']) security.declarePrivate('list_skelton') def list_skelton(self): skelton = [] for item in os.listdir(os.path.join(package_home(globals()), 'skelton')): skelton.append(item) return skelton # Searchable interface security.declareProtected(View, '__call__') def __call__(self, REQUEST=None, internal=0, **kw): brains = apply(self.searchResults,(REQUEST,),kw) if internal: return map(lambda x: x.getObject(), brains) return brains
def search(self, path, default_level=0, depth=-1, navtree=0, navtree_start=0): """ path is either a string representing a relative URL or a part of a relative URL or a tuple (path,level). level >= 0 starts searching at the given level level < 0 not implemented yet """ if isinstance(path, basestring): startlevel = default_level else: startlevel = int(path[1]) path = path[0] absolute_path = isinstance(path, basestring) and path.startswith('/') comps = filter(None, path.split('/')) orig_comps = [''] + comps[:] if depth > 0: raise ValueError("Can't do depth searches anymore") if not comps: comps = ['dmd'] startlevel = 1 else: if comps[0] == getCSEConf().get('virtualroot', '').replace('/', ''): comps = comps[1:] if comps[0] == 'zport': comps = comps[1:] if comps[0] != 'dmd': raise ValueError("Depth searches must start with 'dmd'") startlevel = len(comps) if len(comps) == 0: if depth == -1 and not navtree: return IISet(self._unindex.keys()) # Make sure that we get depth = 1 if in navtree mode # unless specified otherwise orig_depth = depth if depth == -1: depth = 0 or navtree # Optimized navtree starting with absolute path if absolute_path and navtree and depth == 1 and default_level == 0: set_list = [] # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: set_list.append(self._index_parents[parent_path]) except KeyError: pass return multiunion(set_list) # Optimized breadcrumbs elif absolute_path and navtree and depth == 0 and default_level == 0: item_list = IISet() # Insert root element if navtree_start >= len(orig_comps): navtree_start = 0 # create a set of parent paths to search for i in range(len(orig_comps), navtree_start, -1): parent_path = '/'.join(orig_comps[:i]) parent_path = parent_path and parent_path or '/' try: item_list.insert(self._index_items[parent_path]) except KeyError: pass return item_list # Specific object search elif absolute_path and orig_depth == 0 and default_level == 0: try: return IISet([self._index_items[path]]) except KeyError: return IISet() # Single depth search elif absolute_path and orig_depth == 1 and default_level == 0: # only get objects contained in requested folder try: return self._index_parents[path] except KeyError: return IISet() # Sitemaps, relative paths, and depth queries elif startlevel >= 0: pathset = None # Same as pathindex navset = None # For collecting siblings along the way depthset = None # For limiting depth if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(startlevel): navset = self._index[None][startlevel] for level in range(startlevel, startlevel + len(comps)): if level <= len(comps): comp = "/".join(comps[:level]) if (not self._index.has_key(comp) or not self._index[comp].has_key(level)): # Navtree is inverse, keep going even for # nonexisting paths if navtree: pathset = IISet() else: return IISet() else: return self._index[comp][level] if navtree and depth and \ self._index.has_key(None) and \ self._index[None].has_key(level+depth): navset = union( navset, intersection(pathset, self._index[None][level + depth])) if level - startlevel >= len(comps) or navtree: if (self._index.has_key(None) and self._index[None].has_key(level)): depthset = union( depthset, intersection(pathset, self._index[None][level])) if navtree: return union(depthset, navset) or IISet() elif depth: return depthset or IISet() else: return pathset or IISet() else: results = IISet() for level in range(0, self._depth + 1): ids = None error = 0 for cn in range(0, len(comps)): comp = comps[cn] try: ids = intersection(ids, self._index[comp][level + cn]) except KeyError: error = 1 if error == 0: results = union(results, ids) return results