def __init__(self):
# These keep track of symbolic label and branch names that
# have been used to ensure that they don't collide.
self._branches = OIBTree()
self._branches['mainline'] = 1
self._labels = OIBTree()
self._histories = OOBTree()
self._created = time.time()
def setupAnnotations(context):
"""
set up the annotations if they haven't been set up
already. The rest of the functions in here assume that
this has already been set up
"""
annotations = IAnnotations(context)
if yays not in annotations:
annotations[yays] = OIBTree()
if nays not in annotations:
annotations[nays] = OIBTree()
return annotations
def storage(self):
""" get the counter storage
"""
annotation = get_portal_annotation()
if annotation.get(NUMBER_STORAGE) is None:
annotation[NUMBER_STORAGE] = OIBTree()
return annotation[NUMBER_STORAGE]
def clear(self):
"""Empty the lexicon.
"""
self.length = Length()
self._wid_length_based = False
self._wids = OIBTree() # word -> wid
self._words = IOBTree() # wid -> word
def resetOrdering(ordering):
annotations = IAnnotations(ordering.context)
order = PersistentList()
annotations.__setitem__(ordering.ORDER_KEY, order)
pos = OIBTree()
annotations.__setitem__(ordering.POS_KEY, pos)
return order, pos
def clear(self):
self.data = IOBTree() # {rid -> record as tuple}
self.indexes = {} # {index_name -> OOBTree({value -> IITreeSet})}
self.primary_index = OIBTree() # {primary key -> rid}
for position, column in self.col_info:
if column.indexed:
self.indexes[column.name] = OOBTree()
def convert_to_uuidindex(catalog, index):
if isinstance(index, UUIDIndex):
return
logger.info('Converting index `%s` to UUIDIndex.' % index.getId())
index.__class__ = UUIDIndex
index._p_changed = True
catalog._catalog._p_changed = True
# convert from OOBTree to OIBTree
old_index = index._index
if not isinstance(old_index, OIBTree):
index._index = _index = OIBTree()
for k, v in old_index.items():
if k is None:
continue
if isinstance(v, int):
_index[k] = v
else:
if isinstance(v, (IISet, IITreeSet)):
# inconsistent data, one uid with multiple docids
paths = dict((tuple(catalog.getpath(k).split('/')), k)
for k in v.keys())
shortest = min(paths, key=len)
for path, key in paths.iteritems():
if path[:len(shortest)] != shortest:
raise ValueError(
'Inconsistent UID index, UID %s is associated '
'with multiple docids: %r' % (k, paths))
# All other docids are sub-paths of another
# indicating the UID was just acquired,
# choose the shortest
_index[k] = paths[shortest]
del old_index
transaction.savepoint(optimistic=True)
logger.info('Finished conversion.')
def clear(self):
self._length = Length()
self._index = OIBTree()
self._unindex = IOBTree()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def _cleanup(self):
"""Cleans up errors in the BTrees.
Certain ZODB bugs have caused BTrees to become slightly insane.
Fortunately, there is a way to clean up damaged BTrees that
always seems to work: make a new BTree containing the items()
of the old one.
Returns 1 if no damage was detected, or 0 if damage was
detected and fixed.
"""
from BTrees.check import check
path = '/'.join(self.getPhysicalPath())
try:
check(self._tree)
for key in self._tree.keys():
if key not in self._tree:
raise AssertionError(
"Missing value for key: %s" % repr(key))
check(self._mt_index)
keys = set(self._tree.keys())
for key, value in self._mt_index.items():
if (key not in self._mt_index
or self._mt_index[key] is not value):
raise AssertionError(
"Missing or incorrect meta_type index: %s"
% repr(key))
check(value)
for k in value.keys():
if k not in value or k not in keys:
raise AssertionError(
"Missing values for meta_type index: %s"
% repr(key))
return 1
except AssertionError:
LOG.warn('Detected damage to %s. Fixing now.' % path,
exc_info=sys.exc_info())
try:
self._tree = OOBTree(self._tree)
keys = set(self._tree.keys())
mt_index = OOBTree()
for key, value in self._mt_index.items():
for name in tuple(value.keys()):
if name not in keys:
del value[name]
mt_index[key] = OIBTree(value)
self._mt_index = mt_index
new = len(keys)
if self._count() != new:
self._count.set(new)
except:
LOG.error('Failed to fix %s.' % path,
exc_info=sys.exc_info())
raise
else:
LOG.info('Fixed %s.' % path)
return 0
def initializeAnnotations(obj, event):
"""Ensure that we don't delegate certain annotations by setting them
from the beginning.
"""
annotations = IAnnotations(obj)
annotations.setdefault(DefaultOrdering.ORDER_KEY, PersistentList())
annotations.setdefault(DefaultOrdering.POS_KEY, OIBTree())
annotations.setdefault(CONTENTRULES_KEY, None)
annotations.setdefault(CONTEXT_ASSIGNMENT_KEY, OOBTree())
def noteLock(self, obj, user_id):
mapping = getattr(self, '_locks', None)
if mapping is None:
mapping = self._locks = OOBTree()
path = '/'.join(obj.getPhysicalPath())
items = mapping.get(user_id, None)
if items is None:
items = OIBTree()
mapping[user_id] = items
items[path] = 1
def __init__(self, *pipeline):
self._wids = OIBTree() # word -> wid
self._words = IOBTree() # wid -> word
# wid 0 is reserved for words that aren't in the lexicon (OOV -- out
# of vocabulary). This can happen, e.g., if a query contains a word
# we never saw before, and that isn't a known stopword (or otherwise
# filtered out). Returning a special wid value for OOV words is a
# way to let clients know when an OOV word appears.
self.wordCount = Length()
self._pipeline = pipeline
def clear(self):
""" clear catalog """
self.data = IOBTree() # mapping of rid to meta_data
self.uids = OIBTree() # mapping of uid to rid
self.paths = IOBTree() # mapping of rid to uid
self._length = BTrees.Length.Length()
for index in self.indexes.keys():
self.getIndex(index).clear()
def load_model():
data = DataFS()
keys, label = data.gen_train_data()
btree = OIBTree()
rubbish_k = np.load('6k.npy')
rubbish_v = np.load('6v.npy')
w = np.load('w.npy', allow_pickle=True)
b = np.load('b.npy', allow_pickle=True)
for i in range(len(rubbish_k)):
btree[str(rubbish_k[i])] = int(rubbish_v[i])
return keys, label, w, b, btree
def setupAnnotations(context):
"""
set up the annotations if they haven't been set up
already. The rest of the functions in here assume that
this has already been set up
"""
annotations = IAnnotations(context)
changed = False
if yays not in annotations:
annotations[yays] = OIBTree()
changed = True
if nays not in annotations:
annotations[nays] = OIBTree()
changed = True
if changed:
request = getRequest()
alsoProvides(request, IDisableCSRFProtection)
return annotations
def __init__(self, id="++conversation++default"):
self.id = id
# username -> count of comments; key is removed when count reaches 0
self._commentators = OIBTree()
# id -> comment - find comment by id
self._comments = LOBTree()
# id -> LLSet (children) - find all children for a given comment.
# 0 signifies root.
self._children = LOBTree()
def importNode(self, node, mode=PURGE):
"""Import the object from the DOM node.
"""
pipeline = []
for child in node.childNodes:
if child.nodeName == 'element':
element = element_factory.instantiate(
child.getAttribute('group'), child.getAttribute('name'))
pipeline.append(element)
self.context._pipeline = tuple(pipeline)
#clear lexicon
self.context._wids = OIBTree()
self.context._words = IOBTree()
self.context.length = Length()
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
tree = self._tree
if tree.has_key(id):
raise KeyError('There is already an item named "%s".' % id)
tree[id] = object
self._count.change(1)
# Update the meta type index.
mti = self._mt_index
meta_type = getattr(object, 'meta_type', None)
if meta_type is not None:
ids = mti.get(meta_type, None)
if ids is None:
ids = OIBTree()
mti[meta_type] = ids
ids[id] = 1
def create_token(self, userid, timeout=None, data=None):
payload = {}
payload['sub'] = userid
if timeout is None:
timeout = self.token_timeout
if timeout:
payload['exp'] = datetime.utcnow() + timedelta(seconds=timeout)
if data is not None:
payload.update(data)
token = jwt.encode(payload, self._signing_secret(), algorithm='HS256')
if self.store_tokens:
if self._tokens is None:
self._tokens = OOBTree()
if userid not in self._tokens:
self._tokens[userid] = OIBTree()
self._tokens[userid][token] = int(time.time())
return token
def _getCacheId(self):
"""Return a cache id for preferences.
We use:
- user_id: because preferences are always different by user
- self._preference_cache[user_id] which is increased everytime a user
preference is modified
- self._preference_cache[None] which is increased everytime a global
preference is modified
"""
user_id = getSecurityManager().getUser().getId()
try:
self._preference_cache
except AttributeError:
self._preference_cache = OIBTree()
return self._preference_cache.get(None), self._preference_cache.get(
user_id), user_id
def __init__(self, id=None):
super(LinkCheckTool, self).__init__(id)
# This is the work queue; items in this queue are scheduled
# for link validity check.
self.queue = CompositeQueue()
# Additional queue for internal crawler to revalidate the site
self.crawl_queue = CompositeQueue()
# This is the link database. It maps a hyperlink index to a
# tuple (timestamp, status, referers).
self.checked = IOBTree()
# Indexes
self.index = OIBTree()
self.links = IOBTree()
# This is a counter that allows us to add new hyperlinks and
# provide an indexc quickly.
self.counter = 0
def testCleanup(self):
self.assert_(self.f._cleanup())
key = TrojanKey('a')
self.f._tree[key] = 'b'
self.assert_(self.f._cleanup())
key.value = 'z'
# With a key in the wrong place, there should now be damage.
self.assert_(not self.f._cleanup())
# Now it's fixed.
self.assert_(self.f._cleanup())
from BTrees.OIBTree import OIBTree
tree = self.f._mt_index['d'] = OIBTree()
tree['e'] = 1
self.assert_(not self.f._cleanup())
# Verify the management interface also works,
# but don't test return values.
self.f.manage_cleanup()
key.value = 'a'
self.f.manage_cleanup()
def _convertBTrees(self, threshold=200):
if (type(self._lexicon) is OIBTree and
type(getattr(self, '_inverseLex', None)) is IOBTree):
return
from BTrees.convert import convert
lexicon=self._lexicon
self._lexicon=OIBTree()
self._lexicon._p_jar=self._p_jar
convert(lexicon, self._lexicon, threshold)
try:
inverseLex=self._inverseLex
self._inverseLex=IOBTree()
except AttributeError:
# older lexicons didn't have an inverse lexicon
self._inverseLex=IOBTree()
inverseLex=self._inverseLex
self._inverseLex._p_jar=self._p_jar
convert(inverseLex, self._inverseLex, threshold)
def __init__(self, datafs, writable=0, trans=0, pack=0):
self.trans_limit = trans
self.pack_limit = pack
self.trans_count = 0
self.pack_count = 0
self.stopdict = get_stopdict()
self.mh = mhlib.MH()
self.filestorage = FileStorage(datafs, read_only=(not writable))
self.database = DB(self.filestorage)
self.connection = self.database.open()
self.root = self.connection.root()
try:
self.index = self.root["index"]
except KeyError:
self.index = self.root["index"] = TextIndex()
try:
self.docpaths = self.root["docpaths"]
except KeyError:
self.docpaths = self.root["docpaths"] = IOBTree()
try:
self.doctimes = self.root["doctimes"]
except KeyError:
self.doctimes = self.root["doctimes"] = IIBTree()
try:
self.watchfolders = self.root["watchfolders"]
except KeyError:
self.watchfolders = self.root["watchfolders"] = {}
self.path2docid = OIBTree()
for docid in self.docpaths.keys():
path = self.docpaths[docid]
self.path2docid[path] = docid
try:
self.maxdocid = max(self.docpaths.keys())
except ValueError:
self.maxdocid = 0
print len(self.docpaths), "Document ids"
print len(self.path2docid), "Pathnames"
print self.index.lexicon.length(), "Words"
def fixupPloneLexicon(context):
"""Updates the plone_lexicon pipeline with the new splitter
and case normalizer.
"""
catalog = getToolByName(context, 'portal_catalog', None)
if catalog is not None:
if 'plone_lexicon' in catalog.objectIds():
lexicon = catalog.plone_lexicon
pipeline = list(lexicon._pipeline)
if len(pipeline) >= 2:
if (not isinstance(pipeline[0], Splitter)
or not isinstance(pipeline[1], CaseNormalizer)):
pipeline[0] = Splitter()
pipeline[1] = CaseNormalizer()
lexicon._pipeline = tuple(pipeline)
# Clear the lexicon
from BTrees.OIBTree import OIBTree
from BTrees.IOBTree import IOBTree
from BTrees.Length import Length
lexicon._wids = OIBTree()
lexicon._words = IOBTree()
lexicon.length = Length()
logger.info('Updated plone_lexicon pipeline.')
def initialize_storage(self):
ann = IAnnotations(self.context)
if self.ANNOTATIONS_KEY not in ann:
ann[self.ANNOTATIONS_KEY] = OOBTree()
self._storage = ann[self.ANNOTATIONS_KEY]
# Actual list of actions
if self.STORAGE_ACTIONS_KEY not in self._storage:
self._storage[self.STORAGE_ACTIONS_KEY] = IOBTree()
self._actions = self._storage[self.STORAGE_ACTIONS_KEY]
# Indexes needed for fast lookups
if self.STORAGE_INDEXES_KEY not in self._storage:
self._storage[self.STORAGE_INDEXES_KEY] = OOBTree()
self._indexes = self._storage[self.STORAGE_INDEXES_KEY]
# Index: unique_name -> action_id
if self.IDX_UNIQUE_NAME not in self._indexes:
self._indexes[self.IDX_UNIQUE_NAME] = OIBTree()
# Counter for the next 'action_id'
if self.STORAGE_NEXT_ID_KEY not in self._storage:
self._storage[self.STORAGE_NEXT_ID_KEY] = 0
def create_token(self, userid, timeout=None, data=None):
payload = {}
payload["sub"] = userid
if timeout is None:
timeout = self.token_timeout
if timeout:
payload["exp"] = datetime.utcnow() + timedelta(seconds=timeout)
if data is not None:
payload.update(data)
algorithm = "HS256"
if self.haveRSAKeys():
algorithm = "RS256"
token = jwt.encode(payload,
self._signing_secret(),
algorithm=algorithm)
if not six.PY2:
token = token.decode("utf-8")
if self.store_tokens:
if self._tokens is None:
self._tokens = OOBTree()
if userid not in self._tokens:
self._tokens[userid] = OIBTree()
self._tokens[userid][token] = int(time.time())
return token
def __init__(self):
self.docid_to_address = IOBTree()
self.address_to_docid = OIBTree()
self.docid_to_metadata = IOBTree()
def __init__(self):
""" Ballots attr is an OIBTree, since they can have any object as key.
"""
self.ballots = OIBTree()
def clear(self):
""" Complete reset """
self._index = IOBTree()
self._unindex = OIBTree()
self._length = Length()
