def marshal(obj, keys=None, marshallerName='', objs=None):
"""
Convert an object to a dictionary. keys is an optional list of keys to
include in the returned dictionary. if keys is None then all public
attributes are returned. marshallerName is an optional marshalling
adapter name. if it is an empty string then the default marshaller will be
used.
"""
#to prevent recursing back over something twice, keep track of seen objs
if objs is None:
objs = OOSet()
# obj is itself marshallable, so make a marshaller and marshal away
if IMarshallable.providedBy(obj):
marshaller = component.getAdapter(obj, IMarshaller, marshallerName)
verify.verifyObject(IMarshaller, marshaller)
if IInfo.providedBy(obj):
key = (getattr(obj._object, '_p_oid', id(obj)), obj.__class__)
if key in objs:
raise AlreadySeenException()
else:
objs.insert(key)
try:
return marshal(marshaller.marshal(keys),
keys, marshallerName, objs)
except AlreadySeenException:
pass
finally:
objs.remove(key)
else:
return marshal(marshaller.marshal(keys), keys, marshallerName, objs)
# obj is a dict, so marshal its values recursively
# Zuul.marshal({'foo':1, 'bar':2})
if isinstance(obj, dict):
marshalled_dict = {}
for k in obj:
try:
marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs)
except AlreadySeenException:
pass
return marshalled_dict
# obj is a non-string iterable, so marshal its members recursively
# Zuul.marshal(set([o1, o2]))
elif hasattr(obj, '__iter__'):
marshalled_list = []
for o in obj:
try:
marshalled_list.append(marshal(o, keys, marshallerName, objs))
except AlreadySeenException:
pass
return marshalled_list
elif isinstance(obj, DateTime ):
return str(obj)
# Nothing matched, so it's a string or number or other unmarshallable.
else:
return obj
def add(self, old_path, new_path):
old_path = self._canonical(old_path)
new_path = self._canonical(new_path)
if old_path == new_path:
return
# Forget any existing reverse paths to old_path
existing_target = self._paths.get(old_path, None)
if (existing_target is not None) and (existing_target in self._rpaths):
if len(self._rpaths[existing_target]) == 1:
del self._rpaths[existing_target]
else:
self._rpaths[existing_target].remove(old_path)
# Update any references that pointed to old_path
for p in self.redirects(old_path):
if p != new_path:
self._paths[p] = new_path
self._rpaths.setdefault(new_path, OOSet()).insert(p)
else:
del self._paths[new_path]
# Remove reverse paths for old_path
if old_path in self._rpaths:
del self._rpaths[old_path]
self._paths[old_path] = new_path
self._rpaths.setdefault(new_path, OOSet()).insert(old_path)
class Group(Content, LocalRolesMixin, ContextACLMixin):
type_name = u"Group"
type_title = _(u"Group")
add_permission = "Add %s" % type_name
title = u""
description = u""
css_icon = "glyphicon glyphicon-user" #FIXME no group icon!?
def __init__(self, **kwargs):
#Things like created, creator etc...
super(Group, self).__init__()
self.__members__ = OOSet()
self.update(event = False, **kwargs)
@property
def principal_name(self):
""" The way the security system likes to check names
to avoid collisions with userids.
"""
return u"group:%s" % self.__name__
@property
def members(self):
return self.__members__
@members.setter
def members(self, value):
self.__members__.clear()
self.__members__.update(value)
class Group(Content, LocalRolesMixin, ContextACLMixin):
type_name = u"Group"
type_title = _(u"Group")
add_permission = "Add %s" % type_name
title = u""
description = u""
icon = u"user" #FIXME no group icon!?
def __init__(self, **kwargs):
#Things like created, creator etc...
super(Group, self).__init__()
self.__members__ = OOSet()
self.update(event=False, **kwargs)
@property
def principal_name(self):
""" The way the security system likes to check names
to avoid collisions with userids.
"""
return u"group:%s" % self.__name__
@property
def members(self):
return self.__members__
@members.setter
def members(self, value):
self.__members__.clear()
self.__members__.update(value)
def test_Terms(self): # normalize term ki= self.ki; obj= self.obj1 ki.NormalizeTerm= 'python: lambda value: value-1' self.assertEqual(ki._evaluate(obj).keys(),OOSet((0,1)).keys()) # stop term ki.StopTermPredicate= 'python: value == 1' self.assertEqual(ki._evaluate(obj).keys(),OOSet((1,)).keys())
def __init__(self, name = '', desc = ''):
"""
The BudgetGroup constructor takes a string name and desc as the Name of
the project, and it's Description.
It initialises with an empty set.
"""
self.Name = name
self.Description = desc
self.ItemSet = OOSet()
def read(self):
"""Return messages added and removed from folder.
Two sets of message objects are returned. The first set is
messages that were added to the folder since the last read.
The second set is the messages that were removed from the
folder since the last read.
The code assumes messages are added and removed but not edited.
"""
mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory)
self._stat()
cur = OOSet()
new = OOSet()
while 1:
msg = mbox.next()
if msg is None:
break
msgid = msg["message-id"]
cur.insert(msgid)
if not self.messages.has_key(msgid):
self.messages[msgid] = msg
new.insert(msg)
removed = difference(self.messages, cur)
for msgid in removed.keys():
del self.messages[msgid]
# XXX perhaps just return the OOBTree for removed?
return new, OOSet(removed.values())
def __init__(self, name = '', desc = ''):
"""
The Project constructor takes a string name and desc as the Name of
the project, and it's Description.
The default values of Name and Description would be ''.
It initialises with an empty set.
"""
self.Name = name
self.Description = desc
self.GroupSet = OOSet()
def _addToRoomDayReservationsIndex(self):
roomDayReservationsIndexBTree = Reservation.getRoomDayReservationsIndexRoot()
for period in self.splitToPeriods():
day = period.startDT.date()
key = (self.room.id, day)
resvs = roomDayReservationsIndexBTree.get(key)
if resvs is None:
resvs = OOSet()
resvs.add(self)
roomDayReservationsIndexBTree[key] = resvs
def _addToRoomDayReservationsIndex(self):
roomDayReservationsIndexBTree = Reservation.getRoomDayReservationsIndexRoot()
for period in self.splitToPeriods():
day = period.startDT.date()
key = (self.room.id, day)
resvs = roomDayReservationsIndexBTree.get(key)
if resvs is None:
resvs = OOSet()
resvs.add(self)
roomDayReservationsIndexBTree[key] = resvs
class HashList(Content, ContextACLMixin, LocalRolesMixin):
type_name = "HashList"
type_title = _("HashList")
type_description = _(
"Encrypted text-rows that can be matched to this document")
add_permission = PERM_MANAGE_SYSTEM
css_icon = "glyphicon glyphicon-lock"
nav_visible = False
listing_visible = False
search_visible = False
salt = None
def __init__(self, **kw):
self.salt = bcrypt.gensalt()
self.hashset = OOSet()
self.plaintext_rows = OOSet()
super(HashList, self).__init__(**kw)
def check(self, value):
value = b64encode(value.encode('utf-8'))
if value in self.plaintext_rows:
return True
return bcrypt.hashpw(value, self.salt) in self.hashset
def hash_plaintext(self, limit=100):
rows = list(self.plaintext_rows)[:limit]
for row in rows:
hashed = bcrypt.hashpw(row, self.salt)
if hashed not in self.hashset:
self.hashset.add(hashed)
self.plaintext_rows.remove(row)
return len(self.plaintext_rows)
def hash_and_remove_plaintext(self, limit=100):
rows = list(self.plaintext_rows)[:limit]
for row in rows:
hashed = bcrypt.hashpw(row, self.salt)
if hashed in self.hashset:
self.hashset.remove(hashed)
self.plaintext_rows.remove(row)
return len(self.plaintext_rows)
@property
def plaintext(self):
return ''
@plaintext.setter
def plaintext(self, value):
for row in value.splitlines():
row = row.strip()
if row and row not in self.plaintext_rows:
self.plaintext_rows.add(b64encode(row.encode('utf-8')))
class Content(Base, Folder):
title = ""
description = ""
default_view = u"view"
delegate_view = None
nav_visible = True
nav_title = None
listing_visible = True
search_visible = True
show_byline = False
custom_addable = False
def __init__(self, **kw):
#To set that this should keep track of ordering. See Folder
#If _order is set to None, ordering isn't stored
self._order = ()
Folder.__init__(self)
super(Content, self).__init__(**kw)
def get_nav_title(self):
nav_title = getattr(self, 'nav_title', None)
if nav_title:
return nav_title
title = getattr(self, 'title', '')
return title and title or self.__name__
@property
def tags(self): return getattr(self, '__tags__', ())
@tags.setter
def tags(self, value):
#Is this the right way to mutate objects, or should we simply clear the contents?
if value:
self.__tags__ = OOSet(value)
else:
if hasattr(self, '__tags__'):
delattr(self, '__tags__')
@property
def custom_addable_types(self):
return frozenset(getattr(self, '_custom_addable_types', ()))
@custom_addable_types.setter
def custom_addable_types(self, value):
if not hasattr(self, '_custom_addable_types'):
self._custom_addable_types = OOSet(value)
if set(value) != set(self._custom_addable_types):
self._custom_addable_types.clear()
self._custom_addable_types.update(value)
@custom_addable_types.deleter
def custom_addable_types(self):
if hasattr(self, '_custom_addable_types'):
delattr(self, '_custom_addable_types')
def add_user_roles(self, userid: str, *roles):
""" See kedja.interfaces.ISecurityAware """
if isinstance(userid, str):
userid = int(userid)
if userid not in self._rolesdata:
self._rolesdata[userid] = OOSet()
self._rolesdata[userid].update(roles)
def unread_storage(self):
try:
return self.context.__unread_storage__
except AttributeError: #This is basically init
self.context.__unread_storage__ = OOSet(
find_authorized_userids(self.context, (VIEW, )))
return self.context.__unread_storage__
def html(self, suppress_entries=0):
""" html log for viewing transactions in the ZMI """
out = []
keys = OOSet(self._transactions.keys())
for t_id in keys:
t = self._transactions[t_id]
out.append('''
<h4>Transaction id: %s</h4>
<p>
<em>User:</em> %s<br/>
<em>Description:</em> %s<br/>
</p>
''' % (oid2str(t.id), t.user, t.description))
if suppress_entries:
continue
for entry_id in t._entries.keys():
entry = t._entries[entry_id]
out.append('''
<p>
<em>id:</em> %(id)s<br/>
<em>obj:</em> %(path)s<br/>
<em>method:</em> %(method)s<br/>
<em>args:</em> %(args)s<br/>
</p>
''' % entry)
out = '<hr>'.join(out)
return '<html><body>%s</body></html>' % out
def tags(self, value):
#Is this the right way to mutate objects, or should we simply clear the contents?
if value:
self.__tags__ = OOSet(value)
else:
if hasattr(self, '__tags__'):
delattr(self, '__tags__')
class StarredTasks(grok.Annotation):
grok.context(ifaces.IAccount)
grok.implements(ifaces.IStarredTasks)
def __init__(self):
self._starred_tasks = OOSet()
def getStarredTasks(self):
return list(self._starred_tasks)
def addStarredTask(self, task_intid):
self._starred_tasks.insert(task_intid)
def removeStarredTask(self, task_intid):
if task_intid in self._starred_tasks:
self._starred_tasks.remove(task_intid)
def store(self, domain, key, value, overwrite=False):
"""Store a dictionary in the domain's storage
"""
# Get the storage for the current URL
storage = self.get_storage(domain=domain)
datastore = storage["data"]
indexstore = storage["index"]
# already fetched
if key in datastore and not overwrite:
logger.info("Skipping existing key {}".format(key))
return
# Create some indexes
for index in ["portal_type", "parent_id", "parent_path"]:
index_key = "by_{}".format(index)
if not indexstore.get(index_key):
indexstore[index_key] = OOBTree()
indexvalue = value.get(index)
# Check if the index value, e.g. the portal_type="Sample", is
# already known as a key in the index.
if not indexstore[index_key].get(indexvalue):
indexstore[index_key][indexvalue] = OOSet()
indexstore[index_key][indexvalue].add(key)
# store the data
datastore[key] = value
def __init__(self, key=None, main_lang = None, **kwargs):
super(LangField, self).__init__(key=key, **kwargs)
if not main_lang:
request = get_current_request()
main_lang = request.registry.settings.get('default_locale_name', 'en')
self.main_lang = main_lang
self.fuzzy = OOSet()
self.__data__ = OOBTree()
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index an object 'obj' with integer id 'i'
Ideally, we've been passed a sequence of some sort that we
can iterate over. If however, we haven't, we should do something
useful with the results. In the case of a string, this means
indexing the entire string as a keyword."""
# First we need to see if there's anything interesting to look at
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
newKeywords = self._get_object_keywords(obj, attr)
oldKeywords = self._unindex.get(documentId, None)
if oldKeywords is None:
# we've got a new document, let's not futz around.
try:
for kw in newKeywords:
self.insertForwardIndexEntry(kw, documentId)
if newKeywords:
self._unindex[documentId] = list(newKeywords)
except TypeError:
return 0
else:
# we have an existing entry for this document, and we need
# to figure out if any of the keywords have actually changed
if type(oldKeywords) is not OOSet:
oldKeywords = OOSet(oldKeywords)
newKeywords = OOSet(newKeywords)
fdiff = difference(oldKeywords, newKeywords)
rdiff = difference(newKeywords, oldKeywords)
if fdiff or rdiff:
# if we've got forward or reverse changes
if newKeywords:
self._unindex[documentId] = list(newKeywords)
else:
del self._unindex[documentId]
if fdiff:
self.unindex_objectKeywords(documentId, fdiff)
if rdiff:
for kw in rdiff:
self.insertForwardIndexEntry(kw, documentId)
return 1
def __init__(self,tagger=None,noNounRanksToKeep = 20):
"""
"""
self.noNounRanksToKeep = noNounRanksToKeep
self.trainingDocs = PersistentMapping()
self.allNouns = OOSet()
self.classifier = None
self.trainAfterUpdate = True
def __init__(self, storage, db=None, scan_interval=10):
self.storage = storage
self.db = db
self.next_conn_id = 1
self.conn_oids = IOBTree() # IOBTree({ conn_id -> OOSet([oid]) } })
self.oids = OOSet() # OOSet([oid])
self.lock = allocate_lock()
self.scan_interval = scan_interval
self.next_scan = time() + scan_interval
def consume(buf):
dmd._p_jar.sync()
for docId, uid, ob, metadata in buf:
if uid is not None and docId is None:
global_catalog.uncatalog_object(uid)
continue
if uid not in catalog.uids:
catalog._length.change(1)
catalog.uids[uid] = docId
catalog.paths[docId] = uid
if metadata:
catalog.data[docId] = metadata
for idx, val, uval in ob:
if val is not None:
idx = catalog.indexes[idx]
if isinstance(idx, MultiPathIndex):
if docId in idx._unindex:
unin = idx._unindex[docId]
if isinstance(unin, set):
unin = self._unindex[docId] = OOSet(unin)
for oldpath in list(unin):
if list(oldpath.split('/')) not in val:
idx.unindex_paths(docId, (oldpath, ))
else:
idx._unindex[docId] = OOSet()
idx._length.change(1)
idx.index_paths(docId, val)
else:
oldval = idx._unindex.get(docId)
if uval == oldval: continue
customEq = idx._equalValues
if customEq is not None:
if customEq(val, oldval): continue
update = idx._update
if update is None or oldval is None or val is None:
if oldval is not None:
idx._unindex_object(docId, oldval, val is None)
if val is None: continue
idx._indexValue(docId, val, None)
if oldval is None: idx.numObjects.change(1)
else:
rv = update(docId, val, oldval, None)
if isinstance(rv, tuple): continue
idx._unindex[docId] = uval
def test_AttributeLookup(self):
fi= self.fi; ki= self.ki # field and keyword index
obj1= self.obj1; obj2= self.obj2
kw= OOSet((1,2,))
## Acquisition type
# implicit
self.assertEqual(fi._evaluate(obj2),'id')
self.assertEqual(fi._evaluate(obj1),None)
# "OOSet" does not support intelligent equality test
self.assertEqual(ki._evaluate(obj2).keys(),kw.keys())
self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
# none
ki.kw.AcquisitionType= 'none'
self.assertEqual(ki._evaluate(obj2),None)
self.assertEqual(ki._evaluate(obj1).keys(),[1,2])
# explicit
# same as implicit for non-methods
ki.kw.AcquisitionType= 'explicit'
self.assertEqual(fi._evaluate(obj2),'id')
self.assertEqual(fi._evaluate(obj1),None)
self.assertEqual(ki._evaluate(obj2).keys(),kw.keys())
self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
# now check methods
fi.id_.Name= 'fid'; ki.kw.Name= 'fkw'
self.assertEqual(fi._evaluate(obj2),'id')
self.assertEqual(fi._evaluate(obj1),None)
self.assertEqual(ki._evaluate(obj2),None)
self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
## call types
# call -- already checked
# return
fi.id_.CallType= 'return'
self.assertEqual(fi._evaluate(obj2), obj1.fid)
# ignore
fi.id_.CallType= 'ignore'
self.assertEqual(fi._evaluate(obj2), None)
fi.id_.CallType= 'call'
## IgnoreExceptions
fi.id_.IgnoreExceptions= 0
self.assertRaises(AttributeError,fi._evaluate,obj1)
self.assertEqual(fi._evaluate(obj2),'id')
def clear(self):
self.tagsmap = LOBTree()
# index
self.tag_oids = LOBTree()
self.oid_tags = LOBTree()
# tag weight
self.weights = OOSet()
self.tag_weight = LFBTree()
def __setitem__(self, relation_id, rids):
assert isinstance(relation_id, int)
self.can_create_relation(rids)
if relation_id in self:
del self[relation_id]
for x in rids:
assert isinstance(x, int)
if x not in self.rid_to_relations:
self.rid_to_relations[x] = OOSet()
self.rid_to_relations[x].add(relation_id)
self.relation_to_rids[relation_id] = tuple(rids)
def __setitem__(self, key, value):
if value:
#Make sure it exist
roles_principals = get_roles_registry()
if IRole.providedBy(value):
value = [value]
for role in value:
assert role in roles_principals, "'%s' isn't a role" % role
self.data[key] = OOSet(value)
elif key in self.data:
del self.data[key]
def sort(self, result):
index = self.index
values = self.index.documents_to_values
seq = OOBTree()
for key in result:
idx = values.get(key, None)
if idx is None:
continue
data = seq.get(idx, None)
if data is None:
data = OOSet()
data.insert(key)
seq[idx] = data
result = []
for idx, data in seq.items():
result.extend(data)
return result
def set_connection_oids(self, conn_id, oids):
"""Records the OIDs a connection is using and periodically scans.
"""
changed = 0
new_oids = OOSet()
self.lock.acquire()
try:
if oids:
self.conn_oids[conn_id] = OOSet(oids)
else:
if self.conn_oids.has_key(conn_id):
del self.conn_oids[conn_id]
for set in self.conn_oids.values():
new_oids.update(set)
if self.oids != new_oids:
self.oids = new_oids
changed = 1
finally:
self.lock.release()
if changed:
self.storage.scanner.set_oids(new_oids)
def __init__(self, parent=None):
super(ItemModel, self).__init__(parent)
self.labels = self.HEADERS.split(',')
root = parent.conn.root()
if not "item" in root:
root["item"] = PersistentList()
transaction.commit()
if not "hash" in root:
root["hash"] = OOSet()
transaction.commit()
self._data = root["item"]
self._hash = root["hash"]
def set_connection_oids(self, conn_id, oids):
"""Records the OIDs a connection is using and periodically scans.
"""
changed = 0
new_oids = OOSet()
self.lock.acquire()
try:
if oids:
self.conn_oids[conn_id] = OOSet(oids)
else:
if self.conn_oids.has_key(conn_id):
del self.conn_oids[conn_id]
for set in self.conn_oids.values():
new_oids.update(set)
if self.oids != new_oids:
self.oids = new_oids
changed = 1
finally:
self.lock.release()
if changed:
self.storage.scanner.set_oids(new_oids)
def read(self):
"""Return messages added and removed from folder.
Two sets of message objects are returned. The first set is
messages that were added to the folder since the last read.
The second set is the messages that were removed from the
folder since the last read.
The code assumes messages are added and removed but not edited.
"""
mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory)
self._stat()
cur = OOSet()
new = OOSet()
while 1:
msg = mbox.next()
if msg is None:
break
msgid = msg["message-id"]
cur.insert(msgid)
if not self.messages.has_key(msgid):
self.messages[msgid] = msg
new.insert(msg)
removed = difference(self.messages, cur)
for msgid in removed.keys():
del self.messages[msgid]
# XXX perhaps just return the OOBTree for removed?
return new, OOSet(removed.values())
def test_ExpressionEvaluator(self): ki= self.ki; obj2= self.obj2 ee= ExpressionEvaluator(); ee.id= 'ee' ki._setObject(ee.id,ee); ee= ki._getOb(ee.id) ee.manage_changeProperties(Expression= 'python: (3,4,)') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,3,4)).keys()) # ignore ee.manage_changeProperties(IgnorePredicate= 'python: 3 in value') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys()) # ignore - call it ee.manage_changeProperties(IgnorePredicate= 'python: lambda v: 3 in v') # normalize ee.manage_changeProperties(Expression= 'python: (4,)') ee.manage_changeProperties(Normalizer= 'python: (0,) + value') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys()) # normalize - call it ee.manage_changeProperties(Normalizer= 'python: lambda v: (0,) + v') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys()) # method ee.manage_changeProperties(Expression= "python: lambda object: object.kw") self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,)).keys()) ## combine # 'union' - already tested # 'useFirst' ki.CombineType= 'useFirst' self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys())
def marshal(obj, keys=None, marshallerName='', objs=None):
"""
Convert an object to a dictionary. keys is an optional list of keys to
include in the returned dictionary. if keys is None then all public
attributes are returned. marshallerName is an optional marshalling
adapter name. if it is an empty string then the default marshaller will be
used.
"""
#to prevent recursing back over something twice, keep track of seen objs
if objs is None:
objs = OOSet()
# obj is itself marshallable, so make a marshaller and marshal away
if IMarshallable.providedBy(obj):
marshaller = component.getAdapter(obj, IMarshaller, marshallerName)
verify.verifyObject(IMarshaller, marshaller)
if IInfo.providedBy(obj):
key = (obj._object._p_oid, obj.__class__)
if key in objs:
raise AlreadySeenException()
else:
objs.insert(key)
try:
return marshal(marshaller.marshal(keys),
keys, marshallerName, objs)
except AlreadySeenException:
pass
finally:
objs.remove(key)
else:
return marshal(marshaller.marshal(keys), keys, marshallerName, objs)
# obj is a dict, so marshal its values recursively
# Zuul.marshal({'foo':1, 'bar':2})
if isinstance(obj, dict):
marshalled_dict = {}
for k in obj:
try:
marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs)
except AlreadySeenException:
pass
return marshalled_dict
# obj is a non-string iterable, so marshal its members recursively
# Zuul.marshal(set([o1, o2]))
elif hasattr(obj, '__iter__'):
marshalled_list = []
for o in obj:
try:
marshalled_list.append(marshal(o, keys, marshallerName, objs))
except AlreadySeenException:
pass
return marshalled_list
elif isinstance(obj, DateTime ):
return str(obj)
# Nothing matched, so it's a string or number or other unmarshallable.
else:
return obj
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
if isinstance(self._unindex[docid], set):
self._unindex[docid] = OOSet(self._unindex[docid])
unin = set(self._unindex[docid])
paths_set = {'/'.join(x) for x in paths}
for oldpath in unin - paths_set:
self.unindex_paths(docid, (oldpath, ))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def add(self, tag, userid):
if not isinstance(tag, basestring):
raise TypeError('tag must be a string. Was: %s' % tag)
if not TAG_PATTERN.match(tag):
raise ValueError("'tag' doesn't conform to tag standard: %s" %
_TAG_STRING)
if tag not in self.tags_storage:
self.tags_storage[tag] = OOSet()
if userid not in self.tags_storage[tag]:
self.tags_storage[tag].add(userid)
if tag == 'like':
_notify(self.context)
def test_doesnt_cause_redirect_loop_on_bogus_storage_entries(self):
storage = queryUtility(IRedirectionStorage)
storage._paths['/plone/same'] = '/plone/same'
storage._rpaths['/plone/same'] = OOSet(['/plone/same'])
transaction.commit()
response = requests.get(
self.portal_url + '/same/@@view',
headers={'Accept': 'application/json'},
auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD),
allow_redirects=False,
)
self.assertEqual(404, response.status_code)
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
unin = self._unindex[docid]
# Migrate old versions of the index to use OOSet
if isinstance(unin, set):
unin = self._unindex[docid] = OOSet(unin)
for oldpath in list(unin):
if list(oldpath.split('/')) not in paths:
self.unindex_paths(docid, (oldpath, ))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def register_status_tracking(self, username, rest_id):
"""
Register `username` for tracking states of given `rest_id`.
Args:
username (str): Name of the user.
rest_id (str): Unique identificator of given REST request.
"""
self.log("Registering user '%s' to track '%s'." % (username, rest_id))
# handle id->username mapping
self.id_to_username[rest_id] = username
# handle username->ids mapping
uname_to_ids = self.username_to_ids.get(username, None)
if uname_to_ids is None:
uname_to_ids = OOSet()
self.username_to_ids[username] = uname_to_ids
# add new rest_id to set
uname_to_ids.add(rest_id)
self.status_db[rest_id] = StatusInfo(rest_id=rest_id)
def indexConf(self, conf):
# Note: conf can be any object which has getEndDate() and getStartDate() methods
self._idxDay._p_changed = True
days = (conf.getEndDate().date() - conf.getStartDate().date()).days
startDate = datetime(conf.getStartDate().year, conf.getStartDate().month, conf.getStartDate().day)
for day in range(days + 1):
key = int(datetimeToUnixTime(startDate + timedelta(day)))
#checking if 2038 problem occurs
if key > BTREE_MAX_INT:
continue
if self._idxDay.has_key(key):
self._idxDay[key].add(conf)
else:
self._idxDay[key] = OOSet([conf])
def add(self, context):
container = self._find_container(context)
assert container
if container.uid not in self:
self[container.uid] = OOBTree()
if context.type_name not in self[container.uid]:
type_uids = self[container.uid][context.type_name] = OOSet()
else:
type_uids = self[container.uid][context.type_name]
if context.uid not in type_uids:
type_uids.add(context.uid)
#counter_context = self._find_counter(container)
#self._update_counter(counter_context.uid, context.type_name, 1)
return context
class LangField(BaseField):
""" Language sensitive field. """
def __init__(self, key=None, main_lang = None, **kwargs):
super(LangField, self).__init__(key=key, **kwargs)
if not main_lang:
request = get_current_request()
main_lang = request.registry.settings.get('default_locale_name', 'en')
self.main_lang = main_lang
self.fuzzy = OOSet()
self.__data__ = OOBTree()
@property
def langs(self):
return set(self.__data__.keys())
@property
def translated(self):
return self.langs - set([self.main_lang])
def get(self, default=None, langs = None, **kwargs):
if not langs:
request = get_current_request()
langs = (get_locale_name(request),)
return dict((lang, self.__data__.get(lang, default)) for lang in langs)
def set(self, value, **kwargs):
if not isinstance(value, dict):
raise TypeError("Must be a dict")
updated = value.keys()
main_updated = self.main_lang in updated
for (lang, value) in value.items():
self.__data__[lang] = value
if lang in self.fuzzy:
self.fuzzy.remove(lang)
if main_updated:
others = self.translated - set(updated)
self.fuzzy.update(others)
def remove(self, key):
self.__data__.pop(key, None)
if key in self.fuzzy:
self.fuzzy.remove(key)
def __len__(self):
return len(self.__data__)
def setTags(self, pid, oid, tags):
self.removeTags(pid, oid)
if not tags:
return
t = []
for tag in tags:
tag = tag.lower().strip()
if tag:
t.append(tag)
if not t:
return
tags = t
pdata = self.tags.get(pid)
if pdata is None:
pdata = IOBTree()
self.tags[pid] = pdata
oids = pdata.get(oid)
if oids is None:
oids = OOSet()
pdata[oid] = oids
oids.update(tags)
# insert oid -> pid reference
oids = self.oids.get(oid)
if oids is None:
oids = OOSet()
self.oids[oid] = oids
oids.insert(pid)
#update tagging engine
engine = self.getEngine(pid)
try:
engine.update(oid, tags)
except:
engine.clear()
engine.update(oid, tags)
class NounBayesClassifier(Persistent):
"""
"""
implements(IContentClassifier)
def __init__(self,tagger=None,noNounRanksToKeep = 20):
"""
"""
self.noNounRanksToKeep = noNounRanksToKeep
self.trainingDocs = PersistentMapping()
self.allNouns = OOSet()
self.classifier = None
self.trainAfterUpdate = True
def addTrainingDocument(self,doc_id,tags):
"""
"""
storage = getUtility(INounPhraseStorage)
importantNouns = storage.getNounTerms(doc_id,self.noNounRanksToKeep)
self.trainingDocs[doc_id] = (importantNouns,tags)
self.allNouns = union(self.allNouns,OOSet(importantNouns))
def train(self):
"""
"""
presentNouns = dict()
trainingData = []
if not self.allNouns:
storage = getUtility(INounPhraseStorage)
for key in self.trainingDocs.keys():
importantNouns = storage.getNounTerms(
key,
self.noNounRanksToKeep)
self.allNouns = union(self.allNouns,OOSet(importantNouns))
for item in self.allNouns:
presentNouns.setdefault(item,0)
for (nouns,tags) in self.trainingDocs.values():
nounPresence = presentNouns.copy()
for noun in nouns:
nounPresence[noun] = 1
for tag in tags:
trainingData.append((nounPresence,tag,))
if trainingData:
self.classifier = NaiveBayesClassifier.train(trainingData)
def classify(self,doc_id):
"""
"""
if not self.classifier:
return []
presentNouns = dict()
for item in self.allNouns:
presentNouns.setdefault(item,0)
storage = getUtility(INounPhraseStorage)
importantNouns = storage.getNounTerms(doc_id,self.noNounRanksToKeep)
for noun in importantNouns:
if noun in presentNouns.keys():
presentNouns[noun] = 1
return self.classifier.classify(presentNouns)
def probabilityClassify(self,doc_id):
"""
"""
if not self.classifier:
return []
presentNouns = dict()
for item in self.allNouns:
presentNouns.setdefault(item,0)
storage = getUtility(INounPhraseStorage)
importantNouns = storage.getNounTerms(doc_id,self.noNounRanksToKeep)
for noun in importantNouns:
if noun in presentNouns.keys():
presentNouns[noun] = 1
return self.classifier.prob_classify(presentNouns)
def clear(self):
"""Wipes the classifier's data.
"""
self.allNouns.clear()
self.trainingDocs.clear()
def tags(self):
if not self.classifier:
return []
return self.classifier.labels()
class BaseBlog(ContentContainer):
interface.implements(IBlog, IBlogPostListing)
pageSize = FieldProperty(IBlog["pageSize"])
def __init__(self, *args, **kw):
super(BaseBlog, self).__init__(*args, **kw)
self.dates = OOSet()
self.index = IFBTree()
def entries(self, index=None):
if index is None:
result = self.index
else:
_t, result = weightedIntersection(index, self.index)
return result.byValue(0)
def update(self, post):
uid = getUtility(IIntIds).getId(post)
if uid in self.index:
self.remove(post)
date = post.date.timetuple()
idx = time.mktime(date)
self.index[uid] = idx
# update in dates
yearName = str(date[0])
year = self.get(yearName)
if year is None:
year = Year()
event.notify(ObjectCreatedEvent(year))
self[yearName] = year
self.dates.insert(yearName)
year.update(date, uid, idx)
# update categories
self["category"].update(post, uid, idx)
# update tags info
self["tags"].update(post, uid)
def remove(self, post):
uid = getUtility(IIntIds).getId(post)
if uid not in self.index:
return
date = time.localtime(self.index[uid])
# remove from dates
yearName = str(date[0])
year = self.get(yearName)
if year is not None:
year.remove(date, uid)
if not len(year):
if yearName in self.dates:
self.dates.remove(yearName)
del self[yearName]
# remove from category
self["category"].remove(uid)
# remove tags info
self["tags"].remove(post, uid)
# remove from index
del self.index[uid]
def __init__(self, *args, **kw):
super(BaseBlog, self).__init__(*args, **kw)
self.dates = OOSet()
self.index = IFBTree()
class BudgetGroup(Persistent):
"""
The BudgetGroup class contains a set of BudgetItem objects
and is contained by Project.
It has Name and Description string attributes, and a set of BudgetItems.
It has methods to add and delete BudgetItem instances from the set.
BudgetGroup has functions to calculate the subtotal, vat, and total of
the BudgetItems in the set.
The class is hashable and has a toString method.
"""
VAT = 0.14
def __init__(self, name = '', desc = ''):
"""
The BudgetGroup constructor takes a string name and desc as the Name of
the project, and it's Description.
It initialises with an empty set.
"""
self.Name = name
self.Description = desc
self.ItemSet = OOSet()
def add(self, item):
"""The add method adds a BudgetItem object to the set."""
self.ItemSet.add(item)
def delete(self, item):
"""
The delete method removes a BudgetGroup object from the set.
Before removing an object it makes sure it is in the set.
"""
if (item in self.ItemSet):
self.ItemSet.remove(item)
return "Confirmed."
else:
return "Not in set."
def subtotal(self):
"""
The subtotal function returns the total of the subtotal of all
the BudgetItem objects.
"""
stotal = 0
for group in self.ItemSet:
stotal+=group.subtotal()
return stotal
def vat(self):
"""
The vat function calculates the VAT percentage on the subtotal.
Computing the percentage on the subtotal is more efficient than
computing the percentage of each item and adding it.
"""
#return (self.subtotal()*VAT)
return (self.subtotal()*0.14)
def total (self):
"""
The total function computes the cost of the BudgetGroup plus VAT.
Computing the subtotal increasing it with the VAT percentage is
more efficient than computing the subtotal and VAT separately.
"""
#return (self.subtotal()*(1+VAT))
return (self.subtotal()*(1+0.14))
def __hash__(self):
"""This enables the class to be hashable"""
return hash(self.Name)
def __eq__(self, other):
"""This enables the class to be hashable"""
return self.Name == other.Name
def __str__(self):
"""
The toString method returns a string of the name and
description of the class.
If the set is not empty thereafter it prints all
the BudgetItems in the set.
"""
output = self.Name + ": " + self.Description
if self.ItemSet is not None:
for item in self.ItemSet:
output+=("\n\t\t"+str(item))
return output
def custom_addable_types(self, value):
if not hasattr(self, '_custom_addable_types'):
self._custom_addable_types = OOSet(value)
if set(value) != set(self._custom_addable_types):
self._custom_addable_types.clear()
self._custom_addable_types.update(value)
def __init__(self, **kwargs):
#Things like created, creator etc...
super(Group, self).__init__()
self.__members__ = OOSet()
self.update(event = False, **kwargs)
def learn(self, rid, value):
# Validate
# ========
if not isinstance(value, basestring):
raise TypeError("value is not a string: '%s'" % value)
if not self.case_sensitive:
value = value.lower()
# Add to the (one:many) mapping, value to rids.
# =============================================
if value in self.values:
self.values[value].insert(rid)
else:
self.values[value] = IITreeSet([rid])
# Add to the (one:many) beginnings, middles, and endings indices.
# ==================================================================
# These are for the startswith, contains, and endswith searches, which
# function basically like the is search, but we have to learn all
# substrings of the string.
substrings = OOSet()
for i in range(len(value)):
j = i + 1
# beginnings/startswith
# =====================
part = value[:j]
if part in self.beginnings:
self.beginnings[part].insert(rid)
else:
self.beginnings[part] = IITreeSet([rid])
# middles/contains
# ================
for k in range(len(value)-i):
part = value[k:k+j]
if part in self.middles:
self.middles[part].insert(rid)
else:
self.middles[part] = IITreeSet([rid])
substrings.insert(part)
# endings/endswith
# ================
part = value[i:]
if part in self.endings:
self.endings[part].insert(rid)
else:
self.endings[part] = IITreeSet([rid])
# Add to the (one:one) mapping of rid to value.
# =============================================
# This exists so we know where the rid is located when the time comes
# to forget it.
self.rids[rid] = substrings
class TaggingEngine(Persistent):
interface.implements(ITaggingEngine)
def __init__(self):
self.clear()
@property
def tagsCount(self):
return len(self.tag_oids)
@property
def itemsCount(self):
return len(self.oid_tags)
def clear(self):
self.tagsmap = LOBTree()
# index
self.tag_oids = LOBTree()
self.oid_tags = LOBTree()
# tag weight
self.weights = OOSet()
self.tag_weight = LFBTree()
def getHash(self, str):
if not str:
return 0 # empty
res = hash(str)
# NOTE: workaround for 64bit
if struct.calcsize("P") * 8 == 64:
res = ord(str[0]) << 7
for char in str:
res = c_mul(1000003, res) ^ ord(char)
res = res ^ len(str)
if res == -1:
res = -2
if res >= 2**31:
res -= 2**32
return res
def update(self, oid, tags):
self.remove(oid)
for tag in tags:
htag = self.getHash(tag)
self.tagsmap[htag] = tag
# add oid -> tag
oids = self.tag_oids.get(htag)
if oids is None:
oids = LFSet()
self.tag_oids[htag] = oids
if oid not in oids:
oids.insert(oid)
# add tag -> oid
oid_tags = self.oid_tags.get(oid)
if oid_tags is None:
oid_tags = LLSet()
self.oid_tags[oid] = oid_tags
if htag not in oid_tags:
oid_tags.insert(htag)
# culculate weight
weight = self.tag_weight.get(htag)
if weight is not None:
key = (weight, htag)
if key in self.weights:
self.weights.remove(key)
weight = float(len(oids))
self.tag_weight[htag] = weight
self.weights.insert((weight, htag))
def remove(self, oid):
for tag in self.oid_tags.get(oid, ()):
# remove oid from tag -> oids reference
oids = self.tag_oids.get(tag)
if oid in oids:
oids.remove(oid)
oids_len = float(len(oids))
# reculculate weight
weight = self.tag_weight.get(tag)
if weight is not None:
key = (weight, tag)
if key in self.weights:
self.weights.remove(key)
if oids_len:
self.tag_weight[tag] = oids_len
self.weights.insert((oids_len, tag))
# remove tag
if not oids_len:
del self.tag_oids[tag]
del self.tagsmap[tag]
if oid in self.oid_tags:
del self.oid_tags[oid]
def getItems(self, tags):
oids = self.tag_oids
weights = self.tag_weight
weight, result = 0, LFBTree()
for tag in tags:
htag = self.getHash(tag)
if htag in oids:
weight, result = weightedUnion(
oids.get(htag), result, weights.get(htag), weight)
return IFBucket(result)
def getUniqueItems(self, tags):
oids = self.tag_oids
weights = self.tag_weight
weight, result = 1.0, None
for tag in tags:
htag = self.getHash(tag)
if htag in oids:
if result is None:
weight, result = weightedUnion(
oids.get(htag), LFBTree(), weights.get(htag), weight)
else:
weight, result = weightedIntersection(
result, oids.get(htag), weight, weights.get(htag))
return IFBucket(result)
def getTagCloud(self, reverse=False):
total = len(self.oid_tags)
if not total:
return
tags = self.tagsmap
data = self.weights.keys()
percent = total / 100.0
if reverse:
first = len(data)-1
for idx in xrange(first, -1, -1):
weight, htag = data[idx]
yield weight / percent, tags.get(htag)
else:
for weight, htag in data:
yield weight / percent, tags.get(htag)
def getItemsTagCloud(self, items):
oid_tags = self.oid_tags
tags = [oid_tags[oid] for oid in items if oid in oid_tags]
if tags:
tags = multiunion(tags)
else:
return
total = len(oid_tags)
data = self.weights.keys()
weights = self.tag_weight
percent = total / 100.0
for tag in tags:
yield weights[tag] / percent, self.tagsmap.get(tag)
def getFrequency(self, tags):
tagsmap = self.tagsmap
tag_weight = self.tag_weight
for tag in tags:
yield (tag, tag_weight.get(self.getHash(tag), 0))
def __nonzero__(self):
return bool(self.tag_oids)
def __contains__(self, tag):
return self.getHash(tag) in self.tagsmap
def __init__(self):
self._starred_tasks = OOSet()
