def add(self, old_path, new_path):
old_path = self._canonical(old_path)
new_path = self._canonical(new_path)
if old_path == new_path:
return
# Forget any existing reverse paths to old_path
existing_target = self._paths.get(old_path, None)
if (existing_target is not None) and (existing_target in self._rpaths):
if len(self._rpaths[existing_target]) == 1:
del self._rpaths[existing_target]
else:
self._rpaths[existing_target].remove(old_path)
# Update any references that pointed to old_path
for p in self.redirects(old_path):
if p != new_path:
self._paths[p] = new_path
self._rpaths.setdefault(new_path, OOSet()).insert(p)
else:
del self._paths[new_path]
# Remove reverse paths for old_path
if old_path in self._rpaths:
del self._rpaths[old_path]
self._paths[old_path] = new_path
self._rpaths.setdefault(new_path, OOSet()).insert(old_path)
def read(self):
"""Return messages added and removed from folder.
Two sets of message objects are returned. The first set is
messages that were added to the folder since the last read.
The second set is the messages that were removed from the
folder since the last read.
The code assumes messages are added and removed but not edited.
"""
mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory)
self._stat()
cur = OOSet()
new = OOSet()
while 1:
msg = mbox.next()
if msg is None:
break
msgid = msg["message-id"]
cur.insert(msgid)
if not self.messages.has_key(msgid):
self.messages[msgid] = msg
new.insert(msg)
removed = difference(self.messages, cur)
for msgid in removed.keys():
del self.messages[msgid]
# XXX perhaps just return the OOBTree for removed?
return new, OOSet(removed.values())
def test_Terms(self): # normalize term ki= self.ki; obj= self.obj1 ki.NormalizeTerm= 'python: lambda value: value-1' self.assertEqual(ki._evaluate(obj).keys(),OOSet((0,1)).keys()) # stop term ki.StopTermPredicate= 'python: value == 1' self.assertEqual(ki._evaluate(obj).keys(),OOSet((1,)).keys())
def tags(self, value):
#Is this the right way to mutate objects, or should we simply clear the contents?
if value:
self.__tags__ = OOSet(value)
else:
if hasattr(self, '__tags__'):
delattr(self, '__tags__')
def add_user_roles(self, userid: str, *roles):
""" See kedja.interfaces.ISecurityAware """
if isinstance(userid, str):
userid = int(userid)
if userid not in self._rolesdata:
self._rolesdata[userid] = OOSet()
self._rolesdata[userid].update(roles)
def marshal(obj, keys=None, marshallerName='', objs=None):
"""
Convert an object to a dictionary. keys is an optional list of keys to
include in the returned dictionary. if keys is None then all public
attributes are returned. marshallerName is an optional marshalling
adapter name. if it is an empty string then the default marshaller will be
used.
"""
#to prevent recursing back over something twice, keep track of seen objs
if objs is None:
objs = OOSet()
# obj is itself marshallable, so make a marshaller and marshal away
if IMarshallable.providedBy(obj):
marshaller = component.getAdapter(obj, IMarshaller, marshallerName)
verify.verifyObject(IMarshaller, marshaller)
if IInfo.providedBy(obj):
key = (obj._object._p_oid, obj.__class__)
if key in objs:
raise AlreadySeenException()
else:
objs.insert(key)
try:
return marshal(marshaller.marshal(keys),
keys, marshallerName, objs)
except AlreadySeenException:
pass
finally:
objs.remove(key)
else:
return marshal(marshaller.marshal(keys), keys, marshallerName, objs)
# obj is a dict, so marshal its values recursively
# Zuul.marshal({'foo':1, 'bar':2})
if isinstance(obj, dict):
marshalled_dict = {}
for k in obj:
try:
marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs)
except AlreadySeenException:
pass
return marshalled_dict
# obj is a non-string iterable, so marshal its members recursively
# Zuul.marshal(set([o1, o2]))
elif hasattr(obj, '__iter__'):
marshalled_list = []
for o in obj:
try:
marshalled_list.append(marshal(o, keys, marshallerName, objs))
except AlreadySeenException:
pass
return marshalled_list
elif isinstance(obj, DateTime ):
return str(obj)
# Nothing matched, so it's a string or number or other unmarshallable.
else:
return obj
def html(self, suppress_entries=0):
""" html log for viewing transactions in the ZMI """
out = []
keys = OOSet(self._transactions.keys())
for t_id in keys:
t = self._transactions[t_id]
out.append('''
<h4>Transaction id: %s</h4>
<p>
<em>User:</em> %s<br/>
<em>Description:</em> %s<br/>
</p>
''' % (oid2str(t.id), t.user, t.description))
if suppress_entries:
continue
for entry_id in t._entries.keys():
entry = t._entries[entry_id]
out.append('''
<p>
<em>id:</em> %(id)s<br/>
<em>obj:</em> %(path)s<br/>
<em>method:</em> %(method)s<br/>
<em>args:</em> %(args)s<br/>
</p>
''' % entry)
out = '<hr>'.join(out)
return '<html><body>%s</body></html>' % out
def unread_storage(self):
try:
return self.context.__unread_storage__
except AttributeError: #This is basically init
self.context.__unread_storage__ = OOSet(
find_authorized_userids(self.context, (VIEW, )))
return self.context.__unread_storage__
def store(self, domain, key, value, overwrite=False):
"""Store a dictionary in the domain's storage
"""
# Get the storage for the current URL
storage = self.get_storage(domain=domain)
datastore = storage["data"]
indexstore = storage["index"]
# already fetched
if key in datastore and not overwrite:
logger.info("Skipping existing key {}".format(key))
return
# Create some indexes
for index in ["portal_type", "parent_id", "parent_path"]:
index_key = "by_{}".format(index)
if not indexstore.get(index_key):
indexstore[index_key] = OOBTree()
indexvalue = value.get(index)
# Check if the index value, e.g. the portal_type="Sample", is
# already known as a key in the index.
if not indexstore[index_key].get(indexvalue):
indexstore[index_key][indexvalue] = OOSet()
indexstore[index_key][indexvalue].add(key)
# store the data
datastore[key] = value
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index an object 'obj' with integer id 'i'
Ideally, we've been passed a sequence of some sort that we
can iterate over. If however, we haven't, we should do something
useful with the results. In the case of a string, this means
indexing the entire string as a keyword."""
# First we need to see if there's anything interesting to look at
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
newKeywords = self._get_object_keywords(obj, attr)
oldKeywords = self._unindex.get(documentId, None)
if oldKeywords is None:
# we've got a new document, let's not futz around.
try:
for kw in newKeywords:
self.insertForwardIndexEntry(kw, documentId)
if newKeywords:
self._unindex[documentId] = list(newKeywords)
except TypeError:
return 0
else:
# we have an existing entry for this document, and we need
# to figure out if any of the keywords have actually changed
if type(oldKeywords) is not OOSet:
oldKeywords = OOSet(oldKeywords)
newKeywords = OOSet(newKeywords)
fdiff = difference(oldKeywords, newKeywords)
rdiff = difference(newKeywords, oldKeywords)
if fdiff or rdiff:
# if we've got forward or reverse changes
if newKeywords:
self._unindex[documentId] = list(newKeywords)
else:
del self._unindex[documentId]
if fdiff:
self.unindex_objectKeywords(documentId, fdiff)
if rdiff:
for kw in rdiff:
self.insertForwardIndexEntry(kw, documentId)
return 1
def __init__(self, name = '', desc = ''):
"""
The BudgetGroup constructor takes a string name and desc as the Name of
the project, and it's Description.
It initialises with an empty set.
"""
self.Name = name
self.Description = desc
self.ItemSet = OOSet()
def __init__(self, storage, db=None, scan_interval=10):
self.storage = storage
self.db = db
self.next_conn_id = 1
self.conn_oids = IOBTree() # IOBTree({ conn_id -> OOSet([oid]) } })
self.oids = OOSet() # OOSet([oid])
self.lock = allocate_lock()
self.scan_interval = scan_interval
self.next_scan = time() + scan_interval
def consume(buf):
dmd._p_jar.sync()
for docId, uid, ob, metadata in buf:
if uid is not None and docId is None:
global_catalog.uncatalog_object(uid)
continue
if uid not in catalog.uids:
catalog._length.change(1)
catalog.uids[uid] = docId
catalog.paths[docId] = uid
if metadata:
catalog.data[docId] = metadata
for idx, val, uval in ob:
if val is not None:
idx = catalog.indexes[idx]
if isinstance(idx, MultiPathIndex):
if docId in idx._unindex:
unin = idx._unindex[docId]
if isinstance(unin, set):
unin = self._unindex[docId] = OOSet(unin)
for oldpath in list(unin):
if list(oldpath.split('/')) not in val:
idx.unindex_paths(docId, (oldpath, ))
else:
idx._unindex[docId] = OOSet()
idx._length.change(1)
idx.index_paths(docId, val)
else:
oldval = idx._unindex.get(docId)
if uval == oldval: continue
customEq = idx._equalValues
if customEq is not None:
if customEq(val, oldval): continue
update = idx._update
if update is None or oldval is None or val is None:
if oldval is not None:
idx._unindex_object(docId, oldval, val is None)
if val is None: continue
idx._indexValue(docId, val, None)
if oldval is None: idx.numObjects.change(1)
else:
rv = update(docId, val, oldval, None)
if isinstance(rv, tuple): continue
idx._unindex[docId] = uval
def __setitem__(self, key, value):
if value:
#Make sure it exist
roles_principals = get_roles_registry()
if IRole.providedBy(value):
value = [value]
for role in value:
assert role in roles_principals, "'%s' isn't a role" % role
self.data[key] = OOSet(value)
elif key in self.data:
del self.data[key]
def __init__(self, name = '', desc = ''):
"""
The Project constructor takes a string name and desc as the Name of
the project, and it's Description.
The default values of Name and Description would be ''.
It initialises with an empty set.
"""
self.Name = name
self.Description = desc
self.GroupSet = OOSet()
def __setitem__(self, relation_id, rids):
assert isinstance(relation_id, int)
self.can_create_relation(rids)
if relation_id in self:
del self[relation_id]
for x in rids:
assert isinstance(x, int)
if x not in self.rid_to_relations:
self.rid_to_relations[x] = OOSet()
self.rid_to_relations[x].add(relation_id)
self.relation_to_rids[relation_id] = tuple(rids)
def _addToRoomDayReservationsIndex(self):
roomDayReservationsIndexBTree = Reservation.getRoomDayReservationsIndexRoot()
for period in self.splitToPeriods():
day = period.startDT.date()
key = (self.room.id, day)
resvs = roomDayReservationsIndexBTree.get(key)
if resvs is None:
resvs = OOSet()
resvs.add(self)
roomDayReservationsIndexBTree[key] = resvs
def set_connection_oids(self, conn_id, oids):
"""Records the OIDs a connection is using and periodically scans.
"""
changed = 0
new_oids = OOSet()
self.lock.acquire()
try:
if oids:
self.conn_oids[conn_id] = OOSet(oids)
else:
if self.conn_oids.has_key(conn_id):
del self.conn_oids[conn_id]
for set in self.conn_oids.values():
new_oids.update(set)
if self.oids != new_oids:
self.oids = new_oids
changed = 1
finally:
self.lock.release()
if changed:
self.storage.scanner.set_oids(new_oids)
def __init__(self, parent=None):
super(ItemModel, self).__init__(parent)
self.labels = self.HEADERS.split(',')
root = parent.conn.root()
if not "item" in root:
root["item"] = PersistentList()
transaction.commit()
if not "hash" in root:
root["hash"] = OOSet()
transaction.commit()
self._data = root["item"]
self._hash = root["hash"]
def test_ExpressionEvaluator(self): ki= self.ki; obj2= self.obj2 ee= ExpressionEvaluator(); ee.id= 'ee' ki._setObject(ee.id,ee); ee= ki._getOb(ee.id) ee.manage_changeProperties(Expression= 'python: (3,4,)') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,3,4)).keys()) # ignore ee.manage_changeProperties(IgnorePredicate= 'python: 3 in value') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys()) # ignore - call it ee.manage_changeProperties(IgnorePredicate= 'python: lambda v: 3 in v') # normalize ee.manage_changeProperties(Expression= 'python: (4,)') ee.manage_changeProperties(Normalizer= 'python: (0,) + value') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys()) # normalize - call it ee.manage_changeProperties(Normalizer= 'python: lambda v: (0,) + v') self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys()) # method ee.manage_changeProperties(Expression= "python: lambda object: object.kw") self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,)).keys()) ## combine # 'union' - already tested # 'useFirst' ki.CombineType= 'useFirst' self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys())
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
if isinstance(self._unindex[docid], set):
self._unindex[docid] = OOSet(self._unindex[docid])
unin = set(self._unindex[docid])
paths_set = {'/'.join(x) for x in paths}
for oldpath in unin - paths_set:
self.unindex_paths(docid, (oldpath, ))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def add(self, tag, userid):
if not isinstance(tag, basestring):
raise TypeError('tag must be a string. Was: %s' % tag)
if not TAG_PATTERN.match(tag):
raise ValueError("'tag' doesn't conform to tag standard: %s" %
_TAG_STRING)
if tag not in self.tags_storage:
self.tags_storage[tag] = OOSet()
if userid not in self.tags_storage[tag]:
self.tags_storage[tag].add(userid)
if tag == 'like':
_notify(self.context)
def test_doesnt_cause_redirect_loop_on_bogus_storage_entries(self):
storage = queryUtility(IRedirectionStorage)
storage._paths['/plone/same'] = '/plone/same'
storage._rpaths['/plone/same'] = OOSet(['/plone/same'])
transaction.commit()
response = requests.get(
self.portal_url + '/same/@@view',
headers={'Accept': 'application/json'},
auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD),
allow_redirects=False,
)
self.assertEqual(404, response.status_code)
def index_object(self, docid, obj, threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
paths = f()
except AttributeError:
return 0
else:
paths = f
else:
try:
paths = obj.getPhysicalPath()
except AttributeError:
return 0
if not paths: return 0
paths = _recursivePathSplit(paths)
if not _isSequenceOfSequences(paths):
paths = [paths]
if docid in self._unindex:
unin = self._unindex[docid]
# Migrate old versions of the index to use OOSet
if isinstance(unin, set):
unin = self._unindex[docid] = OOSet(unin)
for oldpath in list(unin):
if list(oldpath.split('/')) not in paths:
self.unindex_paths(docid, (oldpath, ))
else:
self._unindex[docid] = OOSet()
self._length.change(1)
self.index_paths(docid, paths)
return 1
def add(self, context):
container = self._find_container(context)
assert container
if container.uid not in self:
self[container.uid] = OOBTree()
if context.type_name not in self[container.uid]:
type_uids = self[container.uid][context.type_name] = OOSet()
else:
type_uids = self[container.uid][context.type_name]
if context.uid not in type_uids:
type_uids.add(context.uid)
#counter_context = self._find_counter(container)
#self._update_counter(counter_context.uid, context.type_name, 1)
return context
def indexConf(self, conf):
# Note: conf can be any object which has getEndDate() and getStartDate() methods
self._idxDay._p_changed = True
days = (conf.getEndDate().date() - conf.getStartDate().date()).days
startDate = datetime(conf.getStartDate().year, conf.getStartDate().month, conf.getStartDate().day)
for day in range(days + 1):
key = int(datetimeToUnixTime(startDate + timedelta(day)))
#checking if 2038 problem occurs
if key > BTREE_MAX_INT:
continue
if self._idxDay.has_key(key):
self._idxDay[key].add(conf)
else:
self._idxDay[key] = OOSet([conf])
def _index_object(self, documentId, obj, threshold=None, attr=''):
# get permuted keywords
newKeywords = self._get_permuted_keywords(obj)
oldKeywords = self._unindex.get(documentId, None)
if oldKeywords is None:
# we've got a new document, let's not futz around.
try:
for kw in newKeywords:
self.insertForwardIndexEntry(kw, documentId)
if newKeywords:
self._unindex[documentId] = list(newKeywords)
except TypeError:
return 0
else:
# we have an existing entry for this document, and we need
# to figure out if any of the keywords have actually changed
if type(oldKeywords) is not OOSet:
oldKeywords = OOSet(oldKeywords)
newKeywords = OOSet(newKeywords)
fdiff = difference(oldKeywords, newKeywords)
rdiff = difference(newKeywords, oldKeywords)
if fdiff or rdiff:
# if we've got forward or reverse changes
if newKeywords:
self._unindex[documentId] = list(newKeywords)
else:
del self._unindex[documentId]
if fdiff:
self.unindex_objectKeywords(documentId, fdiff)
if rdiff:
for kw in rdiff:
self.insertForwardIndexEntry(kw, documentId)
return 1
def index_paths(self, docid, paths):
for path in paths:
if isinstance(path, (list, tuple)):
path = '/' + '/'.join(path[1:])
comps = filter(None, path.split('/'))
parent_path = '/' + '/'.join(comps[:-1])
for i in range(len(comps)):
comp = "/".join(comps[1:i + 1])
if comp:
self.insertEntry(comp, docid, i)
# Add terminator
self.insertEntry(None, docid, len(comps) - 1, parent_path, path)
self._unindex.setdefault(docid, OOSet()).insert(path)
def replicate(self):
""" replicate log to targets """
keys = OOSet(self._transactions.keys())
for key in keys:
transaction = self._transactions[key]
try:
transaction.replicate(self)
del self._transactions[key]
except:
from sys import exc_info
import traceback
info = exc_info()
zlog('Replication',
'Could not replicate transaction %s to %s'%(
oid2str(transaction.id), self.id))
break
def load_all_data():
# For first launch (if types don't exist)
dbtype_lists = ["Account", "Stock", "GLOBAL_STOCK_SCREEN_DICT"]
dbtype_singletons = [
"GLOBAL_ATTRIBUTE_SET",
"GLOBAL_TICKER_LIST",
"SCREEN_NAME_AND_TIME_CREATED_TUPLE_LIST",
]
dbtypes = dbtype_lists + dbtype_singletons
commit = False
for dbtype in dbtypes:
try:
assert (hasattr(root, dbtype))
except:
logging.info("Creating DB for: {}".format(dbtype))
if dbtype in dbtype_lists:
setattr(root, dbtype, BTrees.OOBTree.BTree())
commit = True
elif dbtype in dbtype_singletons:
if dbtype.endswith("_SET"):
setattr(root, dbtype, OOSet())
commit = True
elif dbtype.endswith("_LIST"):
setattr(root, dbtype, persistent.list.PersistentList())
commit = True
else:
logging.error(
"DB type {} does not conform. Exiting...".format(
dbtype))
else:
logging.error("DB types have been changed")
sys.exit()
if commit:
config.LAST_PACKED_DB_SIZE = os.path.getsize(
os.path.join('DO_NOT_COPY', 'mydata.fs'))
commit_db()
# now load db data if they exist
config.LAST_PACKED_DB_SIZE = os.path.getsize(
os.path.join('DO_NOT_COPY', 'mydata.fs'))
load_GLOBAL_STOCK_DICT()
load_all_portfolio_objects()
load_GLOBAL_STOCK_SCREEN_DICT()
load_SCREEN_NAME_AND_TIME_CREATED_TUPLE_LIST()
load_filenames_imported_files()
pack_if_necessary()
