def _BTreeContainer__len(self):
l = Length()
ol = len(self._SampleContainer__data)
if ol > 0:
l.change(ol)
self._p_changed = True
return l
def _BTreeContainer__len(self):
l = Length()
ol = len(self.__data)
if ol > 0:
l.change(ol)
self._p_changed = True
return l
class DateBookingList(Persistent):
""" Simple set of booking objects with a count attribute.
"""
def __init__(self):
self._bookings = OOBTree.OOTreeSet()
self._count = Length(0)
def addBooking(self, booking):
self._bookings.insert(booking)
self._count.change(1)
def removeBooking(self, booking):
self._bookings.remove(booking)
self._count.change(-1)
def getCount(self):
return self._count()
def iterbookings(self):
""" Iterator over the bookings
"""
return self._bookings.__iter__()
def getBookingsPerConf(self):
""" Returns a dictionary where the keys are Conference objects
and the values are the number of Vidyo bookings of that conference.
"""
result = {}
for b in self._bookings:
result[b.getConference()] = result.setdefault(
b.getConference(), 0) + 1
return result
def _Folder__len(self):
l = Length()
ol = len(self.__data)
if ol > 0:
l.change(ol)
self._p_changed = True
return l
class DateBookingList(Persistent):
""" Simple set of booking objects with a count attribute.
"""
def __init__(self):
self._bookings = OOBTree.OOTreeSet()
self._count = Length(0)
def addBooking(self, booking):
self._bookings.insert(booking)
self._count.change(1)
def removeBooking(self, booking):
self._bookings.remove(booking)
self._count.change(-1)
def getCount(self):
return self._count()
def iterbookings(self):
""" Iterator over the bookings
"""
return self._bookings.__iter__()
def getBookingsPerConf(self):
""" Returns a dictionary where the keys are Conference objects
and the values are the number of Vidyo bookings of that conference.
"""
result = {}
for b in self._bookings:
result[b.getConference()] = result.setdefault(b.getConference(), 0) + 1
return result
def _QuestionRecord__len(self):
l=Length()
ol = len(self._tree)
if ol>0:
l.change(ol)
self._p_changed=True
return l
def _Registry__len(self):
l = Length()
ol = len(self.__data)
if ol > 0:
l.change(ol)
self._p_changed = True
return l
def _PersitentOOBTree__len(self):
l = Length()
ol = len(self._data)
if ol > 0:
l.change(ol)
self._p_changed = True
return l
def _get_next_number(self, prefix):
last = getattr(self, '_autoname_last_' + prefix, None)
if last is None:
last = Length()
setattr(self, '_autoname_last_' + prefix, last)
number = last.value
last.change(1)
return number
def _get_next_number(self, prefix):
last = getattr(self, '_autoname_last_' + prefix, None)
if last is None:
last = Length()
setattr(self, '_autoname_last_' + prefix, last)
number = last.value
last.change(1)
return number
class MailDataStorage(PersistentItem):
interface.implements(IMailDataStorage)
def __init__(self, **kw):
self.count = Length(0)
super(MailDataStorage, self).__init__(**kw)
def append(self, form, record, request):
mail = getMultiAdapter((form, request), IMailTemplate)
mail.send((self.emailto,), record=record, storage=self)
self.count.change(1)
class MessageService(Persistent, Location):
interface.implements(IMessageService)
def __init__(self, storage):
self.__parent__ = storage
self.index = OIBTree()
self.unread = Length(0)
def __len__(self):
return len(self.index)
def __iter__(self):
return iter(self.index.values())
def __contains__(self, key):
msg = self.__parent__.getMessage(key)
if msg is not None:
return True
else:
return False
def get(self, msgId, default=None):
msg = self.__parent__.getMessage(msgId)
if msg is not None:
if msg.__date__ in self.index:
return msg
return default
def append(self, message):
message.__parent__ = self
if self.__parent__.readStatus(message):
self.unread.change(1)
self.index[message.__date__] = message.__id__
def remove(self, message):
id = message.__date__
if id in self.index:
del self.index[id]
if self.__parent__.readStatus(message) and self.unread() > 0:
self.unread.change(-1)
def create(self, **data):
raise NotImplemented('create')
class MessageQueues(persistent.dict.PersistentDict):
interface.implements(interfaces.IMessageQueues)
def __init__(self, *args, **kwargs):
super(MessageQueues, self).__init__(*args, **kwargs)
for status in interfaces.MESSAGE_STATES:
self[status] = queue.CompositeQueue()
self._messages_sent = Length()
@property
def messages_sent(self):
return self._messages_sent()
def dispatch(self):
try:
lock = zc.lockfile.LockFile(LOCKFILE_NAME)
except zc.lockfile.LockError:
logger.info("Dispatching is locked by another process.")
return (0, 0)
try:
return self._dispatch()
finally:
lock.close()
def _dispatch(self):
sent = 0
failed = 0
for name in 'new', 'retry':
queue = self[name]
while True:
try:
message = queue.pull()
except IndexError:
break
else:
status, message = dispatch(message)
if status == 'sent':
sent += 1
else:
failed += 1
self._messages_sent.change(sent)
return sent, failed
def clear(self, queue_names=('error', 'sent')):
for name in queue_names:
self[name] = self[name].__class__()
class MessageQueues(persistent.dict.PersistentDict):
interface.implements(interfaces.IMessageQueues)
def __init__(self, *args, **kwargs):
super(MessageQueues, self).__init__(*args, **kwargs)
for status in interfaces.MESSAGE_STATES:
self[status] = queue.CompositeQueue()
self._messages_sent = Length()
@property
def messages_sent(self):
return self._messages_sent()
def dispatch(self):
try:
lock = zc.lockfile.LockFile(LOCKFILE_NAME)
except zc.lockfile.LockError:
logger.info("Dispatching is locked by another process.")
return (0, 0)
try:
return self._dispatch()
finally:
lock.close()
def _dispatch(self):
sent = 0
failed = 0
for name in 'new', 'retry':
queue = self[name]
while True:
try:
message = queue.pull()
except IndexError:
break
else:
status, message = dispatch(message)
if status == 'sent':
sent += 1
else:
failed += 1
self._messages_sent.change(sent)
return sent, failed
def clear(self, queue_names=('error', 'sent')):
for name in queue_names:
self[name] = self[name].__class__()
class PollRecord(BTreeContainer):
implements(IPollRecord, IContentContainer)
voteCount = None
firstVote = None
lastVote = None
def __init__(self, *kv, **kw):
super(PollRecord, self).__init__(*kv, **kw)
self._results = OOBTree()
self.voteCount = Length()
def add(self, record):
polling = getUtility(IPolling)
for key, value in record.choices.items():
item = self._results.get(key)
if item is None:
item = QuestionRecord()
notify(ObjectCreatedEvent(item))
self._results[key] = item
for id in value:
self.voteCount.change(1)
polling.voteCount.change(1)
item.voteCount.change(1)
if item.firstVote is None:
item.firstVote = record
item.lastVote = record
answer = item.get(id)
if answer:
answer.change(1)
else:
item[id] = Length(1)
if self.firstVote is None:
self.firstVote = record
self.lastVote = record
self._p_changed = 1
def getResults(self):
res = {}
for question, answers in self._results.items():
res[question] = {}
size = float(answers.voteCount.value)
for answer, votes in answers.items():
res[question][answer] = (votes.value, votes.value/size)
return res, self
class UniqueIdGeneratorTool(UniqueObject, SimpleItem, ActionProviderBase):
"""Generator of unique ids.
"""
__implements__ = (
SimpleItem.__implements__,
IUniqueIdGenerator,
)
id = 'portal_uidgenerator'
alternative_id = 'portal_standard_uidgenerator'
meta_type = 'Unique Id Generator Tool'
# make AnnotatedUniqueId class available through the tool
# not meant to be altered on runtime !!!
_uid_implementation = AnnotatedUniqueId
security = ClassSecurityInfo()
security.declarePrivate('__init__')
def __init__(self):
"""Initialize the generator
"""
# Using the Length implementation of the BTree.Length module as
# counter handles zodb conflicts for us.
self._uid_counter = Length(0)
def reinitialize(self):
"""Reinitialze the uid generator.
Avoids already existing unique ids beeing generated again.
To be called e.g. after having imported content objects.
"""
# XXX to be implemented by searching the max value in the catalog
raise NotImplementedError
security.declarePrivate('__call__')
def __call__(self):
"""See IUniqueIdGenerator.
"""
self._uid_counter.change(+1)
return self._uid_implementation(self._uid_counter())
class DateBookingList(Persistent):
""" Simple set of booking objects with a count attribute.
"""
def __init__(self):
self._bookings = OOBTree.OOTreeSet()
self._count = Length(0)
def addBooking(self, booking):
self._bookings.insert(booking)
self._count.change(1)
def removeBooking(self, booking):
self._bookings.remove(booking)
self._count.change(-1)
def getCount(self):
return self._count()
def iterbookings(self):
""" Iterator over the bookings
"""
return self._bookings.__iter__()
class UniqueIdGeneratorTool(UniqueObject, SimpleItem, ActionProviderBase):
"""Generator of unique ids.
"""
__implements__ = (
IUniqueIdGenerator,
ActionProviderBase.__implements__,
SimpleItem.__implements__,
)
id = 'portal_uidgenerator'
alternative_id = 'portal_standard_uidgenerator'
meta_type = 'Unique Id Generator Tool'
security = ClassSecurityInfo()
security.declarePrivate('__init__')
def __init__(self):
"""Initialize the generator
"""
# Using the Length implementation of the BTree.Length module as
# counter handles zodb conflicts for us.
self._uid_counter = Length(0)
security.declarePrivate('__call__')
def __call__(self):
"""See IUniqueIdGenerator.
"""
self._uid_counter.change(+1)
return self._uid_counter()
security.declarePrivate('convert')
def convert(self, uid):
"""See IUniqueIdGenerator.
"""
return int(uid)
class DateBookingList(Persistent):
""" Simple set of booking objects with a count attribute.
"""
def __init__(self):
self._bookings = OOBTree.OOTreeSet()
self._count = Length(0)
def addBooking(self, booking):
self._bookings.insert(booking)
self._count.change(1)
def removeBooking(self, booking):
self._bookings.remove(booking)
self._count.change(-1)
def getCount(self):
return self._count()
def iterbookings(self):
""" Iterator over the bookings
"""
return self._bookings.__iter__()
class UniqueIdGeneratorTool(UniqueObject, SimpleItem, ActionProviderBase):
"""Generator of unique ids.
"""
__implements__ = (
IUniqueIdGenerator,
ActionProviderBase.__implements__,
SimpleItem.__implements__,
)
id = 'portal_uidgenerator'
alternative_id = 'portal_standard_uidgenerator'
meta_type = 'Unique Id Generator Tool'
security = ClassSecurityInfo()
security.declarePrivate('__init__')
def __init__(self):
"""Initialize the generator
"""
# Using the Length implementation of the BTree.Length module as
# counter handles zodb conflicts for us.
self._uid_counter = Length(0)
security.declarePrivate('__call__')
def __call__(self):
"""See IUniqueIdGenerator.
"""
self._uid_counter.change(+1)
return self._uid_counter()
security.declarePrivate('convert')
def convert(self, uid):
"""See IUniqueIdGenerator.
"""
return int(uid)
class KeywordIndex(Persistent):
"""
Keyword index.
Implements :class:`zope.index.interfaces.IInjection`,
:class:`zope.index.interfaces.IStatistics`,
:class:`zope.index.interfaces.IIndexSearch` and
:class:`zope.index.keyword.interfaces.IKeywordQuerying`.
"""
family = BTrees.family32
# If a word is referenced by at least tree_threshold docids,
# use a TreeSet for that word instead of a Set.
tree_threshold = 64
def __init__(self, family=None):
if family is not None:
self.family = family
self.clear()
def clear(self):
"""Initialize forward and reverse mappings."""
# The forward index maps index keywords to a sequence of docids
self._fwd_index = self.family.OO.BTree()
# The reverse index maps a docid to its keywords
# TODO: Using a vocabulary might be the better choice to store
# keywords since it would allow use to use integers instead of strings
self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
def documentCount(self):
"""Return the number of documents in the index."""
return self._num_docs()
def wordCount(self):
"""Return the number of indexed words"""
return len(self._fwd_index)
def has_doc(self, docid):
return bool(docid in self._rev_index)
def normalize(self, seq):
"""Perform normalization on sequence of keywords.
Return normalized sequence. This method may be
overriden by subclasses.
"""
return seq
def index_doc(self, docid, seq):
if isinstance(seq, six.string_types):
raise TypeError('seq argument must be a list/tuple of strings')
old_kw = self._rev_index.get(docid, None)
if not seq:
if old_kw:
self.unindex_doc(docid)
return
seq = self.normalize(seq)
new_kw = self.family.OO.Set(seq)
if old_kw is None:
self._insert_forward(docid, new_kw)
self._insert_reverse(docid, new_kw)
self._num_docs.change(1)
else:
# determine added and removed keywords
kw_added = self.family.OO.difference(new_kw, old_kw)
kw_removed = self.family.OO.difference(old_kw, new_kw)
# removed keywords are removed from the forward index
for word in kw_removed:
fwd = self._fwd_index[word]
fwd.remove(docid)
if not fwd:
del self._fwd_index[word]
# now update reverse and forward indexes
self._insert_forward(docid, kw_added)
self._insert_reverse(docid, new_kw)
def unindex_doc(self, docid):
idx = self._fwd_index
try:
for word in self._rev_index[docid]:
idx[word].remove(docid)
if not idx[word]:
del idx[word]
except KeyError:
# 'WAAA! Inconsistent'
return
try:
del self._rev_index[docid]
except KeyError: # pragma: no cover
# 'WAAA! Inconsistent'
pass
self._num_docs.change(-1)
def _insert_forward(self, docid, words):
"""insert a sequence of words into the forward index """
idx = self._fwd_index
get_word_idx = idx.get
IF = self.family.IF
Set = IF.Set
TreeSet = IF.TreeSet
for word in words:
word_idx = get_word_idx(word)
if word_idx is None:
idx[word] = word_idx = Set()
word_idx.insert(docid)
if (not isinstance(word_idx, TreeSet)
and len(word_idx) >= self.tree_threshold):
# Convert to a TreeSet.
idx[word] = TreeSet(word_idx)
def _insert_reverse(self, docid, words):
""" add words to forward index """
if words:
self._rev_index[docid] = words
def search(self, query, operator='and'):
"""Execute a search given by 'query'."""
if isinstance(query, six.string_types):
query = [query]
query = self.normalize(query)
sets = []
for word in query:
docids = self._fwd_index.get(word, self.family.IF.Set())
sets.append(docids)
if operator == 'or':
rs = self.family.IF.multiunion(sets)
elif operator == 'and':
# sort smallest to largest set so we intersect the smallest
# number of document identifiers possible
sets.sort(key=len)
rs = None
for set in sets:
rs = self.family.IF.intersection(rs, set)
if not rs:
break
else:
raise TypeError('Keyword index only supports `and` and `or` '
'operators, not `%s`.' % operator)
if rs:
return rs
return self.family.IF.Set()
def apply(self, query):
operator = 'and'
if isinstance(query, dict):
if 'operator' in query:
operator = query['operator']
query = query['query']
return self.search(query, operator=operator)
def optimize(self):
"""Optimize the index. Call this after changing tree_threshold.
This converts internal data structures between
Sets and TreeSets based on tree_threshold.
"""
idx = self._fwd_index
IF = self.family.IF
Set = IF.Set
TreeSet = IF.TreeSet
items = list(self._fwd_index.items())
for word, word_idx in items:
if len(word_idx) >= self.tree_threshold:
if not isinstance(word_idx, TreeSet):
# Convert to a TreeSet.
idx[word] = TreeSet(word_idx)
else:
if isinstance(word_idx, TreeSet):
# Convert to a Set.
idx[word] = Set(word_idx)
class FormSaveDataAdapter(FormActionAdapter):
"""A form action adapter that will save form input data and
return it in csv- or tab-delimited format."""
schema = FormAdapterSchema.copy() + Schema((
LinesField('showFields',
required=0,
searchable=0,
vocabulary='allFieldDisplayList',
widget=PicklistWidget(
label=_(u'label_savefields_text', default=u"Saved Fields"),
description=_(u'help_savefields_text', default=u"""
Pick the fields whose inputs you'd like to include in
the saved data. If empty, all fields will be saved.
"""),
),
),
LinesField('ExtraData',
widget=MultiSelectionWidget(
label=_(u'label_savedataextra_text', default='Extra Data'),
description=_(u'help_savedataextra_text', default=u"""
Pick any extra data you'd like saved with the form input.
"""),
format='checkbox',
),
vocabulary='vocabExtraDataDL',
),
StringField('DownloadFormat',
searchable=0,
required=1,
default='csv',
vocabulary='vocabFormatDL',
widget=SelectionWidget(
label=_(u'label_downloadformat_text', default=u'Download Format'),
),
),
BooleanField("UseColumnNames",
required=False,
searchable=False,
widget=BooleanWidget(
label=_(u'label_usecolumnnames_text', default=u"Include Column Names"),
description=_(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"),
),
),
ExLinesField('SavedFormInput',
edit_accessor='getSavedFormInputForEdit',
mutator='setSavedFormInput',
searchable=0,
required=0,
primary=1,
schemata="saved data",
read_permission=DOWNLOAD_SAVED_PERMISSION,
widget=TextAreaWidget(
label=_(u'label_savedatainput_text', default=u"Saved Form Input"),
description=_(u'help_savedatainput_text'),
),
),
))
schema.moveField('execCondition', pos='bottom')
meta_type = 'FormSaveDataAdapter'
portal_type = 'FormSaveDataAdapter'
archetype_name = 'Save Data Adapter'
immediate_view = 'fg_savedata_view_p3'
default_view = 'fg_savedata_view_p3'
suppl_views = ('fg_savedata_tabview_p3', 'fg_savedata_recview_p3',)
security = ClassSecurityInfo()
def _migrateStorage(self):
# we're going to use an LOBTree for storage. we need to
# consider the possibility that self is from an
# older version that uses the native Archetypes storage
# or the former IOBTree (<= 1.6.0b2 )
# in the SavedFormInput field.
updated = base_hasattr(self, '_inputStorage') and \
base_hasattr(self, '_inputItems') and \
base_hasattr(self, '_length')
if not updated:
try:
saved_input = self.getSavedFormInput()
except AttributeError:
saved_input = []
self._inputStorage = SavedDataBTree()
i = 0
self._inputItems = 0
self._length = Length()
if len(saved_input):
for row in saved_input:
self._inputStorage[i] = row
i += 1
self.SavedFormInput = []
self._inputItems = i
self._length.set(i)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInput')
def getSavedFormInput(self):
""" returns saved input as an iterable;
each row is a sequence of fields.
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.values()
else:
return self.SavedFormInput
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputItems')
def getSavedFormInputItems(self):
""" returns saved input as an iterable;
each row is an (id, sequence of fields) tuple
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.items()
else:
return enumerate(self.SavedFormInput)
security.declareProtected(ModifyPortalContent, 'getSavedFormInputForEdit')
def getSavedFormInputForEdit(self, **kwargs):
""" returns saved as CSV text """
delimiter = self.csvDelimiter()
sbuf = StringIO()
writer = csv.writer(sbuf, delimiter=delimiter)
for row in self.getSavedFormInput():
writer.writerow(row)
res = sbuf.getvalue()
sbuf.close()
return res
security.declareProtected(ModifyPortalContent, 'setSavedFormInput')
def setSavedFormInput(self, value, **kwargs):
""" expects value as csv text string, stores as list of lists """
self._migrateStorage()
self._inputStorage.clear()
i = 0
self._inputItems = 0
self._length.set(0)
if len(value):
delimiter = self.csvDelimiter()
sbuf = StringIO(value)
reader = csv.reader(sbuf, delimiter=delimiter)
for row in reader:
if row:
self._inputStorage[i] = row
i += 1
self._inputItems = i
self._length.set(i)
sbuf.close()
# logger.debug("setSavedFormInput: %s items" % self._inputItems)
def _clearSavedFormInput(self):
# convenience method to clear input buffer
self._migrateStorage()
self._inputStorage.clear()
self._inputItems = 0
self._length.set(0)
security.declareProtected(ModifyPortalContent, 'clearSavedFormInput')
def clearSavedFormInput(self, **kwargs):
""" clear input buffer TTW """
plone.protect.CheckAuthenticator(self.REQUEST)
plone.protect.PostOnly(self.REQUEST)
self._clearSavedFormInput()
self.REQUEST.response.redirect(self.absolute_url())
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputById')
def getSavedFormInputById(self, id):
""" Return the data stored for record with 'id' """
lst = [field.replace('\r', '').replace('\n', r'\n') for field in self._inputStorage[id]]
return lst
security.declareProtected(ModifyPortalContent, 'manage_saveData')
def manage_saveData(self, id, data):
""" Save the data for record with 'id' """
plone.protect.CheckAuthenticator(self.REQUEST)
plone.protect.PostOnly(self.REQUEST)
self._migrateStorage()
lst = list()
for i in range(0, len(self.getColumnNames())):
lst.append(getattr(data, 'item-%d' % i, '').replace(r'\n', '\n'))
self._inputStorage[id] = lst
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
security.declareProtected(ModifyPortalContent, 'manage_deleteData')
def manage_deleteData(self, id):
""" Delete the data for record with 'id' """
self._migrateStorage()
del self._inputStorage[id]
self._inputItems -= 1
self._length.change(-1)
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
def _addDataRow(self, value):
self._migrateStorage()
if isinstance(self._inputStorage, IOBTree):
# 32-bit IOBTree; use a key which is more likely to conflict
# but which won't overflow the key's bits
id = self._inputItems
self._inputItems += 1
else:
# 64-bit LOBTree
id = int(time.time() * 1000)
while id in self._inputStorage: # avoid collisions during testing
id += 1
self._inputStorage[id] = value
self._length.change(1)
security.declareProtected(ModifyPortalContent, 'addDataRow')
def addDataRow(self, value):
# """ a wrapper for the _addDataRow method """
self._addDataRow(value)
security.declarePrivate('onSuccess')
def onSuccess(self, fields, REQUEST=None, loopstop=False):
# """
# saves data.
# """
if LP_SAVE_TO_CANONICAL and not loopstop:
# LinguaPlone functionality:
# check to see if we're in a translated
# form folder, but not the canonical version.
parent = self.aq_parent
if safe_hasattr(parent, 'isTranslation') and \
parent.isTranslation() and not parent.isCanonical():
# look in the canonical version to see if there is
# a matching (by id) save-data adapter.
# If so, call its onSuccess method
cf = parent.getCanonical()
target = cf.get(self.getId())
if target is not None and target.meta_type == 'FormSaveDataAdapter':
target.onSuccess(fields, REQUEST, loopstop=True)
return
from ZPublisher.HTTPRequest import FileUpload
data = []
for f in fields:
showFields = getattr(self, 'showFields', [])
if showFields and f.id not in showFields:
continue
if f.isFileField():
file = REQUEST.form.get('%s_file' % f.fgField.getName())
if isinstance(file, FileUpload) and file.filename != '':
file.seek(0)
fdata = file.read()
filename = file.filename
mimetype, enc = guess_content_type(filename, fdata, None)
if mimetype.find('text/') >= 0:
# convert to native eols
fdata = fdata.replace('\x0d\x0a', '\n').replace('\x0a', '\n').replace('\x0d', '\n')
data.append('%s:%s:%s:%s' % (filename, mimetype, enc, fdata))
else:
data.append('%s:%s:%s:Binary upload discarded' % (filename, mimetype, enc))
else:
data.append('NO UPLOAD')
elif not f.isLabel():
val = REQUEST.form.get(f.fgField.getName(), '')
if not type(val) in StringTypes:
# Zope has marshalled the field into
# something other than a string
val = str(val)
data.append(val)
if self.ExtraData:
for f in self.ExtraData:
if f == 'dt':
data.append(str(DateTime()))
else:
data.append(getattr(REQUEST, f, ''))
self._addDataRow(data)
security.declareProtected(ModifyPortalContent, 'allFieldDisplayList')
def allFieldDisplayList(self):
# """ returns a DisplayList of all fields """
return self.fgFieldsDisplayList()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames')
def getColumnNames(self, excludeServerSide=True):
# """Returns a list of column names"""
showFields = getattr(self, 'showFields', [])
names = [field.getName() for field
in self.fgFields(displayOnly=True, excludeServerSide=excludeServerSide)
if not showFields or field.getName() in showFields]
for f in self.ExtraData:
names.append(f)
return names
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles')
def getColumnTitles(self, excludeServerSide=True):
# """Returns a list of column titles"""
names = [field.widget.label for field
in self.fgFields(displayOnly=True, excludeServerSide=excludeServerSide)]
for f in self.ExtraData:
names.append(self.vocabExtraDataDL().getValue(f, ''))
return names
def _cleanInputForTSV(self, value):
# make data safe to store in tab-delimited format
return str(value).replace('\x0d\x0a', r'\n').replace('\x0a', r'\n').replace('\x0d', r'\n').replace('\t', r'\t')
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_tsv')
def download_tsv(self, REQUEST=None, RESPONSE=None):
# """Download the saved data
# """
filename = self.id
if filename.find('.') < 0:
filename = '%s.tsv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/tab-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
res = "%s\n" % '\t'.join(self.getColumnNames(excludeServerSide=False))
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
for row in self.getSavedFormInput():
res = '%s%s\n' % (res, '\t'.join([self._cleanInputForTSV(col) for col in row]))
return res
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_csv')
def download_csv(self, REQUEST=None, RESPONSE=None):
# """Download the saved data
# """
filename = self.id
if filename.find('.') < 0:
filename = '%s.csv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/comma-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
delimiter = self.csvDelimiter()
res = "%s\n" % delimiter.join(self.getColumnNames(excludeServerSide=False))
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
return '%s%s' % (res, self.getSavedFormInputForEdit())
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_xls')
def download_xls(self, REQUEST=None, RESPONSE=None):
# """Download the saved data
# """
filename = self.id
if filename.find('.') < 0:
filename = '%s.xls' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'application/vnd.ms-excel')
xldoc = xlwt.Workbook(encoding=self.getCharset())
sheet = xldoc.add_sheet(self.Title())
row_num = 0
if getattr(self, 'UseColumnNames', False):
col_names = self.getColumnNames(excludeServerSide=False)
for idx, label in enumerate(col_names):
sheet.write(0, idx, label.encode(self.getCharset()))
row_num += 1
for row in self.getSavedFormInput():
for col_num, col in enumerate(row):
if type(col) is unicode:
col = col.encode(self.getCharset())
if urlparse(col).scheme in ('http', 'https'):
col = xlwt.Formula('HYPERLINK("%(url)s")' % dict(url=col))
else:
for format in (int, float):
try:
col = format(col)
break
except ValueError:
pass
sheet.write(row_num, col_num, col)
row_num += 1
string_buffer = StringIO()
xldoc.save(string_buffer)
return string_buffer.getvalue()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download')
def download(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return self.download_tsv(REQUEST, RESPONSE)
if format == 'xls':
assert has_xls, 'xls download not available'
return self.download_xls(REQUEST, RESPONSE)
else:
assert format == 'csv', 'Unknown download format'
return self.download_csv(REQUEST, RESPONSE)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'rowAsColDict')
def rowAsColDict(self, row, cols):
# """ Where row is a data sequence and cols is a column name sequence,
# returns a dict of colname:column. This is a convenience method
# used in the record view.
# """
colcount = len(cols)
rdict = {}
for i in range(0, len(row)):
if i < colcount:
rdict[cols[i]] = row[i]
else:
rdict['column-%s' % i] = row[i]
return rdict
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'inputAsDictionaries')
def inputAsDictionaries(self):
# """returns saved data as an iterable of dictionaries
# """
cols = self.getColumnNames()
for row in self.getSavedFormInput():
yield self.rowAsColDict(row, cols)
# alias for old mis-naming
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'InputAsDictionaries')
InputAsDictionaries = inputAsDictionaries
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'formatMIME')
def formatMIME(self):
# """MIME format selected for download
# """
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return 'text/tab-separated-values'
if format == 'xls':
return 'application/vnd.ms-excel'
else:
assert format == 'csv', 'Unknown download format'
return 'text/comma-separated-values'
security.declarePrivate('csvDelimiter')
def csvDelimiter(self):
# """Delimiter character for CSV downloads
# """
fgt = getToolByName(self, 'formgen_tool')
return fgt.getCSVDelimiter()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'itemsSaved')
def itemsSaved(self):
# """Download the saved data
# """
if base_hasattr(self, '_length'):
return self._length()
elif base_hasattr(self, '_inputItems'):
return self._inputItems
else:
return len(self.SavedFormInput)
def vocabExtraDataDL(self):
# """ returns vocabulary for extra data """
return DisplayList((
('dt',
self.translate(msgid='vocabulary_postingdt_text',
domain='ploneformgen',
default='Posting Date/Time')
),
('HTTP_X_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR',),
('REMOTE_ADDR', 'REMOTE_ADDR',),
('HTTP_USER_AGENT', 'HTTP_USER_AGENT',),
))
def vocabFormatDL(self):
# """ returns vocabulary for format """
formats = [
('tsv',
self.translate(msgid='vocabulary_tsv_text',
domain='ploneformgen',
default='Tab-Separated Values')
),
('csv',
self.translate(msgid='vocabulary_csv_text',
domain='ploneformgen',
default='Comma-Separated Values')
),
]
if has_xls:
formats.append(
('xls',
self.translate(msgid='vocabulary_xls_doc',
domain='ploneformgen',
default='Excel document')
),
)
return DisplayList(formats)
class OkapiIndex(BaseIndex):
"""
Full text index with relevance ranking, using an Okapi BM25 rank.
"""
# BM25 free parameters.
K1 = 1.2
B = 0.75
assert K1 >= 0.0
assert 0.0 <= B <= 1.0
def __init__(self, lexicon, family=None):
BaseIndex.__init__(self, lexicon, family=family)
# ._wordinfo for Okapi is
# wid -> {docid -> frequency}; t -> D -> f(D, t)
# ._docweight for Okapi is
# docid -> # of words in the doc
# This is just len(self._docwords[docid]), but _docwords is stored
# in compressed form, so uncompressing it just to count the list
# length would be ridiculously expensive.
# sum(self._docweight.values()), the total # of words in all docs
# This is a long for "better safe than sorry" reasons. It isn't
# used often enough that speed should matter.
self._totaldoclen = Length(0)
def index_doc(self, docid, text):
count = BaseIndex.index_doc(self, docid, text)
self._change_doc_len(count)
return count
def _reindex_doc(self, docid, text):
self._change_doc_len(-self._docweight[docid])
return BaseIndex._reindex_doc(self, docid, text)
def unindex_doc(self, docid):
if docid not in self._docwords:
return
self._change_doc_len(-self._docweight[docid])
BaseIndex.unindex_doc(self, docid)
def _change_doc_len(self, delta):
# Change total doc length used for scoring
delta = int(delta)
try:
self._totaldoclen.change(delta)
except AttributeError:
# Opportunistically upgrade _totaldoclen attribute to Length object
self._totaldoclen = Length(int(self._totaldoclen + delta))
# The workhorse. Return a list of (IFBucket, weight) pairs, one pair
# for each wid t in wids. The IFBucket, times the weight, maps D to
# TF(D,t) * IDF(t) for every docid D containing t.
# As currently written, the weights are always 1, and the IFBucket maps
# D to TF(D,t)*IDF(t) directly, where the product is computed as a float.
# NOTE: This may be overridden below, by a function that computes the
# same thing but with the inner scoring loop in C.
def _python_search_wids(self, wids):
if not wids:
return []
N = float(self.documentCount()) # total # of docs
try:
doclen = self._totaldoclen()
except TypeError:
# _totaldoclen has not yet been upgraded
doclen = self._totaldoclen
meandoclen = doclen / N
K1 = self.K1
B = self.B
K1_plus1 = K1 + 1.0
B_from1 = 1.0 - B
# f(D, t) * (k1 + 1)
# TF(D, t) = -------------------------------------------
# f(D, t) + k1 * ((1-b) + b*len(D)/E(len(D)))
L = []
docid2len = self._docweight
for t in wids:
d2f = self._wordinfo[t] # map {docid -> f(docid, t)}
idf = inverse_doc_frequency(len(d2f), N) # an unscaled float
result = self.family.IF.Bucket()
for docid, f in d2f.items():
lenweight = B_from1 + B * docid2len[docid] / meandoclen
tf = f * K1_plus1 / (f + K1 * lenweight)
result[docid] = tf * idf
L.append((result, 1))
return L
# Note about the above: the result is tf * idf. tf is
# small -- it can't be larger than k1+1 = 2.2. idf is
# formally unbounded, but is less than 14 for a term that
# appears in only 1 of a million documents. So the
# product is probably less than 32, or 5 bits before the
# radix point. If we did the scaled-int business on both
# of them, we'd be up to 25 bits. Add 64 of those and
# we'd be in overflow territory. That's pretty unlikely,
# so we *could* just store scaled_int(tf) in
# result[docid], and use scaled_int(idf) as an invariant
# weight across the whole result. But besides skating
# near the edge, it's not a speed cure, since the
# computation of tf would still be done at Python speed,
# and it's a lot more work than just multiplying by idf.
# The same function as _search_wids above, but with the inner scoring
# loop written in C (module okascore, function score()).
# Cautions: okascore hardcodes the values of K, B1, and the scaled_int
# function.
def _c_search_wids(self, wids):
if not wids:
return []
N = float(self.documentCount()) # total # of docs
try:
doclen = self._totaldoclen()
except TypeError:
# _totaldoclen has not yet been upgraded
doclen = self._totaldoclen
meandoclen = doclen / N
#K1 = self.K1
#B = self.B
#K1_plus1 = K1 + 1.0
#B_from1 = 1.0 - B
# f(D, t) * (k1 + 1)
# TF(D, t) = -------------------------------------------
# f(D, t) + k1 * ((1-b) + b*len(D)/E(len(D)))
L = []
docid2len = self._docweight
for t in wids:
d2f = self._wordinfo[t] # map {docid -> f(docid, t)}
idf = inverse_doc_frequency(len(d2f), N) # an unscaled float
result = self.family.IF.Bucket()
items = d2f.items() if PY2 else list(d2f.items())
score(result, items, docid2len, idf, meandoclen)
L.append((result, 1))
return L
_search_wids = _python_search_wids if score is None else _c_search_wids
def query_weight(self, terms):
# Get the wids.
wids = []
for term in terms:
termwids = self._lexicon.termToWordIds(term)
wids.extend(termwids)
# The max score for term t is the maximum value of
# TF(D, t) * IDF(Q, t)
# We can compute IDF directly, and as noted in the comments below
# TF(D, t) is bounded above by 1+K1.
N = float(len(self._docweight))
tfmax = 1.0 + self.K1
sum = 0
for t in self._remove_oov_wids(wids):
idf = inverse_doc_frequency(len(self._wordinfo[t]), N)
sum += idf * tfmax
return sum
def _get_frequencies(self, wids):
d = {}
dget = d.get
for wid in wids:
d[wid] = dget(wid, 0) + 1
return d, len(wids)
class UnIndex(SimpleItem):
"""Simple forward and reverse index.
"""
implements(ILimitedResultIndex, IUniqueValueIndex, ISortIndex)
def __init__(self,
id,
ignore_ex=None,
call_methods=None,
extra=None,
caller=None):
"""Create an unindex
UnIndexes are indexes that contain two index components, the
forward index (like plain index objects) and an inverted
index. The inverted index is so that objects can be unindexed
even when the old value of the object is not known.
e.g.
self._index = {datum:[documentId1, documentId2]}
self._unindex = {documentId:datum}
The arguments are:
'id' -- the name of the item attribute to index. This is
either an attribute name or a record key.
'ignore_ex' -- should be set to true if you want the index
to ignore exceptions raised while indexing instead of
propagating them.
'call_methods' -- should be set to true if you want the index
to call the attribute 'id' (note: 'id' should be callable!)
You will also need to pass in an object in the index and
uninded methods for this to work.
'extra' -- a mapping object that keeps additional
index-related parameters - subitem 'indexed_attrs'
can be string with comma separated attribute names or
a list
'caller' -- reference to the calling object (usually
a (Z)Catalog instance
"""
def _get(o, k, default):
""" return a value for a given key of a dict/record 'o' """
if isinstance(o, dict):
return o.get(k, default)
else:
return getattr(o, k, default)
self.id = id
self.ignore_ex = ignore_ex # currently unimplemented
self.call_methods = call_methods
self.operators = ('or', 'and')
self.useOperator = 'or'
# allow index to index multiple attributes
ia = _get(extra, 'indexed_attrs', id)
if isinstance(ia, str):
self.indexed_attrs = ia.split(',')
else:
self.indexed_attrs = list(ia)
self.indexed_attrs = [
attr.strip() for attr in self.indexed_attrs if attr
]
if not self.indexed_attrs:
self.indexed_attrs = [id]
self.clear()
def __len__(self):
return self._length()
def getId(self):
return self.id
def clear(self):
self._length = Length()
self._index = OOBTree()
self._unindex = IOBTree()
def __nonzero__(self):
return not not self._unindex
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
for item in self._index.items():
if isinstance(item, int):
entry = 1 # "set" length is 1
else:
key, value = item
entry = len(value)
histogram[entry] = histogram.get(entry, 0) + 1
return histogram
def referencedObjects(self):
"""Generate a list of IDs for which we have referenced objects."""
return self._unindex.keys()
def getEntryForObject(self, documentId, default=_marker):
"""Takes a document ID and returns all the information we have
on that specific object.
"""
if default is _marker:
return self._unindex.get(documentId)
return self._unindex.get(documentId, default)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
indexRow = self._index.get(entry, _marker)
if indexRow is not _marker:
try:
indexRow.remove(documentId)
if not indexRow:
del self._index[entry]
self._length.change(-1)
except ConflictError:
raise
except AttributeError:
# index row is an int
try:
del self._index[entry]
except KeyError:
# XXX swallow KeyError because it was probably
# removed and then _length AttributeError raised
pass
if isinstance(self.__len__, Length):
self._length = self.__len__
del self.__len__
self._length.change(-1)
except Exception:
LOG.error(
'%s: unindex_object could not remove '
'documentId %s from index %s. This '
'should not happen.' %
(self.__class__.__name__, str(documentId), str(self.id)),
exc_info=sys.exc_info())
else:
LOG.error('%s: unindex_object tried to retrieve set %s '
'from index %s but couldn\'t. This '
'should not happen.' %
(self.__class__.__name__, repr(entry), str(self.id)))
def insertForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and put it in the correct place
in the forward index.
This will also deal with creating the entire row if necessary.
"""
indexRow = self._index.get(entry, _marker)
# Make sure there's actually a row there already. If not, create
# a set and stuff it in first.
if indexRow is _marker:
# We always use a set to avoid getting conflict errors on
# multiple threads adding a new row at the same time
self._index[entry] = IITreeSet((documentId, ))
self._length.change(1)
else:
try:
indexRow.insert(documentId)
except AttributeError:
# Inline migration: index row with one element was an int at
# first (before Zope 2.13).
indexRow = IITreeSet((indexRow, documentId))
self._index[entry] = indexRow
def index_object(self, documentId, obj, threshold=None):
""" wrapper to handle indexing of multiple attributes """
fields = self.getIndexSourceNames()
res = 0
for attr in fields:
res += self._index_object(documentId, obj, threshold, attr)
return res > 0
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId)
if datum is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.error('Should not happen: oldDatum was there, '
'now its not, for document: %s' % documentId)
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
self._unindex[documentId] = datum
returnStatus = 1
return returnStatus
def _get_object_datum(self, obj, attr):
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
try:
datum = getattr(obj, attr)
if safe_callable(datum):
datum = datum()
except (AttributeError, TypeError):
datum = _marker
return datum
def numObjects(self):
"""Return the number of indexed objects."""
return len(self._unindex)
def indexSize(self):
"""Return the size of the index in terms of distinct values."""
return len(self)
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.debug('Attempt to unindex nonexistent document'
' with id %s' % documentId,
exc_info=True)
def _apply_not(self, not_parm, resultset=None):
index = self._index
setlist = []
for k in not_parm:
s = index.get(k, None)
if s is None:
continue
elif isinstance(s, int):
s = IISet((s, ))
setlist.append(s)
return multiunion(setlist)
def _convert(self, value, default=None):
return value
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the request arg.
The request argument should be a mapping object.
If the request does not have a key which matches the "id" of
the index instance, then None is returned.
If the request *does* have a key which matches the "id" of
the index instance, one of a few things can happen:
- if the value is a blank string, None is returned (in
order to support requests from web forms where
you can't tell a blank string from empty).
- if the value is a nonblank string, turn the value into
a single-element sequence, and proceed.
- if the value is a sequence, return a union search.
- If the value is a dict and contains a key of the form
'<index>_operator' this overrides the default method
('or') to combine search results. Valid values are "or"
and "and".
If None is not returned as a result of the abovementioned
constraints, two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
FAQ answer: to search a Field Index for documents that
have a blank string as their value, wrap the request value
up in a tuple ala: request = {'id':('',)}
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
index = self._index
r = None
opr = None
# not / exclude parameter
not_parm = record.get('not', None)
if not record.keys and not_parm:
# convert into indexed format
not_parm = map(self._convert, not_parm)
# we have only a 'not' query
record.keys = [k for k in index.keys() if k not in not_parm]
else:
# convert query arguments into indexed format
record.keys = map(self._convert, record.keys)
# experimental code for specifing the operator
operator = record.get('operator', self.useOperator)
if not operator in self.operators:
raise RuntimeError("operator not valid: %s" % escape(operator))
# Range parameter
range_parm = record.get('range', None)
if range_parm:
opr = "range"
opr_args = []
if range_parm.find("min") > -1:
opr_args.append("min")
if range_parm.find("max") > -1:
opr_args.append("max")
if record.get('usage', None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args = opr[0], opr[1:]
if opr == "range": # range search
if 'min' in opr_args:
lo = min(record.keys)
else:
lo = None
if 'max' in opr_args:
hi = max(record.keys)
else:
hi = None
if hi:
setlist = index.values(lo, hi)
else:
setlist = index.values(lo)
# If we only use one key, intersect and return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result, ))
if not_parm:
exclude = self._apply_not(not_parm, resultset)
result = difference(result, exclude)
return result, (self.id, )
if operator == 'or':
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s, ))
tmp.append(s)
r = multiunion(tmp)
else:
# For intersection, sort with smallest data set first
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s, ))
tmp.append(s)
if len(tmp) > 2:
setlist = sorted(tmp, key=len)
else:
setlist = tmp
r = resultset
for s in setlist:
# the result is bound by the resultset
r = intersection(r, s)
else: # not a range search
# Filter duplicates
setlist = []
for k in record.keys:
s = index.get(k, None)
# If None, try to bail early
if s is None:
if operator == 'or':
# If union, we can't possibly get a bigger result
continue
# If intersection, we can't possibly get a smaller result
return IISet(), (self.id, )
elif isinstance(s, int):
s = IISet((s, ))
setlist.append(s)
# If we only use one key return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result, ))
if not_parm:
exclude = self._apply_not(not_parm, resultset)
result = difference(result, exclude)
return result, (self.id, )
if operator == 'or':
# If we already get a small result set passed in, intersecting
# the various indexes with it and doing the union later is
# faster than creating a multiunion first.
if resultset is not None and len(resultset) < 200:
smalllist = []
for s in setlist:
smalllist.append(intersection(resultset, s))
r = multiunion(smalllist)
else:
r = multiunion(setlist)
else:
# For intersection, sort with smallest data set first
if len(setlist) > 2:
setlist = sorted(setlist, key=len)
r = resultset
for s in setlist:
r = intersection(r, s)
if isinstance(r, int):
r = IISet((r, ))
if r is None:
return IISet(), (self.id, )
if not_parm:
exclude = self._apply_not(not_parm, resultset)
r = difference(r, exclude)
return r, (self.id, )
def hasUniqueValuesFor(self, name):
"""has unique values for column name"""
if name == self.id:
return 1
return 0
def getIndexSourceNames(self):
""" return sequence of indexed attributes """
# BBB: older indexes didn't have 'indexed_attrs'
return getattr(self, 'indexed_attrs', [self.id])
def getIndexQueryNames(self):
"""Indicate that this index applies to queries for the index's name."""
return (self.id, )
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
raise StopIteration
if not withLengths:
for key in self._index.keys():
yield key
else:
for key, value in self._index.items():
if isinstance(value, int):
yield (key, 1)
else:
yield (key, len(value))
def keyForDocument(self, id):
# This method is superseded by documentToKeyMap
return self._unindex[id]
def documentToKeyMap(self):
return self._unindex
def items(self):
items = []
for k, v in self._index.items():
if isinstance(v, int):
v = IISet((v, ))
items.append((k, v))
return items
class UUIDIndex(UnIndex):
"""Index for uuid fields with an unique value per key.
The internal structure is:
self._index = {datum:documentId]}
self._unindex = {documentId:datum}
For each datum only one documentId can exist.
"""
meta_type = "UUIDIndex"
manage_options = (
{
'label': 'Settings',
'action': 'manage_main'
},
{
'label': 'Browse',
'action': 'manage_browse'
},
)
query_options = ["query", "range"]
manage = manage_main = DTMLFile('dtml/manageUUIDIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
def clear(self):
self._length = Length()
self._index = OIBTree()
self._unindex = IOBTree()
def numObjects(self):
"""Return the number of indexed objects. Since we have a 1:1 mapping
from documents to values, we can reuse the stored length.
"""
return self.indexSize()
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
return []
if not withLengths:
return tuple(self._index.keys())
# We know the length for each value is one
return [(k, 1) for k in self._index.keys()]
def insertForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and put it in the correct place
in the forward index.
"""
if entry is None:
return
old_docid = self._index.get(entry, _marker)
if old_docid is _marker:
self._index[entry] = documentId
self._length.change(1)
elif old_docid != documentId:
logger.error("A different document with value '%s' already "
"exists in the index.'" % entry)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
old_docid = self._index.get(entry, _marker)
if old_docid is not _marker:
del self._index[entry]
self._length.change(-1)
def _get_object_datum(self, obj, attr):
# for a uuid it never makes sense to acquire a parent value via
# Acquisition
has_attr = getattr(aq_base(obj), attr, _marker)
if has_attr is _marker:
return _marker
return super(UUIDIndex, self)._get_object_datum(obj, attr)
class FieldIndex(SortingIndexMixin, persistent.Persistent):
"""
A field index.
Implements :class:`zope.index.interfaces.IInjection`,
:class:`zope.index.interfaces.IStatistics` and
:class:`zope.index.interfaces.IIndexSearch`.
"""
family = BTrees.family32
def __init__(self, family=None):
if family is not None:
self.family = family
self.clear()
def clear(self):
"""Initialize forward and reverse mappings."""
# The forward index maps indexed values to a sequence of docids
self._fwd_index = self.family.OO.BTree()
# The reverse index maps a docid to its index value
self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
def documentCount(self):
"""See interface IStatistics"""
return self._num_docs()
def wordCount(self):
"""See interface IStatistics"""
return len(self._fwd_index)
def index_doc(self, docid, value):
"""See interface IInjection"""
rev_index = self._rev_index
if docid in rev_index:
if docid in self._fwd_index.get(value, ()):
# no need to index the doc, its already up to date
return
self.unindex_doc(docid)
# Insert into forward index.
set = self._fwd_index.get(value)
if set is None:
set = self.family.IF.TreeSet()
self._fwd_index[value] = set
set.insert(docid)
# increment doc count
self._num_docs.change(1)
# Insert into reverse index.
rev_index[docid] = value
def unindex_doc(self, docid):
"""See interface IInjection"""
rev_index = self._rev_index
value = rev_index.get(docid, _MARKER)
if value is _MARKER:
return # not in index
del rev_index[docid]
try:
set = self._fwd_index[value]
set.remove(docid)
except KeyError: #pragma NO COVERAGE
# This is fishy, but we don't want to raise an error.
# We should probably log something.
# but keep it from throwing a dirty exception
set = 1
if not set:
del self._fwd_index[value]
self._num_docs.change(-1)
def apply(self, query):
if len(query) != 2 or not isinstance(query, tuple):
raise TypeError("two-length tuple expected", query)
return self.family.IF.multiunion(
self._fwd_index.values(*query))
class CachingCatalog(Catalog):
implements(ICatalog)
os = os # for unit tests
generation = None # b/c
def __init__(self):
super(CachingCatalog, self).__init__()
self.generation = Length(0)
def clear(self):
self.invalidate()
super(CachingCatalog, self).clear()
def index_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).index_doc(*arg, **kw)
def unindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).unindex_doc(*arg, **kw)
def reindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).reindex_doc(*arg, **kw)
def __setitem__(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).__setitem__(*arg, **kw)
def search(self, *arg, **kw):
use_cache = True
if 'use_cache' in kw:
use_cache = kw.pop('use_cache')
if 'NO_CATALOG_CACHE' in self.os.environ:
use_cache = False
if 'tags' in kw:
# The tags index changes without invalidating the catalog,
# so don't cache any query involving the tags index.
use_cache = False
if not use_cache:
return self._search(*arg, **kw)
cache = queryUtility(ICatalogSearchCache)
if cache is None:
return self._search(*arg, **kw)
key = cPickle.dumps((arg, kw))
generation = self.generation
if generation is None:
generation = Length(0)
genval = generation.value
if (genval == 0) or (genval > cache.generation):
# an update in another process requires that the local cache be
# invalidated
cache.clear()
cache.generation = genval
if cache.get(key) is None:
num, docids = self._search(*arg, **kw)
# We don't cache large result sets because the time it takes to
# unroll the result set turns out to be far more time than it
# takes to run the search. In a particular instance using OSI's
# catalog a search that took 0.015s but returned nearly 35,295
# results took over 50s to unroll the result set for caching,
# significantly slowing search performance.
if num > LARGE_RESULT_SET:
return num, docids
# we need to unroll here; a btree-based structure may have
# a reference to its connection
docids = list(docids)
cache[key] = (num, docids)
return cache.get(key)
def _search(self, *arg, **kw):
start = time.time()
res = super(CachingCatalog, self).search(*arg, **kw)
duration = time.time() - start
notify(CatalogQueryEvent(self, kw, duration, res))
return res
def invalidate(self):
# Increment the generation; this tells *another process* that
# its catalog cache needs to be cleared
generation = self.generation
if generation is None:
generation = self.generation = Length(0)
if generation.value >= sys.maxint:
# don't keep growing the generation integer; wrap at sys.maxint
self.generation.set(0)
else:
self.generation.change(1)
# Clear the cache for *this process*
cache = queryUtility(ICatalogSearchCache)
if cache is not None:
cache.clear()
cache.generation = self.generation.value
class UnIndex(SimpleItem):
"""Simple forward and reverse index.
"""
implements(ILimitedResultIndex, IUniqueValueIndex, ISortIndex)
_counter = None
def __init__(self, id, ignore_ex=None, call_methods=None,
extra=None, caller=None):
"""Create an unindex
UnIndexes are indexes that contain two index components, the
forward index (like plain index objects) and an inverted
index. The inverted index is so that objects can be unindexed
even when the old value of the object is not known.
e.g.
self._index = {datum:[documentId1, documentId2]}
self._unindex = {documentId:datum}
The arguments are:
'id' -- the name of the item attribute to index. This is
either an attribute name or a record key.
'ignore_ex' -- should be set to true if you want the index
to ignore exceptions raised while indexing instead of
propagating them.
'call_methods' -- should be set to true if you want the index
to call the attribute 'id' (note: 'id' should be callable!)
You will also need to pass in an object in the index and
uninded methods for this to work.
'extra' -- a mapping object that keeps additional
index-related parameters - subitem 'indexed_attrs'
can be string with comma separated attribute names or
a list
'caller' -- reference to the calling object (usually
a (Z)Catalog instance
"""
def _get(o, k, default):
""" return a value for a given key of a dict/record 'o' """
if isinstance(o, dict):
return o.get(k, default)
else:
return getattr(o, k, default)
self.id = id
self.ignore_ex = ignore_ex # currently unimplemented
self.call_methods = call_methods
self.operators = ('or', 'and')
self.useOperator = 'or'
# allow index to index multiple attributes
ia = _get(extra, 'indexed_attrs', id)
if isinstance(ia, str):
self.indexed_attrs = ia.split(',')
else:
self.indexed_attrs = list(ia)
self.indexed_attrs = [
attr.strip() for attr in self.indexed_attrs if attr]
if not self.indexed_attrs:
self.indexed_attrs = [id]
self.clear()
def __len__(self):
return self._length()
def getId(self):
return self.id
def clear(self):
self._length = Length()
self._index = OOBTree()
self._unindex = IOBTree()
self._counter = Length()
def __nonzero__(self):
return not not self._unindex
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
for item in self._index.items():
if isinstance(item, int):
entry = 1 # "set" length is 1
else:
key, value = item
entry = len(value)
histogram[entry] = histogram.get(entry, 0) + 1
return histogram
def referencedObjects(self):
"""Generate a list of IDs for which we have referenced objects."""
return self._unindex.keys()
def getEntryForObject(self, documentId, default=_marker):
"""Takes a document ID and returns all the information we have
on that specific object.
"""
if default is _marker:
return self._unindex.get(documentId)
return self._unindex.get(documentId, default)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
indexRow = self._index.get(entry, _marker)
if indexRow is not _marker:
try:
indexRow.remove(documentId)
if not indexRow:
del self._index[entry]
self._length.change(-1)
except ConflictError:
raise
except AttributeError:
# index row is an int
try:
del self._index[entry]
except KeyError:
# XXX swallow KeyError because it was probably
# removed and then _length AttributeError raised
pass
if isinstance(self.__len__, Length):
self._length = self.__len__
del self.__len__
self._length.change(-1)
except Exception:
LOG.error('%s: unindex_object could not remove '
'documentId %s from index %s. This '
'should not happen.' %
(self.__class__.__name__,
str(documentId), str(self.id)),
exc_info=sys.exc_info())
else:
LOG.error('%s: unindex_object tried to retrieve set %s '
'from index %s but couldn\'t. This '
'should not happen.' %
(self.__class__.__name__,
repr(entry), str(self.id)))
def insertForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and put it in the correct place
in the forward index.
This will also deal with creating the entire row if necessary.
"""
indexRow = self._index.get(entry, _marker)
# Make sure there's actually a row there already. If not, create
# a set and stuff it in first.
if indexRow is _marker:
# We always use a set to avoid getting conflict errors on
# multiple threads adding a new row at the same time
self._index[entry] = IITreeSet((documentId, ))
self._length.change(1)
else:
try:
indexRow.insert(documentId)
except AttributeError:
# Inline migration: index row with one element was an int at
# first (before Zope 2.13).
indexRow = IITreeSet((indexRow, documentId))
self._index[entry] = indexRow
def index_object(self, documentId, obj, threshold=None):
""" wrapper to handle indexing of multiple attributes """
fields = self.getIndexSourceNames()
res = 0
for attr in fields:
res += self._index_object(documentId, obj, threshold, attr)
if res > 0:
self._increment_counter()
return res > 0
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
if datum is None:
# Prevent None from being indexed. None doesn't have a valid
# ordering definition compared to any other object.
# BTrees 4.0+ will throw a TypeError
# "object has default comparison" and won't let it be indexed.
raise TypeError('None cannot be indexed.')
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId)
if datum is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.error('Should not happen: oldDatum was there, '
'now its not, for document: %s' % documentId)
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
self._unindex[documentId] = datum
returnStatus = 1
return returnStatus
def _get_object_datum(self, obj, attr):
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
try:
datum = getattr(obj, attr)
if safe_callable(datum):
datum = datum()
except (AttributeError, TypeError):
datum = _marker
return datum
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
def getCounter(self):
"""Return a counter which is increased on index changes"""
return self._counter is not None and self._counter() or 0
def numObjects(self):
"""Return the number of indexed objects."""
return len(self._unindex)
def indexSize(self):
"""Return the size of the index in terms of distinct values."""
return len(self)
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self._increment_counter()
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.debug('Attempt to unindex nonexistent document'
' with id %s' % documentId, exc_info=True)
def _apply_not(self, not_parm, resultset=None):
index = self._index
setlist = []
for k in not_parm:
s = index.get(k, None)
if s is None:
continue
elif isinstance(s, int):
s = IISet((s, ))
setlist.append(s)
return multiunion(setlist)
def _convert(self, value, default=None):
return value
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the request arg.
The request argument should be a mapping object.
If the request does not have a key which matches the "id" of
the index instance, then None is returned.
If the request *does* have a key which matches the "id" of
the index instance, one of a few things can happen:
- if the value is a blank string, None is returned (in
order to support requests from web forms where
you can't tell a blank string from empty).
- if the value is a nonblank string, turn the value into
a single-element sequence, and proceed.
- if the value is a sequence, return a union search.
- If the value is a dict and contains a key of the form
'<index>_operator' this overrides the default method
('or') to combine search results. Valid values are "or"
and "and".
If None is not returned as a result of the abovementioned
constraints, two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
FAQ answer: to search a Field Index for documents that
have a blank string as their value, wrap the request value
up in a tuple ala: request = {'id':('',)}
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
index = self._index
r = None
opr = None
# not / exclude parameter
not_parm = record.get('not', None)
if not record.keys and not_parm:
# convert into indexed format
not_parm = map(self._convert, not_parm)
# we have only a 'not' query
record.keys = [k for k in index.keys() if k not in not_parm]
else:
# convert query arguments into indexed format
record.keys = map(self._convert, record.keys)
# experimental code for specifing the operator
operator = record.get('operator', self.useOperator)
if not operator in self.operators:
raise RuntimeError("operator not valid: %s" % escape(operator))
# Range parameter
range_parm = record.get('range', None)
if range_parm:
opr = "range"
opr_args = []
if range_parm.find("min") > -1:
opr_args.append("min")
if range_parm.find("max") > -1:
opr_args.append("max")
if record.get('usage', None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args = opr[0], opr[1:]
if opr == "range": # range search
if 'min' in opr_args:
lo = min(record.keys)
else:
lo = None
if 'max' in opr_args:
hi = max(record.keys)
else:
hi = None
if hi:
setlist = index.values(lo, hi)
else:
setlist = index.values(lo)
# If we only use one key, intersect and return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
if not_parm:
exclude = self._apply_not(not_parm, resultset)
result = difference(result, exclude)
return result, (self.id,)
if operator == 'or':
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s,))
tmp.append(s)
r = multiunion(tmp)
else:
# For intersection, sort with smallest data set first
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s,))
tmp.append(s)
if len(tmp) > 2:
setlist = sorted(tmp, key=len)
else:
setlist = tmp
r = resultset
for s in setlist:
# the result is bound by the resultset
r = intersection(r, s)
else: # not a range search
# Filter duplicates
setlist = []
for k in record.keys:
if k is None:
raise TypeError('None cannot be in an index.')
s = index.get(k, None)
# If None, try to bail early
if s is None:
if operator == 'or':
# If union, we can't possibly get a bigger result
continue
# If intersection, we can't possibly get a smaller result
return IISet(), (self.id,)
elif isinstance(s, int):
s = IISet((s,))
setlist.append(s)
# If we only use one key return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
if not_parm:
exclude = self._apply_not(not_parm, resultset)
result = difference(result, exclude)
return result, (self.id,)
if operator == 'or':
# If we already get a small result set passed in, intersecting
# the various indexes with it and doing the union later is
# faster than creating a multiunion first.
if resultset is not None and len(resultset) < 200:
smalllist = []
for s in setlist:
smalllist.append(intersection(resultset, s))
r = multiunion(smalllist)
else:
r = multiunion(setlist)
else:
# For intersection, sort with smallest data set first
if len(setlist) > 2:
setlist = sorted(setlist, key=len)
r = resultset
for s in setlist:
r = intersection(r, s)
if isinstance(r, int):
r = IISet((r, ))
if r is None:
return IISet(), (self.id,)
if not_parm:
exclude = self._apply_not(not_parm, resultset)
r = difference(r, exclude)
return r, (self.id,)
def hasUniqueValuesFor(self, name):
"""has unique values for column name"""
if name == self.id:
return 1
return 0
def getIndexSourceNames(self):
""" return sequence of indexed attributes """
# BBB: older indexes didn't have 'indexed_attrs'
return getattr(self, 'indexed_attrs', [self.id])
def getIndexQueryNames(self):
"""Indicate that this index applies to queries for the index's name."""
return (self.id,)
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
raise StopIteration
if not withLengths:
for key in self._index.keys():
yield key
else:
for key, value in self._index.items():
if isinstance(value, int):
yield (key, 1)
else:
yield (key, len(value))
def keyForDocument(self, id):
# This method is superseded by documentToKeyMap
return self._unindex[id]
def documentToKeyMap(self):
return self._unindex
def items(self):
items = []
for k, v in self._index.items():
if isinstance(v, int):
v = IISet((v,))
items.append((k, v))
return items
class BTreeFolder2Base(Persistent):
"""Base for BTree-based folders.
"""
security = ClassSecurityInfo()
manage_options = (({
'label': 'Contents',
'action': 'manage_main',
}, ) + Folder.manage_options[1:])
security.declareProtected(view_management_screens, 'manage_main')
manage_main = DTMLFile('contents', globals())
_tree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
_mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } }
title = ''
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._tree = OOBTree()
self._count = Length()
self._mt_index = OOBTree()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name in source.objectIds():
value = source._getOb(name, None)
if value is not None:
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount()
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in BTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in BTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self):
"""Checks if the value of self._count disagrees with
len(self.objectIds()). If so, corrects self._count. Returns the
old and new count values. If old==new, no correction was
performed.
"""
old = self._count()
new = len(self.objectIds())
if old != new:
self._count.set(new)
return old, new
security.declareProtected(view_management_screens, 'manage_cleanup')
def manage_cleanup(self):
"""Calls self._cleanup() and reports the result as text.
"""
v = self._cleanup()
path = '/'.join(self.getPhysicalPath())
if v:
return "No damage detected in BTreeFolder2 at %s." % path
else:
return ("Fixed BTreeFolder2 at %s. "
"See the log for more details." % path)
def _cleanup(self):
"""Cleans up errors in the BTrees.
Certain ZODB bugs have caused BTrees to become slightly insane.
Fortunately, there is a way to clean up damaged BTrees that
always seems to work: make a new BTree containing the items()
of the old one.
Returns 1 if no damage was detected, or 0 if damage was
detected and fixed.
"""
from BTrees.check import check
path = '/'.join(self.getPhysicalPath())
try:
check(self._tree)
for key in self._tree.keys():
if not self._tree.has_key(key):
raise AssertionError("Missing value for key: %s" %
repr(key))
check(self._mt_index)
for key, value in self._mt_index.items():
if (not self._mt_index.has_key(key)
or self._mt_index[key] is not value):
raise AssertionError(
"Missing or incorrect meta_type index: %s" % repr(key))
check(value)
for k in value.keys():
if not value.has_key(k):
raise AssertionError(
"Missing values for meta_type index: %s" %
repr(key))
return 1
except AssertionError:
LOG.warn('Detected damage to %s. Fixing now.' % path,
exc_info=sys.exc_info())
try:
self._tree = OOBTree(self._tree)
mt_index = OOBTree()
for key, value in self._mt_index.items():
mt_index[key] = OIBTree(value)
self._mt_index = mt_index
except:
LOG.error('Failed to fix %s.' % path, exc_info=sys.exc_info())
raise
else:
LOG.info('Fixed %s.' % path)
return 0
def _getOb(self, id, default=_marker):
"""Return the named object from the folder.
"""
tree = self._tree
if default is _marker:
ob = tree[id]
return ob.__of__(self)
else:
ob = tree.get(id, _marker)
if ob is _marker:
return default
else:
return ob.__of__(self)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
tree = self._tree
if tree.has_key(id):
raise KeyError('There is already an item named "%s".' % id)
tree[id] = object
self._count.change(1)
# Update the meta type index.
mti = self._mt_index
meta_type = getattr(object, 'meta_type', None)
if meta_type is not None:
ids = mti.get(meta_type, None)
if ids is None:
ids = OIBTree()
mti[meta_type] = ids
ids[id] = 1
def _delOb(self, id):
"""Remove the named object from the folder.
"""
tree = self._tree
meta_type = getattr(tree[id], 'meta_type', None)
del tree[id]
self._count.change(-1)
# Update the meta type index.
if meta_type is not None:
mti = self._mt_index
ids = mti.get(meta_type, None)
if ids is not None and ids.has_key(id):
del ids[id]
if not ids:
# Removed the last object of this meta_type.
# Prune the index.
del mti[meta_type]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
idlist = self.objectIds() # Pre-sorted.
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = []
formatted.append(listtext0 % pref_rows)
for i in range(b_start - 1, b_end):
optID = escape(idlist[i])
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {
'b_start': b_start,
'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)
}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect('%s/%s/manage_workspace' %
(self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information, 'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information, 'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
return self._tree.has_key(id)
security.declareProtected(access_contents_information, 'objectIds')
def objectIds(self, spec=None):
# Returns a list of subobject ids of the current object.
# If 'spec' is specified, returns objects whose meta_type
# matches 'spec'.
mti = self._mt_index
if spec is None:
spec = mti.keys() #all meta types
if isinstance(spec, StringType):
spec = [spec]
set = None
for meta_type in spec:
ids = mti.get(meta_type, None)
if ids is not None:
set = union(set, ids)
if set is None:
return ()
else:
return set.keys()
security.declareProtected(access_contents_information, 'objectValues')
def objectValues(self, spec=None):
# Returns a list of actual subobjects of the current object.
# If 'spec' is specified, returns only objects whose meta_type
# match 'spec'.
return LazyMap(self._getOb, self.objectIds(spec))
security.declareProtected(access_contents_information, 'objectItems')
def objectItems(self, spec=None):
# Returns a list of (id, subobject) tuples of the current object.
# If 'spec' is specified, returns only objects whose meta_type match
# 'spec'
return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)),
self.objectIds(spec))
security.declareProtected(access_contents_information, 'objectMap')
def objectMap(self):
# Returns a tuple of mappings containing subobject meta-data.
return LazyMap(
lambda (k, v): {
'id': k,
'meta_type': getattr(v, 'meta_type', None)
}, self._tree.items(), self._count())
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information, 'objectIds_d')
def objectIds_d(self, t=None):
ids = self.objectIds(t)
res = {}
for id in ids:
res[id] = 1
return res
security.declareProtected(access_contents_information, 'objectMap_d')
def objectMap_d(self, t=None):
return self.objectMap()
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self,
id,
object,
roles=None,
user=None,
set_owner=1,
suppress_events=False):
ob = object # better name, keep original function signature
v = self._checkId(id)
if v is not None:
id = v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
if not suppress_events:
notify(ObjectWillBeAddedEvent(ob, self, id))
self._setOb(id, ob)
ob = self._getOb(id)
if set_owner:
# TODO: eventify manage_fixupOwnershipAfterAdd
# This will be called for a copy/clone, or a normal _setObject.
ob.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if getattr(ob, '__ac_local_roles__', _marker) is None:
user = getSecurityManager().getUser()
if user is not None:
userid = user.getId()
if userid is not None:
ob.manage_setLocalRoles(userid, ['Owner'])
if not suppress_events:
notify(ObjectAddedEvent(ob, self, id))
notifyContainerModified(self)
OFS.subscribers.compatibilityCall('manage_afterAdd', ob, ob, self)
return id
def _delObject(self, id, dp=1, suppress_events=False):
ob = self._getOb(id)
OFS.subscribers.compatibilityCall('manage_beforeDelete', ob, ob, self)
if not suppress_events:
notify(ObjectWillBeRemovedEvent(ob, self, id))
self._delOb(id)
if not suppress_events:
notify(ObjectRemovedEvent(ob, self, id))
notifyContainerModified(self)
# Aliases for mapping-like access.
__len__ = objectCount
keys = objectIds
values = objectValues
items = objectItems
# backward compatibility
hasObject = has_key
security.declareProtected(access_contents_information, 'get')
def get(self, name, default=None):
return self._getOb(name, default)
# Utility for generating unique IDs.
security.declareProtected(access_contents_information, 'generateId')
def generateId(self, prefix='item', suffix='', rand_ceiling=999999999):
"""Returns an ID not used yet by this folder.
The ID is unlikely to collide with other threads and clients.
The IDs are sequential to optimize access to objects
that are likely to have some relation.
"""
tree = self._tree
n = self._v_nextid
attempt = 0
while 1:
if n % 4000 != 0 and n <= rand_ceiling:
id = '%s%d%s' % (prefix, n, suffix)
if not tree.has_key(id):
break
n = randint(1, rand_ceiling)
attempt = attempt + 1
if attempt > MAX_UNIQUEID_ATTEMPTS:
# Prevent denial of service
raise ExhaustedUniqueIdsError
self._v_nextid = n + 1
return id
def __getattr__(self, name):
# Boo hoo hoo! Zope 2 prefers implicit acquisition over traversal
# to subitems, and __bobo_traverse__ hooks don't work with
# restrictedTraverse() unless __getattr__() is also present.
# Oh well.
res = self._tree.get(name)
if res is None:
raise AttributeError, name
return res
class GranularIndex(CatalogFieldIndex):
"""Indexes integer values using multiple granularity levels.
The multiple levels of granularity make it possible to query large
ranges without loading many IFTreeSets from the forward index.
"""
implements(
ICatalogIndex,
IStatistics,
)
def __init__(self, discriminator, levels=(1000,)):
"""Create an index.
levels is a sequence of integer coarseness levels.
The default is (1000,).
"""
self._levels = tuple(levels)
super(GranularIndex, self).__init__(discriminator)
def clear(self):
"""Initialize all mappings."""
# The forward index maps an indexed value to IFSet(docids)
self._fwd_index = self.family.IO.BTree()
# The reverse index maps a docid to its index value
self._rev_index = self.family.II.BTree()
self._num_docs = Length(0)
# self._granular_indexes: [(level, BTree(value -> IFSet([docid])))]
self._granular_indexes = [(level, self.family.IO.BTree())
for level in self._levels]
def index_doc(self, docid, obj):
if callable(self.discriminator):
value = self.discriminator(obj, _marker)
else:
value = getattr(obj, self.discriminator, _marker)
if value is _marker:
# unindex the previous value
self.unindex_doc(docid)
return
if not isinstance(value, int):
raise ValueError(
'GranularIndex cannot index non-integer value %s' % value)
rev_index = self._rev_index
if docid in rev_index:
if docid in self._fwd_index.get(value, ()):
# There's no need to index the doc; it's already up to date.
return
# unindex doc if present
self.unindex_doc(docid)
# Insert into forward index.
set = self._fwd_index.get(value)
if set is None:
set = self.family.IF.TreeSet()
self._fwd_index[value] = set
set.insert(docid)
# increment doc count
self._num_docs.change(1)
# Insert into reverse index.
rev_index[docid] = value
for level, ndx in self._granular_indexes:
v = value // level
set = ndx.get(v)
if set is None:
set = self.family.IF.TreeSet()
ndx[v] = set
set.insert(docid)
def unindex_doc(self, docid):
rev_index = self._rev_index
value = rev_index.get(docid)
if value is None:
return # not in index
del rev_index[docid]
self._num_docs.change(-1)
ndx = self._fwd_index
try:
set = ndx[value]
set.remove(docid)
if not set:
del ndx[value]
except KeyError:
pass
for level, ndx in self._granular_indexes:
v = value // level
try:
set = ndx[v]
set.remove(docid)
if not set:
del ndx[v]
except KeyError:
pass
def search(self, queries, operator='or'):
sets = []
for query in queries:
if isinstance(query, Range):
query = query.as_tuple()
else:
query = (query, query)
set = self.family.IF.multiunion(self.docids_in_range(*query))
sets.append(set)
result = None
if len(sets) == 1:
result = sets[0]
elif operator == 'and':
sets.sort()
for set in sets:
result = self.family.IF.intersection(set, result)
else:
result = self.family.IF.multiunion(sets)
return result
def docids_in_range(self, min, max):
"""List the docids for an integer range, inclusive on both ends.
min or max can be None, making them unbounded.
Returns an iterable of IFSets.
"""
for level, ndx in sorted(self._granular_indexes, reverse=True):
# Try to fill the range using coarse buckets first.
# Use only buckets that completely fill the range.
# For example, if start is 2 and level is 10, then we can't
# use bucket 0; only buckets 1 and greater are useful.
# Similarly, if end is 18 and level is 10, then we can't use
# bucket 1; only buckets 0 and less are useful.
if min is not None:
a = (min + level - 1) // level
else:
a = None
if max is not None:
b = (max - level + 1) // level
else:
b = None
# a and b are now coarse bucket values (or None).
if a is None or b is None or a <= b:
sets = []
if a is not None and min < a * level:
# include the gap before
sets.extend(self.docids_in_range(min, a * level - 1))
sets.extend(ndx.values(a, b))
if b is not None and (b + 1) * level - 1 < max:
# include the gap after
sets.extend(self.docids_in_range((b + 1) * level, max))
return sets
return self._fwd_index.values(min, max)
class FormSaveDataAdapter(FormActionAdapter):
"""A form action adapter that will save form input data and
return it in csv- or tab-delimited format."""
schema = FormAdapterSchema.copy() + Schema((
LinesField('ExtraData',
widget=MultiSelectionWidget(
label=_(u'label_savedataextra_text', default='Extra Data'),
description=_(u'help_savedataextra_text', default=u"""
Pick any extra data you'd like saved with the form input.
"""),
format='checkbox',
),
vocabulary = 'vocabExtraDataDL',
),
StringField('DownloadFormat',
searchable=0,
required=1,
default='csv',
vocabulary = 'vocabFormatDL',
widget=SelectionWidget(
label=_(u'label_downloadformat_text', default=u'Download Format'),
),
),
BooleanField("UseColumnNames",
required=False,
searchable=False,
widget=BooleanWidget(
label = _(u'label_usecolumnnames_text', default=u"Include Column Names"),
description = _(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"),
),
),
ExLinesField('SavedFormInput',
edit_accessor='getSavedFormInputForEdit',
mutator='setSavedFormInput',
searchable=0,
required=0,
primary=1,
schemata="saved data",
read_permission=DOWNLOAD_SAVED_PERMISSION,
widget=TextAreaWidget(
label=_(u'label_savedatainput_text', default=u"Saved Form Input"),
description=_(u'help_savedatainput_text'),
),
),
))
schema.moveField('execCondition', pos='bottom')
meta_type = 'FormSaveDataAdapter'
portal_type = 'FormSaveDataAdapter'
archetype_name = 'Save Data Adapter'
immediate_view = 'fg_savedata_view_p3'
default_view = 'fg_savedata_view_p3'
suppl_views = ('fg_savedata_tabview_p3', 'fg_savedata_recview_p3',)
security = ClassSecurityInfo()
def _migrateStorage(self):
# we're going to use an LOBTree for storage. we need to
# consider the possibility that self is from an
# older version that uses the native Archetypes storage
# or the former IOBTree (<= 1.6.0b2 )
# in the SavedFormInput field.
updated = base_hasattr(self, '_inputStorage') and \
base_hasattr(self, '_inputItems') and \
base_hasattr(self, '_length')
if not updated:
try:
saved_input = self.getSavedFormInput()
except AttributeError:
saved_input = []
self._inputStorage = SavedDataBTree()
i = 0
self._inputItems = 0
self._length = Length()
if len(saved_input):
for row in saved_input:
self._inputStorage[i] = row
i += 1
self.SavedFormInput = []
self._inputItems = i
self._length.set(i)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInput')
def getSavedFormInput(self):
""" returns saved input as an iterable;
each row is a sequence of fields.
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.values()
else:
return self.SavedFormInput
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputItems')
def getSavedFormInputItems(self):
""" returns saved input as an iterable;
each row is an (id, sequence of fields) tuple
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.items()
else:
return enumerate(self.SavedFormInput)
security.declareProtected(ModifyPortalContent, 'getSavedFormInputForEdit')
def getSavedFormInputForEdit(self, **kwargs):
""" returns saved as CSV text """
delimiter = self.csvDelimiter()
sbuf = StringIO()
writer = csv.writer(sbuf, delimiter=delimiter)
for row in self.getSavedFormInput():
writer.writerow( row )
res = sbuf.getvalue()
sbuf.close()
return res
security.declareProtected(ModifyPortalContent, 'setSavedFormInput')
def setSavedFormInput(self, value, **kwargs):
""" expects value as csv text string, stores as list of lists """
self._migrateStorage()
self._inputStorage.clear()
i = 0
self._inputItems = 0
self._length.set(0)
if len(value):
delimiter = self.csvDelimiter()
sbuf = StringIO( value )
reader = csv.reader(sbuf, delimiter=delimiter)
for row in reader:
if row:
self._inputStorage[i] = row
i += 1
self._inputItems = i
self._length.set(i)
sbuf.close()
# logger.debug("setSavedFormInput: %s items" % self._inputItems)
security.declareProtected(ModifyPortalContent, 'clearSavedFormInput')
def clearSavedFormInput(self, **kwargs):
""" convenience method to clear input buffer """
REQUEST = kwargs.get('request', self.REQUEST)
if REQUEST.form.has_key('clearSavedFormInput'):
# we're processing a request from the web;
# check for CSRF
plone.protect.CheckAuthenticator(REQUEST)
plone.protect.PostOnly(REQUEST)
self._migrateStorage()
self._inputStorage.clear()
self._inputItems = 0
self._length.set(0)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputById')
def getSavedFormInputById(self, id):
""" Return the data stored for record with 'id' """
lst = [field.replace('\r','').replace('\n', r'\n') for field in self._inputStorage[id]]
return lst
security.declareProtected(ModifyPortalContent, 'manage_saveData')
def manage_saveData(self, id, data):
""" Save the data for record with 'id' """
self._migrateStorage()
lst = list()
for i in range(0, len(self.getColumnNames())):
lst.append(getattr(data, 'item-%d' % i, '').replace(r'\n', '\n'))
self._inputStorage[id] = lst
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
security.declareProtected(ModifyPortalContent, 'manage_deleteData')
def manage_deleteData(self, id):
""" Delete the data for record with 'id' """
self._migrateStorage()
del self._inputStorage[id]
self._inputItems -= 1
self._length.change(-1)
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
def _addDataRow(self, value):
self._migrateStorage()
if isinstance(self._inputStorage, IOBTree):
# 32-bit IOBTree; use a key which is more likely to conflict
# but which won't overflow the key's bits
id = self._inputItems
self._inputItems += 1
else:
# 64-bit LOBTree
id = int(time.time() * 1000)
while id in self._inputStorage: # avoid collisions during testing
id += 1
self._inputStorage[id] = value
self._length.change(1)
security.declareProtected(ModifyPortalContent, 'addDataRow')
def addDataRow(self, value):
""" a wrapper for the _addDataRow method """
self._addDataRow(value)
def onSuccess(self, fields, REQUEST=None, loopstop=False):
"""
saves data.
"""
if LP_SAVE_TO_CANONICAL and not loopstop:
# LinguaPlone functionality:
# check to see if we're in a translated
# form folder, but not the canonical version.
parent = self.aq_parent
if safe_hasattr(parent, 'isTranslation') and \
parent.isTranslation() and not parent.isCanonical():
# look in the canonical version to see if there is
# a matching (by id) save-data adapter.
# If so, call its onSuccess method
cf = parent.getCanonical()
target = cf.get(self.getId())
if target is not None and target.meta_type == 'FormSaveDataAdapter':
target.onSuccess(fields, REQUEST, loopstop=True)
return
from ZPublisher.HTTPRequest import FileUpload
data = []
for f in fields:
if f.isFileField():
file = REQUEST.form.get('%s_file' % f.fgField.getName())
if isinstance(file, FileUpload) and file.filename != '':
file.seek(0)
fdata = file.read()
filename = file.filename
mimetype, enc = guess_content_type(filename, fdata, None)
if mimetype.find('text/') >= 0:
# convert to native eols
fdata = fdata.replace('\x0d\x0a', '\n').replace('\x0a', '\n').replace('\x0d', '\n')
data.append( '%s:%s:%s:%s' % (filename, mimetype, enc, fdata) )
else:
data.append( '%s:%s:%s:Binary upload discarded' % (filename, mimetype, enc) )
else:
data.append( 'NO UPLOAD' )
elif not f.isLabel():
val = REQUEST.form.get(f.fgField.getName(),'')
if not type(val) in StringTypes:
# Zope has marshalled the field into
# something other than a string
val = str(val)
data.append(val)
if self.ExtraData:
for f in self.ExtraData:
if f == 'dt':
data.append( str(DateTime()) )
else:
data.append( getattr(REQUEST, f, '') )
self._addDataRow( data )
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames')
def getColumnNames(self):
"""Returns a list of column names"""
names = [field.getName() for field in self.fgFields(displayOnly=True)]
for f in self.ExtraData:
names.append(f)
return names
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles')
def getColumnTitles(self):
"""Returns a list of column titles"""
names = [field.widget.label for field in self.fgFields(displayOnly=True)]
for f in self.ExtraData:
names.append(self.vocabExtraDataDL().getValue(f, ''))
return names
def _cleanInputForTSV(self, value):
# make data safe to store in tab-delimited format
return str(value).replace('\x0d\x0a', r'\n').replace('\x0a', r'\n').replace('\x0d', r'\n').replace('\t', r'\t')
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_tsv')
def download_tsv(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
filename = self.id
if filename.find('.') < 0:
filename = '%s.tsv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/tab-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
res = "%s\n" % '\t'.join( self.getColumnNames() )
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
for row in self.getSavedFormInput():
res = '%s%s\n' % (res, '\t'.join( [self._cleanInputForTSV(col) for col in row] ))
return res
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_csv')
def download_csv(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
filename = self.id
if filename.find('.') < 0:
filename = '%s.csv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/comma-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
delimiter = self.csvDelimiter()
res = "%s\n" % delimiter.join( self.getColumnNames() )
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
return '%s%s' % (res, self.getSavedFormInputForEdit())
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download')
def download(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return self.download_tsv(REQUEST, RESPONSE)
else:
assert format == 'csv', 'Unknown download format'
return self.download_csv(REQUEST, RESPONSE)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'rowAsColDict')
def rowAsColDict(self, row, cols):
""" Where row is a data sequence and cols is a column name sequence,
returns a dict of colname:column. This is a convenience method
used in the record view.
"""
colcount = len(cols)
rdict = {}
for i in range(0, len(row)):
if i < colcount:
rdict[cols[i]] = row[i]
else:
rdict['column-%s' % i] = row[i]
return rdict
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'inputAsDictionaries')
def inputAsDictionaries(self):
"""returns saved data as an iterable of dictionaries
"""
cols = self.getColumnNames()
for row in self.getSavedFormInput():
yield self.rowAsColDict(row, cols)
# alias for old mis-naming
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'InputAsDictionaries')
InputAsDictionaries = inputAsDictionaries
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'formatMIME')
def formatMIME(self):
"""MIME format selected for download
"""
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return 'text/tab-separated-values'
else:
assert format == 'csv', 'Unknown download format'
return 'text/comma-separated-values'
security.declarePrivate('csvDelimiter')
def csvDelimiter(self):
"""Delimiter character for CSV downloads
"""
fgt = getToolByName(self, 'formgen_tool')
return fgt.getCSVDelimiter()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'itemsSaved')
def itemsSaved(self):
"""Download the saved data
"""
if base_hasattr(self, '_length'):
return self._length()
elif base_hasattr(self, '_inputItems'):
return self._inputItems
else:
return len(self.SavedFormInput)
def vocabExtraDataDL(self):
""" returns vocabulary for extra data """
return DisplayList( (
('dt',
self.translate( msgid='vocabulary_postingdt_text',
domain='ploneformgen',
default='Posting Date/Time')
),
('HTTP_X_FORWARDED_FOR','HTTP_X_FORWARDED_FOR',),
('REMOTE_ADDR','REMOTE_ADDR',),
('HTTP_USER_AGENT','HTTP_USER_AGENT',),
) )
def vocabFormatDL(self):
""" returns vocabulary for format """
return DisplayList( (
('tsv',
self.translate( msgid='vocabulary_tsv_text',
domain='ploneformgen',
default='Tab-Separated Values')
),
('csv',
self.translate( msgid='vocabulary_csv_text',
domain='ploneformgen',
default='Comma-Separated Values')
),
) )
class HypatiaDateRecurringIndex(KeywordIndex, FieldIndex):
def discriminate(self, obj, default):
""" See interface IIndexInjection """
if callable(self.discriminator):
value = self.discriminator(obj, _marker)
else:
value = getattr(obj, self.discriminator, _marker)
if value is _marker:
return default
if isinstance(value, Persistent):
raise ValueError('Catalog cannot index persistent object %s' %
value)
if isinstance(value, Broken):
raise ValueError('Catalog cannot index broken object %s' % value)
if not isinstance(value, dict):
raise ValueError(
'Catalog can only index dict with '
'attr and date keys, or date and recurdef keys, given %s' %
value)
# examples:
# {'attr': 'dates',
# 'date': datetime.datetime.now()}
# will get dates_recurrence attribute on the obj to get iCal string
# for recurrence definition
# or
# {'date': datetime.datetime.now(),
# 'recurdef': ICALSTRING}
# no access to obj attributes at all
date = value.get('date')
default_recurdef = value.get('recurdef', _marker)
if default_recurdef is not _marker:
recurdef = default_recurdef
else:
attr_recurdef = value.get('attr') + '_recurrence'
recurdef = getattr(obj, attr_recurdef, None)
if callable(recurdef):
recurdef = recurdef()
if not recurdef:
dates = [date]
else:
dates = recurrence_sequence_ical(date, recrule=recurdef)
# dates is a generator
return tuple(dates)
def normalize(self, dates):
return [dt2int(date) for date in dates]
# below is the same implementation as Keyword Index, but replacing
# self._fwd_index = self.family.OO.BTree() by self._fwd_index = self.family.IO.BTree()
# family.OO.Set by family.II.Set
# family.OO.difference by family.II.difference
def reset(self):
"""Initialize forward and reverse mappings."""
# The forward index maps index keywords to a sequence of docids
self._fwd_index = self.family.IO.BTree()
# The reverse index maps a docid to its keywords
self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
self._not_indexed = self.family.IF.TreeSet()
def index_doc(self, docid, obj):
seq = self.discriminate(obj, _marker)
if seq is _marker:
if not (docid in self._not_indexed):
# unindex the previous value
self.unindex_doc(docid)
# Store docid in set of unindexed docids
self._not_indexed.add(docid)
return None
if docid in self._not_indexed:
# Remove from set of unindexed docs if it was in there.
self._not_indexed.remove(docid)
if isinstance(seq, string_types):
raise TypeError('seq argument must be a list/tuple of strings')
old_kw = self._rev_index.get(docid, None)
if not seq:
if old_kw:
self.unindex_doc(docid)
return
seq = self.normalize(seq)
new_kw = self.family.II.Set(seq)
if old_kw is None:
self._insert_forward(docid, new_kw)
self._insert_reverse(docid, new_kw)
self._num_docs.change(1)
else:
# determine added and removed keywords
kw_added = self.family.II.difference(new_kw, old_kw)
kw_removed = self.family.II.difference(old_kw, new_kw)
if not (kw_added or kw_removed):
return
# removed keywords are removed from the forward index
for word in kw_removed:
fwd = self._fwd_index[word]
fwd.remove(docid)
if not fwd:
del self._fwd_index[word]
# now update reverse and forward indexes
self._insert_forward(docid, kw_added)
self._insert_reverse(docid, new_kw)
def applyInRange(self, start, end, excludemin=False, excludemax=False):
if start is not None:
start = dt2int(start)
if end is not None:
end = dt2int(end)
return self.family.IF.multiunion(
self._fwd_index.values(start,
end,
excludemin=excludemin,
excludemax=excludemax))
def document_repr(self, docid, default=None):
result = self._rev_index.get(docid, default)
if result is not default:
return ', '.join([int2dt(r).isoformat() for r in result])
# return repr(result)
return default
def inrange_with_not_indexed(self,
start,
end,
excludemin=False,
excludemax=False):
return InRangeWithNotIndexed(self, start, end, excludemin, excludemax)
def sort(self,
docids,
reverse=False,
limit=None,
sort_type=None,
raise_unsortable=True,
from_=None,
until=None):
if from_ is not None:
from_ = dt2int(from_)
if until is not None:
until = dt2int(until)
if limit is not None:
limit = int(limit)
if limit < 1:
raise ValueError('limit must be 1 or greater')
if not docids:
return []
numdocs = self._num_docs.value
if not numdocs:
if raise_unsortable:
raise Unsortable(docids)
return []
if sort_type == interfaces.STABLE:
sort_type = interfaces.TIMSORT
elif sort_type == interfaces.OPTIMAL:
sort_type = None
if reverse:
raise NotImplementedError
else:
return self.sort_forward(docids,
limit,
numdocs,
from_=from_,
until=until,
sort_type=sort_type,
raise_unsortable=raise_unsortable)
def sort_forward(self,
docids,
limit,
numdocs,
from_,
until,
sort_type=None,
raise_unsortable=True):
rlen = len(docids)
# See http://www.zope.org/Members/Caseman/ZCatalog_for_2.6.1
# for an overview of why we bother doing all this work to
# choose the right sort algorithm.
if sort_type is None:
if limit and nbest_ascending_wins(limit, rlen, numdocs):
# nbest beats timsort reliably if this is true
sort_type = interfaces.NBEST
else:
sort_type = interfaces.TIMSORT
if sort_type == interfaces.NBEST:
if limit is None:
raise ValueError('nbest requires a limit')
return self.nbest_ascending(docids,
limit,
from_=from_,
until=until,
raise_unsortable=raise_unsortable)
elif sort_type == interfaces.TIMSORT:
return self.timsort_ascending(docids,
limit,
from_=from_,
until=until,
raise_unsortable=raise_unsortable)
else:
raise ValueError('Unknown sort type %s' % sort_type)
def nbest_ascending(self,
docids,
limit,
from_,
until,
raise_unsortable=False):
if limit is None: #pragma NO COVERAGE
raise RuntimeError('n-best used without limit')
# lifted from heapq.nsmallest
h = nsort(docids, self._rev_index, ASC, from_, until)
it = iter(h)
result = sorted(islice(it, 0, limit))
if not result: #pragma NO COVERAGE
raise StopIteration
insort = bisect.insort
pop = result.pop
los = result[-1] # los --> Largest of the nsmallest
for elem in it:
if los <= elem:
continue
insort(result, elem)
pop()
los = result[-1]
missing_docids = []
for value, docid in result:
if value is ASC:
missing_docids.append(docid)
else:
yield docid
if raise_unsortable and missing_docids:
raise Unsortable(missing_docids)
def timsort_ascending(self,
docids,
limit,
from_,
until,
raise_unsortable=True):
return self._timsort(
docids,
from_=from_,
until=until,
limit=limit,
reverse=False,
raise_unsortable=raise_unsortable,
)
def _timsort(
self,
docids,
from_,
until,
limit=None,
reverse=False,
raise_unsortable=True,
):
n = 0
missing_docids = []
def get(k, rev_index=self._rev_index):
v = rev_index.get(k, ASC)
if v is ASC:
missing_docids.append(k)
else:
v = get_first_occurence(v, from_=from_, until=until)
return v
for docid in sorted(docids, key=get, reverse=reverse):
if docid in missing_docids:
# skip docids not in this index
continue
n += 1
yield docid
if limit and n >= limit:
raise StopIteration
if raise_unsortable and missing_docids:
raise Unsortable(missing_docids)
class PathIndex(Persistent, SimpleItem):
"""Index for paths returned by getPhysicalPath.
A path index stores all path components of the physical path of an object.
Internal datastructure:
- a physical path of an object is split into its components
- every component is kept as a key of a OOBTree in self._indexes
- the value is a mapping 'level of the path component' to
'all docids with this path component on this level'
"""
__implements__ = (PluggableIndex.UniqueValueIndex,)
implements(IPathIndex, IUniqueValueIndex)
meta_type="PathIndex"
manage_options= (
{'label': 'Settings',
'action': 'manage_main',
'help': ('PathIndex','PathIndex_Settings.stx')},
)
query_options = ("query", "level", "operator")
def __init__(self,id,caller=None):
self.id = id
self.operators = ('or','and')
self.useOperator = 'or'
self.clear()
def clear(self):
self._depth = 0
self._index = OOBTree()
self._unindex = IOBTree()
self._length = Length(0)
def insertEntry(self, comp, id, level):
"""Insert an entry.
comp is a path component
id is the docid
level is the level of the component inside the path
"""
if not self._index.has_key(comp):
self._index[comp] = IOBTree()
if not self._index[comp].has_key(level):
self._index[comp][level] = IITreeSet()
self._index[comp][level].insert(id)
if level > self._depth:
self._depth = level
def index_object(self, docid, obj ,threshold=100):
""" hook for (Z)Catalog """
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (StringType, TupleType)):
raise TypeError('path value must be string or tuple of strings')
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (ListType, TupleType)):
path = '/'+ '/'.join(path[1:])
comps = filter(None, path.split('/'))
if not self._unindex.has_key(docid):
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
self._unindex[docid] = path
return 1
def unindex_object(self, docid):
""" hook for (Z)Catalog """
if not self._unindex.has_key(docid):
LOG.debug('Attempt to unindex nonexistent document with id %s'
% docid)
return
comps = self._unindex[docid].split('/')
for level in range(len(comps[1:])):
comp = comps[level+1]
try:
self._index[comp][level].remove(docid)
if not self._index[comp][level]:
del self._index[comp][level]
if not self._index[comp]:
del self._index[comp]
except KeyError:
LOG.debug('Attempt to unindex document with id %s failed'
% docid)
self._length.change(-1)
del self._unindex[docid]
def search(self, path, default_level=0):
"""
path is either a string representing a
relative URL or a part of a relative URL or
a tuple (path,level).
level >= 0 starts searching at the given level
level < 0 not implemented yet
"""
if isinstance(path, StringType):
level = default_level
else:
level = int(path[1])
path = path[0]
comps = filter(None, path.split('/'))
if len(comps) == 0:
return IISet(self._unindex.keys())
if level >= 0:
results = []
for i in range(len(comps)):
comp = comps[i]
if not self._index.has_key(comp): return IISet()
if not self._index[comp].has_key(level+i): return IISet()
results.append( self._index[comp][level+i] )
res = results[0]
for i in range(1,len(results)):
res = intersection(res,results[i])
return res
else:
results = IISet()
for level in range(0,self._depth + 1):
ids = None
error = 0
for cn in range(0,len(comps)):
comp = comps[cn]
try:
ids = intersection(ids,self._index[comp][level+cn])
except KeyError:
error = 1
if error==0:
results = union(results,ids)
return results
def numObjects(self):
""" return the number distinct values """
return len(self._unindex)
def indexSize(self):
""" return the number of indexed objects"""
return len(self)
def __len__(self):
return self._length()
def _apply_index(self, request, cid=''):
""" hook for (Z)Catalog
'request' -- mapping type (usually {"path": "..." }
additionaly a parameter "path_level" might be passed
to specify the level (see search())
'cid' -- ???
"""
record = parseIndexRequest(request,self.id,self.query_options)
if record.keys==None: return None
level = record.get("level",0)
operator = record.get('operator',self.useOperator).lower()
# depending on the operator we use intersection of union
if operator == "or": set_func = union
else: set_func = intersection
res = None
for k in record.keys:
rows = self.search(k,level)
res = set_func(res,rows)
if res:
return res, (self.id,)
else:
return IISet(), (self.id,)
def hasUniqueValuesFor(self, name):
"""has unique values for column name"""
return name == self.id
def uniqueValues(self, name=None, withLength=0):
""" needed to be consistent with the interface """
return self._index.keys()
def getIndexSourceNames(self):
""" return names of indexed attributes """
return ('getPhysicalPath', )
def getEntryForObject(self, docid, default=_marker):
""" Takes a document ID and returns all the information
we have on that specific object.
"""
try:
return self._unindex[docid]
except KeyError:
# XXX Why is default ignored?
return None
manage = manage_main = DTMLFile('dtml/managePathIndex', globals())
manage_main._setName('manage_main')
class IdStore(persistent.Persistent):
"""
Persistent storage of objects which generates non-conflictiong unique IDs
for them
"""
_v_nextid = None
family = family32
def __init__(self, family=family32):
"""
:param family: Family of BTrees to use
"""
self.tree = family.IO.BTree()
self.length = Length()
def _generateId(self):
"""Generate an id which is not yet taken.
This tries to allocate sequential ids so they fall into the same BTree
bucket, and randomizes if it stumbles upon a used one.
This algorithm is taken from zope.intid but it will cause performance
degradation due to fragmentation if used too often, we need something
better eventually
"""
nextid = self._v_nextid
while True:
if nextid is None:
nextid = random.randrange(0, self.family.maxint)
uid = nextid
if uid not in self.tree:
nextid += 1
if nextid > self.family.maxint:
nextid = None
self._v_nextid = nextid
return uid
nextid = None
def add(self, obj):
"""
Add object to the storage
:param obj: Object to store (persistent.Persistent but not necessarily)
:return: Unique ID
:rtype: int
"""
if not hasattr(self, "length"):
self.length = Length(len(self.tree))
while True:
uid = self._generateId()
if self.tree.insert(
uid,
obj): # We use this feature of BTrees to avoid conflicts
# _v_* and _p_* are not saved in the database
# However assigning _v_* unghostifies the object while _p_* doesn't
# We don't want to force-unghostify the object in some cases
obj._p_uid = uid
self.length.change(1)
return uid
def remove(self, iobj):
"""
Remove object from the storage
:param obj: Object or its integer unique id
"""
if not hasattr(self, "length"):
self.length = Length(len(self.tree))
if type(iobj) in (int, long):
del self.tree[iobj]
self.length.change(-1)
elif hasattr(iobj, "_p_uid"):
del self.tree[iobj._p_uid]
iobj._p_uid = None
self.length.change(-1)
else:
raise TypeError("Argument should be either uid or object itself")
def __getitem__(self, uid):
"""
:param int uid: Get object by its unique ID
"""
return self.tree[uid]
def __delitem__(self, uid):
"""
:param int uid: Get object by its unique ID
"""
self.remove(uid)
def __len__(self):
if not hasattr(self, "length"):
self.length = Length(len(self.tree))
return self.length.value
class PersistentWaitingQueue(Persistent):
"""
A Waiting queue, implemented using a map structure (BTree...)
It is persistent, but very vulnerable to conflicts. This is due to the
fact that sets are used as container, and there can happen a situation
where two different sets are assigned to the same timestamp. This will
for sure result in conflict.
That said, the commits of objects like these have to be carefully
synchronized. See `indico.modules.scheduler.controllers` for more info
(particularly the way we use the 'spool').
"""
def __init__(self):
super(PersistentWaitingQueue, self).__init__()
self._reset()
def _reset(self):
# this counter keeps the number of elements
self._elem_counter = Length(0)
self._container = IOBTree()
def _gc_bin(self, t):
"""
'garbage-collect' bins
"""
if len(self._container[t]) == 0:
del self._container[t]
def _check_gc_consistency(self):
"""
'check that there are no empty bins'
"""
for t in self._container:
if len(self._container[t]) == 0:
return False
return True
def enqueue(self, t, obj):
"""
Add an element to the queue
"""
if t not in self._container:
self._container[t] = OOTreeSet()
if obj in self._container[t]:
raise DuplicateElementException(obj)
self._container[t].add(obj)
self._elem_counter.change(1)
def dequeue(self, t, obj):
"""
Remove an element from the queue
"""
self._container[t].remove(obj)
self._gc_bin(t)
self._elem_counter.change(-1)
def _next_timestamp(self):
"""
Return the next 'priority' to be served
"""
i = iter(self._container)
try:
t = i.next()
return t
except StopIteration:
return None
def peek(self):
"""
Return the next element
"""
t = self._next_timestamp()
if t:
# just to be sure
assert (len(self._container[t]) != 0)
# find the next element
i = iter(self._container[t])
# store it
elem = i.next()
# return the element
return t, elem
else:
return None
def pop(self):
"""
Remove and return the next set of elements to be processed
"""
pair = self.peek()
if pair:
self.dequeue(*pair)
# return the element
return pair
else:
return None
def nbins(self):
"""
Return the number of 'bins' (map entries) currently used
"""
# get 'real' len()
return len(self._container)
def __len__(self):
return self._elem_counter()
def __getitem__(self, param):
return self._container.__getitem__(param)
def __iter__(self):
# tree iterator
for tstamp in iter(self._container):
cur_set = self._container[tstamp]
try:
# set iterator
for elem in cur_set:
yield tstamp, elem
except StopIteration:
pass
class Lexicon(Persistent):
implements(ILexicon)
def __init__(self, *pipeline):
self._wids = OIBTree() # word -> wid
self._words = IOBTree() # wid -> word
# wid 0 is reserved for words that aren't in the lexicon (OOV -- out
# of vocabulary). This can happen, e.g., if a query contains a word
# we never saw before, and that isn't a known stopword (or otherwise
# filtered out). Returning a special wid value for OOV words is a
# way to let clients know when an OOV word appears.
self.length = Length()
self._pipeline = pipeline
def length(self):
"""Return the number of unique terms in the lexicon."""
# Overridden in instances
return len(self._wids)
def words(self):
return self._wids.keys()
def wids(self):
return self._words.keys()
def items(self):
return self._wids.items()
def sourceToWordIds(self, text):
last = _text2list(text)
for element in self._pipeline:
last = element.process(last)
if not hasattr(self.length, 'change'):
# Make sure length is overridden with a BTrees.Length.Length
self.length = Length(self.length())
# Strategically unload the length value so that we get the most
# recent value written to the database to minimize conflicting wids
# Because length is independent, this will load the most
# recent value stored, regardless of whether MVCC is enabled
self.length._p_deactivate()
return map(self._getWordIdCreate, last)
def termToWordIds(self, text):
last = _text2list(text)
for element in self._pipeline:
process = getattr(element, "process_post_glob", element.process)
last = process(last)
wids = []
for word in last:
wids.append(self._wids.get(word, 0))
return wids
def parseTerms(self, text):
last = _text2list(text)
for element in self._pipeline:
process = getattr(element, "processGlob", element.process)
last = process(last)
return last
def isGlob(self, word):
return "*" in word or "?" in word
def get_word(self, wid):
return self._words[wid]
def get_wid(self, word):
return self._wids.get(word, 0)
def globToWordIds(self, pattern):
# Implement * and ? just as in the shell, except the pattern
# must not start with either of these
prefix = ""
while pattern and pattern[0] not in "*?":
prefix += pattern[0]
pattern = pattern[1:]
if not pattern:
# There were no globbing characters in the pattern
wid = self._wids.get(prefix, 0)
if wid:
return [wid]
else:
return []
if not prefix:
# The pattern starts with a globbing character.
# This is too efficient, so we raise an exception.
raise QueryError("pattern %r shouldn't start with glob character" %
pattern)
pat = prefix
for c in pattern:
if c == "*":
pat += ".*"
elif c == "?":
pat += "."
else:
pat += re.escape(c)
pat += "$"
prog = re.compile(pat)
keys = self._wids.keys(prefix) # Keys starting at prefix
wids = []
for key in keys:
if not key.startswith(prefix):
break
if prog.match(key):
wids.append(self._wids[key])
return wids
def _getWordIdCreate(self, word):
wid = self._wids.get(word)
if wid is None:
wid = self._new_wid()
self._wids[word] = wid
self._words[wid] = word
return wid
def _new_wid(self):
self.length.change(1)
while self._words.has_key(self.length()): # just to be safe
self.length.change(1)
return self.length()
class KeywordIndex(Persistent):
"""Keyword index"""
family = BTrees.family32
# If a word is referenced by at least tree_threshold docids,
# use a TreeSet for that word instead of a Set.
tree_threshold = 64
def __init__(self, family=None):
if family is not None:
self.family = family
self.clear()
def clear(self):
"""Initialize forward and reverse mappings."""
# The forward index maps index keywords to a sequence of docids
self._fwd_index = self.family.OO.BTree()
# The reverse index maps a docid to its keywords
# TODO: Using a vocabulary might be the better choice to store
# keywords since it would allow use to use integers instead of strings
self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
def documentCount(self):
"""Return the number of documents in the index."""
return self._num_docs()
def wordCount(self):
"""Return the number of indexed words"""
return len(self._fwd_index)
def has_doc(self, docid):
return bool(docid in self._rev_index)
def normalize(self, seq):
"""Perform normalization on sequence of keywords.
Return normalized sequence. This method may be
overriden by subclasses.
"""
return seq
def index_doc(self, docid, seq):
if isinstance(seq, six.string_types):
raise TypeError('seq argument must be a list/tuple of strings')
old_kw = self._rev_index.get(docid, None)
if not seq:
if old_kw:
self.unindex_doc(docid)
return
seq = self.normalize(seq)
new_kw = self.family.OO.Set(seq)
if old_kw is None:
self._insert_forward(docid, new_kw)
self._insert_reverse(docid, new_kw)
self._num_docs.change(1)
else:
# determine added and removed keywords
kw_added = self.family.OO.difference(new_kw, old_kw)
kw_removed = self.family.OO.difference(old_kw, new_kw)
# removed keywords are removed from the forward index
for word in kw_removed:
fwd = self._fwd_index[word]
fwd.remove(docid)
if not fwd:
del self._fwd_index[word]
# now update reverse and forward indexes
self._insert_forward(docid, kw_added)
self._insert_reverse(docid, new_kw)
def unindex_doc(self, docid):
idx = self._fwd_index
try:
for word in self._rev_index[docid]:
idx[word].remove(docid)
if not idx[word]:
del idx[word]
except KeyError:
msg = 'WAAA! Inconsistent'
return
try:
del self._rev_index[docid]
except KeyError: #pragma NO COVERAGE
msg = 'WAAA! Inconsistent'
self._num_docs.change(-1)
def _insert_forward(self, docid, words):
"""insert a sequence of words into the forward index """
idx = self._fwd_index
get_word_idx = idx.get
IF = self.family.IF
Set = IF.Set
TreeSet = IF.TreeSet
for word in words:
word_idx = get_word_idx(word)
if word_idx is None:
idx[word] = word_idx = Set()
word_idx.insert(docid)
if (not isinstance(word_idx, TreeSet) and
len(word_idx) >= self.tree_threshold):
# Convert to a TreeSet.
idx[word] = TreeSet(word_idx)
def _insert_reverse(self, docid, words):
""" add words to forward index """
if words:
self._rev_index[docid] = words
def search(self, query, operator='and'):
"""Execute a search given by 'query'."""
if isinstance(query, six.string_types):
query = [query]
query = self.normalize(query)
sets = []
for word in query:
docids = self._fwd_index.get(word, self.family.IF.Set())
sets.append(docids)
if operator == 'or':
rs = self.family.IF.multiunion(sets)
elif operator == 'and':
# sort smallest to largest set so we intersect the smallest
# number of document identifiers possible
sets.sort(key=len)
rs = None
for set in sets:
rs = self.family.IF.intersection(rs, set)
if not rs:
break
else:
raise TypeError('Keyword index only supports `and` and `or` '
'operators, not `%s`.' % operator)
if rs:
return rs
else:
return self.family.IF.Set()
def apply(self, query):
operator = 'and'
if isinstance(query, dict):
if 'operator' in query:
operator = query['operator']
query = query['query']
return self.search(query, operator=operator)
def optimize(self):
"""Optimize the index. Call this after changing tree_threshold.
This converts internal data structures between
Sets and TreeSets based on tree_threshold.
"""
idx = self._fwd_index
IF = self.family.IF
Set = IF.Set
TreeSet = IF.TreeSet
items = list(self._fwd_index.items())
for word, word_idx in items:
if len(word_idx) >= self.tree_threshold:
if not isinstance(word_idx, TreeSet):
# Convert to a TreeSet.
idx[word] = TreeSet(word_idx)
else:
if isinstance(word_idx, TreeSet):
# Convert to a Set.
idx[word] = Set(word_idx)
class CatalogTool(PloneBaseTool, BaseTool):
"""Plone's catalog tool"""
meta_type = 'Plone Catalog Tool'
security = ClassSecurityInfo()
toolicon = 'skins/plone_images/book_icon.png'
_counter = None
manage_catalogAdvanced = DTMLFile('www/catalogAdvanced', globals())
manage_options = (
{
'action': 'manage_main',
'label': 'Contents'
},
{
'action': 'manage_catalogView',
'label': 'Catalog'
},
{
'action': 'manage_catalogIndexes',
'label': 'Indexes'
},
{
'action': 'manage_catalogSchema',
'label': 'Metadata'
},
{
'action': 'manage_catalogAdvanced',
'label': 'Advanced'
},
{
'action': 'manage_catalogReport',
'label': 'Query Report'
},
{
'action': 'manage_catalogPlan',
'label': 'Query Plan'
},
{
'action': 'manage_propertiesForm',
'label': 'Properties'
},
)
def __init__(self):
ZCatalog.__init__(self, self.getId())
def _removeIndex(self, index):
# Safe removal of an index.
try:
self.manage_delIndex(index)
except:
pass
def _listAllowedRolesAndUsers(self, user):
# Makes sure the list includes the user's groups.
result = user.getRoles()
if 'Anonymous' in result:
# The anonymous user has no further roles
return ['Anonymous']
result = list(result)
if hasattr(aq_base(user), 'getGroups'):
groups = ['user:%s' % x for x in user.getGroups()]
if groups:
result = result + groups
# Order the arguments from small to large sets
result.insert(0, 'user:%s' % user.getId())
result.append('Anonymous')
return result
@security.private
def indexObject(self, object, idxs=None):
# Add object to catalog.
# The optional idxs argument is a list of specific indexes
# to populate (all of them by default).
if idxs is None:
idxs = []
self.reindexObject(object, idxs)
@security.protected(ManageZCatalogEntries)
def catalog_object(self,
object,
uid=None,
idxs=None,
update_metadata=1,
pghandler=None):
if idxs is None:
idxs = []
self._increment_counter()
w = object
if not IIndexableObject.providedBy(object):
# This is the CMF 2.2 compatible approach, which should be used
# going forward
wrapper = queryMultiAdapter((object, self), IIndexableObject)
if wrapper is not None:
w = wrapper
ZCatalog.catalog_object(self,
w,
uid,
idxs,
update_metadata,
pghandler=pghandler)
@security.protected(ManageZCatalogEntries)
def uncatalog_object(self, *args, **kwargs):
self._increment_counter()
return BaseTool.uncatalog_object(self, *args, **kwargs)
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
@security.private
def getCounter(self):
processQueue()
return self._counter is not None and self._counter() or 0
@security.private
def allow_inactive(self, query_kw):
"""Check, if the user is allowed to see inactive content.
First, check if the user is allowed to see inactive content site-wide.
Second, if there is a 'path' key in the query, check if the user is
allowed to see inactive content for these paths.
Conservative check: as soon as one path is disallowed, return False.
If a path cannot be traversed, ignore it.
"""
allow_inactive = _checkPermission(AccessInactivePortalContent, self)
if allow_inactive:
return True
paths = query_kw.get('path', False)
if not paths:
return False
if isinstance(paths, dict):
# Like: {'path': {'depth': 0, 'query': ['/Plone/events/']}}
# Or: {'path': {'depth': 0, 'query': '/Plone/events/'}}
paths = paths.get('query', [])
if isinstance(paths, six.string_types):
paths = [paths]
objs = []
site = getSite()
for path in list(paths):
path = path.encode('utf-8') # paths must not be unicode
try:
site_path = '/'.join(site.getPhysicalPath())
parts = path[len(site_path) + 1:].split('/')
parent = site.unrestrictedTraverse('/'.join(parts[:-1]))
objs.append(parent.restrictedTraverse(parts[-1]))
except (KeyError, AttributeError, Unauthorized):
# When no object is found don't raise an error
pass
if not objs:
return False
allow = True
for ob in objs:
allow = allow and\
_checkPermission(AccessInactivePortalContent, ob)
return allow
@security.protected(SearchZCatalog)
def searchResults(self, query=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
# Make sure any pending index tasks have been processed
processQueue()
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(query, dict) and not show_inactive:
show_inactive = 'show_inactive' in query
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not self.allow_inactive(kw):
kw['effectiveRange'] = DateTime()
sort_on = kw.get('sort_on')
if sort_on and sort_on not in self.indexes():
# I get crazy sort_ons like '194' or 'null'.
kw.pop('sort_on')
return ZCatalog.searchResults(self, query, **kw)
__call__ = searchResults
def search(self, query, sort_index=None, reverse=0, limit=None, merge=1):
# Wrap search() the same way that searchResults() is
# Make sure any pending index tasks have been processed
processQueue()
user = _getAuthenticatedUser(self)
query['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not self.allow_inactive(query):
query['effectiveRange'] = DateTime()
return super(CatalogTool, self).search(query, sort_index, reverse,
limit, merge)
@security.protected(ManageZCatalogEntries)
def clearFindAndRebuild(self):
# Empties catalog, then finds all contentish objects (i.e. objects
# with an indexObject method), and reindexes them.
# This may take a long time.
def indexObject(obj, path):
if (base_hasattr(obj, 'indexObject')
and safe_callable(obj.indexObject)):
try:
obj.indexObject()
# index conversions from plone.app.discussion
annotions = IAnnotations(obj)
catalog = getToolByName(obj, "portal_catalog")
if DISCUSSION_ANNOTATION_KEY in annotions:
conversation = annotions[DISCUSSION_ANNOTATION_KEY]
conversation = conversation.__of__(obj)
for comment in conversation.getComments():
try:
if catalog:
catalog.indexObject(comment)
except StopIteration: # pragma: no cover
pass
except TypeError:
# Catalogs have 'indexObject' as well, but they
# take different args, and will fail
pass
self.manage_catalogClear()
portal = aq_parent(aq_inner(self))
portal.ZopeFindAndApply(portal,
search_sub=True,
apply_func=indexObject)
@security.protected(ManageZCatalogEntries)
def manage_catalogRebuild(self, RESPONSE=None, URL1=None):
"""Clears the catalog and indexes all objects with an 'indexObject'
method. This may take a long time.
"""
elapse = time.time()
c_elapse = time.clock()
self.clearFindAndRebuild()
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
msg = ('Catalog Rebuilt\n'
'Total time: %s\n'
'Total CPU time: %s' % (repr(elapse), repr(c_elapse)))
logger.info(msg)
if RESPONSE is not None:
RESPONSE.redirect(URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
urllib.parse.quote(msg))
class GranularIndex(CatalogFieldIndex):
"""Indexes integer values using multiple granularity levels.
The multiple levels of granularity make it possible to query large
ranges without loading many IFTreeSets from the forward index.
"""
implements(
ICatalogIndex,
IStatistics,
)
def __init__(self, discriminator, levels=(1000, )):
"""Create an index.
levels is a sequence of integer coarseness levels.
The default is (1000,).
"""
self._levels = tuple(levels)
super(GranularIndex, self).__init__(discriminator)
def clear(self):
"""Initialize all mappings."""
# The forward index maps an indexed value to IFSet(docids)
self._fwd_index = self.family.IO.BTree()
# The reverse index maps a docid to its index value
self._rev_index = self.family.II.BTree()
self._num_docs = Length(0)
# self._granular_indexes: [(level, BTree(value -> IFSet([docid])))]
self._granular_indexes = [(level, self.family.IO.BTree())
for level in self._levels]
def index_doc(self, docid, obj):
if callable(self.discriminator):
value = self.discriminator(obj, _marker)
else:
value = getattr(obj, self.discriminator, _marker)
if value is _marker:
# unindex the previous value
self.unindex_doc(docid)
return
if not isinstance(value, int):
raise ValueError(
'GranularIndex cannot index non-integer value %s' % value)
rev_index = self._rev_index
if docid in rev_index:
if docid in self._fwd_index.get(value, ()):
# There's no need to index the doc; it's already up to date.
return
# unindex doc if present
self.unindex_doc(docid)
# Insert into forward index.
set = self._fwd_index.get(value)
if set is None:
set = self.family.IF.TreeSet()
self._fwd_index[value] = set
set.insert(docid)
# increment doc count
self._num_docs.change(1)
# Insert into reverse index.
rev_index[docid] = value
for level, ndx in self._granular_indexes:
v = value // level
set = ndx.get(v)
if set is None:
set = self.family.IF.TreeSet()
ndx[v] = set
set.insert(docid)
def unindex_doc(self, docid):
rev_index = self._rev_index
value = rev_index.get(docid)
if value is None:
return # not in index
del rev_index[docid]
self._num_docs.change(-1)
ndx = self._fwd_index
try:
set = ndx[value]
set.remove(docid)
if not set:
del ndx[value]
except KeyError:
pass
for level, ndx in self._granular_indexes:
v = value // level
try:
set = ndx[v]
set.remove(docid)
if not set:
del ndx[v]
except KeyError:
pass
def search(self, queries, operator='or'):
sets = []
for query in queries:
if isinstance(query, Range):
query = query.as_tuple()
else:
query = (query, query)
set = self.family.IF.multiunion(self.docids_in_range(*query))
sets.append(set)
result = None
if len(sets) == 1:
result = sets[0]
elif operator == 'and':
sets.sort()
for set in sets:
result = self.family.IF.intersection(set, result)
else:
result = self.family.IF.multiunion(sets)
return result
def docids_in_range(self, min, max):
"""List the docids for an integer range, inclusive on both ends.
min or max can be None, making them unbounded.
Returns an iterable of IFSets.
"""
for level, ndx in sorted(self._granular_indexes, reverse=True):
# Try to fill the range using coarse buckets first.
# Use only buckets that completely fill the range.
# For example, if start is 2 and level is 10, then we can't
# use bucket 0; only buckets 1 and greater are useful.
# Similarly, if end is 18 and level is 10, then we can't use
# bucket 1; only buckets 0 and less are useful.
if min is not None:
a = (min + level - 1) // level
else:
a = None
if max is not None:
b = (max - level + 1) // level
else:
b = None
# a and b are now coarse bucket values (or None).
if a is None or b is None or a <= b:
sets = []
if a is not None and min < a * level:
# include the gap before
sets.extend(self.docids_in_range(min, a * level - 1))
sets.extend(ndx.values(a, b))
if b is not None and (b + 1) * level - 1 < max:
# include the gap after
sets.extend(self.docids_in_range((b + 1) * level, max))
return sets
return self._fwd_index.values(min, max)
class Folder(Persistent):
"""
A folder implementation which acts much like a Python dictionary.
keys are Unicode strings; values are arbitrary Python objects.
"""
# _num_objects=None below is b/w compat for older instances of
# folders which don't have a BTrees.Length object as a
# _num_objects attribute.
_num_objects = None
__name__ = None
__parent__ = None
# Default uses ordering of underlying BTree.
_order = None
def _get_order(self):
if self._order is not None:
return list(self._order)
return self.data.keys()
def _set_order(self, value):
# XXX: should we test against self.data.keys()?
self._order = tuple([unicodify(x) for x in value])
def _del_order(self):
del self._order
order = property(_get_order, _set_order, _del_order)
def __init__(self, data=None):
if data is None:
data = {}
self.data = OOBTree(data)
self._num_objects = Length(len(data))
def keys(self):
"""See IFolder."""
return self.order
def __iter__(self):
return iter(self.order)
def values(self):
"""See IFolder."""
if self._order is not None:
return [self.data[name] for name in self.order]
return self.data.values()
def items(self):
"""See IFolder."""
if self._order is not None:
return [(name, self.data[name]) for name in self.order]
return self.data.items()
def __len__(self):
"""See IFolder."""
if self._num_objects is None:
# can be arbitrarily expensive
return len(self.data)
return self._num_objects()
def __nonzero__(self):
"""See IFolder."""
return True
def __getitem__(self, name):
"""See IFolder."""
name = unicodify(name)
return self.data[name]
def get(self, name, default=None):
"""See IFolder."""
name = unicodify(name)
return self.data.get(name, default)
def __contains__(self, name):
"""See IFolder."""
name = unicodify(name)
return name in self.data
def __setitem__(self, name, other):
"""See IFolder."""
return self.add(name, other)
def add(self, name, other, send_events=True):
"""See IFolder."""
if not isinstance(name, basestring):
raise TypeError("Name must be a string rather than a %s" %
name.__class__.__name__)
if not name:
raise TypeError("Name must not be empty")
name = unicodify(name)
if name in self.data:
raise KeyError('An object named %s already exists' % name)
if send_events:
objectEventNotify(ObjectWillBeAddedEvent(other, self, name))
other.__parent__ = self
other.__name__ = name
# backwards compatibility: add a Length _num_objects to folders that
# have none
if self._num_objects is None:
self._num_objects = Length(len(self.data))
self.data[name] = other
self._num_objects.change(1)
if self._order is not None:
self._order += (name, )
if send_events:
objectEventNotify(ObjectAddedEvent(other, self, name))
def __delitem__(self, name):
"""See IFolder."""
return self.remove(name)
def remove(self, name, send_events=True):
"""See IFolder."""
name = unicodify(name)
other = self.data[name]
if send_events:
objectEventNotify(ObjectWillBeRemovedEvent(other, self, name))
if hasattr(other, '__parent__'):
del other.__parent__
if hasattr(other, '__name__'):
del other.__name__
# backwards compatibility: add a Length _num_objects to folders that
# have none
if self._num_objects is None:
self._num_objects = Length(len(self.data))
del self.data[name]
self._num_objects.change(-1)
if self._order is not None:
self._order = tuple([x for x in self._order if x != name])
if send_events:
objectEventNotify(ObjectRemovedEvent(other, self, name))
return other
def pop(self, name, default=marker):
"""See IFolder."""
try:
result = self.remove(name)
except KeyError:
if default is marker:
raise
return default
return result
def __repr__(self):
klass = self.__class__
classname = '%s.%s' % (klass.__module__, klass.__name__)
return '<%s object %r at %#x>' % (classname, self.__name__, id(self))
class Folder(Persistent):
""" A folder implementation which acts much like a Python dictionary.
Keys must be Unicode strings; values must be arbitrary Python objects.
"""
family = BTrees.family64
__name__ = None
__parent__ = None
# Default uses ordering of underlying BTree.
_order = None # tuple of names
_order_oids = None # tuple of oids
_reorderable = None
def __init__(self, data=None, family=None):
""" Constructor. Data may be an initial dictionary mapping object
name to object. """
if family is not None:
self.family = family
if data is None:
data = {}
self.data = self.family.OO.BTree(data)
self._num_objects = Length(len(data))
def set_order(self, names, reorderable=None):
""" Sets the folder order. ``names`` is a list of names for existing
folder items, in the desired order. All names that currently exist in
the folder must be mentioned in ``names``, or a :exc:`ValueError` will
be raised.
If ``reorderable`` is passed, value, it must be ``None``, ``True`` or
``False``. If it is ``None``, the reorderable flag will not be reset
from its current value. If it is anything except ``None``, it will be
treated as a boolean and the reorderable flag will be set to that
value. The ``reorderable`` value of a folder will be returned by that
folder's :meth:`~substanced.folder.Folder.is_reorderable` method. The
:meth:`~substanced.folder.Folder.is_reorderable` method is used by the
SDI folder contents view to indicate that the folder can or cannot be
reordered via the web UI.
If ``reorderable`` is set to ``True``, the
:meth:`~substanced.folder.Folder.reorder` method will work properly,
otherwise it will raise a :exc:`ValueError` when called.
"""
nameset = set(names)
if len(self) != len(nameset):
raise ValueError('Must specify all names when calling set_order')
if len(names) != len(nameset):
raise ValueError('No repeated items allowed in names')
order = []
order_oids = []
for name in names:
assert (isinstance(name, string_types))
name = u(name)
oid = get_oid(self[name])
order.append(name)
order_oids.append(oid)
self._order = tuple(order)
self._order_oids = tuple(order_oids)
assert (len(self._order) == len(self._order_oids))
if reorderable is not None:
self._reorderable = bool(reorderable)
def unset_order(self):
""" Remove set order from a folder, making it unordered, and
non-reorderable."""
if self._order is not None:
del self._order
if self._order_oids is not None:
del self._order_oids
if self._reorderable is not None:
del self._reorderable
def reorder(self, names, before):
""" Move one or more items from a folder into new positions inside that
folder. ``names`` is a list of ids of existing folder subobject names,
which will be inserted in order before the item named ``before``. All
other items are left in the original order. If ``before`` is ``None``,
the items will be appended after the last item in the current order. If
this method is called on a folder which does not have an order set, or
which is not reorderable, a :exc:`ValueError` will be raised. A
:exc:`KeyError` is raised, if ``before`` does not correspond to any
item, and is not ``None``."""
if not self._reorderable:
raise ValueError('Folder is not reorderable')
before_idx = None
if len(set(names)) != len(names):
raise ValueError('No repeated values allowed in names')
if before is not None:
if not before in self._order:
raise FolderKeyError(before)
before_idx = self._order.index(before)
assert (len(self._order) == len(self._order_oids))
order_names = list(self._order)
order_oids = list(self._order_oids)
reorder_names = []
reorder_oids = []
for name in names:
assert (isinstance(name, string_types))
name = u(name)
if not name in order_names:
raise FolderKeyError(name)
idx = order_names.index(name)
oid = order_oids[idx]
order_names[idx] = None
order_oids[idx] = None
reorder_names.append(name)
reorder_oids.append(oid)
assert (len(reorder_names) == len(reorder_oids))
# NB: technically we could use filter(None, oids) and filter(None,
# names) because names cannot be empty string and oid 0 is disallowed,
# but just in case this becomes untrue later we define "filt" instead
def filt(L):
return [x for x in L if x is not None]
if before_idx is None:
order_names = filt(order_names)
order_names.extend(reorder_names)
order_oids = filt(order_oids)
order_oids.extend(reorder_oids)
else:
before_idx_names = filt(order_names[:before_idx])
after_idx_names = filt(order_names[before_idx:])
before_idx_oids = filt(order_oids[:before_idx])
after_idx_oids = filt(order_oids[before_idx:])
assert (len(before_idx_names +
after_idx_names) == len(before_idx_oids +
after_idx_oids))
order_names = before_idx_names + reorder_names + after_idx_names
order_oids = before_idx_oids + reorder_oids + after_idx_oids
for oid, name in zip(order_oids, order_names):
# belt and suspenders check
assert oid == get_oid(self[name])
self._order = tuple(order_names)
self._order_oids = tuple(order_oids)
def is_ordered(self):
""" Return true if the folder has a manually set ordering, false
otherwise."""
return self._order is not None
def is_reorderable(self):
""" Return true if the folder can be reordered, false otherwise."""
return self._reorderable
def sort(self, oids, reverse=False, limit=None, **kw):
# used by the hypatia resultset "sort" method when the folder contents
# view uses us as a "sort index"
if self._order_oids is not None:
ids = [oid for oid in self._order_oids if oid in oids]
else:
ids = []
for resource in self.values():
oid = get_oid(resource)
if oid in oids:
ids.append(oid)
if reverse:
ids = ids[::-1]
if limit is not None:
ids = ids[:limit]
return ids
def find_service(self, service_name):
""" Return a service named by ``service_name`` in this folder *or any
parent service folder* or ``None`` if no such service exists. A
shortcut for :func:`substanced.service.find_service`."""
return find_service(self, service_name)
def find_services(self, service_name):
""" Returns a sequence of service objects named by ``service_name``
in this folder's lineage or an empty sequence if no such service
exists. A shortcut for :func:`substanced.service.find_services`"""
return find_services(self, service_name)
def add_service(self, name, obj, registry=None, **kw):
""" Add a service to this folder named ``name``."""
if registry is None:
registry = get_current_registry()
kw['registry'] = registry
self.add(name, obj, **kw)
alsoProvides(obj, IService)
def keys(self):
""" Return an iterable sequence of object names present in the folder.
Respect order, if set.
"""
if self._order is not None:
return self._order
return self.data.keys()
order = property(keys, set_order, unset_order) # b/c
def __iter__(self):
""" An alias for ``keys``
"""
return iter(self.keys())
def values(self):
""" Return an iterable sequence of the values present in the folder.
Respect ``order``, if set.
"""
if self._order is not None:
return [self.data[name] for name in self.keys()]
return self.data.values()
def items(self):
""" Return an iterable sequence of (name, value) pairs in the folder.
Respect ``order``, if set.
"""
if self._order is not None:
return [(name, self.data[name]) for name in self.keys()]
return self.data.items()
def __len__(self):
""" Return the number of objects in the folder.
"""
return self._num_objects()
def __nonzero__(self):
""" Return ``True`` unconditionally.
"""
return True
__bool__ = __nonzero__
def __repr__(self):
klass = self.__class__
classname = '%s.%s' % (klass.__module__, klass.__name__)
return '<%s object %r at %#x>' % (classname, self.__name__, id(self))
def __getitem__(self, name):
""" Return the object named ``name`` added to this folder or raise
``KeyError`` if no such object exists. ``name`` must be a Unicode
object or directly decodeable to Unicode using the system default
encoding.
"""
with statsd_timer('folder.get'):
name = u(name)
return wrap_if_broken(self.data[name])
def get(self, name, default=None):
""" Return the object named by ``name`` or the default.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
"""
with statsd_timer('folder.get'):
name = u(name)
return wrap_if_broken(self.data.get(name, default))
def __contains__(self, name):
""" Does the container contains an object named by name?
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
"""
name = u(name)
return name in self.data
def __setitem__(self, name, other):
""" Set object ``other`` into this folder under the name ``name``.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
``name`` cannot be the empty string.
When ``other`` is seated into this folder, it will also be decorated
with a ``__parent__`` attribute (a reference to the folder into which
it is being seated) and ``__name__`` attribute (the name passed in to
this function. It must not already have a ``__parent__`` attribute
before being seated into the folder, or an exception will be raised.
If a value already exists in the foldr under the name ``name``, raise
:exc:`KeyError`.
When this method is called, the object will be added to the objectmap,
an :class:`substanced.event.ObjectWillBeAdded` event will be emitted
before the object obtains a ``__name__`` or ``__parent__`` value, then
a :class:`substanced.event.ObjectAdded` will be emitted after the
object obtains a ``__name__`` and ``__parent__`` value.
"""
return self.add(name, other)
def validate_name(self, name, reserved_names=()):
"""
Validate the ``name`` passed to ensure that it's addable to the folder.
Returns the name decoded to Unicode if it passes all addable checks.
It's not addable if:
- the name is not decodeable to Unicode.
- the name starts with ``@@`` (conflicts with explicit view names).
- the name has slashes in it (WSGI limitation).
- the name is empty.
If any of these conditions are untrue, raise a :exc:`ValueError`. If
the name passed is in the list of ``reserved_names``, raise a
:exc:`ValueError`.
"""
if not isinstance(name, STRING_TYPES):
raise ValueError("Name must be a string rather than a %s" %
name.__class__.__name__)
if not name:
raise ValueError("Name must not be empty")
try:
name = u(name)
except UnicodeDecodeError: #pragma NO COVER (on Py3k)
raise ValueError('Name "%s" not decodeable to unicode' % name)
if name in reserved_names:
raise ValueError('%s is a reserved name' % name)
if name.startswith('@@'):
raise ValueError('Names which start with "@@" are not allowed')
if '/' in name:
raise ValueError('Names which contain a slash ("/") are not '
'allowed')
return name
def check_name(self, name, reserved_names=()):
""" Perform all the validation checks implied by
:meth:`~substanced.folder.Folder.validate_name` against the ``name``
supplied but also fail with a
:class:`~substanced.folder.FolderKeyError` if an object with the name
``name`` already exists in the folder."""
name = self.validate_name(name, reserved_names=reserved_names)
if name in self.data:
raise FolderKeyError('An object named %s already exists' % name)
return name
def add(self,
name,
other,
send_events=True,
reserved_names=(),
duplicating=None,
moving=None,
loading=False,
registry=None):
""" Same as ``__setitem__``.
If ``send_events`` is False, suppress the sending of folder events.
Don't allow names in the ``reserved_names`` sequence to be added.
If ``duplicating`` not ``None``, it must be the object which is being
duplicated; a result of a non-``None`` duplicating means that oids will
be replaced in objectmap. If ``moving`` is not ``None``, it must be
the folder from which the object is moving; this will be the ``moving``
attribute of events sent by this function too. If ``loading`` is
``True``, the ``loading`` attribute of events sent as a result of
calling this method will be ``True`` too.
This method returns the name used to place the subobject in the
folder (a derivation of ``name``, usually the result of
``self.check_name(name)``).
"""
if registry is None:
registry = get_current_registry()
name = self.check_name(name, reserved_names)
if getattr(other, '__parent__', None):
raise ValueError(
'obj %s added to folder %s already has a __parent__ attribute, '
'please remove it completely from its existing parent (%s) '
'before trying to readd it to this one' %
(other, self, self.__parent__))
with statsd_timer('folder.add'):
objectmap = find_objectmap(self)
if objectmap is not None:
basepath = resource_path_tuple(self)
for node in postorder(other):
node_path = node_path_tuple(node)
path_tuple = basepath + (name, ) + node_path[1:]
# the below gives node an objectid; if the will-be-added
# event is the result of a duplication, replace the oid of
# the node with a new one
objectmap.add(
node,
path_tuple,
duplicating=duplicating is not None,
moving=moving is not None,
)
if send_events:
event = ObjectWillBeAdded(
other,
self,
name,
duplicating=duplicating,
moving=moving,
loading=loading,
)
self._notify(event, registry)
other.__parent__ = self
other.__name__ = name
self.data[name] = other
self._num_objects.change(1)
if self._order is not None:
oid = get_oid(other)
self._order += (name, )
self._order_oids += (oid, )
if send_events:
event = ObjectAdded(
other,
self,
name,
duplicating=duplicating,
moving=moving,
loading=loading,
)
self._notify(event, registry)
return name
def pop(self, name, default=marker, registry=None):
""" Remove the item stored in the under ``name`` and return it.
If ``name`` doesn't exist in the folder, and ``default`` **is not**
passed, raise a :exc:`KeyError`.
If ``name`` doesn't exist in the folder, and ``default`` **is**
passed, return ``default``.
When the object stored under ``name`` is removed from this folder,
remove its ``__parent__`` and ``__name__`` values.
When this method is called, emit an
:class:`substanced.event.ObjectWillBeRemoved` event before the
object loses its ``__name__`` or ``__parent__`` values. Emit an
:class:`substanced.event.ObjectRemoved` after the object loses its
``__name__`` and ``__parent__`` value,
"""
if registry is None:
registry = get_current_registry()
try:
result = self.remove(name, registry=registry)
except KeyError:
if default is marker:
raise
return default
return result
def _notify(self, event, registry=None):
if registry is None:
registry = get_current_registry()
registry.subscribers((event, event.object, self), None)
def __delitem__(self, name):
""" Remove the object from this folder stored under ``name``.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
If no object is stored in the folder under ``name``, raise a
:exc:`KeyError`.
When the object stored under ``name`` is removed from this folder,
remove its ``__parent__`` and ``__name__`` values.
When this method is called, the removed object will be removed from the
objectmap, a :class:`substanced.event.ObjectWillBeRemoved` event will
be emitted before the object loses its ``__name__`` or ``__parent__``
values and a :class:`substanced.event.ObjectRemoved` will be emitted
after the object loses its ``__name__`` and ``__parent__`` value,
"""
return self.remove(name)
def remove(self,
name,
send_events=True,
moving=None,
loading=False,
registry=None):
""" Same thing as ``__delitem__``.
If ``send_events`` is false, suppress the sending of folder events.
If ``moving`` is not ``None``, the ``moving`` argument must be the
folder to which the named object will be moving. This value will be
passed along as the ``moving`` attribute of the events sent as the
result of this action. If ``loading`` is ``True``, the ``loading``
attribute of events sent as a result of calling this method will be
``True`` too.
"""
name = u(name)
other = wrap_if_broken(self.data[name])
oid = get_oid(other, None)
if registry is None:
registry = get_current_registry()
with statsd_timer('folder.remove'):
if send_events:
event = ObjectWillBeRemoved(other,
self,
name,
moving=moving,
loading=loading)
self._notify(event, registry)
if hasattr(other, '__parent__'):
try:
del other.__parent__
except AttributeError:
# this might be a broken object
pass
if hasattr(other, '__name__'):
try:
del other.__name__
except AttributeError:
# this might be a broken object
pass
del self.data[name]
self._num_objects.change(-1)
if self._order is not None:
assert (len(self._order) == len(self._order_oids))
idx = self._order.index(name)
order = list(self._order)
order.pop(idx)
order_oids = list(self._order_oids)
order_oids.pop(idx)
self._order = tuple(order)
self._order_oids = tuple(order_oids)
objectmap = find_objectmap(self)
removed_oids = set([oid])
if objectmap is not None and oid is not None:
removed_oids = objectmap.remove(oid, moving=moving is not None)
if send_events:
event = ObjectRemoved(other,
self,
name,
removed_oids,
moving=moving,
loading=loading)
self._notify(event, registry)
return other
def copy(self, name, other, newname=None, registry=None):
"""
Copy a subobject named ``name`` from this folder to the folder
represented by ``other``. If ``newname`` is not none, it is used as
the target object name; otherwise the existing subobject name is
used.
"""
if newname is None:
newname = name
if registry is None:
registry = get_current_registry()
with statsd_timer('folder.copy'):
obj = self[name]
newobj = copy(obj)
return other.add(newname,
newobj,
duplicating=obj,
registry=registry)
def move(self, name, other, newname=None, registry=None):
"""
Move a subobject named ``name`` from this folder to the folder
represented by ``other``. If ``newname`` is not none, it is used as
the target object name; otherwise the existing subobject name is
used.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events as well as the Added and WillBeAdded events
sent will indicate that the object is moving.
"""
if newname is None:
newname = name
if registry is None:
registry = get_current_registry()
ob = self.remove(name, moving=other, registry=registry)
other.add(newname, ob, moving=self, registry=registry)
return ob
def rename(self, oldname, newname, registry=None):
"""
Rename a subobject from oldname to newname.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events sent will indicate that the object is
moving.
"""
if registry is None:
registry = get_current_registry()
return self.move(oldname, self, newname, registry=registry)
def replace(self, name, newobject, send_events=True, registry=None):
""" Replace an existing object named ``name`` in this folder with a
new object ``newobject``. If there isn't an object named ``name`` in
this folder, an exception will *not* be raised; instead, the new
object will just be added.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events will be sent for the old object, and the
WillBeAdded and Added events will be sent for the new object.
"""
if registry is None:
registry = get_current_registry()
if name in self:
self.remove(name, send_events=send_events)
self.add(name, newobject, send_events=send_events, registry=registry)
def load(self, name, newobject, registry=None):
""" A replace method used by the code that loads an existing dump.
Events sent during this replace will have a true ``loading`` flag."""
if registry is None:
registry = get_current_registry()
if name in self:
self.remove(name, loading=True)
self.add(name, newobject, loading=True, registry=registry)
def clear(self, registry=None):
""" Clear all items from the folder. This is the equivalent of calling
``.remove`` with each key that exists in the folder. """
if registry is None:
registry = get_current_registry()
for name in self:
self.remove(name, registry=registry)
class EventEndDateIndex(Persistent):
""" List of bookings ordered by their event's ending date
"""
def __init__(self):
self._tree = LOBTree.LOBTree()
self._count = Length(0)
## private class methods ##
@classmethod
def _dateToKey(cls, date):
if date:
return datetimeToUnixTimeInt(date)
else:
return None
@classmethod
def _keyToDate(cls, key):
if key:
return unixTimeToDatetime(key)
else:
return None
@classmethod
def _bookingToKey(cls, booking):
return cls._dateToKey(
booking.getConference().getAdjustedEndDate(tz='UTC'))
## public instance methods ##
def clear(self):
""" Clears all the information stored
"""
self._tree = LOBTree.LOBTree()
self._count = Length(0)
def getCount(self):
""" Returns the number of bookings (not keys) stored
"""
return self._count(
) #to get the value of a Length object, one has to "call" the object
def indexBooking(self, booking):
""" Stores a booking in the index
"""
key = EventEndDateIndex._bookingToKey(booking)
if not key in self._tree:
self._tree[key] = DateBookingList()
self._tree[key].addBooking(booking)
self._count.change(1)
def unindexBooking(self, booking):
""" Removes a booking from the index
"""
key = EventEndDateIndex._bookingToKey(booking)
try:
self._tree[key].removeBooking(booking)
if self._tree[key].getCount() == 0:
del self._tree[key]
self._count.change(-1)
except KeyError:
Logger.get('Vidyo').warning(
"Could not unindex booking: (confId=%s, id=%s) from Vidyo's GlobalData. Tried with key: %s."
% (booking.getConference().getId(), booking.getId(), str(key)))
def moveBooking(self, booking, oldDate):
""" Changes the position of a booking in the index
"""
oldKey = EventEndDateIndex._dateToKey(oldDate)
newKey = EventEndDateIndex._bookingToKey(booking)
try:
self._tree[oldKey].removeBooking(booking)
if self._tree[oldKey].getCount() == 0:
del self._tree[oldKey]
if not newKey in self._tree:
self._tree[newKey] = DateBookingList()
self._tree[newKey].addBooking(booking)
except KeyError:
Logger.get('Vidyo').warning(
"Could not move booking: (confId=%s, id=%s) from Vidyo's GlobalData. Tried moving from key: %s to key: %s."
% (booking.getConference().getId(), booking.getId(),
str(oldKey), str(newKey)))
def iterbookings(self, minDate=None, maxDate=None):
""" Will return an iterator over Vidyo bookings attached to conferences whose
end date is between minDate and maxDate
"""
minKey = EventEndDateIndex._dateToKey(minDate)
maxKey = EventEndDateIndex._dateToKey(maxDate)
for bookingList in self._tree.itervalues(min=minKey, max=maxKey):
for b in bookingList.iterbookings():
yield b
def deleteKeys(self, minDate=None, maxDate=None):
"""
"""
minKey = EventEndDateIndex._dateToKey(minDate)
maxKey = EventEndDateIndex._dateToKey(maxDate)
for key in list(self._tree.keys(
min=minKey,
max=maxKey)): #we want a copy because we are going to modify
self._deleteKey(key)
def _deleteKey(self, key):
Logger.get("Vidyo").info(
"Vidyo EventEndDateIndex: deleting key %s (%s)" %
(str(key), str(EventEndDateIndex._keyToDate(key)) + " (UTC)"))
self._count.change(-self._tree[key].getCount())
del self._tree[key]
def initialize(self, dbi=None):
""" Cleans the indexes, and then indexes all the vidyo bookings from all the conferences
WARNING: obviously, this can potentially take a while
"""
i = 0
self.clear()
for conf in ConferenceHolder().getList():
csbm = Catalog.getIdx("cs_bookingmanager_conference").get(
conf.getId())
for booking in csbm.getBookingList():
if booking.getType() == "Vidyo" and booking.isCreated():
self.indexBooking(booking)
i += 1
if dbi and i % 100 == 0:
dbi.commit()
if dbi:
dbi.commit()
class CatalogTool(PloneBaseTool, BaseTool):
"""Plone's catalog tool"""
meta_type = 'Plone Catalog Tool'
security = ClassSecurityInfo()
toolicon = 'skins/plone_images/book_icon.png'
_counter = None
manage_catalogAdvanced = DTMLFile('www/catalogAdvanced', globals())
manage_options = (
{'action': 'manage_main', 'label': 'Contents'},
{'action': 'manage_catalogView', 'label': 'Catalog'},
{'action': 'manage_catalogIndexes', 'label': 'Indexes'},
{'action': 'manage_catalogSchema', 'label': 'Metadata'},
{'action': 'manage_catalogAdvanced', 'label': 'Advanced'},
{'action': 'manage_catalogReport', 'label': 'Query Report'},
{'action': 'manage_catalogPlan', 'label': 'Query Plan'},
{'action': 'manage_propertiesForm', 'label': 'Properties'},
)
def __init__(self):
ZCatalog.__init__(self, self.getId())
def _removeIndex(self, index):
# Safe removal of an index.
try:
self.manage_delIndex(index)
except:
pass
def _listAllowedRolesAndUsers(self, user):
# Makes sure the list includes the user's groups.
result = user.getRoles()
if 'Anonymous' in result:
# The anonymous user has no further roles
return ['Anonymous']
result = list(result)
if hasattr(aq_base(user), 'getGroups'):
groups = ['user:%s' % x for x in user.getGroups()]
if groups:
result = result + groups
# Order the arguments from small to large sets
result.insert(0, 'user:%s' % user.getId())
result.append('Anonymous')
return result
@security.private
def indexObject(self, object, idxs=None):
# Add object to catalog.
# The optional idxs argument is a list of specific indexes
# to populate (all of them by default).
if idxs is None:
idxs = []
self.reindexObject(object, idxs)
@security.protected(ManageZCatalogEntries)
def catalog_object(self, object, uid=None, idxs=None,
update_metadata=1, pghandler=None):
if idxs is None:
idxs = []
self._increment_counter()
w = object
if not IIndexableObject.providedBy(object):
# This is the CMF 2.2 compatible approach, which should be used
# going forward
wrapper = queryMultiAdapter((object, self), IIndexableObject)
if wrapper is not None:
w = wrapper
ZCatalog.catalog_object(self, w, uid, idxs,
update_metadata, pghandler=pghandler)
@security.protected(ManageZCatalogEntries)
def uncatalog_object(self, *args, **kwargs):
self._increment_counter()
return BaseTool.uncatalog_object(self, *args, **kwargs)
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
@security.private
def getCounter(self):
processQueue()
return self._counter is not None and self._counter() or 0
@security.private
def allow_inactive(self, query_kw):
"""Check, if the user is allowed to see inactive content.
First, check if the user is allowed to see inactive content site-wide.
Second, if there is a 'path' key in the query, check if the user is
allowed to see inactive content for these paths.
Conservative check: as soon as one path is disallowed, return False.
If a path cannot be traversed, ignore it.
"""
allow_inactive = _checkPermission(AccessInactivePortalContent, self)
if allow_inactive:
return True
paths = query_kw.get('path', False)
if not paths:
return False
if isinstance(paths, dict):
# Like: {'path': {'depth': 0, 'query': ['/Plone/events/']}}
# Or: {'path': {'depth': 0, 'query': '/Plone/events/'}}
paths = paths.get('query', [])
if isinstance(paths, six.string_types):
paths = [paths]
objs = []
site = getSite()
for path in list(paths):
if six.PY2:
path = path.encode('utf-8') # paths must not be unicode
try:
site_path = '/'.join(site.getPhysicalPath())
parts = path[len(site_path) + 1:].split('/')
parent = site.unrestrictedTraverse('/'.join(parts[:-1]))
objs.append(parent.restrictedTraverse(parts[-1]))
except (KeyError, AttributeError, Unauthorized):
# When no object is found don't raise an error
pass
if not objs:
return False
allow = True
for ob in objs:
allow = allow and\
_checkPermission(AccessInactivePortalContent, ob)
return allow
@security.protected(SearchZCatalog)
def searchResults(self, query=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
# Make sure any pending index tasks have been processed
processQueue()
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(query, dict) and not show_inactive:
show_inactive = 'show_inactive' in query
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not self.allow_inactive(kw):
kw['effectiveRange'] = DateTime()
# filter out invalid sort_on indexes
sort_on = kw.get('sort_on') or []
if isinstance(sort_on, six.string_types):
sort_on = [sort_on]
valid_indexes = self.indexes()
try:
sort_on = [idx for idx in sort_on if idx in valid_indexes]
except TypeError:
# sort_on is not iterable
sort_on = []
if not sort_on:
kw.pop('sort_on', None)
else:
kw['sort_on'] = sort_on
return ZCatalog.searchResults(self, query, **kw)
__call__ = searchResults
def search(self, query,
sort_index=None, reverse=0, limit=None, merge=1):
# Wrap search() the same way that searchResults() is
# Make sure any pending index tasks have been processed
processQueue()
user = _getAuthenticatedUser(self)
query['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not self.allow_inactive(query):
query['effectiveRange'] = DateTime()
return super(CatalogTool, self).search(
query, sort_index, reverse, limit, merge)
@security.protected(ManageZCatalogEntries)
def clearFindAndRebuild(self):
# Empties catalog, then finds all contentish objects (i.e. objects
# with an indexObject method), and reindexes them.
# This may take a long time.
idxs = list(self.indexes())
def indexObject(obj, path):
if (base_hasattr(obj, 'reindexObject') and
safe_callable(obj.reindexObject)):
try:
self.reindexObject(obj, idxs=idxs)
# index conversions from plone.app.discussion
annotions = IAnnotations(obj)
if DISCUSSION_ANNOTATION_KEY in annotions:
conversation = annotions[DISCUSSION_ANNOTATION_KEY]
conversation = conversation.__of__(obj)
for comment in conversation.getComments():
try:
self.indexObject(comment, idxs=idxs)
except StopIteration: # pragma: no cover
pass
except TypeError:
# Catalogs have 'indexObject' as well, but they
# take different args, and will fail
pass
self.manage_catalogClear()
portal = aq_parent(aq_inner(self))
portal.ZopeFindAndApply(
portal,
search_sub=True,
apply_func=indexObject
)
@security.protected(ManageZCatalogEntries)
def manage_catalogRebuild(self, RESPONSE=None, URL1=None):
"""Clears the catalog and indexes all objects with an 'indexObject'
method. This may take a long time.
"""
elapse = time.time()
c_elapse = time.clock()
self.clearFindAndRebuild()
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
msg = ('Catalog Rebuilt\n'
'Total time: %s\n'
'Total CPU time: %s' % (repr(elapse), repr(c_elapse)))
logger.info(msg)
if RESPONSE is not None:
RESPONSE.redirect(
URL1 + '/manage_catalogAdvanced?manage_tabs_message=' +
urllib.parse.quote(msg))
class FieldIndex(BaseIndexMixin, persistent.Persistent):
""" Field indexing.
Query types supported:
- Eq
- NotEq
- Gt
- Ge
- Lt
- Le
- In
- NotIn
- Any
- NotAny
- InRange
- NotInRange
"""
def __init__(self, discriminator, family=None):
if family is not None:
self.family = family
if not callable(discriminator):
if not isinstance(discriminator, string_types):
raise ValueError('discriminator value must be callable or a '
'string')
self.discriminator = discriminator
self.reset()
def reset(self):
"""Initialize forward and reverse mappings."""
# The forward index maps indexed values to a sequence of docids
self._fwd_index = self.family.OO.BTree()
# The reverse index maps a docid to its index value
self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
self._not_indexed = self.family.IF.TreeSet()
def not_indexed(self):
return self._not_indexed
def not_indexed_count(self):
return len(self._not_indexed)
def indexed(self):
return self._rev_index.keys()
def indexed_count(self):
return self._num_docs()
def word_count(self):
"""See interface IIndexStatistics"""
return len(self._fwd_index)
def document_repr(self, docid, default=None):
result = self._rev_index.get(docid, default)
if result is not default:
return repr(result)
return default
def index_doc(self, docid, value):
"""See interface IIndexInjection"""
value = self.discriminate(value, _marker)
if value is _marker:
if not (docid in self._not_indexed):
# unindex the previous value
self.unindex_doc(docid)
# Store docid in set of unindexed docids
self._not_indexed.add(docid)
return None
if docid in self._not_indexed:
# Remove from set of unindexed docs if it was in there.
self._not_indexed.remove(docid)
rev_index = self._rev_index
if docid in rev_index:
if docid in self._fwd_index.get(value, ()):
# no need to index the doc, its already up to date
return
# unindex doc if present
self.unindex_doc(docid)
# Insert into forward index.
set = self._fwd_index.get(value)
if set is None:
set = self.family.IF.TreeSet()
self._fwd_index[value] = set
set.insert(docid)
# increment doc count
self._num_docs.change(1)
# Insert into reverse index.
rev_index[docid] = value
def unindex_doc(self, docid):
"""See interface IIndexInjection.
"""
_not_indexed = self._not_indexed
if docid in _not_indexed:
_not_indexed.remove(docid)
rev_index = self._rev_index
value = rev_index.get(docid, _marker)
if value is _marker:
return # not in index
del rev_index[docid]
try:
set = self._fwd_index[value]
set.remove(docid)
except KeyError: #pragma NO COVERAGE
# This is fishy, but we don't want to raise an error.
# We should probably log something.
# but keep it from throwing a dirty exception
set = 1
if not set:
del self._fwd_index[value]
self._num_docs.change(-1)
def reindex_doc(self, docid, value):
""" See interface IIndexInjection """
# the base index's index_doc method special-cases a reindex
return self.index_doc(docid, value)
def sort(
self,
docids,
reverse=False,
limit=None,
sort_type=None,
raise_unsortable=True,
):
if limit is not None:
limit = int(limit)
if limit < 1:
raise ValueError('limit must be 1 or greater')
if not docids:
return []
numdocs = self._num_docs.value
if not numdocs:
return []
if sort_type == interfaces.STABLE:
sort_type = interfaces.TIMSORT
elif sort_type == interfaces.OPTIMAL:
sort_type = None
if reverse:
return self.sort_reverse(
docids,
limit,
numdocs,
sort_type,
raise_unsortable,
)
else:
return self.sort_forward(
docids,
limit,
numdocs,
sort_type,
raise_unsortable,
)
def sort_forward(
self,
docids,
limit,
numdocs,
sort_type=None,
raise_unsortable=True,
):
rlen = len(docids)
# See http://www.zope.org/Members/Caseman/ZCatalog_for_2.6.1
# for an overview of why we bother doing all this work to
# choose the right sort algorithm.
if sort_type is None:
if fwscan_wins(limit, rlen, numdocs):
# forward scan beats both n-best and timsort reliably
# if this is true
sort_type = interfaces.FWSCAN
elif limit and nbest_ascending_wins(limit, rlen, numdocs):
# nbest beats timsort reliably if this is true
sort_type = interfaces.NBEST
else:
sort_type = interfaces.TIMSORT
if sort_type == interfaces.FWSCAN:
return self.scan_forward(docids, limit, raise_unsortable)
elif sort_type == interfaces.NBEST:
if limit is None:
raise ValueError('nbest requires a limit')
return self.nbest_ascending(docids, limit, raise_unsortable)
elif sort_type == interfaces.TIMSORT:
return self.timsort_ascending(docids, limit, raise_unsortable)
else:
raise ValueError('Unknown sort type %s' % sort_type)
def sort_reverse(
self,
docids,
limit,
numdocs,
sort_type=None,
raise_unsortable=True,
):
if sort_type is None:
rlen = len(docids)
if limit:
if (limit < 300) or (limit/float(rlen) > 0.09):
sort_type = interfaces.NBEST
else:
sort_type = interfaces.TIMSORT
else:
sort_type = interfaces.TIMSORT
if sort_type == interfaces.NBEST:
if limit is None:
raise ValueError('nbest requires a limit')
return self.nbest_descending(docids, limit, raise_unsortable)
elif sort_type == interfaces.TIMSORT:
return self.timsort_descending(docids, limit, raise_unsortable)
else:
raise ValueError('Unknown sort type %s' % sort_type)
def scan_forward(self, docids, limit=None, raise_unsortable=True):
fwd_index = self._fwd_index
# make a copy so we don't mutate what we're passed.
docids = self.family.IF.TreeSet(docids)
n = 0
for set in fwd_index.values():
for docid in set:
if docid in docids:
n+=1
docids.remove(docid)
yield docid
if limit and n >= limit:
raise StopIteration
if raise_unsortable and docids:
raise Unsortable(docids)
def nbest_ascending(self, docids, limit, raise_unsortable=False):
if limit is None: #pragma NO COVERAGE
raise RuntimeError('n-best used without limit')
# lifted from heapq.nsmallest
h = nsort(docids, self._rev_index, ASC)
it = iter(h)
result = sorted(islice(it, 0, limit))
if not result: #pragma NO COVERAGE
raise StopIteration
insort = bisect.insort
pop = result.pop
los = result[-1] # los --> Largest of the nsmallest
for elem in it:
if los <= elem:
continue
insort(result, elem)
pop()
los = result[-1]
missing_docids = []
for value, docid in result:
if value is ASC:
missing_docids.append(docid)
else:
yield docid
if raise_unsortable and missing_docids:
raise Unsortable(missing_docids)
def nbest_descending(self, docids, limit, raise_unsortable=True):
if limit is None: #pragma NO COVERAGE
raise RuntimeError('N-Best used without limit')
iterable = nsort(docids, self._rev_index, DESC)
missing_docids = []
for value, docid in heapq.nlargest(limit, iterable):
if value is DESC:
missing_docids.append(docid)
else:
yield docid
if raise_unsortable and missing_docids:
raise Unsortable(missing_docids)
def timsort_ascending(self, docids, limit, raise_unsortable=True):
return self._timsort(
docids,
limit,
reverse=False,
raise_unsortable=raise_unsortable,
)
def timsort_descending(self, docids, limit, raise_unsortable=True):
return self._timsort(
docids,
limit,
reverse=True,
raise_unsortable=raise_unsortable,
)
def _timsort(
self,
docids,
limit=None,
reverse=False,
raise_unsortable=True,
):
n = 0
missing_docids = []
def get(k, rev_index=self._rev_index):
v = rev_index.get(k, ASC)
if v is ASC:
missing_docids.append(k)
return v
for docid in sorted(docids, key=get, reverse=reverse):
if docid in missing_docids:
# skip docids not in this index
continue
n += 1
yield docid
if limit and n >= limit:
raise StopIteration
if raise_unsortable and missing_docids:
raise Unsortable(missing_docids)
def search(self, queries, operator='or'):
sets = []
for q in queries:
if isinstance(q, RangeValue):
q = q.as_tuple()
else:
q = (q, q)
set = self.family.IF.multiunion(self._fwd_index.values(*q))
sets.append(set)
result = None
if len(sets) == 1:
result = sets[0]
elif operator == 'and':
for _, set in sorted([(len(x), x) for x in sets]):
result = self.family.IF.intersection(set, result)
else:
result = self.family.IF.multiunion(sets)
return result
def apply(self, q):
if isinstance(q, dict):
val = q['query']
if isinstance(val, RangeValue):
val = [val]
elif not isinstance(val, (list, tuple)):
val = [val]
operator = q.get('operator', 'or')
result = self.search(val, operator)
else:
if isinstance(q, tuple) and len(q) == 2:
# b/w compat stupidity; this needs to die
q = RangeValue(*q)
q = [q]
elif not isinstance(q, (list, tuple)):
q = [q]
result = self.search(q, 'or')
return result
def applyEq(self, value):
return self.apply(value)
def eq(self, value):
return query.Eq(self, value)
def applyNotEq(self, *args, **kw):
return self._negate(self.applyEq, *args, **kw)
def noteq(self, value):
return query.NotEq(self, value)
def applyGe(self, min_value):
return self.applyInRange(min_value, None)
def ge(self, value):
return query.Ge(self, value)
def applyLe(self, max_value):
return self.applyInRange(None, max_value)
def le(self, value):
return query.Le(self, value)
def applyGt(self, min_value):
return self.applyInRange(min_value, None, excludemin=True)
def gt(self, value):
return query.Gt(self, value)
def applyLt(self, max_value):
return self.applyInRange(None, max_value, excludemax=True)
def lt(self, value):
return query.Lt(self, value)
def applyAny(self, values):
queries = list(values)
return self.search(queries, operator='or')
def any(self, value):
return query.Any(self, value)
def applyNotAny(self, *args, **kw):
return self._negate(self.applyAny, *args, **kw)
def notany(self, value):
return query.NotAny(self, value)
def applyInRange(self, start, end, excludemin=False, excludemax=False):
return self.family.IF.multiunion(
self._fwd_index.values(
start, end, excludemin=excludemin, excludemax=excludemax)
)
def inrange(self, start, end, excludemin=False, excludemax=False):
return query.InRange(self, start, end, excludemin, excludemax)
def applyNotInRange(self, *args, **kw):
return self._negate(self.applyInRange, *args, **kw)
def notinrange(self, start, end, excludemin=False, excludemax=False):
return query.NotInRange(self, start, end, excludemin, excludemax)
class UUIDIndex(UnIndex):
"""Index for uuid fields with an unique value per key.
The internal structure is:
self._index = {datum:documentId]}
self._unindex = {documentId:datum}
For each datum only one documentId can exist.
"""
meta_type = "UUIDIndex"
manage_options = (
{'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
)
query_options = ["query", "range"]
manage = manage_main = DTMLFile('dtml/manageUUIDIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
def clear(self):
self._length = Length()
self._index = OIBTree()
self._unindex = IOBTree()
self._counter = Length()
def numObjects(self):
"""Return the number of indexed objects. Since we have a 1:1 mapping
from documents to values, we can reuse the stored length.
"""
return self.indexSize()
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
raise StopIteration
if not withLengths:
for key in self._index.keys():
yield key
else:
# We know the length for each value is one
for key in self._index.keys():
yield (key, 1)
def insertForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and put it in the correct place
in the forward index.
"""
if entry is None:
return
old_docid = self._index.get(entry, _marker)
if old_docid is _marker:
self._index[entry] = documentId
self._length.change(1)
elif old_docid != documentId:
logger.error("A different document with value '%s' already "
"exists in the index.'" % entry)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
old_docid = self._index.get(entry, _marker)
if old_docid is not _marker:
del self._index[entry]
self._length.change(-1)
def _get_object_datum(self, obj, attr):
# for a uuid it never makes sense to acquire a parent value via
# Acquisition
has_attr = getattr(aq_base(obj), attr, _marker)
if has_attr is _marker:
return _marker
return super(UUIDIndex, self)._get_object_datum(obj, attr)
class CachingCatalog(Catalog):
implements(ICatalog)
os = os # for unit tests
generation = None # b/c
def __init__(self):
super(CachingCatalog, self).__init__()
self.generation = Length(0)
def clear(self):
self.invalidate()
super(CachingCatalog, self).clear()
def index_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).index_doc(*arg, **kw)
def unindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).unindex_doc(*arg, **kw)
def reindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).reindex_doc(*arg, **kw)
def __setitem__(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).__setitem__(*arg, **kw)
@MetricMod('CS.%s')
@metricmethod
def search(self, *arg, **kw):
use_cache = True
if 'use_cache' in kw:
use_cache = kw.pop('use_cache')
if 'NO_CATALOG_CACHE' in self.os.environ:
use_cache = False
if 'tags' in kw:
# The tags index changes without invalidating the catalog,
# so don't cache any query involving the tags index.
use_cache = False
if not use_cache:
return self._search(*arg, **kw)
cache = queryUtility(ICatalogSearchCache)
if cache is None:
return self._search(*arg, **kw)
key = cPickle.dumps((arg, kw))
generation = self.generation
if generation is None:
generation = Length(0)
genval = generation.value
if (genval == 0) or (genval > cache.generation):
# an update in another process requires that the local cache be
# invalidated
cache.clear()
cache.generation = genval
if cache.get(key) is None:
start = time.time()
num, docids = self._search(*arg, **kw)
# We don't cache large result sets because the time it takes to
# unroll the result set turns out to be far more time than it
# takes to run the search. In a particular instance using OSI's
# catalog a search that took 0.015s but returned nearly 35,295
# results took over 50s to unroll the result set for caching,
# significantly slowing search performance.
if num > LARGE_RESULT_SET:
return num, docids
# we need to unroll here; a btree-based structure may have
# a reference to its connection
docids = list(docids)
cache[key] = (num, docids)
return cache.get(key)
@metricmethod
def _search(self, *arg, **kw):
start = time.time()
res = super(CachingCatalog, self).search(*arg, **kw)
duration = time.time() - start
notify(CatalogQueryEvent(self, kw, duration, res))
return res
def invalidate(self):
# Increment the generation; this tells *another process* that
# its catalog cache needs to be cleared
generation = self.generation
if generation is None:
generation = self.generation = Length(0)
if generation.value >= sys.maxint:
# don't keep growing the generation integer; wrap at sys.maxint
self.generation.set(0)
else:
self.generation.change(1)
# Clear the cache for *this process*
cache = queryUtility(ICatalogSearchCache)
if cache is not None:
cache.clear()
cache.generation = self.generation.value
class BTreeFolder2Base (Persistent):
"""Base for BTree-based folders.
"""
security = ClassSecurityInfo()
manage_options=(
({'label':'Contents', 'action':'manage_main',},
) + Folder.manage_options[1:]
)
security.declareProtected(view_management_screens,
'manage_main')
manage_main = DTMLFile('contents', globals())
_tree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
_mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } }
title = ''
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._tree = OOBTree()
self._count = Length()
self._mt_index = OOBTree()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name in source.objectIds():
value = source._getOb(name, None)
if value is not None:
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount()
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in BTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in BTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self):
"""Checks if the value of self._count disagrees with
len(self.objectIds()). If so, corrects self._count. Returns the
old and new count values. If old==new, no correction was
performed.
"""
old = self._count()
new = len(self.objectIds())
if old != new:
self._count.set(new)
return old, new
security.declareProtected(view_management_screens, 'manage_cleanup')
def manage_cleanup(self):
"""Calls self._cleanup() and reports the result as text.
"""
v = self._cleanup()
path = '/'.join(self.getPhysicalPath())
if v:
return "No damage detected in BTreeFolder2 at %s." % path
else:
return ("Fixed BTreeFolder2 at %s. "
"See the log for more details." % path)
def _cleanup(self):
"""Cleans up errors in the BTrees.
Certain ZODB bugs have caused BTrees to become slightly insane.
Fortunately, there is a way to clean up damaged BTrees that
always seems to work: make a new BTree containing the items()
of the old one.
Returns 1 if no damage was detected, or 0 if damage was
detected and fixed.
"""
from BTrees.check import check
path = '/'.join(self.getPhysicalPath())
try:
check(self._tree)
for key in self._tree.keys():
if not self._tree.has_key(key):
raise AssertionError(
"Missing value for key: %s" % repr(key))
check(self._mt_index)
for key, value in self._mt_index.items():
if (not self._mt_index.has_key(key)
or self._mt_index[key] is not value):
raise AssertionError(
"Missing or incorrect meta_type index: %s"
% repr(key))
check(value)
for k in value.keys():
if not value.has_key(k):
raise AssertionError(
"Missing values for meta_type index: %s"
% repr(key))
return 1
except AssertionError:
LOG.warn( 'Detected damage to %s. Fixing now.' % path,
exc_info=True)
try:
self._tree = OOBTree(self._tree)
mt_index = OOBTree()
for key, value in self._mt_index.items():
mt_index[key] = OIBTree(value)
self._mt_index = mt_index
except:
LOG.error('Failed to fix %s.' % path,
exc_info=True)
raise
else:
LOG.info('Fixed %s.' % path)
return 0
def _getOb(self, id, default=_marker):
"""Return the named object from the folder.
"""
tree = self._tree
if default is _marker:
ob = tree[id]
return ob.__of__(self)
else:
ob = tree.get(id, _marker)
if ob is _marker:
return default
else:
return ob.__of__(self)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
tree = self._tree
if tree.has_key(id):
raise KeyError('There is already an item named "%s".' % id)
tree[id] = object
self._count.change(1)
# Update the meta type index.
mti = self._mt_index
meta_type = getattr(object, 'meta_type', None)
if meta_type is not None:
ids = mti.get(meta_type, None)
if ids is None:
ids = OIBTree()
mti[meta_type] = ids
ids[id] = 1
def _delOb(self, id):
"""Remove the named object from the folder.
"""
tree = self._tree
meta_type = getattr(tree[id], 'meta_type', None)
del tree[id]
self._count.change(-1)
# Update the meta type index.
if meta_type is not None:
mti = self._mt_index
ids = mti.get(meta_type, None)
if ids is not None and ids.has_key(id):
del ids[id]
if not ids:
# Removed the last object of this meta_type.
# Prune the index.
del mti[meta_type]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
idlist = self.objectIds() # Pre-sorted.
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = []
formatted.append(listtext0 % pref_rows)
for i in range(b_start - 1, b_end):
optID = escape(idlist[i])
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {'b_start': b_start, 'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect(
'%s/%s/manage_workspace' % (
self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information,
'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information,
'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
return self._tree.has_key(id)
security.declareProtected(access_contents_information,
'objectIds')
def objectIds(self, spec=None):
# Returns a list of subobject ids of the current object.
# If 'spec' is specified, returns objects whose meta_type
# matches 'spec'.
if spec is not None:
if isinstance(spec, StringType):
spec = [spec]
mti = self._mt_index
set = None
for meta_type in spec:
ids = mti.get(meta_type, None)
if ids is not None:
set = union(set, ids)
if set is None:
return ()
else:
return set.keys()
else:
return self._tree.keys()
security.declareProtected(access_contents_information,
'objectValues')
def objectValues(self, spec=None):
# Returns a list of actual subobjects of the current object.
# If 'spec' is specified, returns only objects whose meta_type
# match 'spec'.
return LazyMap(self._getOb, self.objectIds(spec))
security.declareProtected(access_contents_information,
'objectItems')
def objectItems(self, spec=None):
# Returns a list of (id, subobject) tuples of the current object.
# If 'spec' is specified, returns only objects whose meta_type match
# 'spec'
return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)),
self.objectIds(spec))
security.declareProtected(access_contents_information,
'objectMap')
def objectMap(self):
# Returns a tuple of mappings containing subobject meta-data.
return LazyMap(lambda (k, v):
{'id': k, 'meta_type': getattr(v, 'meta_type', None)},
self._tree.items(), self._count())
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information,
'objectIds_d')
def objectIds_d(self, t=None):
ids = self.objectIds(t)
res = {}
for id in ids:
res[id] = 1
return res
security.declareProtected(access_contents_information,
'objectMap_d')
def objectMap_d(self, t=None):
return self.objectMap()
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self, id, object, roles=None, user=None, set_owner=1,
suppress_events=False):
ob = object # better name, keep original function signature
v = self._checkId(id)
if v is not None:
id = v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
if not suppress_events:
notify(ObjectWillBeAddedEvent(ob, self, id))
self._setOb(id, ob)
ob = self._getOb(id)
if set_owner:
# TODO: eventify manage_fixupOwnershipAfterAdd
# This will be called for a copy/clone, or a normal _setObject.
ob.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if getattr(ob, '__ac_local_roles__', _marker) is None:
user = getSecurityManager().getUser()
if user is not None:
userid = user.getId()
if userid is not None:
ob.manage_setLocalRoles(userid, ['Owner'])
if not suppress_events:
notify(ObjectAddedEvent(ob, self, id))
notifyContainerModified(self)
OFS.subscribers.compatibilityCall('manage_afterAdd', ob, ob, self)
return id
def _delObject(self, id, dp=1, suppress_events=False):
ob = self._getOb(id)
OFS.subscribers.compatibilityCall('manage_beforeDelete', ob, ob, self)
if not suppress_events:
notify(ObjectWillBeRemovedEvent(ob, self, id))
self._delOb(id)
if not suppress_events:
notify(ObjectRemovedEvent(ob, self, id))
notifyContainerModified(self)
# Aliases for mapping-like access.
__len__ = objectCount
keys = objectIds
values = objectValues
items = objectItems
# backward compatibility
hasObject = has_key
security.declareProtected(access_contents_information, 'get')
def get(self, name, default=None):
return self._getOb(name, default)
# Utility for generating unique IDs.
security.declareProtected(access_contents_information, 'generateId')
def generateId(self, prefix='item', suffix='', rand_ceiling=999999999):
"""Returns an ID not used yet by this folder.
The ID is unlikely to collide with other threads and clients.
The IDs are sequential to optimize access to objects
that are likely to have some relation.
"""
tree = self._tree
n = self._v_nextid
attempt = 0
while 1:
if n % 4000 != 0 and n <= rand_ceiling:
id = '%s%d%s' % (prefix, n, suffix)
if not tree.has_key(id):
break
n = randint(1, rand_ceiling)
attempt = attempt + 1
if attempt > MAX_UNIQUEID_ATTEMPTS:
# Prevent denial of service
raise ExhaustedUniqueIdsError
self._v_nextid = n + 1
return id
def __getattr__(self, name):
# Boo hoo hoo! Zope 2 prefers implicit acquisition over traversal
# to subitems, and __bobo_traverse__ hooks don't work with
# restrictedTraverse() unless __getattr__() is also present.
# Oh well.
res = self._tree.get(name)
if res is None:
raise AttributeError, name
return res
class UnIndex(SimpleItem):
"""Simple forward and reverse index.
"""
zmi_icon = 'fas fa-info-circle'
_counter = None
operators = ('or', 'and')
useOperator = 'or'
query_options = ()
def __init__(self, id, ignore_ex=None, call_methods=None,
extra=None, caller=None):
"""Create an unindex
UnIndexes are indexes that contain two index components, the
forward index (like plain index objects) and an inverted
index. The inverted index is so that objects can be unindexed
even when the old value of the object is not known.
e.g.
self._index = {datum:[documentId1, documentId2]}
self._unindex = {documentId:datum}
The arguments are:
'id' -- the name of the item attribute to index. This is
either an attribute name or a record key.
'ignore_ex' -- should be set to true if you want the index
to ignore exceptions raised while indexing instead of
propagating them.
'call_methods' -- should be set to true if you want the index
to call the attribute 'id' (note: 'id' should be callable!)
You will also need to pass in an object in the index and
uninded methods for this to work.
'extra' -- a mapping object that keeps additional
index-related parameters - subitem 'indexed_attrs'
can be string with comma separated attribute names or
a list
'caller' -- reference to the calling object (usually
a (Z)Catalog instance
"""
def _get(o, k, default):
""" return a value for a given key of a dict/record 'o' """
if isinstance(o, dict):
return o.get(k, default)
else:
return getattr(o, k, default)
self.id = id
self.ignore_ex = ignore_ex # currently unimplemented
self.call_methods = call_methods
# allow index to index multiple attributes
ia = _get(extra, 'indexed_attrs', id)
if isinstance(ia, str):
self.indexed_attrs = ia.split(',')
else:
self.indexed_attrs = list(ia)
self.indexed_attrs = [
attr.strip() for attr in self.indexed_attrs if attr]
if not self.indexed_attrs:
self.indexed_attrs = [id]
self.clear()
def __len__(self):
return self._length()
def getId(self):
return self.id
def clear(self):
self._length = Length()
self._index = OOBTree()
self._unindex = IOBTree()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def __nonzero__(self):
return not not self._unindex
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
for item in self._index.items():
if isinstance(item, int):
entry = 1 # "set" length is 1
else:
key, value = item
entry = len(value)
histogram[entry] = histogram.get(entry, 0) + 1
return histogram
def referencedObjects(self):
"""Generate a list of IDs for which we have referenced objects."""
return self._unindex.keys()
def getEntryForObject(self, documentId, default=_marker):
"""Takes a document ID and returns all the information we have
on that specific object.
"""
if default is _marker:
return self._unindex.get(documentId)
return self._unindex.get(documentId, default)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
indexRow = self._index.get(entry, _marker)
if indexRow is not _marker:
try:
indexRow.remove(documentId)
if not indexRow:
del self._index[entry]
self._length.change(-1)
except ConflictError:
raise
except AttributeError:
# index row is an int
try:
del self._index[entry]
except KeyError:
# swallow KeyError because it was probably
# removed and then _length AttributeError raised
pass
if isinstance(self.__len__, Length):
self._length = self.__len__
del self.__len__
self._length.change(-1)
except Exception:
LOG.error('%(context)s: unindex_object could not remove '
'documentId %(doc_id)s from index %(index)r. This '
'should not happen.', dict(
context=self.__class__.__name__,
doc_id=documentId,
index=self.id),
exc_info=sys.exc_info())
else:
LOG.error('%(context)s: unindex_object tried to '
'retrieve set %(entry)r from index %(index)r '
'but couldn\'t. This should not happen.', dict(
context=self.__class__.__name__,
entry=entry,
index=self.id))
def insertForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and put it in the correct place
in the forward index.
This will also deal with creating the entire row if necessary.
"""
indexRow = self._index.get(entry, _marker)
# Make sure there's actually a row there already. If not, create
# a set and stuff it in first.
if indexRow is _marker:
# We always use a set to avoid getting conflict errors on
# multiple threads adding a new row at the same time
self._index[entry] = IITreeSet((documentId, ))
self._length.change(1)
else:
try:
indexRow.insert(documentId)
except AttributeError:
# Inline migration: index row with one element was an int at
# first (before Zope 2.13).
indexRow = IITreeSet((indexRow, documentId))
self._index[entry] = indexRow
def index_object(self, documentId, obj, threshold=None):
""" wrapper to handle indexing of multiple attributes """
fields = self.getIndexSourceNames()
res = 0
for attr in fields:
res += self._index_object(documentId, obj, threshold, attr)
if res > 0:
self._increment_counter()
return res > 0
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
if datum is None:
# Prevent None from being indexed. None doesn't have a valid
# ordering definition compared to any other object.
# BTrees 4.0+ will throw a TypeError
# "object has default comparison" and won't let it be indexed.
return 0
datum = self._convert(datum, default=_marker)
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId)
if datum is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.error('%(context)s: oldDatum was there, '
'now it\'s not for documentId %(doc_id)s '
'from index %(index)r. This '
'should not happen.', dict(
context=self.__class__.__name__,
doc_id=documentId,
index=self.id),
exc_info=sys.exc_info())
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
self._unindex[documentId] = datum
returnStatus = 1
return returnStatus
def _get_object_datum(self, obj, attr):
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
try:
datum = getattr(obj, attr)
if safe_callable(datum):
datum = datum()
except (AttributeError, TypeError):
datum = _marker
return datum
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
def getCounter(self):
"""Return a counter which is increased on index changes"""
return self._counter is not None and self._counter() or 0
def numObjects(self):
"""Return the number of indexed objects."""
return len(self._unindex)
def indexSize(self):
"""Return the size of the index in terms of distinct values."""
return len(self)
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self._increment_counter()
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.debug('%(context)s: attempt to unindex nonexistent '
'documentId %(doc_id)s from index %(index)r. This '
'should not happen.', dict(
context=self.__class__.__name__,
doc_id=documentId,
index=self.id),
exc_info=True)
def _apply_not(self, not_parm, resultset=None):
index = self._index
setlist = []
for k in not_parm:
s = index.get(k, None)
if s is None:
continue
elif isinstance(s, int):
s = IISet((s, ))
setlist.append(s)
return multiunion(setlist)
def _convert(self, value, default=None):
return value
def getRequestCache(self):
"""returns dict for caching per request for interim results
of an index search. Returns 'None' if no REQUEST attribute
is available"""
cache = None
REQUEST = aq_get(self, 'REQUEST', None)
if REQUEST is not None:
catalog = aq_parent(aq_parent(aq_inner(self)))
if catalog is not None:
# unique catalog identifier
key = '_catalogcache_{0}_{1}'.format(
catalog.getId(), id(catalog))
cache = REQUEST.get(key, None)
if cache is None:
cache = REQUEST[key] = RequestCache()
return cache
def getRequestCacheKey(self, record, resultset=None):
"""returns an unique key of a search record"""
params = []
# record operator (or, and)
params.append(('operator', record.operator))
# not / exclude operator
not_value = record.get('not', None)
if not_value is not None:
not_value = frozenset(not_value)
params.append(('not', not_value))
# record options
for op in ['range', 'usage']:
op_value = record.get(op, None)
if op_value is not None:
params.append((op, op_value))
# record keys
rec_keys = frozenset(record.keys)
params.append(('keys', rec_keys))
# build record identifier
rid = frozenset(params)
# unique index identifier
iid = '_{0}_{1}_{2}'.format(self.__class__.__name__,
self.id, self.getCounter())
return (iid, rid)
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the request arg.
If the query does not match the index, return None, otherwise
return a tuple of (result, used_attributes), where used_attributes
is again a tuple with the names of all used data fields.
If not `None`, the resultset argument
indicates that the search result is relevant only on this set,
i.e. everything outside resultset is of no importance.
The index can use this information for optimizations.
"""
record = IndexQuery(request, self.id, self.query_options,
self.operators, self.useOperator)
if record.keys is None:
return None
return (self.query_index(record, resultset=resultset), (self.id, ))
def query_index(self, record, resultset=None):
"""Search the index with the given IndexQuery object.
If not `None`, the resultset argument
indicates that the search result is relevant only on this set,
i.e. everything outside resultset is of no importance.
The index can use this information for optimizations.
"""
index = self._index
r = None
opr = None
# not / exclude parameter
not_parm = record.get('not', None)
operator = record.operator
cachekey = None
cache = self.getRequestCache()
if cache is not None:
cachekey = self.getRequestCacheKey(record)
if cachekey is not None:
cached = None
if operator == 'or':
cached = cache.get(cachekey, None)
else:
cached_setlist = cache.get(cachekey, None)
if cached_setlist is not None:
r = resultset
for s in cached_setlist:
# the result is bound by the resultset
r = intersection(r, s)
# If intersection, we can't possibly get a
# smaller result
if not r:
break
cached = r
if cached is not None:
if isinstance(cached, int):
cached = IISet((cached, ))
if not_parm:
not_parm = list(map(self._convert, not_parm))
exclude = self._apply_not(not_parm, resultset)
cached = difference(cached, exclude)
return cached
if not record.keys and not_parm:
# convert into indexed format
not_parm = list(map(self._convert, not_parm))
# we have only a 'not' query
record.keys = [k for k in index.keys() if k not in not_parm]
else:
# convert query arguments into indexed format
record.keys = list(map(self._convert, record.keys))
# Range parameter
range_parm = record.get('range', None)
if range_parm:
opr = 'range'
opr_args = []
if range_parm.find('min') > -1:
opr_args.append('min')
if range_parm.find('max') > -1:
opr_args.append('max')
if record.get('usage', None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args = opr[0], opr[1:]
if opr == 'range': # range search
if 'min' in opr_args:
lo = min(record.keys)
else:
lo = None
if 'max' in opr_args:
hi = max(record.keys)
else:
hi = None
if hi:
setlist = index.values(lo, hi)
else:
setlist = index.values(lo)
# If we only use one key, intersect and return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
if cachekey is not None:
if operator == 'or':
cache[cachekey] = result
else:
cache[cachekey] = [result]
if not_parm:
exclude = self._apply_not(not_parm, resultset)
result = difference(result, exclude)
return result
if operator == 'or':
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s,))
tmp.append(s)
r = multiunion(tmp)
if cachekey is not None:
cache[cachekey] = r
else:
# For intersection, sort with smallest data set first
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s,))
tmp.append(s)
if len(tmp) > 2:
setlist = sorted(tmp, key=len)
else:
setlist = tmp
# 'r' is not invariant of resultset. Thus, we
# have to remember 'setlist'
if cachekey is not None:
cache[cachekey] = setlist
r = resultset
for s in setlist:
# the result is bound by the resultset
r = intersection(r, s)
# If intersection, we can't possibly get a smaller result
if not r:
break
else: # not a range search
# Filter duplicates
setlist = []
for k in record.keys:
if k is None:
# Prevent None from being looked up. None doesn't
# have a valid ordering definition compared to any
# other object. BTrees 4.0+ will throw a TypeError
# "object has default comparison".
continue
try:
s = index.get(k, None)
except TypeError:
# key is not valid for this Btree so the value is None
LOG.error(
'%(context)s: query_index tried '
'to look up key %(key)r from index %(index)r '
'but key was of the wrong type.', dict(
context=self.__class__.__name__,
key=k,
index=self.id,
)
)
s = None
# If None, try to bail early
if s is None:
if operator == 'or':
# If union, we can possibly get a bigger result
continue
# If intersection, we can't possibly get a smaller result
if cachekey is not None:
# If operator is 'and', we have to cache a list of
# IISet objects
cache[cachekey] = [IISet()]
return IISet()
elif isinstance(s, int):
s = IISet((s,))
setlist.append(s)
# If we only use one key return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
if cachekey is not None:
if operator == 'or':
cache[cachekey] = result
else:
cache[cachekey] = [result]
if not_parm:
exclude = self._apply_not(not_parm, resultset)
result = difference(result, exclude)
return result
if operator == 'or':
# If we already get a small result set passed in, intersecting
# the various indexes with it and doing the union later is
# faster than creating a multiunion first.
if resultset is not None and len(resultset) < 200:
smalllist = []
for s in setlist:
smalllist.append(intersection(resultset, s))
r = multiunion(smalllist)
# 'r' is not invariant of resultset. Thus, we
# have to remember the union of 'setlist'. But
# this is maybe a performance killer. So we do not cache.
# if cachekey is not None:
# cache[cachekey] = multiunion(setlist)
else:
r = multiunion(setlist)
if cachekey is not None:
cache[cachekey] = r
else:
# For intersection, sort with smallest data set first
if len(setlist) > 2:
setlist = sorted(setlist, key=len)
# 'r' is not invariant of resultset. Thus, we
# have to remember the union of 'setlist'
if cachekey is not None:
cache[cachekey] = setlist
r = resultset
for s in setlist:
r = intersection(r, s)
# If intersection, we can't possibly get a smaller result
if not r:
break
if isinstance(r, int):
r = IISet((r, ))
if r is None:
return IISet()
if not_parm:
exclude = self._apply_not(not_parm, resultset)
r = difference(r, exclude)
return r
def hasUniqueValuesFor(self, name):
"""has unique values for column name"""
if name == self.id:
return 1
return 0
def getIndexSourceNames(self):
"""Return sequence of indexed attributes."""
return getattr(self, 'indexed_attrs', [self.id])
def getIndexQueryNames(self):
"""Indicate that this index applies to queries for the index's name."""
return (self.id,)
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
return
if not withLengths:
for key in self._index.keys():
yield key
else:
for key, value in self._index.items():
if isinstance(value, int):
yield (key, 1)
else:
yield (key, len(value))
def keyForDocument(self, id):
# This method is superseded by documentToKeyMap
return self._unindex[id]
def documentToKeyMap(self):
return self._unindex
def items(self):
items = []
for k, v in self._index.items():
if isinstance(v, int):
v = IISet((v,))
items.append((k, v))
return items
class HypatiaDateRecurringIndex(KeywordIndex, FieldIndex):
def discriminate(self, obj, default):
""" See interface IIndexInjection """
if callable(self.discriminator):
value = self.discriminator(obj, _marker)
else:
value = getattr(obj, self.discriminator, _marker)
if value is _marker:
return default
if isinstance(value, Persistent):
raise ValueError('Catalog cannot index persistent object %s' %
value)
if isinstance(value, Broken):
raise ValueError('Catalog cannot index broken object %s' % value)
if not isinstance(value, dict):
raise ValueError(
'Catalog can only index dict with '
'attr and date keys, or date and recurdef keys, given %s' %
value)
# examples:
# {'attr': 'dates',
# 'date': datetime.datetime.now()}
# will get dates_recurrence attribute on the obj to get iCal string
# for recurrence definition
# or
# {'date': datetime.datetime.now(),
# 'recurdef': ICALSTRING}
# no access to obj attributes at all
date = value.get('date')
default_recurdef = value.get('recurdef', _marker)
if default_recurdef is not _marker:
recurdef = default_recurdef
else:
attr_recurdef = value.get('attr') + '_recurrence'
recurdef = getattr(obj, attr_recurdef, None)
if callable(recurdef):
recurdef = recurdef()
if not recurdef:
dates = [date]
else:
dates = recurrence_sequence_ical(date, recrule=recurdef)
# dates is a generator
return tuple(dates)
def normalize(self, dates):
return [dt2int(date) for date in dates]
# below is the same implementation as Keyword Index, but replacing
# self._fwd_index = self.family.OO.BTree() by self._fwd_index = self.family.IO.BTree()
# family.OO.Set by family.II.Set
# family.OO.difference by family.II.difference
def reset(self):
"""Initialize forward and reverse mappings."""
# The forward index maps index keywords to a sequence of docids
self._fwd_index = self.family.IO.BTree()
# The reverse index maps a docid to its keywords
self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
self._not_indexed = self.family.IF.TreeSet()
def index_doc(self, docid, obj):
seq = self.discriminate(obj, _marker)
if seq is _marker:
if not (docid in self._not_indexed):
# unindex the previous value
self.unindex_doc(docid)
# Store docid in set of unindexed docids
self._not_indexed.add(docid)
return None
if docid in self._not_indexed:
# Remove from set of unindexed docs if it was in there.
self._not_indexed.remove(docid)
if isinstance(seq, string_types):
raise TypeError('seq argument must be a list/tuple of strings')
old_kw = self._rev_index.get(docid, None)
if not seq:
if old_kw:
self.unindex_doc(docid)
return
seq = self.normalize(seq)
new_kw = self.family.II.Set(seq)
if old_kw is None:
self._insert_forward(docid, new_kw)
self._insert_reverse(docid, new_kw)
self._num_docs.change(1)
else:
# determine added and removed keywords
kw_added = self.family.II.difference(new_kw, old_kw)
kw_removed = self.family.II.difference(old_kw, new_kw)
if not (kw_added or kw_removed):
return
# removed keywords are removed from the forward index
for word in kw_removed:
fwd = self._fwd_index[word]
fwd.remove(docid)
if not fwd:
del self._fwd_index[word]
# now update reverse and forward indexes
self._insert_forward(docid, kw_added)
self._insert_reverse(docid, new_kw)
def applyInRange(self, start, end, excludemin=False, excludemax=False):
if start is not None:
start = dt2int(start)
if end is not None:
end = dt2int(end)
return self.family.IF.multiunion(
self._fwd_index.values(start,
end,
excludemin=excludemin,
excludemax=excludemax))
def document_repr(self, docid, default=None):
result = self._rev_index.get(docid, default)
if result is not default:
return ', '.join([int2dt(r).isoformat() for r in result])
# return repr(result)
return default
def inrange_with_not_indexed(self,
start,
end,
excludemin=False,
excludemax=False):
return InRangeWithNotIndexed(self, start, end, excludemin, excludemax)
class CatalogTool(PloneBaseTool, BaseTool):
"""Plone's catalog tool"""
implements(IPloneCatalogTool)
meta_type = 'Plone Catalog Tool'
security = ClassSecurityInfo()
toolicon = 'skins/plone_images/book_icon.png'
_counter = None
manage_catalogAdvanced = DTMLFile('www/catalogAdvanced', globals())
manage_options = (
{'action': 'manage_main', 'label': 'Contents'},
{'action': 'manage_catalogView', 'label': 'Catalog'},
{'action': 'manage_catalogIndexes', 'label': 'Indexes'},
{'action': 'manage_catalogSchema', 'label': 'Metadata'},
{'action': 'manage_catalogAdvanced', 'label': 'Advanced'},
{'action': 'manage_catalogReport', 'label': 'Query Report'},
{'action': 'manage_catalogPlan', 'label': 'Query Plan'},
{'action': 'manage_propertiesForm', 'label': 'Properties'},
)
def __init__(self):
ZCatalog.__init__(self, self.getId())
def _removeIndex(self, index):
"""Safe removal of an index.
"""
try:
self.manage_delIndex(index)
except:
pass
def _listAllowedRolesAndUsers(self, user):
"""Makes sure the list includes the user's groups.
"""
result = user.getRoles()
if 'Anonymous' in result:
# The anonymous user has no further roles
return ['Anonymous']
result = list(result)
if hasattr(aq_base(user), 'getGroups'):
# remove the AuthenticatedUsers group, the Authenticated role is
# already included in the user.getRoles() list
groups = ['user:%s' % x for x in user.getGroups() if
x != 'AuthenticatedUsers']
if groups:
result = result + groups
result.append('Anonymous')
result.append('user:%s' % user.getId())
return result
security.declarePrivate('indexObject')
def indexObject(self, object, idxs=[]):
"""Add object to catalog.
The optional idxs argument is a list of specific indexes
to populate (all of them by default).
"""
self.reindexObject(object, idxs)
security.declareProtected(ManageZCatalogEntries, 'catalog_object')
def catalog_object(self, object, uid=None, idxs=[],
update_metadata=1, pghandler=None):
self._increment_counter()
w = object
if not IIndexableObject.providedBy(object):
# This is the CMF 2.2 compatible approach, which should be used
# going forward
wrapper = queryMultiAdapter((object, self), IIndexableObject)
if wrapper is not None:
w = wrapper
ZCatalog.catalog_object(self, w, uid, idxs,
update_metadata, pghandler=pghandler)
security.declareProtected(ManageZCatalogEntries, 'catalog_object')
def uncatalog_object(self, *args, **kwargs):
self._increment_counter()
return BaseTool.uncatalog_object(self, *args, **kwargs)
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
security.declarePrivate('getCounter')
def getCounter(self):
return self._counter is not None and self._counter() or 0
security.declareProtected(SearchZCatalog, 'searchResults')
def searchResults(self, REQUEST=None, **kw):
"""Calls ZCatalog.searchResults with extra arguments that
limit the results to what the user is allowed to see.
This version uses the 'effectiveRange' DateRangeIndex.
It also accepts a keyword argument show_inactive to disable
effectiveRange checking entirely even for those without portal
wide AccessInactivePortalContent permission.
"""
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(REQUEST, dict) and not show_inactive:
show_inactive = 'show_inactive' in REQUEST
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not _checkPermission(
AccessInactivePortalContent, self):
kw['effectiveRange'] = DateTime()
return ZCatalog.searchResults(self, REQUEST, **kw)
__call__ = searchResults
security.declareProtected(ManageZCatalogEntries, 'clearFindAndRebuild')
def clearFindAndRebuild(self):
"""Empties catalog, then finds all contentish objects (i.e. objects
with an indexObject method), and reindexes them.
This may take a long time.
"""
def indexObject(obj, path):
if (base_hasattr(obj, 'indexObject') and
safe_callable(obj.indexObject)):
try:
obj.indexObject()
except TypeError:
# Catalogs have 'indexObject' as well, but they
# take different args, and will fail
pass
self.manage_catalogClear()
portal = aq_parent(aq_inner(self))
portal.ZopeFindAndApply(portal, search_sub=True,
apply_func=indexObject)
security.declareProtected(ManageZCatalogEntries, 'manage_catalogRebuild')
def manage_catalogRebuild(self, RESPONSE=None, URL1=None):
"""Clears the catalog and indexes all objects with an 'indexObject'
method. This may take a long time.
"""
elapse = time.time()
c_elapse = time.clock()
self.clearFindAndRebuild()
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
if RESPONSE is not None:
RESPONSE.redirect(
URL1 + '/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote('Catalog Rebuilt\n'
'Total time: %s\n'
'Total CPU time: %s' % (`elapse`, `c_elapse`)))
class CatalogTool(PloneBaseTool, BaseTool):
"""Plone's catalog tool"""
meta_type = 'Plone Catalog Tool'
security = ClassSecurityInfo()
toolicon = 'skins/plone_images/book_icon.png'
_counter = None
manage_catalogAdvanced = DTMLFile('www/catalogAdvanced', globals())
manage_options = (
{'action': 'manage_main', 'label': 'Contents'},
{'action': 'manage_catalogView', 'label': 'Catalog'},
{'action': 'manage_catalogIndexes', 'label': 'Indexes'},
{'action': 'manage_catalogSchema', 'label': 'Metadata'},
{'action': 'manage_catalogAdvanced', 'label': 'Advanced'},
{'action': 'manage_catalogReport', 'label': 'Query Report'},
{'action': 'manage_catalogPlan', 'label': 'Query Plan'},
{'action': 'manage_propertiesForm', 'label': 'Properties'},
)
def __init__(self):
ZCatalog.__init__(self, self.getId())
def _removeIndex(self, index):
# Safe removal of an index.
try:
self.manage_delIndex(index)
except:
pass
def _listAllowedRolesAndUsers(self, user):
# Makes sure the list includes the user's groups.
result = user.getRoles()
if 'Anonymous' in result:
# The anonymous user has no further roles
return ['Anonymous']
result = list(result)
if hasattr(aq_base(user), 'getGroups'):
groups = ['user:%s' % x for x in user.getGroups()]
if groups:
result = result + groups
# Order the arguments from small to large sets
result.insert(0, 'user:%s' % user.getId())
result.append('Anonymous')
return result
@security.private
def indexObject(self, object, idxs=None):
# Add object to catalog.
# The optional idxs argument is a list of specific indexes
# to populate (all of them by default).
if idxs is None:
idxs = []
self.reindexObject(object, idxs)
@security.protected(ManageZCatalogEntries)
def catalog_object(self, object, uid=None, idxs=None,
update_metadata=1, pghandler=None):
if idxs is None:
idxs = []
self._increment_counter()
w = object
if not IIndexableObject.providedBy(object):
# This is the CMF 2.2 compatible approach, which should be used
# going forward
wrapper = queryMultiAdapter((object, self), IIndexableObject)
if wrapper is not None:
w = wrapper
ZCatalog.catalog_object(self, w, uid, idxs,
update_metadata, pghandler=pghandler)
@security.protected(ManageZCatalogEntries)
def uncatalog_object(self, *args, **kwargs):
self._increment_counter()
return BaseTool.uncatalog_object(self, *args, **kwargs)
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
@security.private
def getCounter(self):
return self._counter is not None and self._counter() or 0
@security.protected(SearchZCatalog)
def searchResults(self, REQUEST=None, **kw):
# Calls ZCatalog.searchResults with extra arguments that
# limit the results to what the user is allowed to see.
#
# This version uses the 'effectiveRange' DateRangeIndex.
#
# It also accepts a keyword argument show_inactive to disable
# effectiveRange checking entirely even for those without portal
# wide AccessInactivePortalContent permission.
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(REQUEST, dict) and not show_inactive:
show_inactive = 'show_inactive' in REQUEST
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive \
and not _checkPermission(AccessInactivePortalContent, self):
kw['effectiveRange'] = DateTime()
return ZCatalog.searchResults(self, REQUEST, **kw)
__call__ = searchResults
def search(self, *args, **kw):
# Wrap search() the same way that searchResults() is
query = {}
if args:
query = args[0]
elif 'query_request' in kw:
query = kw.get('query_request')
kw['query_request'] = query.copy()
user = _getAuthenticatedUser(self)
query['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not _checkPermission(AccessInactivePortalContent, self):
query['effectiveRange'] = DateTime()
kw['query_request'] = query
return super(CatalogTool, self).search(**kw)
@security.protected(ManageZCatalogEntries)
def clearFindAndRebuild(self):
# Empties catalog, then finds all contentish objects (i.e. objects
# with an indexObject method), and reindexes them.
# This may take a long time.
def indexObject(obj, path):
if (base_hasattr(obj, 'indexObject') and
safe_callable(obj.indexObject)):
try:
obj.indexObject()
# index conversions from plone.app.discussion
annotions = IAnnotations(obj)
catalog = getToolByName(obj, "portal_catalog")
if DISCUSSION_ANNOTATION_KEY in annotions:
conversation = annotions[DISCUSSION_ANNOTATION_KEY]
conversation = conversation.__of__(obj)
for comment in conversation.getComments():
try:
if catalog:
catalog.indexObject(comment)
except StopIteration: # pragma: no cover
pass
except TypeError:
# Catalogs have 'indexObject' as well, but they
# take different args, and will fail
pass
self.manage_catalogClear()
portal = aq_parent(aq_inner(self))
portal.ZopeFindAndApply(
portal,
search_sub=True,
apply_func=indexObject
)
@security.protected(ManageZCatalogEntries)
def manage_catalogRebuild(self, RESPONSE=None, URL1=None):
"""Clears the catalog and indexes all objects with an 'indexObject'
method. This may take a long time.
"""
elapse = time.time()
c_elapse = time.clock()
self.clearFindAndRebuild()
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
msg = ('Catalog Rebuilt\n'
'Total time: %s\n'
'Total CPU time: %s' % (repr(elapse), repr(c_elapse)))
logger.info(msg)
if RESPONSE is not None:
RESPONSE.redirect(
URL1 + '/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote(msg))
class CatalogTool(PloneBaseTool, BaseTool):
"""Plone's catalog tool"""
implements(IPloneCatalogTool)
meta_type = 'Plone Catalog Tool'
security = ClassSecurityInfo()
toolicon = 'skins/plone_images/book_icon.png'
_counter = None
manage_catalogAdvanced = DTMLFile('www/catalogAdvanced', globals())
manage_options = (
{
'action': 'manage_main',
'label': 'Contents'
},
{
'action': 'manage_catalogView',
'label': 'Catalog'
},
{
'action': 'manage_catalogIndexes',
'label': 'Indexes'
},
{
'action': 'manage_catalogSchema',
'label': 'Metadata'
},
{
'action': 'manage_catalogAdvanced',
'label': 'Advanced'
},
{
'action': 'manage_catalogReport',
'label': 'Query Report'
},
{
'action': 'manage_catalogPlan',
'label': 'Query Plan'
},
{
'action': 'manage_propertiesForm',
'label': 'Properties'
},
)
def __init__(self):
ZCatalog.__init__(self, self.getId())
def _removeIndex(self, index):
"""Safe removal of an index.
"""
try:
self.manage_delIndex(index)
except:
pass
def _listAllowedRolesAndUsers(self, user):
"""Makes sure the list includes the user's groups.
"""
result = user.getRoles()
if 'Anonymous' in result:
# The anonymous user has no further roles
return ['Anonymous']
result = list(result)
if hasattr(aq_base(user), 'getGroups'):
groups = ['user:%s' % x for x in user.getGroups()]
if groups:
result = result + groups
# Order the arguments from small to large sets
result.insert(0, 'user:%s' % user.getId())
result.append('Anonymous')
return result
security.declarePrivate('indexObject')
def indexObject(self, object, idxs=None):
"""Add object to catalog.
The optional idxs argument is a list of specific indexes
to populate (all of them by default).
"""
if idxs is None:
idxs = []
self.reindexObject(object, idxs)
security.declareProtected(ManageZCatalogEntries, 'catalog_object')
def catalog_object(self,
object,
uid=None,
idxs=None,
update_metadata=1,
pghandler=None):
if idxs is None:
idxs = []
self._increment_counter()
w = object
if not IIndexableObject.providedBy(object):
# This is the CMF 2.2 compatible approach, which should be used
# going forward
wrapper = queryMultiAdapter((object, self), IIndexableObject)
if wrapper is not None:
w = wrapper
ZCatalog.catalog_object(self,
w,
uid,
idxs,
update_metadata,
pghandler=pghandler)
security.declareProtected(ManageZCatalogEntries, 'catalog_object')
def uncatalog_object(self, *args, **kwargs):
self._increment_counter()
return BaseTool.uncatalog_object(self, *args, **kwargs)
def _increment_counter(self):
if self._counter is None:
self._counter = Length()
self._counter.change(1)
security.declarePrivate('getCounter')
def getCounter(self):
return self._counter is not None and self._counter() or 0
security.declareProtected(SearchZCatalog, 'searchResults')
def searchResults(self, REQUEST=None, **kw):
"""Calls ZCatalog.searchResults with extra arguments that
limit the results to what the user is allowed to see.
This version uses the 'effectiveRange' DateRangeIndex.
It also accepts a keyword argument show_inactive to disable
effectiveRange checking entirely even for those without portal
wide AccessInactivePortalContent permission.
"""
kw = kw.copy()
show_inactive = kw.get('show_inactive', False)
if isinstance(REQUEST, dict) and not show_inactive:
show_inactive = 'show_inactive' in REQUEST
user = _getAuthenticatedUser(self)
kw['allowedRolesAndUsers'] = self._listAllowedRolesAndUsers(user)
if not show_inactive and not _checkPermission(
AccessInactivePortalContent, self):
kw['effectiveRange'] = DateTime()
return ZCatalog.searchResults(self, REQUEST, **kw)
__call__ = searchResults
security.declareProtected(ManageZCatalogEntries, 'clearFindAndRebuild')
def clearFindAndRebuild(self):
"""Empties catalog, then finds all contentish objects (i.e. objects
with an indexObject method), and reindexes them.
This may take a long time.
"""
def indexObject(obj, path):
if (base_hasattr(obj, 'indexObject')
and safe_callable(obj.indexObject)):
try:
obj.indexObject()
except TypeError:
# Catalogs have 'indexObject' as well, but they
# take different args, and will fail
pass
self.manage_catalogClear()
portal = aq_parent(aq_inner(self))
portal.ZopeFindAndApply(portal,
search_sub=True,
apply_func=indexObject)
security.declareProtected(ManageZCatalogEntries, 'manage_catalogRebuild')
def manage_catalogRebuild(self, RESPONSE=None, URL1=None):
"""Clears the catalog and indexes all objects with an 'indexObject'
method. This may take a long time.
"""
elapse = time.time()
c_elapse = time.clock()
self.clearFindAndRebuild()
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
if RESPONSE is not None:
RESPONSE.redirect(URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote('Catalog Rebuilt\n'
'Total time: %s\n'
'Total CPU time: %s' %
(repr(elapse), repr(c_elapse))))
class BTreeOrderStorage(Persistent):
implements(IOrderStorage)
title = u"BTree Storage"
def __init__(self):
self._orderStorage = BTreeImplementation()
self._orderCount = 0
self._length = Length()
def getOrder(self, id):
return self._orderStorage[id]
def getAllOrders(self):
return list(self._orderStorage.values())
def _addDataRow(self, value):
"""Adds a row of data to the internal storage
"""
# Get id for order to be added and increment internal counter
id = self._orderCount + 1
self._orderCount += 1
self._orderStorage[id] = value
self._length.change(1)
return id
security.declareProtected(ModifyPortalContent, 'createOrder')
def createOrder(self, status=None, date=None, customer_data=None,
shipping_data=None, cart_data=None, total=None):
""" a wrapper for the _addDataRow method """
new_order = Order()
new_order.status = status
new_order.date = date
new_order.total = Decimal(total)
order_id = self._addDataRow(new_order)
new_order.order_id = order_id
# calc order number
now = DateTime()
order_prefix = '%03d%s' % (now.dayOfYear() + 500, now.yy())
order_number = '%s%04d' % (order_prefix, order_id)
new_order.title = order_number
for key in customer_data.keys():
setattr(new_order, "customer_%s" % key, customer_data[key])
for key in shipping_data.keys():
setattr(new_order, "shipping_%s" % key, shipping_data[key])
# store cart in order
all_cart_items = []
vat_amount_total = Decimal('0.0')
for key in cart_data.keys():
cart_items = CartItems()
cart_items.sku_code = cart_data[key]['skucode']
cart_items.quantity = cart_data[key]['quantity']
cart_items.title = cart_data[key]['title']
cart_items.price = Decimal(cart_data[key]['price'])
cart_items.show_price = cart_data[key]['show_price']
cart_items.total = Decimal(cart_data[key]['total'])
cart_items.supplier_name = cart_data[key]['supplier_name']
cart_items.supplier_email = cart_data[key]['supplier_email']
cart_items.vat_rate = cart_data[key]['vat_rate']
cart_items.vat_amount = Decimal(cart_data[key]['vat_amount'])
cart_items.order_id = order_id
cart_items.order = new_order
all_cart_items.append(cart_items)
vat_amount_total += cart_items.vat_amount
new_order.cartitems = all_cart_items
new_order.vat_amount = vat_amount_total
return order_id
def cancelOrder(self, order_id):
del self._orderStorage[order_id]
self._length.change(-1)
def flush(self):
pass
def getFieldNames(self):
o = Order()
field_names = [f for f in dir(o) if not (f.startswith('_') or \
f.startswith('get') or \
f == 'cartitems')]
return field_names
class UserRatingStorage(Contained, Persistent):
"""BTree-based storage for user ratings, keeps a running
statistics tally for efficiency."""
implements(IUserRating, IRatingStorage)
_average = 0.0
_anon_average = 0.0
_most_recent = None
_penultimate = None
_length = None
_anon_length = None
# BBB
scale = 5
def __init__(self):
"""Setup our data structures"""
self._anon_ratings = IOBTree()
self._ratings = OOBTree()
self._sessions = OOBTree()
self._length = Length()
self._anon_length = Length()
def rate(self, rating, username=None, session_key=None):
"""Set a rating for a particular user"""
# We keep a running average for efficiency, we need to
# have the current statistics to do so
orig_total = self._average * self.numberOfRatings
orig_rating = self._ratings.get(username, 0.0)
rating = Rating(rating, username)
if username:
self._ratings[username] = rating
if not orig_rating:
# If the user hadn't set a rating yet, the number of
# ratings grew
self._length.change(1)
else:
# For anonymous users, we use a sequential key, which
# may lead to conflicts. There's probably a better way
anon_total = self._anon_average * self._anon_count
# Update the corresponding BTree length to get the key
self._anon_length.change(1)
self._anon_ratings[self._anon_count] = rating
self._anon_average = (anon_total + rating)/self._anon_count
# If a session key was passed in for an anonymous user
# store it with a datestamp
if session_key:
self._sessions[session_key] = datetime.utcnow()
# Calculate the updated average
self._average = (orig_total + rating - orig_rating)/self.numberOfRatings
# If this isn't just a change in the last rating update the
# most recent rating
if self._most_recent and username != self._most_recent.userid:
self._penultimate = self._most_recent
# Mark this new rating as the most recent
self._most_recent = rating
return rating
def userRating(self, username=None):
"""Retreive the rating for the specified user, or the average
anonymous rating if no user is specified"""
if username is not None:
return self._ratings.get(username, None)
else:
if self._anon_count:
return Rating(self._anon_average)
else:
return None
def remove_rating(self, username):
"""Remove the rating for a given user"""
orig_total = self._average * self.numberOfRatings
rating = self._ratings[username]
del self._ratings[username]
self._length.change(-1)
# Since we want to keep track of the most recent rating, we
# need to replace it with the second most recent if the most
# recent was deleted
if rating is self.most_recent:
self._most_recent = self._penultimate
self._penultimate = None
if self._most_recent is None:
ordered = sorted(self.all_user_ratings(True),
key=lambda x: x.timestamp)
if ordered:
self._most_recent = ordered[-1]
if len(ordered > 1):
self._penultimate = ordered[-2]
else:
self._penultimate = None
else:
self._most_recent = None
self._penultimate = None
# Update the average
self._average = float(self.numberOfRatings and
(orig_total - rating)/self.numberOfRatings)
@property
def _anon_count(self):
# Dynamic Migration
if self._anon_length is None:
self._anon_length = Length(len(self._anon_ratings))
return self._anon_length()
@property
def most_recent(self):
return self._most_recent
def all_user_ratings(self, include_anon=False):
ratings = self._ratings.values()
if include_anon:
ratings = chain(ratings, self._anon_ratings.values())
return ratings
def all_raters(self):
return self._ratings.keys()
@property
def numberOfRatings(self):
# Dynamic Migration
if self._length is None:
self._length = Length(len(self._ratings))
return self._length() + self._anon_count
@property
def averageRating(self):
return self._average
def last_anon_rating(self, session_key):
"""Returns a timestamp indicating the last time the anonymous user
with the given session_key rated the object."""
return self._sessions.get(session_key, None)
class GraphDB(Persistent):
def __init__(self):
self._init()
self.node_catalog = Catalog()
self.edge_catalog = Catalog()
def _init(self):
self.nodes = IOBTree()
self.edges = IOBTree()
self.edgedata = IOBTree()
self.outgoing = IOBTree()
self.incoming = IOBTree()
self.typeids = PObject()
self._nodeid = Length(0)
self._edgeid = Length(0)
self._typeid = Length(0)
def nodeid(self):
self._nodeid.change(1)
return self._nodeid.value
def edgeid(self):
self._edgeid.change(1)
return self._edgeid.value
def typeid(self, name):
if not hasattr(self.typeids, name):
self._typeid.change(1)
setattr(self.typeids, name, self._typeid.value)
self.revtypes[self._typeid.value] = name
return getattr(self.typeids, name)
@property
def revtypes(self):
if (not hasattr(self, '_v_revtypes')) or (not bool(self._v_revtypes)):
dir(self.typeids)
dir(self.typeids)
self._v_revtypes = dict([
(v, k) for k, v in list(self.typeids.__dict__.items())
])
return self._v_revtypes
def getType(self, typeid):
if type(typeid) != int:
#lets assume an edge
typeid = typeid[2]
return self.revtypes[typeid]
def addNode(self, **kwargs):
if '_id' not in kwargs:
_id = self.nodeid()
else:
_id = kwargs.pop('_id')
self.nodes[_id] = kwargs
ln = self.lightNode(_id, kwargs)
self.node_catalog.index_doc(_id, ln)
return ln
def lightNode(self, _id, node=None):
"{'_id':nodeid, ...other attributes...}"
if node == None:
node = self.nodes[_id]
out = dict(node)
out['_id'] = _id
return out
def addEdge(self, start, end, edgetype, **kwargs):
_id = self.edgeid()
if type(edgetype) != int:
edgetype = self.typeid(edgetype)
if type(start) == dict:
start = start['_id']
if type(end) == dict:
end = end['_id']
edge = [start, end, edgetype]
self.edges[_id] = edge
if kwargs:
self.edgedata[_id] = kwargs
le = self.lightEdge(_id, edge)
self.edge_catalog.index_doc(_id, le)
le = self.lightEdge(_id, edge)
# edgeid:nodeid
data = self.outgoing.setdefault(edgetype,
IOBTree()).setdefault(start, {})
data[_id] = end
self.outgoing[edgetype][start] = data
data = self.incoming.setdefault(edgetype,
IOBTree()).setdefault(end, {})
data[_id] = start
self.incoming[edgetype][end] = data
return le
def lightEdge(self, _id, edge=None):
'[sourceid targetid typeid kwargs edgeid]'
if edge == None:
edge = self.edges[_id]
out = list(edge)
out.append(self.edgedata.get(_id, {}))
out.append(_id)
return out
def delEdge(self, edge):
if type(edge) == int:
edge = self.lightEdge(edge)
start, end, edgetype, props, edgeid = edge
data = self.outgoing[edgetype][start]
del (data[edgeid])
self.outgoing[edgetype][start] = data
data = self.incoming[edgetype][end]
del (data[edgeid])
self.incoming[edgetype][end] = data
del (self.edges[edgeid])
if edgeid in self.edgedata:
self.edge_catalog.unindex_doc(edgeid)
#del(self.edges[edgeid])
def delNode(self, node):
if type(node) == int:
node = self.lightNode(node)
nodeid = node['_id']
for edgetype in list(self.outgoing.keys()):
if len(self.outgoing[edgetype].get(nodeid, {})) > 0:
raise StillConnected('outgoing',
self.outgoing[edgetype][nodeid])
for edgetype in list(self.incoming.keys()):
if len(self.incoming[edgetype].get(nodeid, {})) > 0:
raise StillConnected('incoming',
self.incoming[edgetype][nodeid])
#all good, lets delete
for edgetype in list(self.outgoing.keys()):
if nodeid in self.outgoing[edgetype]:
del (self.outgoing[edgetype][nodeid])
for edgetype in list(self.incoming.keys()):
if nodeid in self.incoming[edgetype]:
del (self.incoming[edgetype][nodeid])
self.node_catalog.unindex_doc(nodeid)
del (self.nodes[nodeid])
def updateNode(self, lightnode):
nodeid = lightnode['_id']
data = dict(lightnode)
self.nodes[nodeid] = data
self.node_catalog.reindex_doc(nodeid, lightnode)
def updateEdge(self, lightedge):
edgeid = lightedge[4]
edge = list(lightedge[:4])
data = lightedge[3]
self.edges[edgeid] = edge
if data:
self.edgedata[edgeid] = data
self.edge_catalog.reindex_doc(edgeid, lightedge)
elif edgeid in self.edgedata:
del (self.edgedata[edgeid])
self.edge_catalog.unindex_doc(edgeid)
def kwQuery(self, **kwargs):
kwitems = list(kwargs.items())
key, value = kwitems[0]
query = rc_query.Eq(key, value)
for k, v in kwitems[1:]:
query = query & rc_query.Eq(k, v)
return query
def queryNode(self, **kwargs):
result = self.node_catalog.query(self.kwQuery(**kwargs))
return [self.lightNode(i) for i in result[1]]
def queryEdge(self, **kwargs):
result = self.edge_catalog.query(self.kwQuery(**kwargs))
return [self.lightEdge(i) for i in result[1]]
def prepareTypes(self, types=None):
if types is None:
return types
else:
if type(types) not in (list, tuple):
types = [types]
out = []
for t in types:
if type(t) == str:
t = self.typeid(t)
out.append(t)
return out
################## Higher Level API, functionality > speed ###################
def getAllEdges(self, nodeids, directions=None, types=None, returnIds=0):
if not islisttype(nodeids):
nodeids = [nodeids]
if directions == None:
directions = ['i', 'o']
elif type(directions) not in (list, tuple):
directions = [directions]
types = self.prepareTypes(types)
tmp = []
for n in nodeids:
if type(n) != int:
n = n['_id']
tmp.append(n)
nodeids = tmp
out = EdgeDict()
for direction in directions:
if direction.startswith('i'):
d = 'incoming'
elif direction.startswith('o'):
d = 'outgoing'
else:
raise 'unknown'
result = []
container = getattr(self, d)
for edgetype in list(container.keys()):
if types != None and edgetype not in types:
continue
for n in nodeids:
edges = container[edgetype].get(n, {})
if returnIds:
result.extend(list(edges.keys()))
else:
for key in list(edges.keys()):
result.append(self.edge(key))
out[direction] = result
if len(directions) == 1:
return result
else:
return out
# XXX work in progress
def getEdges(self, start, end, edgetype):
#import ipdb; ipdb.set_trace()
if type(edgetype) != int:
edgetype = self.typeid(edgetype)
if type(start) != int:
start = start['_id']
if type(end) != int:
end = end['_id']
out = []
targets = self.outgoing.get(edgetype, {}).get(start, {})
for edgeid, nodeid in list(targets.items()):
if nodeid == end:
out.append(self.lightEdge(edgeid))
return out
# XXX work in progress
def addUniqueEdge(self, start, end, edgetype, **kwargs):
if not self.getEdges(start, end, edgetype):
return self.addEdge(start, end, edgetype, **kwargs)
def clean(self):
#import ipdb; ipdb.set_trace()
for k in list(self.edges.keys()):
self.delEdge(k)
for k in list(self.nodes.keys()):
self.delNode(k)
self._init()
def render(self, filename='graphagus', label='name', source=False):
from graphviz import Digraph
dot = Digraph('Graphagus dump', format='svg')
for k in list(self.nodes.keys()):
n = self.lightNode(k)
dot.node(str(k), n[label])
for k in list(self.edges.keys()):
e = self.lightEdge(k)
dot.edge(str(e[0]), str(e[1]), self.getType(e))
if source:
return dot.source
else:
dot.render(filename, cleanup=True)
def edge(self, lightEdge):
if type(lightEdge) == int:
lightEdge = self.lightEdge(lightEdge)
return Edge(self, lightEdge)
def node(self, lightNode):
if type(lightNode) == int:
lightNode = self.lightNode(lightNode)
return Node(self, lightNode)
class Folder(Persistent):
""" A folder implementation which acts much like a Python dictionary.
Keys must be Unicode strings; values must be arbitrary Python objects.
"""
family = BTrees.family64
__name__ = None
__parent__ = None
# Default uses ordering of underlying BTree.
_order = None # tuple of names
_order_oids = None # tuple of oids
_reorderable = None
def __init__(self, data=None, family=None):
""" Constructor. Data may be an initial dictionary mapping object
name to object. """
if family is not None:
self.family = family
if data is None:
data = {}
self.data = self.family.OO.BTree(data)
self._num_objects = Length(len(data))
def set_order(self, names, reorderable=None):
""" Sets the folder order. ``names`` is a list of names for existing
folder items, in the desired order. All names that currently exist in
the folder must be mentioned in ``names``, or a :exc:`ValueError` will
be raised.
If ``reorderable`` is passed, value, it must be ``None``, ``True`` or
``False``. If it is ``None``, the reorderable flag will not be reset
from its current value. If it is anything except ``None``, it will be
treated as a boolean and the reorderable flag will be set to that
value. The ``reorderable`` value of a folder will be returned by that
folder's :meth:`~substanced.folder.Folder.is_reorderable` method. The
:meth:`~substanced.folder.Folder.is_reorderable` method is used by the
SDI folder contents view to indicate that the folder can or cannot be
reordered via the web UI.
If ``reorderable`` is set to ``True``, the
:meth:`~substanced.folder.Folder.reorder` method will work properly,
otherwise it will raise a :exc:`ValueError` when called.
"""
nameset = set(names)
if len(self) != len(nameset):
raise ValueError('Must specify all names when calling set_order')
if len(names) != len(nameset):
raise ValueError('No repeated items allowed in names')
order = []
order_oids = []
for name in names:
assert(isinstance(name, string_types))
name = u(name)
oid = get_oid(self[name])
order.append(name)
order_oids.append(oid)
self._order = tuple(order)
self._order_oids = tuple(order_oids)
assert(len(self._order) == len(self._order_oids))
if reorderable is not None:
self._reorderable = bool(reorderable)
def unset_order(self):
""" Remove set order from a folder, making it unordered, and
non-reorderable."""
if self._order is not None:
del self._order
if self._order_oids is not None:
del self._order_oids
if self._reorderable is not None:
del self._reorderable
def reorder(self, names, before):
""" Move one or more items from a folder into new positions inside that
folder. ``names`` is a list of ids of existing folder subobject names,
which will be inserted in order before the item named ``before``. All
other items are left in the original order. If ``before`` is ``None``,
the items will be appended after the last item in the current order. If
this method is called on a folder which does not have an order set, or
which is not reorderable, a :exc:`ValueError` will be raised. A
:exc:`KeyError` is raised, if ``before`` does not correspond to any
item, and is not ``None``."""
if not self._reorderable:
raise ValueError('Folder is not reorderable')
before_idx = None
if len(set(names)) != len(names):
raise ValueError('No repeated values allowed in names')
if before is not None:
if not before in self._order:
raise FolderKeyError(before)
before_idx = self._order.index(before)
assert(len(self._order) == len(self._order_oids))
order_names = list(self._order)
order_oids = list(self._order_oids)
reorder_names = []
reorder_oids = []
for name in names:
assert(isinstance(name, string_types))
name = u(name)
if not name in order_names:
raise FolderKeyError(name)
idx = order_names.index(name)
oid = order_oids[idx]
order_names[idx] = None
order_oids[idx] = None
reorder_names.append(name)
reorder_oids.append(oid)
assert(len(reorder_names) == len(reorder_oids))
# NB: technically we could use filter(None, oids) and filter(None,
# names) because names cannot be empty string and oid 0 is disallowed,
# but just in case this becomes untrue later we define "filt" instead
def filt(L):
return [x for x in L if x is not None]
if before_idx is None:
order_names = filt(order_names)
order_names.extend(reorder_names)
order_oids = filt(order_oids)
order_oids.extend(reorder_oids)
else:
before_idx_names = filt(order_names[:before_idx])
after_idx_names = filt(order_names[before_idx:])
before_idx_oids = filt(order_oids[:before_idx])
after_idx_oids = filt(order_oids[before_idx:])
assert(
len(before_idx_names+after_idx_names) ==
len(before_idx_oids+after_idx_oids)
)
order_names = before_idx_names + reorder_names + after_idx_names
order_oids = before_idx_oids + reorder_oids + after_idx_oids
for oid, name in zip(order_oids, order_names):
# belt and suspenders check
assert oid == get_oid(self[name])
self._order = tuple(order_names)
self._order_oids = tuple(order_oids)
def is_ordered(self):
""" Return true if the folder has a manually set ordering, false
otherwise."""
return self._order is not None
def is_reorderable(self):
""" Return true if the folder can be reordered, false otherwise."""
return self._reorderable
def sort(self, oids, reverse=False, limit=None, **kw):
# used by the hypatia resultset "sort" method when the folder contents
# view uses us as a "sort index"
if self._order_oids is not None:
ids = [oid for oid in self._order_oids if oid in oids]
else:
ids = []
for resource in self.values():
oid = get_oid(resource)
if oid in oids:
ids.append(oid)
if reverse:
ids = ids[::-1]
if limit is not None:
ids = ids[:limit]
return ids
def find_service(self, service_name):
""" Return a service named by ``service_name`` in this folder *or any
parent service folder* or ``None`` if no such service exists. A
shortcut for :func:`substanced.service.find_service`."""
return find_service(self, service_name)
def find_services(self, service_name):
""" Returns a sequence of service objects named by ``service_name``
in this folder's lineage or an empty sequence if no such service
exists. A shortcut for :func:`substanced.service.find_services`"""
return find_services(self, service_name)
def add_service(self, name, obj, registry=None, **kw):
""" Add a service to this folder named ``name``."""
if registry is None:
registry = get_current_registry()
kw['registry'] = registry
self.add(name, obj, **kw)
obj.__is_service__ = True
def keys(self):
""" Return an iterable sequence of object names present in the folder.
Respect order, if set.
"""
if self._order is not None:
return self._order
return self.data.keys()
order = property(keys, set_order, unset_order) # b/c
def __iter__(self):
""" An alias for ``keys``
"""
return iter(self.keys())
def values(self):
""" Return an iterable sequence of the values present in the folder.
Respect ``order``, if set.
"""
if self._order is not None:
return [self.data[name] for name in self.keys()]
return self.data.values()
def items(self):
""" Return an iterable sequence of (name, value) pairs in the folder.
Respect ``order``, if set.
"""
if self._order is not None:
return [(name, self.data[name]) for name in self.keys()]
return self.data.items()
def __len__(self):
""" Return the number of objects in the folder.
"""
return self._num_objects()
def __nonzero__(self):
""" Return ``True`` unconditionally.
"""
return True
__bool__ = __nonzero__
def __repr__(self):
klass = self.__class__
classname = '%s.%s' % (klass.__module__, klass.__name__)
return '<%s object %r at %#x>' % (classname,
self.__name__,
id(self))
def __getitem__(self, name):
""" Return the object named ``name`` added to this folder or raise
``KeyError`` if no such object exists. ``name`` must be a Unicode
object or directly decodeable to Unicode using the system default
encoding.
"""
with statsd_timer('folder.get'):
name = u(name)
return wrap_if_broken(self.data[name])
def get(self, name, default=None):
""" Return the object named by ``name`` or the default.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
"""
with statsd_timer('folder.get'):
name = u(name)
return wrap_if_broken(self.data.get(name, default))
def __contains__(self, name):
""" Does the container contains an object named by name?
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
"""
name = u(name)
return name in self.data
def __setitem__(self, name, other):
""" Set object ``other' into this folder under the name ``name``.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
``name`` cannot be the empty string.
When ``other`` is seated into this folder, it will also be decorated
with a ``__parent__`` attribute (a reference to the folder into which
it is being seated) and ``__name__`` attribute (the name passed in to
this function. It must not already have a ``__parent__`` attribute
before being seated into the folder, or an exception will be raised.
If a value already exists in the foldr under the name ``name``, raise
:exc:`KeyError`.
When this method is called, the object will be added to the objectmap,
an :class:`substanced.event.ObjectWillBeAdded` event will be emitted
before the object obtains a ``__name__`` or ``__parent__`` value, then
a :class:`substanced.event.ObjectAdded` will be emitted after the
object obtains a ``__name__`` and ``__parent__`` value.
"""
return self.add(name, other)
def validate_name(self, name, reserved_names=()):
"""
Validate the ``name`` passed to ensure that it's addable to the folder.
Returns the name decoded to Unicode if it passes all addable checks.
It's not addable if:
- the name is not decodeable to Unicode.
- the name starts with ``@@`` (conflicts with explicit view names).
- the name has slashes in it (WSGI limitation).
- the name is empty.
If any of these conditions are untrue, raise a :exc:`ValueError`. If
the name passed is in the list of ``reserved_names``, raise a
:exc:`ValueError`.
"""
if not isinstance(name, STRING_TYPES):
raise ValueError("Name must be a string rather than a %s" %
name.__class__.__name__)
if not name:
raise ValueError("Name must not be empty")
try:
name = u(name)
except UnicodeDecodeError: #pragma NO COVER (on Py3k)
raise ValueError('Name "%s" not decodeable to unicode' % name)
if name in reserved_names:
raise ValueError('%s is a reserved name' % name)
if name.startswith('@@'):
raise ValueError('Names which start with "@@" are not allowed')
if '/' in name:
raise ValueError('Names which contain a slash ("/") are not '
'allowed')
return name
def check_name(self, name, reserved_names=()):
""" Perform all the validation checks implied by
:meth:`~substanced.folder.Folder.validate_name` against the ``name``
supplied but also fail with a
:class:`~substanced.folder.FolderKeyError` if an object with the name
``name`` already exists in the folder."""
name = self.validate_name(name, reserved_names=reserved_names)
if name in self.data:
raise FolderKeyError('An object named %s already exists' % name)
return name
def add(self, name, other, send_events=True, reserved_names=(),
duplicating=None, moving=None, loading=False, registry=None):
""" Same as ``__setitem__``.
If ``send_events`` is False, suppress the sending of folder events.
Don't allow names in the ``reserved_names`` sequence to be added.
If ``duplicating`` not ``None``, it must be the object which is being
duplicated; a result of a non-``None`` duplicating means that oids will
be replaced in objectmap. If ``moving`` is not ``None``, it must be
the folder from which the object is moving; this will be the ``moving``
attribute of events sent by this function too. If ``loading`` is
``True``, the ``loading`` attribute of events sent as a result of
calling this method will be ``True`` too.
This method returns the name used to place the subobject in the
folder (a derivation of ``name``, usually the result of
``self.check_name(name)``).
"""
if registry is None:
registry = get_current_registry()
name = self.check_name(name, reserved_names)
if getattr(other, '__parent__', None):
raise ValueError(
'obj %s added to folder %s already has a __parent__ attribute, '
'please remove it completely from its existing parent (%s) '
'before trying to readd it to this one' % (
other, self, self.__parent__)
)
with statsd_timer('folder.add'):
objectmap = find_objectmap(self)
if objectmap is not None:
basepath = resource_path_tuple(self)
for node in postorder(other):
node_path = node_path_tuple(node)
path_tuple = basepath + (name,) + node_path[1:]
# the below gives node an objectid; if the will-be-added
# event is the result of a duplication, replace the oid of
# the node with a new one
objectmap.add(
node,
path_tuple,
duplicating=duplicating is not None,
moving=moving is not None,
)
if send_events:
event = ObjectWillBeAdded(
other, self, name, duplicating=duplicating, moving=moving,
loading=loading,
)
self._notify(event, registry)
other.__parent__ = self
other.__name__ = name
self.data[name] = other
self._num_objects.change(1)
if self._order is not None:
oid = get_oid(other)
self._order += (name,)
self._order_oids += (oid,)
if send_events:
event = ObjectAdded(
other, self, name, duplicating=duplicating, moving=moving,
loading=loading,
)
self._notify(event, registry)
return name
def pop(self, name, default=marker, registry=None):
""" Remove the item stored in the under ``name`` and return it.
If ``name`` doesn't exist in the folder, and ``default`` **is not**
passed, raise a :exc:`KeyError`.
If ``name`` doesn't exist in the folder, and ``default`` **is**
passed, return ``default``.
When the object stored under ``name`` is removed from this folder,
remove its ``__parent__`` and ``__name__`` values.
When this method is called, emit an
:class:`substanced.event.ObjectWillBeRemoved` event before the
object loses its ``__name__`` or ``__parent__`` values. Emit an
:class:`substanced.event.ObjectRemoved` after the object loses its
``__name__`` and ``__parent__`` value,
"""
if registry is None:
registry = get_current_registry()
try:
result = self.remove(name, registry=registry)
except KeyError:
if default is marker:
raise
return default
return result
def _notify(self, event, registry=None):
if registry is None:
registry = get_current_registry()
registry.subscribers((event, event.object, self), None)
def __delitem__(self, name):
""" Remove the object from this folder stored under ``name``.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
If no object is stored in the folder under ``name``, raise a
:exc:`KeyError`.
When the object stored under ``name`` is removed from this folder,
remove its ``__parent__`` and ``__name__`` values.
When this method is called, the removed object will be removed from the
objectmap, a :class:`substanced.event.ObjectWillBeRemoved` event will
be emitted before the object loses its ``__name__`` or ``__parent__``
values and a :class:`substanced.event.ObjectRemoved` will be emitted
after the object loses its ``__name__`` and ``__parent__`` value,
"""
return self.remove(name)
def remove(self, name, send_events=True, moving=None, loading=False,
registry=None):
""" Same thing as ``__delitem__``.
If ``send_events`` is false, suppress the sending of folder events.
If ``moving`` is not ``None``, the ``moving`` argument must be the
folder to which the named object will be moving. This value will be
passed along as the ``moving`` attribute of the events sent as the
result of this action. If ``loading`` is ``True``, the ``loading``
attribute of events sent as a result of calling this method will be
``True`` too.
"""
name = u(name)
other = wrap_if_broken(self.data[name])
oid = get_oid(other, None)
if registry is None:
registry = get_current_registry()
with statsd_timer('folder.remove'):
if send_events:
event = ObjectWillBeRemoved(
other, self, name, moving=moving, loading=loading
)
self._notify(event, registry)
if hasattr(other, '__parent__'):
try:
del other.__parent__
except AttributeError:
# this might be a broken object
pass
if hasattr(other, '__name__'):
try:
del other.__name__
except AttributeError:
# this might be a broken object
pass
del self.data[name]
self._num_objects.change(-1)
if self._order is not None:
assert(len(self._order) == len(self._order_oids))
idx = self._order.index(name)
order = list(self._order)
order.pop(idx)
order_oids = list(self._order_oids)
order_oids.pop(idx)
self._order = tuple(order)
self._order_oids = tuple(order_oids)
objectmap = find_objectmap(self)
removed_oids = set([oid])
if objectmap is not None and oid is not None:
removed_oids = objectmap.remove(oid, moving=moving is not None)
if send_events:
event = ObjectRemoved(other, self, name, removed_oids,
moving=moving, loading=loading)
self._notify(event, registry)
return other
def copy(self, name, other, newname=None, registry=None):
"""
Copy a subobject named ``name`` from this folder to the folder
represented by ``other``. If ``newname`` is not none, it is used as
the target object name; otherwise the existing subobject name is
used.
"""
if newname is None:
newname = name
if registry is None:
registry = get_current_registry()
with statsd_timer('folder.copy'):
obj = self[name]
newobj = copy(obj)
return other.add(
newname, newobj, duplicating=obj, registry=registry
)
def move(self, name, other, newname=None, registry=None):
"""
Move a subobject named ``name`` from this folder to the folder
represented by ``other``. If ``newname`` is not none, it is used as
the target object name; otherwise the existing subobject name is
used.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events as well as the Added and WillBeAdded events
sent will indicate that the object is moving.
"""
if newname is None:
newname = name
if registry is None:
registry = get_current_registry()
ob = self.remove(
name,
moving=other,
registry=registry
)
other.add(
newname,
ob,
moving=self,
registry=registry
)
return ob
def rename(self, oldname, newname, registry=None):
"""
Rename a subobject from oldname to newname.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events sent will indicate that the object is
moving.
"""
if registry is None:
registry = get_current_registry()
return self.move(oldname, self, newname, registry=registry)
def replace(self, name, newobject, send_events=True, registry=None):
""" Replace an existing object named ``name`` in this folder with a
new object ``newobject``. If there isn't an object named ``name`` in
this folder, an exception will *not* be raised; instead, the new
object will just be added.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events will be sent for the old object, and the
WillBeAdded and Added events will be sent for the new object.
"""
if registry is None:
registry = get_current_registry()
if name in self:
self.remove(name, send_events=send_events)
self.add(name, newobject, send_events=send_events, registry=registry)
def load(self, name, newobject, registry=None):
""" A replace method used by the code that loads an existing dump.
Events sent during this replace will have a true ``loading`` flag."""
if registry is None:
registry = get_current_registry()
if name in self:
self.remove(name, loading=True)
self.add(name, newobject, loading=True, registry=registry)
class Folder(Persistent):
""" A folder implementation which acts much like a Python dictionary.
Keys must be Unicode strings; values must be arbitrary Python objects.
"""
family = BTrees.family64
__name__ = None
__parent__ = None
# Default uses ordering of underlying BTree.
_order = None
def _get_order(self):
if self._order is not None:
return list(self._order)
return self.data.keys()
def _set_order(self, value):
# XXX: should we test against self.data.keys()?
self._order = tuple([unicode(x) for x in value])
def _del_order(self):
del self._order
order = property(_get_order, _set_order, _del_order)
def __init__(self, data=None, family=None):
""" Constructor. Data may be an initial dictionary mapping object
name to object. """
if family is not None:
self.family = family
if data is None:
data = {}
self.data = self.family.OO.BTree(data)
self._num_objects = Length(len(data))
def find_service(self, service_name):
""" Return a service named by ``service_name`` in this folder's
``__services__`` folder *or any parent service folder* or ``None`` if
no such service exists. A shortcut for
:func:`substanced.service.find_service`."""
return find_service(self, service_name)
def find_services(self, service_name):
""" Returns a sequence of service objects named by ``service_name``
in this folder's lineage or an empty sequence if no such service
exists. A shortcut for :func:`substanced.service.find_services`"""
return find_services(self, service_name)
def add_service(self, name, obj, registry=None):
""" Add a service to this folder's ``__services__`` folder named
``name``."""
if registry is None:
registry = get_current_registry()
services = self.get(SERVICES_NAME)
if services is None:
services = registry.content.create('Services')
self.add(SERVICES_NAME, services, reserved_names=())
services.add(name, obj)
def keys(self):
""" Return an iterable sequence of object names present in the folder.
Respect ``order``, if set.
"""
return self.order
def __iter__(self):
""" An alias for ``keys``
"""
return iter(self.order)
def values(self):
""" Return an iterable sequence of the values present in the folder.
Respect ``order``, if set.
"""
if self._order is not None:
return [self.data[name] for name in self.order]
return self.data.values()
def items(self):
""" Return an iterable sequence of (name, value) pairs in the folder.
Respect ``order``, if set.
"""
if self._order is not None:
return [(name, self.data[name]) for name in self.order]
return self.data.items()
def __len__(self):
""" Return the number of objects in the folder.
"""
return self._num_objects()
def __nonzero__(self):
""" Return ``True`` unconditionally.
"""
return True
def __repr__(self):
klass = self.__class__
classname = '%s.%s' % (klass.__module__, klass.__name__)
return '<%s object %r at %#x>' % (classname,
self.__name__,
id(self))
def __getitem__(self, name):
""" Return the object named ``name`` added to this folder or raise
``KeyError`` if no such object exists. ``name`` must be a Unicode
object or directly decodeable to Unicode using the system default
encoding.
"""
name = unicode(name)
return self.data[name]
def get(self, name, default=None):
""" Return the object named by ``name`` or the default.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
"""
name = unicode(name)
return self.data.get(name, default)
def __contains__(self, name):
""" Does the container contains an object named by name?
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
"""
name = unicode(name)
return name in self.data
def __setitem__(self, name, other):
""" Set object ``other' into this folder under the name ``name``.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
``name`` cannot be the empty string.
When ``other`` is seated into this folder, it will also be
decorated with a ``__parent__`` attribute (a reference to the
folder into which it is being seated) and ``__name__``
attribute (the name passed in to this function.
If a value already exists in the foldr under the name ``name``, raise
:exc:`KeyError`.
When this method is called, emit an
:class:`substanced.event.ObjectWillBeAdded` event before the
object obtains a ``__name__`` or ``__parent__`` value. Emit an
:class:`substanced.event.ObjectAdded` after the object obtains a
``__name__`` and ``__parent__`` value.
"""
return self.add(name, other)
def check_name(self, name, reserved_names=RESERVED_NAMES):
"""
Check the ``name`` passed to ensure that it's addable to the folder.
Returns the name decoded to Unicode if it passes all addable checks.
It's not addable if:
- an object by the name already exists in the folder
- the name is not decodeable to Unicode.
- the name starts with ``@@`` (conflicts with explicit view names).
- the name has slashes in it (WSGI limitation).
- the name is empty.
If any of these conditions are untrue, raise a :exc:`ValueError`. If
the name passed is in the list of ``reserved_names``, raise a
:exc:`ValueError`.
"""
if not isinstance(name, basestring):
raise ValueError("Name must be a string rather than a %s" %
name.__class__.__name__)
if not name:
raise ValueError("Name must not be empty")
try:
name = unicode(name)
except UnicodeDecodeError:
raise ValueError('Name "%s" not decodeable to unicode' % name)
if name in reserved_names:
raise ValueError('%s is a reserved name' % name)
if name.startswith('@@'):
raise ValueError('Names which start with "@@" are not allowed')
if '/' in name:
raise ValueError('Names which contain a slash ("/") are not '
'allowed')
if name in self.data:
raise FolderKeyError('An object named %s already exists' % name)
return name
def add(self, name, other, send_events=True, reserved_names=RESERVED_NAMES,
duplicating=False, registry=None):
""" Same as ``__setitem__``.
If ``send_events`` is False, suppress the sending of folder events.
Don't allow names in the ``reserved_names`` sequence to be
added. If ``duplicating`` is True, oids will be replaced in
objectmap.
"""
if registry is None:
registry = get_current_registry()
name = self.check_name(name, reserved_names)
if send_events:
event = ObjectWillBeAdded(other, self, name, duplicating)
self._notify(event, registry)
other.__parent__ = self
other.__name__ = name
self.data[name] = other
self._num_objects.change(1)
if self._order is not None:
self._order += (name,)
if send_events:
event = ObjectAdded(other, self, name)
self._notify(event, registry)
def pop(self, name, default=marker):
""" Remove the item stored in the under ``name`` and return it.
If ``name`` doesn't exist in the folder, and ``default`` **is not**
passed, raise a :exc:`KeyError`.
If ``name`` doesn't exist in the folder, and ``default`` **is**
passed, return ``default``.
When the object stored under ``name`` is removed from this folder,
remove its ``__parent__`` and ``__name__`` values.
When this method is called, emit an
:class:`substanced.event.ObjectWillBeRemoved` event before the
object loses its ``__name__`` or ``__parent__`` values. Emit an
:class:`substanced.event.ObjectRemoved` after the object loses its
``__name__`` and ``__parent__`` value,
"""
try:
result = self.remove(name)
except KeyError:
if default is marker:
raise
return default
return result
def _notify(self, event, registry=None):
if registry is None:
registry = get_current_registry()
registry.subscribers((event, event.object, self), None)
def __delitem__(self, name):
""" Remove the object from this folder stored under ``name``.
``name`` must be a Unicode object or a bytestring object.
If ``name`` is a bytestring object, it must be decodable using the
system default encoding.
If no object is stored in the folder under ``name``, raise a
:exc:`KeyError`.
When the object stored under ``name`` is removed from this folder,
remove its ``__parent__`` and ``__name__`` values.
When this method is called, emit an
:class:`substanced.event.ObjectWillBeRemoved` event before the
object loses its ``__name__`` or ``__parent__`` values. Emit an
:class:`substanced.event.ObjectRemoved` after the object loses
its ``__name__`` and ``__parent__`` value,
"""
return self.remove(name)
def remove(self, name, send_events=True, moving=False):
""" Same thing as ``__delitem__``.
If ``send_events`` is false, suppress the sending of folder events.
If ``moving`` is True, the events sent will indicate that a move is
in process.
"""
name = unicode(name)
other = self.data[name]
if send_events:
event = ObjectWillBeRemoved(other, self, name, moving)
self._notify(event)
if hasattr(other, '__parent__'):
del other.__parent__
if hasattr(other, '__name__'):
del other.__name__
del self.data[name]
self._num_objects.change(-1)
if self._order is not None:
self._order = tuple([x for x in self._order if x != name])
if send_events:
event = ObjectRemoved(other, self, name, moving)
self._notify(event)
return other
def copy(self, name, other, newname=None):
"""
Copy a subobject named ``name`` from this folder to the folder
represented by ``other``. If ``newname`` is not none, it is used as
the target object name; otherwise the existing subobject name is
used.
"""
if newname is None:
newname = name
with tempfile.TemporaryFile() as f:
obj = self.get(name)
obj._p_jar.exportFile(obj._p_oid, f)
f.seek(0)
new_obj = obj._p_jar.importFile(f)
del new_obj.__parent__
obj = other.add(newname, new_obj, duplicating=True)
return obj
def move(self, name, other, newname=None):
"""
Move a subobject named ``name`` from this folder to the folder
represented by ``other``. If ``newname`` is not none, it is used as
the target object name; otherwise the existing subobject name is
used.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events sent will indicate that the object is
moving.
"""
if newname is None:
newname = name
ob = self.remove(name, moving=True)
other.add(newname, ob)
return ob
def rename(self, oldname, newname):
"""
Rename a subobject from oldname to newname.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events sent will indicate that the object is
moving.
"""
return self.move(oldname, self, newname)
def replace(self, name, newobject):
""" Replace an existing object named ``name`` in this folder with a
new object ``newobject``. If there isn't an object named ``name`` in
this folder, an exception will *not* be raised; instead, the new
object will just be added.
This operation is done in terms of a remove and an add. The Removed
and WillBeRemoved events will be sent for the old object, and the
WillBeAdded and Add events will be sent for the new object.
"""
if name in self:
del self[name]
self[name] = newobject
