class TagTimeKey(TimeKey):
def __init__(self, tag, minutes=each10minutes, name='tagtime'):
self.minutes = IIBTree([(i, 1) for i in minutes])
self.tag = tag
self.name = name
def __call__(self, objectId, instance, *arg, **kw):
tag = self.tag.query(instance.context)
if not tag:
return ()
tag = time.localtime(tag)
minute = tag[4]
if not (self.minutes.has_key(-1) or self.minutes.has_key(minute)):
minute = self.minutes.maxKey(minute)
try:
minute2 = self.minutes.minKey(minute+1)
time2 = time.mktime(tag[:4] + (minute2, 0, 0, 0, 0))
except ValueError:
time2 = time.mktime(tag[:4] + (0, 0, 0, 0, 0)) + 3600
if time2 < time.time():
return {'%s:%s'%(self.name, self.tag.name): time2}
return {'%s:%s'%(self.name, self.tag.name):
time.mktime(tag[:4] + (minute, 0, 0, 0, 0))}
def testPairs(self):
t1 = IIBTree([(1, 10), (3, 30), (7, 70)])
t2 = IIBTree([(3, 30), (5, 50), (7, 7), (9, 90)])
allkeys = [1, 3, 5, 7, 9]
b1 = IIBucket(t1)
b2 = IIBucket(t2)
for x in t1, t2, b1, b2:
for key in x.keys():
self.assertEqual(key in allkeys, 1)
for y in t1, t2, b1, b2:
for w1, w2 in (0, 0), (1, 10), (10, 1), (2, 3):
# Test the union.
expected = []
for key in allkeys:
if x.has_key(key) or y.has_key(key):
result = x.get(key, 0) * w1 + y.get(key, 0) * w2
expected.append((key, result))
expected.sort()
got = mass_weightedUnion([(x, w1), (y, w2)])
self.assertEqual(expected, list(got.items()))
got = mass_weightedUnion([(y, w2), (x, w1)])
self.assertEqual(expected, list(got.items()))
# Test the intersection.
expected = []
for key in allkeys:
if x.has_key(key) and y.has_key(key):
result = x[key] * w1 + y[key] * w2
expected.append((key, result))
expected.sort()
got = mass_weightedIntersection([(x, w1), (y, w2)])
self.assertEqual(expected, list(got.items()))
got = mass_weightedIntersection([(y, w2), (x, w1)])
self.assertEqual(expected, list(got.items()))
def convertScores(scores,
type=type,
TupleType=TupleType,
IIBTree=IIBTree):
if type(scores) is not TupleType and type(scores) is not IIBTree():
scores = IIBTree(scores)
return scores
def __init__(self, id=None, **kwargs):
super(TracListing, self).__init__(id, **kwargs)
# manage an index of parent-to-child, int parent ticket id key
# to PersistentList of int ticket id value:
self._children = IOBTree()
# indexes for score and reward-ratio values:
self._scores = IIBTree() # int (ticket#) -> int (sum/score)
self._reward = IOBTree() # int (ticket#) -> float (ratio)
def clear(self):
""" Complete reset """
self._index = IOBTree()
self._unindex = IIBTree()
self._length = Length()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
if self._counter is None:
self._counter = BTrees.Length.Length()
else:
self._increment_counter()
class VerificationCodeUtility(Persistent):
implements(IVerificationCodeUtility)
def __init__(self):
self._verification_codes = IIBTree()
def generate(self, order):
verification_code = random.randint(LOWER, UPPER)
count = 0
while not self.is_unique(verification_code) and count < RETRIES:
count += 1
verification_code = random.randint(LOWER, UPPER)
if count > RETRIES - 1:
raise Exception('Could not find unique verification code.')
self.add(verification_code, order)
return str(verification_code)
def is_unique(self, code):
if code in self._verification_codes.keys():
return False
else:
return True
def add(self, verification_code, order):
order_id = order.getId()
if not isinstance(order_id, IntType):
order_id = int(order_id)
self._verification_codes[verification_code] = order_id
def __init__(self, lexicon):
self._lexicon = lexicon
# wid -> {docid -> weight}; t -> D -> w(D, t)
# Different indexers have different notions of term weight, but we
# expect each indexer to use ._wordinfo to map wids to its notion
# of a docid-to-weight map.
# There are two kinds of OOV words: wid 0 is explicitly OOV,
# and it's possible that the lexicon will return a non-zero wid
# for a word we don't currently know about. For example, if we
# unindex the last doc containing a particular word, that wid
# remains in the lexicon, but is no longer in our _wordinfo map;
# lexicons can also be shared across indices, and some other index
# may introduce a lexicon word we've never seen.
# A word is in-vocabulary for this index if and only if
# _wordinfo.has_key(wid). Note that wid 0 must not be a key.
self._wordinfo = IOBTree()
# docid -> weight
# Different indexers have different notions of doc weight, but we
# expect each indexer to use ._docweight to map docids to its
# notion of what a doc weight is.
self._docweight = IIBTree()
# docid -> WidCode'd list of wids
# Used for un-indexing, and for phrase search.
self._docwords = IOBTree()
# Use a BTree length for efficient length computation w/o conflicts
self.length = Length()
self.document_count = Length()
def search_phrase(self, phrase):
wids = self._lexicon.termToWordIds(phrase)
cleaned_wids = self._remove_oov_wids(wids)
if len(wids) != len(cleaned_wids):
# At least one wid was OOV: can't possibly find it.
return IIBTree()
scores = self._search_wids(wids)
hits = mass_weightedIntersection(scores)
if not hits:
return hits
code = WidCode.encode(wids)
result = IIBTree()
for docid, weight in hits.items():
docwords = self._docwords[docid]
if docwords.find(code) >= 0:
result[docid] = weight
return result
def testIdentity(self):
t = IIBTree([(1, 2)])
b = IIBucket([(1, 2)])
for x in t, b:
for func in mass_weightedUnion, mass_weightedIntersection:
result = func([(x, 1)])
self.assertEqual(len(result), 1)
self.assertEqual(list(result.items()), list(x.items()))
def __init__(self, id=None, **kwargs):
super(TracListing, self).__init__(id, **kwargs)
# manage an index of parent-to-child, int parent ticket id key
# to PersistentList of int ticket id value:
self._children = IOBTree()
# indexes for score and reward-ratio values:
self._scores = IIBTree() # int (ticket#) -> int (sum/score)
self._reward = IOBTree() # int (ticket#) -> float (ratio)
def testMany(self):
import random
N = 15 # number of IIBTrees to feed in
L = []
commonkey = N * 1000
allkeys = {commonkey: 1}
for i in range(N):
t = IIBTree()
t[commonkey] = i
for j in range(N-i):
key = i + j
allkeys[key] = 1
t[key] = N*i + j
L.append((t, i+1))
random.shuffle(L)
allkeys = allkeys.keys()
allkeys.sort()
# Test the union.
expected = []
for key in allkeys:
sum = 0
for t, w in L:
if t.has_key(key):
sum += t[key] * w
expected.append((key, sum))
# print 'union', expected
got = mass_weightedUnion(L)
self.assertEqual(expected, list(got.items()))
# Test the intersection.
expected = []
for key in allkeys:
sum = 0
for t, w in L:
if t.has_key(key):
sum += t[key] * w
else:
break
else:
# We didn't break out of the loop so it's in the intersection.
expected.append((key, sum))
# print 'intersection', expected
got = mass_weightedIntersection(L)
self.assertEqual(expected, list(got.items()))
def testMany(self):
import random
N = 15 # number of IIBTrees to feed in
L = []
commonkey = N * 1000
allkeys = {commonkey: 1}
for i in range(N):
t = IIBTree()
t[commonkey] = i
for j in range(N - i):
key = i + j
allkeys[key] = 1
t[key] = N * i + j
L.append((t, i + 1))
random.shuffle(L)
allkeys = allkeys.keys()
allkeys.sort()
# Test the union.
expected = []
for key in allkeys:
sum = 0
for t, w in L:
if t.has_key(key):
sum += t[key] * w
expected.append((key, sum))
# print 'union', expected
got = mass_weightedUnion(L)
self.assertEqual(expected, list(got.items()))
# Test the intersection.
expected = []
for key in allkeys:
sum = 0
for t, w in L:
if t.has_key(key):
sum += t[key] * w
else:
break
else:
# We didn't break out of the loop so it's in the intersection.
expected.append((key, sum))
# print 'intersection', expected
got = mass_weightedIntersection(L)
self.assertEqual(expected, list(got.items()))
def _apply_index(self, request):
"""Apply the index to query parameters given in 'request'.
The argument should be a mapping object.
If the request does not contain the needed parameters, then
None is returned.
If the request contains a parameter with the name of the
column and this parameter is either a Record or a class
instance then it is assumed that the parameters of this index
are passed as attribute (Note: this is the recommended way to
pass parameters since Zope 2.4)
Otherwise two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
"""
if query_blocker.blocked:
return
record = parseIndexRequest(request, self.id)
if record.keys is None:
return None
template_params = {
'keys': record.keys,
}
query_body = self._apply_template(template_params)
logger.info(query_body)
es_kwargs = dict(
index=index_name(),
body=query_body,
size=BATCH_SIZE,
scroll='1m',
_source_include=['rid'],
)
es = get_query_client()
result = es.search(**es_kwargs)
# initial return value, other batches to be applied
def score(record):
return int(10000 * float(record['_score']))
retval = IIBTree()
for r in result['hits']['hits']:
retval[r['_source']['rid']] = score(r)
total = result['hits']['total']
if total > BATCH_SIZE:
sid = result['_scroll_id']
counter = BATCH_SIZE
while counter < total:
result = es.scroll(scroll_id=sid, scroll='1m')
for record in result['hits']['hits']:
retval[record['_source']['rid']] = score(record)
counter += BATCH_SIZE
return retval, (self.id,)
def clear(self):
""" Complete reset """
self._index = IOBTree()
self._unindex = IIBTree()
self._length = Length()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
if self._counter is None:
self._counter = BTrees.Length.Length()
else:
self._increment_counter()
def __init__(self, name, root):
# m_order maintains a newest-first mapping of int -> version id.
# m_date maintains a mapping of a packed date (int # of minutes
# since the epoch) to a lookup key in m_order. The two structures
# are separate because we only support minute precision for date
# lookups (and multiple versions could be added in a minute).
self.date_created = time.time()
self.m_order = IOBTree()
self.m_date = IIBTree()
self.name = name
self.root = root
def testScalarMultiply(self):
t = IIBTree([(1, 2), (2, 3), (3, 4)])
allkeys = [1, 2, 3]
b = IIBucket(t)
for x in t, b:
self.assertEqual(list(x.keys()), allkeys)
for func in mass_weightedUnion, mass_weightedIntersection:
for factor in 0, 1, 5, 10:
result = func([(x, factor)])
self.assertEqual(allkeys, list(result.keys()))
for key in x.keys():
self.assertEqual(x[key] * factor, result[key])
def test_walk_w_normal_btree(self):
from BTrees.IIBTree import IIBTree
obj = IIBTree()
for i in range(1000):
obj[i] = i
walker = self._makeOne(obj)
path = '/'
parent = object()
is_mapping = True
keys = []
kids = []
lo = 0
hi = None
self.assertRaises(NotImplementedError, walker.walk)
def _mass_add_wordinfo(self, wid2weight, docid):
dicttype = type({})
get_doc2score = self._wordinfo.get
new_word_count = 0
for wid, weight in wid2weight.items():
doc2score = get_doc2score(wid)
if doc2score is None:
doc2score = {}
new_word_count += 1
elif (isinstance(doc2score, dicttype)
and len(doc2score) == self.DICT_CUTOFF):
doc2score = IIBTree(doc2score)
doc2score[docid] = weight
self._wordinfo[wid] = doc2score # not redundant: Persistency!
self.length.change(new_word_count)
def optimize_dateindex(index):
# migrate _unindex from OIBTree to IIBTree
old_unindex = index._unindex
if isinstance(old_unindex, IIBTree):
return
index._unindex = _unindex = IIBTree()
logger.info('Converting to IIBTree for index `%s`.' % index.getId())
for pos, (k, v) in enumerate(old_unindex.items()):
_unindex[k] = v
if pos and pos % 10000 == 0:
transaction.savepoint(optimistic=True)
logger.info('Processed %s items.' % pos)
transaction.savepoint(optimistic=True)
logger.info('Finished conversion.')
def insertDocument(self, docid, widlist):
Storage.insertDocument(self, docid, widlist)
occurences = {} # wid -> #(occurences)
num_wids = float(len(widlist))
for wid in widlist:
if not occurences.has_key(wid):
occurences[wid] = 1
else:
occurences[wid] += 1
self._frequencies[docid] = IIBTree()
tree = self._frequencies[docid]
for wid, num in occurences.items():
tree[wid] = num
def insertForwardIndexEntry(self, entry, documentId, score=1):
"""Uses the information provided to update the indexes.
The basic logic for choice of data structure is based on
the number of entries as follows:
1 tuple
2-3 dictionary
4+ bucket.
"""
index = self._index
indexRow = index.get(entry, None)
if indexRow is not None:
if type(indexRow) is TupleType:
# Tuples are only used for rows which have only
# a single entry. Since we now need more, we'll
# promote it to a mapping object (dictionary).
# First, make sure we're not already in it, if so
# update the score if necessary.
if indexRow[0] == documentId:
if indexRow[1] != score:
indexRow = (documentId, score)
index[entry] = indexRow
else:
indexRow = {
indexRow[0]: indexRow[1],
documentId: score,
}
index[entry] = indexRow
else:
if indexRow.get(documentId, -1) != score:
# score changed (or new entry)
if type(indexRow) is DictType:
indexRow[documentId] = score
if len(indexRow) > 3:
# Big enough to give it's own database record
indexRow = IIBTree(indexRow)
index[entry] = indexRow
else:
indexRow[documentId] = score
else:
# We don't have any information at this point, so we'll
# put our first entry in, and use a tuple to save space
index[entry] = (documentId, score)
def optimize_dateindex(index):
# migrate _unindex from OIBTree to IIBTree
old_unindex = index._unindex
if isinstance(old_unindex, IIBTree):
return
index._unindex = _unindex = IIBTree()
logger.info('Converting to IIBTree for index `%s`.', index.getId())
for pos, (k, v) in enumerate(old_unindex.items()):
_unindex[k] = v
# Note: flake8 erroneously complains about module formatter.
if pos and pos % 10000 == 0: # noqa S001
transaction.savepoint(optimistic=True)
logger.info('Processed %s items.', pos)
transaction.savepoint(optimistic=True)
logger.info('Finished conversion.')
def insert(self, idx, results, relnames=None, treePrefix=None):
unindex = None
for brain in results:
# Use the first brain to get a reference to the index, then reuse
# that reference
unindex = unindex or brain.global_catalog._catalog.indexes[
'path']._unindex
path = brain.getPath()
if treePrefix and not path.startswith(treePrefix):
for p in unindex[brain.getRID()]:
if p.startswith(treePrefix):
path = p
break
path = path.split('/', 3)[-1]
for depth in xrange(path.count('/') + 1):
comp = idx.setdefault(path, IIBTree())
comp[depth] = comp.get(depth, 0) + 1
path = path.rsplit('/', 1)[0]
class database:
def __init__(self):
self.data = []
self.dict = IIBTree()
self.index = -1 #index of the last element in the list
def insert(self, key, value):
if self.dict.has_key(key):
index = self.dict.get(key)
self.data[index] = value
else:
self.data.append(value)
self.index = self.index + 1
index = self.index
self.dict.update({key: index})
def search(self, key):
if self.dict.has_key(key):
value = self.data[self.dict.get(key)]
return value
return "NOT PRESENT"
def delete(self, key):
self.dict.pop(key)
def __init__(self, datafs, writable=0, trans=0, pack=0):
self.trans_limit = trans
self.pack_limit = pack
self.trans_count = 0
self.pack_count = 0
self.stopdict = get_stopdict()
self.mh = mhlib.MH()
self.filestorage = FileStorage(datafs, read_only=(not writable))
self.database = DB(self.filestorage)
self.connection = self.database.open()
self.root = self.connection.root()
try:
self.index = self.root["index"]
except KeyError:
self.index = self.root["index"] = TextIndex()
try:
self.docpaths = self.root["docpaths"]
except KeyError:
self.docpaths = self.root["docpaths"] = IOBTree()
try:
self.doctimes = self.root["doctimes"]
except KeyError:
self.doctimes = self.root["doctimes"] = IIBTree()
try:
self.watchfolders = self.root["watchfolders"]
except KeyError:
self.watchfolders = self.root["watchfolders"] = {}
self.path2docid = OIBTree()
for docid in self.docpaths.keys():
path = self.docpaths[docid]
self.path2docid[path] = docid
try:
self.maxdocid = max(self.docpaths.keys())
except ValueError:
self.maxdocid = 0
print len(self.docpaths), "Document ids"
print len(self.path2docid), "Pathnames"
print self.index.lexicon.length(), "Words"
def _add_wordinfo(self, wid, f, docid):
# Store a wordinfo in a dict as long as there are less than
# DICT_CUTOFF docids in the dict. Otherwise use an IIBTree.
# The pickle of a dict is smaller than the pickle of an
# IIBTree, substantially so for small mappings. Thus, we use
# a dictionary until the mapping reaches DICT_CUTOFF elements.
# The cutoff is chosen based on the implementation
# characteristics of Python dictionaries. The dict hashtable
# always has 2**N slots and is resized whenever it is 2/3s
# full. A pickled dict with 10 elts is half the size of an
# IIBTree with 10 elts, and 10 happens to be 2/3s of 2**4. So
# choose 10 as the cutoff for now.
# The IIBTree has a smaller in-memory representation than a
# dictionary, so pickle size isn't the only consideration when
# choosing the threshold. The pickle of a 500-elt dict is 92%
# of the size of the same IIBTree, but the dict uses more
# space when it is live in memory. An IIBTree stores two C
# arrays of ints, one for the keys and one for the values. It
# holds up to 120 key-value pairs in a single bucket.
doc2score = self._wordinfo.get(wid)
if doc2score is None:
doc2score = {}
self.length.change(1)
else:
# _add_wordinfo() is called for each update. If the map
# size exceeds the DICT_CUTOFF, convert to an IIBTree.
# Obscure: First check the type. If it's not a dict, it
# can't need conversion, and then we can avoid an expensive
# len(IIBTree).
if (isinstance(doc2score, type({}))
and len(doc2score) == self.DICT_CUTOFF):
doc2score = IIBTree(doc2score)
doc2score[docid] = f
self._wordinfo[wid] = doc2score # not redundant: Persistency!
def convert_to_booleanindex(catalog, index):
if isinstance(index, BooleanIndex):
return
logger.info('Converting index `%s` to BooleanIndex.' % index.getId())
index.__class__ = BooleanIndex
index._p_changed = True
catalog._catalog._p_changed = True
# convert _unindex from IOBTree to IIBTree
sets = {0: IITreeSet(), 1: IITreeSet()}
old_unindex = index._unindex
index._unindex = _unindex = IIBTree()
for k, v in old_unindex.items():
# docid to value (True, False)
value = int(bool(v))
_unindex[k] = value
sets[value].add(k)
del old_unindex
# convert _index from OOBTree to IITreeSet and set lengths
false_length = len(sets[0])
true_length = len(sets[1])
index._length = Length(false_length + true_length)
# we put the smaller set into the index
if false_length < true_length:
index._index_value = 0
index._index_length = Length(false_length)
index._index = sets[0]
del sets[1]
else:
index._index_value = 1
index._index_length = Length(true_length)
index._index = sets[1]
del sets[0]
transaction.savepoint(optimistic=True)
logger.info('Finished conversion.')
class TimeKey(object):
interface.implements(ITimeKey)
def __init__(self, hours=eachHour, minutes=each0minute, name='time'):
self.hours = IIBTree([(i, 1) for i in hours])
self.minutes = IIBTree([(i, 1) for i in minutes])
self.name = name
def __call__(self, *args, **kw):
now = time.localtime()
minute = now[4]
if not (self.minutes.has_key(-1) or self.minutes.has_key(minute)):
minute = self.minutes.maxKey(minute)
hour = now[3]
if not (self.hours.has_key(-1) or self.hours.has_key(hour)):
hour = self.hours.maxKey(hour)
return {self.name: time.mktime(now[:3] + (hour, minute, 0, 0, 0, 0))}
def _apply_index(self, request):
"""Apply the index to query parameters given in 'request'.
The argument should be a mapping object.
If the request does not contain the needed parameters, then
None is returned.
If the request contains a parameter with the name of the
column and this parameter is either a Record or a class
instance then it is assumed that the parameters of this index
are passed as attribute (Note: this is the recommended way to
pass parameters since Zope 2.4)
Otherwise two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
"""
config = get_configuration()
timeout = getattr(config, 'request_timeout', 20)
search_fields = getattr(config, 'search_fields', None)
if not search_fields:
search_fields = SEARCH_FIELDS
search_fields = search_fields.split()
logger.info(search_fields)
if query_blocker.blocked:
return
record = parseIndexRequest(request, self.id)
if record.keys is None:
return None
es = get_query_client()
search = Search(using=es, index=index_name())
search = search.params(request_timeout=timeout)
search = search.sort('rid', '_id')
search = search.source(include='rid')
query_string = record.keys[0].decode('utf8')
logger.info(query_string)
if '*' in query_string:
query_string = query_string.replace('*', ' ')
query_string = query_string.strip()
search = search.query('simple_query_string',
query=query_string,
fields=search_fields)
results_count = search.count()
search = search.params(request_timeout=timeout,
size=BATCH_SIZE,
track_scores=True)
# setup highlighting
for field in search_fields:
name = field.split('^')[0]
if name == 'title':
# title shows up in results anyway
continue
search = search.highlight(name, fragment_size=FRAGMENT_SIZE)
# initial return value, other batches to be applied
retval = IIBTree()
highlights = OOBTree()
last_seen = None
count = 0
batch_count = results_count / BATCH_SIZE
if results_count % BATCH_SIZE != 0:
batch_count = batch_count + 1
for i in xrange(batch_count):
if last_seen is not None:
search = search.update_from_dict({'search_after': last_seen})
try:
results = search.execute(ignore_cache=True)
except TransportError:
# No es client, return empty results
logger.exception('ElasticSearch client not available.')
return IIBTree(), (self.id, )
for r in results:
rid = getattr(r, 'rid', None)
if rid is not None:
retval[rid] = int(10000 * float(r.meta.score))
# Index query returns only rids, so we need
# to save highlights for later use
highlight_list = []
if getattr(r.meta, 'highlight', None) is not None:
for key in dir(r.meta.highlight):
highlight_list.extend(r.meta.highlight[key])
highlights[r.meta.id] = highlight_list
last_seen = [rid, r.meta.id]
count = count + 1
# store highlights
try:
annotations = IAnnotations(self.REQUEST)
annotations[HIGHLIGHT_KEY] = highlights
except TypeError:
# maybe we are in a test
pass
return retval, (self.id, )
def __init__(self, hours=eachHour, minutes=each0minute, name='time'):
self.hours = IIBTree([(i, 1) for i in hours])
self.minutes = IIBTree([(i, 1) for i in minutes])
self.name = name
class BooleanIndex(UnIndex):
"""Index for booleans
self._index = set([documentId1, documentId2])
self._unindex = {documentId:[True/False]}
self._length is the length of the unindex
self._index_length is the length of the index
False doesn't have actual entries in _index.
"""
meta_type = "BooleanIndex"
manage_options = (
{'label': 'Settings',
'action': 'manage_main'},
{'label': 'Browse',
'action': 'manage_browse'},
)
query_options = ["query"]
manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
_index_value = 1
_index_length = None
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
if self._counter is None:
self._counter = BTrees.Length.Length()
else:
self._increment_counter()
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
indexed = bool(self._index_value)
histogram[indexed] = self._index_length.value
histogram[not indexed] = self._length.value - self._index_length.value
return histogram
def _invert_index(self, documentId=None):
self._index_value = indexed = int(not self._index_value)
self._index.clear()
length = 0
for rid, value in self._unindex.iteritems():
if value == indexed:
self._index.add(rid)
length += 1
# documentId is the rid of the currently processed object that
# triggered the invert. in the case of unindexing, the rid hasn't
# been removed from the unindex yet. While indexing, the rid will
# be added to the index and unindex after this method is done
if documentId is not None:
self._index.remove(documentId)
length -= 1
self._index_length = BTrees.Length.Length(length)
def insertForwardIndexEntry(self, entry, documentId):
"""If the value matches the indexed one, insert into treeset
"""
# When we get the first entry, decide to index the opposite of what
# we got, as indexing zero items is fewer than one.
if self._length.value == 0:
self._index_value = int(not bool(entry))
# if the added entry value is index value, insert it into index
if bool(entry) is bool(self._index_value):
self._index_length.change(1)
self._index.insert(documentId)
# insert value into global unindex (before computing index invert)
self._unindex[documentId] = entry
self._length.change(1)
# is the index (after adding the current entry) larger than 60%
# of the total length? than switch the indexed value
if bool(entry) is bool(self._index_value):
if (self._index_length.value) >= ((self._length.value) * 0.6):
self._invert_index()
def removeForwardIndexEntry(self, entry, documentId, check=True):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
if bool(entry) is bool(self._index_value):
try:
self._index.remove(documentId)
self._index_length.change(-1)
except ConflictError:
raise
except Exception:
LOG.exception(
'%s: unindex_object could not remove documentId %s '
'from index %s. This should not happen.' % (
self.__class__.__name__,
str(documentId),
str(self.id)))
elif check:
# is the index (after removing the current entry) larger than
# 60% of the total length? than switch the indexed value
if (self._index_length.value) <= ((self._length.value - 1) * 0.6):
self._invert_index(documentId)
return
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# Make it boolean, int as an optimization
if datum is not _marker:
datum = int(bool(datum))
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId, check=False)
if datum is _marker:
try:
del self._unindex[documentId]
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.error('Should not happen: oldDatum was there, now '
'its not, for document with id %s' %
documentId)
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
returnStatus = 1
return returnStatus
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self._increment_counter()
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.debug('Attempt to unindex nonexistent document'
' with id %s' % documentId, exc_info=True)
def query_index(self, record, resultset=None):
index = self._index
indexed = self._index_value
for key in record.keys:
if bool(key) is bool(indexed):
# If we match the indexed value, check index
return intersection(index, resultset)
else:
# Otherwise, remove from resultset or _unindex
if resultset is None:
return union(difference(self._unindex, index), IISet([]))
else:
return difference(resultset, index)
return IISet()
def indexSize(self):
"""Return distinct values, as an optimization we always claim 2."""
return 2
def items(self):
# return a list of value to int set of rid tuples
indexed = self._index_value
items = [(bool(indexed), self._index)]
false = IISet()
for rid, value in self._unindex.iteritems():
if value != indexed:
false.add(rid)
items.append((not bool(indexed), false))
return items
class DateIndex(UnIndex, PropertyManager):
"""Index for dates.
"""
implements(IDateIndex)
meta_type = 'DateIndex'
query_options = ('query', 'range')
index_naive_time_as_local = True # False means index as UTC
_properties=({'id':'index_naive_time_as_local',
'type':'boolean',
'mode':'w'},)
manage = manage_main = DTMLFile( 'dtml/manageDateIndex', globals() )
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_main._setName( 'manage_main' )
manage_options = ( { 'label' : 'Settings'
, 'action' : 'manage_main'
},
{'label': 'Browse',
'action': 'manage_browse',
},
) + PropertyManager.manage_options
def clear( self ):
""" Complete reset """
self._index = IOBTree()
self._unindex = IIBTree()
self._length = Length()
def index_object( self, documentId, obj, threshold=None ):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
"""
returnStatus = 0
try:
date_attr = getattr( obj, self.id )
if safe_callable( date_attr ):
date_attr = date_attr()
ConvertedDate = self._convert( value=date_attr, default=_marker )
except AttributeError:
ConvertedDate = _marker
oldConvertedDate = self._unindex.get( documentId, _marker )
if ConvertedDate != oldConvertedDate:
if oldConvertedDate is not _marker:
self.removeForwardIndexEntry(oldConvertedDate, documentId)
if ConvertedDate is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error("Should not happen: ConvertedDate was there,"
" now it's not, for document with id %s" %
documentId)
if ConvertedDate is not _marker:
self.insertForwardIndexEntry( ConvertedDate, documentId )
self._unindex[documentId] = ConvertedDate
returnStatus = 1
return returnStatus
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the argument
Normalize the 'query' arguments into integer values at minute
precision before querying.
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
keys = map( self._convert, record.keys )
index = self._index
r = None
opr = None
#experimental code for specifing the operator
operator = record.get( 'operator', self.useOperator )
if not operator in self.operators :
raise RuntimeError("operator not valid: %s" % operator)
# depending on the operator we use intersection or union
if operator=="or":
set_func = union
else:
set_func = intersection
# range parameter
range_arg = record.get('range',None)
if range_arg:
opr = "range"
opr_args = []
if range_arg.find("min") > -1:
opr_args.append("min")
if range_arg.find("max") > -1:
opr_args.append("max")
if record.get('usage',None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args = opr[0], opr[1:]
if opr=="range": # range search
if 'min' in opr_args:
lo = min(keys)
else:
lo = None
if 'max' in opr_args:
hi = max(keys)
else:
hi = None
if hi:
setlist = index.values(lo,hi)
else:
setlist = index.values(lo)
r = multiunion(setlist)
else: # not a range search
for key in keys:
set = index.get(key, None)
if set is not None:
if isinstance(set, int):
set = IISet((set,))
else:
# set can't be bigger than resultset
set = intersection(set, resultset)
r = set_func(r, set)
if isinstance(r, int):
r = IISet((r,))
if r is None:
return IISet(), (self.id,)
else:
return r, (self.id,)
def _convert( self, value, default=None ):
"""Convert Date/Time value to our internal representation"""
# XXX: Code patched 20/May/2003 by Kiran Jonnalagadda to
# convert dates to UTC first.
if isinstance(value, DateTime):
t_tup = value.toZone('UTC').parts()
elif isinstance(value, (float, int)):
t_tup = time.gmtime( value )
elif isinstance(value, str) and value:
t_obj = DateTime( value ).toZone('UTC')
t_tup = t_obj.parts()
elif isinstance(value, datetime):
if self.index_naive_time_as_local and value.tzinfo is None:
value = value.replace(tzinfo=Local)
# else if tzinfo is None, naive time interpreted as UTC
t_tup = value.utctimetuple()
elif isinstance(value, date):
t_tup = value.timetuple()
else:
return default
yr = t_tup[0]
mo = t_tup[1]
dy = t_tup[2]
hr = t_tup[3]
mn = t_tup[4]
t_val = ( ( ( ( yr * 12 + mo ) * 31 + dy ) * 24 + hr ) * 60 + mn )
if t_val > MAX32:
# t_val must be integer fitting in the 32bit range
raise OverflowError(
"%s is not within the range of indexable dates (index: %s)"
% (value, self.id))
return t_val
def clear( self ):
""" Complete reset """
self._index = IOBTree()
self._unindex = IIBTree()
self._length = Length()
def __init__(self, tag, minutes=each10minutes, name='tagtime'):
self.minutes = IIBTree([(i, 1) for i in minutes])
self.tag = tag
self.name = name
def clear(self):
self._doc2wid = IOBTree() # docid -> [wordids]
self._wid2doc = IOBTree() # wordid -> [docids]
self._docweight = IIBTree() # docid -> (# terms in document)
self._length = BTrees.Length.Length()
class BooleanIndex(UnIndex):
"""Index for booleans
self._index = set([documentId1, documentId2])
self._unindex = {documentId:[True/False]}
self._length is the length of the unindex
self._index_length is the length of the index
False doesn't have actual entries in _index.
"""
meta_type = "BooleanIndex"
manage_options = (
{
'label': 'Settings',
'action': 'manage_main'
},
{
'label': 'Browse',
'action': 'manage_browse'
},
)
query_options = ["query"]
manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
_index_value = 1
_index_length = None
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
indexed = bool(self._index_value)
histogram[indexed] = self._index_length.value
histogram[not indexed] = self._length.value - self._index_length.value
return histogram
def _invert_index(self, documentId=None):
self._index_value = indexed = int(not self._index_value)
self._index.clear()
length = 0
for rid, value in self._unindex.iteritems():
if value == indexed:
self._index.add(rid)
length += 1
# documentId is the rid of the currently processed object that
# triggered the invert. in the case of unindexing, the rid hasn't
# been removed from the unindex yet. While indexing, the rid will
# be added to the index and unindex after this method is done
if documentId is not None:
self._index.remove(documentId)
length -= 1
self._index_length = BTrees.Length.Length(length)
def _inline_migration(self):
self._length = BTrees.Length.Length(len(self._unindex.keys()))
self._index_length = BTrees.Length.Length(len(self._index))
if self._index_length.value > (self._length.value / 2):
self._index_value = 1
self._invert_index()
else:
# set an instance variable
self._index_value = 1
def insertForwardIndexEntry(self, entry, documentId):
"""If the value matches the indexed one, insert into treeset
"""
# when we get the first entry, decide to index the opposite of what
# we got, as indexing zero items is fewer than one
# BBB inline migration
if self._index_length is None:
self._inline_migration()
if self._length.value == 0:
self._index_value = int(not bool(entry))
# if the added entry value is index value, insert it into index
if bool(entry) is bool(self._index_value):
self._index_length.change(1)
self._index.insert(documentId)
# insert value into global unindex (before computing index invert)
self._unindex[documentId] = entry
self._length.change(1)
# is the index (after adding the current entry) larger than 60%
# of the total length? than switch the indexed value
if bool(entry) is bool(self._index_value):
if (self._index_length.value) >= ((self._length.value) * 0.6):
self._invert_index()
def removeForwardIndexEntry(self, entry, documentId, check=True):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
index_length = self._index_length
if index_length is None:
self._inline_migration()
if bool(entry) is bool(self._index_value):
try:
self._index.remove(documentId)
# BBB inline migration
length = self._index_length
length.change(-1)
except ConflictError:
raise
except Exception:
LOG.exception(
'%s: unindex_object could not remove '
'documentId %s from index %s. This '
'should not happen.' %
(self.__class__.__name__, str(documentId), str(self.id)))
elif check:
length = self._length.value
index_length = self._index_length.value
# is the index (after removing the current entry) larger than
# 60% of the total length? than switch the indexed value
if (index_length) <= ((length - 1) * 0.6):
self._invert_index(documentId)
return
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# Make it boolean, int as an optimization
if datum is not _marker:
datum = int(bool(datum))
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId, check=False)
if datum is _marker:
try:
del self._unindex[documentId]
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.error('Should not happen: oldDatum was there, now '
'its not, for document with id %s' %
documentId)
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
returnStatus = 1
return returnStatus
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.debug('Attempt to unindex nonexistent document'
' with id %s' % documentId,
exc_info=True)
def _apply_index(self, request, resultset=None):
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
index = self._index
indexed = self._index_value
for key in record.keys:
if bool(key) is bool(indexed):
# If we match the indexed value, check index
return (intersection(index, resultset), (self.id, ))
else:
# Otherwise, remove from resultset or _unindex
if resultset is None:
return (union(difference(self._unindex, index),
IISet([])), (self.id, ))
else:
return (difference(resultset, index), (self.id, ))
return (IISet(), (self.id, ))
def indexSize(self):
"""Return distinct values, as an optimization we always claim 2."""
return 2
def items(self):
# return a list of value to int set of rid tuples
indexed = self._index_value
items = [(bool(indexed), self._index)]
false = IISet()
for rid, value in self._unindex.iteritems():
if value != indexed:
false.add(rid)
items.append((not bool(indexed), false))
return items
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
class TracListing(Container):
"""Content class for trac listing"""
implements(ITracListing)
def __init__(self, id=None, **kwargs):
super(TracListing, self).__init__(id, **kwargs)
# manage an index of parent-to-child, int parent ticket id key
# to PersistentList of int ticket id value:
self._children = IOBTree()
# indexes for score and reward-ratio values:
self._scores = IIBTree() # int (ticket#) -> int (sum/score)
self._reward = IOBTree() # int (ticket#) -> float (ratio)
def index_parent_child(self, parent, child=None):
if ITracTicket.providedBy(parent):
parent = parent.getId()
if ITracTicket.providedBy(child):
child = child.getId()
parent = int(parent)
child = int(child) if child else None
self._children[parent] = self._children.get(parent, PersistentList())
if child is not None:
self._children[parent].append(child)
def _adapter(self):
if getattr(self, '_v_trac_adapter', None) is None:
self._v_trac_adapter = TracTickets(self) # adapt once
return self._v_trac_adapter
def select(self, query):
return self._adapter().select(query)
def _add(self, ticket_number, adapter=None):
adapter = adapter or self._adapter()
# generated id from title will be str(ticket_number), we
# then change the title in sync after creation.
content = createContentInContainer(
self,
portal_type='uu.trac.ticket',
title=unicode(ticket_number),
)
content.sync()
# check for and recursively add child tickets:
childq = 'parent=~#%s' % ticket_number
children = adapter.select(childq)
for child_id in children:
self._add(child_id, adapter)
def sync(self):
adapter = self._adapter()
self._children = IOBTree() # reset all parent/child refs
q = 'status!=closed'
for number in adapter.select(q):
if str(number) not in self.objectIds():
if number in (self.visible_tickets or []):
self._add(number, adapter)
else:
self.get(str(number)).sync()
def children_for(self, ticket_number):
return list(self._children.get(int(ticket_number), []))
def result(self, reward=False):
"""
Return sorted listing of tuples with ticket number, score.
If reward is True, use reward-ratio instead of score.
"""
keyfn = lambda t: t[1] # sort function, on score, not ticket
_sorted = lambda l: sorted(l, key=keyfn, reverse=True)
_visible = lambda t: t[0] in self.visible_tickets
if reward:
return _sorted(filter(_visible, self._reward.items()))
return _sorted(filter(_visible, self._scores.items()))
def index(self, ticket):
if not ITracTicket.providedBy(ticket):
ticket = self.get(int(ticket))
tid = int(ticket.getId())
parent = ticket.parent
if parent:
self.index_parent_child(parent, tid)
self._scores[tid] = ticket.score()
self._reward[tid] = ticket.reward_ratio()
class BooleanIndex(UnIndex):
"""Index for booleans
self._index = set([documentId1, documentId2])
self._unindex = {documentId:[True/False]}
self._length is the length of the unindex
self._index_length is the length of the index
self._index_value is the indexed value
The document ids in self._index have self._index_value as their value.
Since there are only two possible values (True/False), the index
only stores a forward index for the less common value.
It starts off with the opposite of value of the first document
and later checks and inverts itself, if more than 60% of all
documents now have the indexed value. It does the inversion
at 60% to avoid inverting itself constantly for an index that
has a roughly equal 50/50 split.
"""
meta_type = 'BooleanIndex'
manage_options = (
{
'label': 'Settings',
'action': 'manage_main'
},
{
'label': 'Browse',
'action': 'manage_browse'
},
)
query_options = ['query']
manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
_index_value = 1
_index_length = None
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
if self._counter is None:
self._counter = BTrees.Length.Length()
else:
self._increment_counter()
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
indexed = bool(self._index_value)
histogram[indexed] = self._index_length.value
histogram[not indexed] = self._length.value - self._index_length.value
return histogram
def _invert_index(self, documentId=None):
self._index_value = indexed = int(not self._index_value)
self._index.clear()
length = 0
for rid, value in self._unindex.iteritems():
if value == indexed:
self._index.add(rid)
length += 1
# documentId is the rid of the currently processed object that
# triggered the invert. in the case of unindexing, the rid hasn't
# been removed from the unindex yet. While indexing, the rid will
# be added to the index and unindex after this method is done
if documentId is not None:
self._index.remove(documentId)
length -= 1
self._index_length = BTrees.Length.Length(length)
def insertForwardIndexEntry(self, entry, documentId):
"""If the value matches the indexed one, insert into treeset
"""
# When we get the first entry, decide to index the opposite of what
# we got, as indexing zero items is fewer than one.
if self._length.value == 0:
self._index_value = int(not bool(entry))
# if the added entry value is index value, insert it into index
if bool(entry) is bool(self._index_value):
self._index_length.change(1)
self._index.insert(documentId)
# insert value into global unindex (before computing index invert)
self._unindex[documentId] = entry
self._length.change(1)
# is the index (after adding the current entry) larger than 60%
# of the total length? than switch the indexed value
if bool(entry) is bool(self._index_value):
if (self._index_length.value) >= ((self._length.value) * 0.6):
self._invert_index()
def removeForwardIndexEntry(self, entry, documentId, check=True):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
if bool(entry) is bool(self._index_value):
try:
self._index.remove(documentId)
self._index_length.change(-1)
except ConflictError:
raise
except Exception:
LOG.exception(
'%(context)s: unindex_object could not '
'remove documentId %(doc_id)s from '
'index %(index)r. This should not '
'happen.',
dict(context=self.__class__.__name__,
doc_id=documentId,
index=self.id))
elif check:
# is the index (after removing the current entry) larger than
# 60% of the total length? than switch the indexed value
if (self._index_length.value) <= ((self._length.value - 1) * 0.6):
self._invert_index(documentId)
return
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# Make it boolean, int as an optimization
if datum is not _marker:
datum = int(bool(datum))
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId, check=False)
if datum is _marker:
try:
del self._unindex[documentId]
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.error(
'Should not happen: oldDatum was there, now '
'its not, for document with id %s', documentId)
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
returnStatus = 1
return returnStatus
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self._increment_counter()
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.debug('Attempt to unindex nonexistent document'
' with id %s',
documentId,
exc_info=True)
def query_index(self, record, resultset=None):
index = self._index
indexed = self._index_value
for key in record.keys:
if bool(key) is bool(indexed):
# If we match the indexed value, check index
return intersection(index, resultset)
else:
# Otherwise, remove from resultset or _unindex
if resultset is None:
return union(difference(self._unindex, index), IISet([]))
else:
return intersection(difference(resultset, index),
self._unindex)
return IISet()
def indexSize(self):
"""Return distinct values, as an optimization we always claim 2."""
return 2
def items(self):
# return a list of value to int set of rid tuples
indexed = self._index_value
items = [(bool(indexed), self._index)]
false = IISet()
for rid, value in self._unindex.iteritems():
if value != indexed:
false.add(rid)
items.append((not bool(indexed), false))
return items
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
return
indexed = bool(self._index_value)
unique_values = (indexed, not indexed)
if not withLengths:
for key in unique_values:
yield key
else:
for key in unique_values:
ilen = len(self._index)
if key is indexed:
yield (key, ilen)
else:
ulen = len(self._unindex)
yield (key, ulen - ilen)
def clear(self):
self._index = IITreeSet()
self._index_length = BTrees.Length.Length()
self._index_value = 1
self._unindex = IIBTree()
self._length = BTrees.Length.Length()
class DateIndex(UnIndex, PropertyManager):
"""Index for dates.
"""
meta_type = 'DateIndex'
query_options = ('query', 'range', 'not')
index_naive_time_as_local = True # False means index as UTC
_properties = ({'id': 'index_naive_time_as_local',
'type': 'boolean',
'mode': 'w'},)
manage = manage_main = DTMLFile('dtml/manageDateIndex', globals())
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_main._setName('manage_main')
manage_options = ({'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
) + PropertyManager.manage_options
def clear(self):
""" Complete reset """
self._index = IOBTree()
self._unindex = IIBTree()
self._length = Length()
if self._counter is None:
self._counter = Length()
else:
self._increment_counter()
def index_object(self, documentId, obj, threshold=None):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
"""
returnStatus = 0
try:
date_attr = getattr(obj, self.id)
if safe_callable(date_attr):
date_attr = date_attr()
ConvertedDate = self._convert(value=date_attr, default=_marker)
except AttributeError:
ConvertedDate = _marker
oldConvertedDate = self._unindex.get(documentId, _marker)
if ConvertedDate != oldConvertedDate:
if oldConvertedDate is not _marker:
self.removeForwardIndexEntry(oldConvertedDate, documentId)
if ConvertedDate is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.error("Should not happen: ConvertedDate was there,"
" now it's not, for document with id %s" %
documentId)
if ConvertedDate is not _marker:
self.insertForwardIndexEntry(ConvertedDate, documentId)
self._unindex[documentId] = ConvertedDate
returnStatus = 1
if returnStatus > 0:
self._increment_counter()
return returnStatus
def _convert(self, value, default=None):
"""Convert Date/Time value to our internal representation"""
if isinstance(value, DateTime):
t_tup = value.toZone('UTC').parts()
elif isinstance(value, (float, int)):
t_tup = time.gmtime(value)
elif isinstance(value, str) and value:
t_obj = DateTime(value).toZone('UTC')
t_tup = t_obj.parts()
elif isinstance(value, datetime):
if self.index_naive_time_as_local and value.tzinfo is None:
value = value.replace(tzinfo=Local)
# else if tzinfo is None, naive time interpreted as UTC
t_tup = value.utctimetuple()
elif isinstance(value, date):
t_tup = value.timetuple()
else:
return default
yr = t_tup[0]
mo = t_tup[1]
dy = t_tup[2]
hr = t_tup[3]
mn = t_tup[4]
t_val = ((((yr * 12 + mo) * 31 + dy) * 24 + hr) * 60 + mn)
if t_val > MAX32:
# t_val must be integer fitting in the 32bit range
raise OverflowError(
"%s is not within the range of indexable dates (index: %s)"
% (value, self.id))
return t_val
def _apply_index(self, request):
"""Apply the index to query parameters given in 'request'.
The argument should be a mapping object.
If the request does not contain the needed parameters, then
None is returned.
If the request contains a parameter with the name of the
column and this parameter is either a Record or a class
instance then it is assumed that the parameters of this index
are passed as attribute (Note: this is the recommended way to
pass parameters since Zope 2.4)
Otherwise two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
"""
config = get_configuration()
timeout = getattr(config, 'request_timeout', 20)
search_fields = getattr(config, 'search_fields', None)
if not search_fields:
search_fields = SEARCH_FIELDS
search_fields = search_fields.split()
if query_blocker.blocked:
return
record = parseIndexRequest(request, self.id)
if record.keys is None:
return None
es = get_query_client()
search = Search(using=es, index=index_name())
search = search.params(
request_timeout=timeout,
size=BATCH_SIZE,
preserve_order=True,
)
search = search.source(include='rid')
query_string = record.keys[0]
if query_string and query_string.startswith('*'):
# plone.app.querystring contains op sends a leading *, remove it
query_string = query_string[1:]
search = search.query('simple_query_string',
query=query_string,
fields=search_fields)
# setup highlighting
for field in search_fields:
name = field.split('^')[0]
if name == 'title':
# title shows up in results anyway
continue
search = search.highlight(name, fragment_size=FRAGMENT_SIZE)
try:
result = search.scan()
except TransportError:
# No es client, return empty results
logger.exception('ElasticSearch client not available.')
return IIBTree(), (self.id, )
# initial return value, other batches to be applied
retval = IIBTree()
highlights = OOBTree()
for r in result:
if getattr(r, 'rid', None) is None:
# something was indexed with no rid. Ignore for now.
# this is only for highlights, so no big deal if we
# skip one
continue
retval[r.rid] = int(10000 * float(r.meta.score))
# Index query returns only rids, so we need
# to save highlights for later use
highlight_list = []
if getattr(r.meta, 'highlight', None) is not None:
for key in dir(r.meta.highlight):
highlight_list.extend(r.meta.highlight[key])
highlights[r.meta.id] = highlight_list
# store highlights
try:
annotations = IAnnotations(self.REQUEST)
annotations[HIGHLIGHT_KEY] = highlights
except TypeError:
# maybe we are in a test
pass
return retval, (self.id, )
def __init__(self):
self._verification_codes = IIBTree()
def _makeOne(self):
from BTrees.IIBTree import IIBTree
return IIBTree()
