def __or__(self, x):
return self.__class__(
weightedUnion(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
return self.__class__(result, self._words+x._words, self._index)
def test_None_is_smallest(self):
t = self._makeOne()
for i in range(999): # Make sure we multiple buckets
t[i] = i*i
t[None] = -1
for i in range(-99,0): # Make sure we multiple buckets
t[i] = i*i
self.assertEqual(list(t), [None] + list(range(-99, 999)))
self.assertEqual(list(t.values()),
[-1] + [i*i for i in range(-99, 999)])
self.assertEqual(t[2], 4)
self.assertEqual(t[-2], 4)
self.assertEqual(t[None], -1)
t[None] = -2
self.assertEqual(t[None], -2)
t2 = t.__class__(t)
del t[None]
self.assertEqual(list(t), list(range(-99, 999)))
if 'Py' in self.__class__.__name__:
return
from BTrees.OOBTree import difference, union, intersection
self.assertEqual(list(difference(t2, t).items()), [(None, -2)])
self.assertEqual(list(union(t, t2)), list(t2))
self.assertEqual(list(intersection(t, t2)), list(t))
def near(self, x):
result = IIBucket()
dict = self._dict
xdict = x._dict
xhas = xdict.has_key
positions = self._index.positions
for id, score in dict.items():
if not xhas(id): continue
p=(map(lambda i: (i,0), positions(id,self._words))+
map(lambda i: (i,1), positions(id,x._words)))
p.sort()
d = lp = 9999
li = None
lsrc = None
for i,src in p:
if i is not li and src is not lsrc and li is not None:
d = min(d,i-li)
li = i
lsrc = src
if d==lp: score = min(score,xdict[id]) # synonyms
else: score = (score+xdict[id])/d
result[id] = score
return self.__class__(
result, union(self._words, x._words), self._index)
def test_None_is_smallest(self):
t = self._makeOne()
for i in range(999): # Make sure we multiple buckets
t[i] = i * i
t[None] = -1
for i in range(-99, 0): # Make sure we multiple buckets
t[i] = i * i
self.assertEqual(list(t), [None] + list(range(-99, 999)))
self.assertEqual(list(t.values()),
[-1] + [i * i for i in range(-99, 999)])
self.assertEqual(t[2], 4)
self.assertEqual(t[-2], 4)
self.assertEqual(t[None], -1)
t[None] = -2
self.assertEqual(t[None], -2)
t2 = t.__class__(t)
del t[None]
self.assertEqual(list(t), list(range(-99, 999)))
if 'Py' in self.__class__.__name__:
return
from BTrees.OOBTree import difference, union, intersection
self.assertEqual(list(difference(t2, t).items()), [(None, -2)])
self.assertEqual(list(union(t, t2)), list(t2))
self.assertEqual(list(intersection(t, t2)), list(t))
def near(self, x):
result = IIBucket()
dict = self._dict
xdict = x._dict
xhas = xdict.has_key
positions = self._index.positions
for id, score in dict.items():
if not xhas(id): continue
p = (map(lambda i: (i, 0), positions(id, self._words)) +
map(lambda i: (i, 1), positions(id, x._words)))
p.sort()
d = lp = 9999
li = None
lsrc = None
for i, src in p:
if i is not li and src is not lsrc and li is not None:
d = min(d, i - li)
li = i
lsrc = src
if d == lp: score = min(score, xdict[id]) # synonyms
else: score = (score + xdict[id]) / d
result[id] = score
return self.__class__(result, union(self._words, x._words),
self._index)
def _combine_union(self, values, object):
if not values: return
set = None
for v in values:
sv = self._standardizeValue(v, object)
if not sv: continue
set = union(set, sv)
return set
def addTrainingDocument(self,doc_id,tags):
"""
"""
storage = getUtility(INounPhraseStorage)
importantNouns = storage.getNounTerms(doc_id,self.noNounRanksToKeep)
self.trainingDocs[doc_id] = (importantNouns,tags)
self.allNouns = union(self.allNouns,OOSet(importantNouns))
def _combine_union(self,values,object):
if not values: return
set= None
for v in values:
sv= self._standardizeValue(v,object)
if not sv: continue
set= union(set,sv)
return set
def testFunkyKeyIteration(self):
# The internal set iteration protocol allows "iterating over" a
# a single key as if it were a set.
N = 100
union, mkset = self.union, self.mkset
slow = mkset()
for i in range(N):
slow = union(slow, mkset([i]))
fast = self.multiunion(range(N)) # acts like N distinct singleton sets
self.assertEqual(len(slow), N)
self.assertEqual(len(fast), N)
self.assertEqual(list(slow), list(fast))
self.assertEqual(list(fast), range(N))
def testFunkyKeyIteration(self):
# The internal set iteration protocol allows "iterating over" a
# a single key as if it were a set.
N = 100
union, mkset = self.union, self.mkset
slow = mkset()
for i in range(N):
slow = union(slow, mkset([i]))
fast = self.multiunion(range(N)) # acts like N distinct singleton sets
self.assertEqual(len(slow), N)
self.assertEqual(len(fast), N)
self.assertEqual(list(slow), list(fast))
self.assertEqual(list(fast), range(N))
def train(self):
"""
"""
presentNouns = dict()
trainingData = []
if not self.allNouns:
storage = getUtility(INounPhraseStorage)
for key in self.trainingDocs.keys():
importantNouns = storage.getNounTerms(
key,
self.noNounRanksToKeep)
self.allNouns = union(self.allNouns,OOSet(importantNouns))
for item in self.allNouns:
presentNouns.setdefault(item,0)
for (nouns,tags) in self.trainingDocs.values():
nounPresence = presentNouns.copy()
for noun in nouns:
nounPresence[noun] = 1
for tag in tags:
trainingData.append((nounPresence,tag,))
if trainingData:
self.classifier = NaiveBayesClassifier.train(trainingData)
def __or__(self, x):
return self.__class__(
weightedUnion(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
def __and__(self, x):
return self.__class__(
weightedIntersection(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
def __and__(self, x):
return self.__class__(
weightedIntersection(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
def __or__(self, other):
return QuerySet(union(OOTreeSet(self), OOTreeSet(other)))
def union(self, *args):
from BTrees.OOBTree import union
return union(*args)
def __or__(self, x):
return self.__class__(
weightedUnion(self._dict, x._dict),
union(self._words, x._words),
self._index,
)
def union(self, *args):
from BTrees.LFBTree import union
return union(*args)
def __or__(self, x):
return self.__class__(
ii_union(self._dict, x._dict),
union(self._words, x._words),
self._index,
)
def __and__(self, x):
return self.__class__(
intersection(self._dict, x._dict),
union(self._words, x._words),
self._index,
)
def __and__(self, x):
return self.__class__(
intersection(self._dict, x._dict),
union(self._words, x._words),
self._index,
)