def get_doc_vectors(self, reader, tweets, num_docs):
""" plot each document in a vector space. """
terms_dict, idf_dict = self.get_dicts(reader, tweets, num_docs)
doc_vectors = []
liveDocs = MultiFields.getLiveDocs(reader)
for i in range(0, reader.maxDoc()):
if liveDocs is not None and not liveDocs.get(i): continue
doc_vectors.append(DocVector(terms_dict, idf_dict))
tfvs = reader.getTermVector(i, "contents")
terms_enum = tfvs.iterator(None)
for bytes_ref in util.BytesRefIterator.cast_(terms_enum):
doc_vectors[i].set_entry(bytes_ref.utf8ToString(), terms_enum.totalTermFreq())
return doc_vectors
def testSetPosition(self):
class _tokenizer(PythonTokenizer):
def __init__(_self, reader):
super(_tokenizer, _self).__init__(reader)
_self.TOKENS = ["1", "2", "3", "4", "5"]
_self.INCREMENTS = [1, 2, 1, 0, 1]
_self.i = 0
_self.posIncrAtt = _self.addAttribute(PositionIncrementAttribute.class_)
_self.termAtt = _self.addAttribute(CharTermAttribute.class_)
_self.offsetAtt = _self.addAttribute(OffsetAttribute.class_)
def incrementToken(_self):
if _self.i == len(_self.TOKENS):
return False
_self.clearAttributes()
_self.termAtt.append(_self.TOKENS[_self.i])
_self.offsetAtt.setOffset(_self.i, _self.i)
_self.posIncrAtt.setPositionIncrement(_self.INCREMENTS[_self.i])
_self.i += 1
return True
def end(_self):
pass
def reset(_self):
pass
def close(_self):
pass
class _analyzer(PythonAnalyzer):
def createComponents(_self, fieldName, reader):
return Analyzer.TokenStreamComponents(_tokenizer(reader))
writer = self.getWriter(analyzer=_analyzer())
d = Document()
d.add(Field("field", "bogus", TextField.TYPE_STORED))
writer.addDocument(d)
writer.commit()
writer.close()
searcher = self.getSearcher()
reader = searcher.getIndexReader()
pos = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "field", BytesRef("1"))
pos.nextDoc()
# first token should be at position 0
self.assertEqual(0, pos.nextPosition())
pos = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "field", BytesRef("2"))
pos.nextDoc()
# second token should be at position 2
self.assertEqual(2, pos.nextPosition())
q = PhraseQuery()
q.add(Term("field", "1"))
q.add(Term("field", "2"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# same as previous, just specify positions explicitely.
q = PhraseQuery()
q.add(Term("field", "1"), 0)
q.add(Term("field", "2"), 1)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# specifying correct positions should find the phrase.
q = PhraseQuery()
q.add(Term("field", "1"), 0)
q.add(Term("field", "2"), 2)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "3"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "3"))
q.add(Term("field", "4"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# phrase query would find it when correct positions are specified.
q = PhraseQuery()
q.add(Term("field", "3"), 0)
q.add(Term("field", "4"), 0)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
# phrase query should fail for non existing searched term
# even if there exist another searched terms in the same searched
# position.
q = PhraseQuery()
q.add(Term("field", "3"), 0)
q.add(Term("field", "9"), 0)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# multi-phrase query should succed for non existing searched term
# because there exist another searched terms in the same searched
# position.
mq = MultiPhraseQuery()
mq.add([Term("field", "3"), Term("field", "9")], 0)
hits = searcher.search(mq, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "4"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "3"))
q.add(Term("field", "5"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "4"))
q.add(Term("field", "5"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "5"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
def testPayloadsPos0(self):
writer = self.getWriter(analyzer=TestPayloadAnalyzer())
doc = Document()
doc.add(Field("content", "a a b c d e a f g h i j a b k k",
TextField.TYPE_STORED))
writer.addDocument(doc)
reader = writer.getReader()
writer.close()
tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
"content", BytesRef("a"))
count = 0
self.assert_(tp.nextDoc() != tp.NO_MORE_DOCS)
# "a" occurs 4 times
self.assertEqual(4, tp.freq())
expected = 0
self.assertEqual(expected, tp.nextPosition())
self.assertEqual(1, tp.nextPosition())
self.assertEqual(3, tp.nextPosition())
self.assertEqual(6, tp.nextPosition())
# only one doc has "a"
self.assert_(tp.nextDoc() == tp.NO_MORE_DOCS)
searcher = self.getSearcher(reader=reader)
stq1 = SpanTermQuery(Term("content", "a"))
stq2 = SpanTermQuery(Term("content", "k"))
sqs = [stq1, stq2]
snq = SpanNearQuery(sqs, 30, False)
count = 0
sawZero = False
pspans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq)
while pspans.next():
payloads = pspans.getPayload()
sawZero |= pspans.start() == 0
it = payloads.iterator()
while it.hasNext():
count += 1
it.next()
self.assertEqual(5, count)
self.assert_(sawZero)
spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq)
count = 0
sawZero = False
while spans.next():
count += 1
sawZero |= spans.start() == 0
self.assertEqual(4, count)
self.assert_(sawZero)
sawZero = False
psu = PayloadSpanUtil(searcher.getTopReaderContext())
pls = psu.getPayloadsForQuery(snq)
count = pls.size()
it = pls.iterator()
while it.hasNext():
bytes = JArray('byte').cast_(it.next())
s = bytes.string_
sawZero |= s == "pos: 0"
self.assertEqual(5, count)
self.assert_(sawZero)
def testSetPosition(self):
class _tokenizer(PythonTokenizer):
def __init__(_self, reader):
super(_tokenizer, _self).__init__(reader)
_self.TOKENS = ["1", "2", "3", "4", "5"]
_self.INCREMENTS = [1, 2, 1, 0, 1]
_self.i = 0
_self.posIncrAtt = _self.addAttribute(
PositionIncrementAttribute.class_)
_self.termAtt = _self.addAttribute(CharTermAttribute.class_)
_self.offsetAtt = _self.addAttribute(OffsetAttribute.class_)
def incrementToken(_self):
if _self.i == len(_self.TOKENS):
return False
_self.clearAttributes()
_self.termAtt.append(_self.TOKENS[_self.i])
_self.offsetAtt.setOffset(_self.i, _self.i)
_self.posIncrAtt.setPositionIncrement(
_self.INCREMENTS[_self.i])
_self.i += 1
return True
def end(_self):
pass
def reset(_self):
pass
def close(_self):
pass
class _analyzer(PythonAnalyzer):
def createComponents(_self, fieldName, reader):
return Analyzer.TokenStreamComponents(_tokenizer(reader))
writer = self.getWriter(analyzer=_analyzer())
d = Document()
d.add(Field("field", "bogus", TextField.TYPE_STORED))
writer.addDocument(d)
writer.commit()
writer.close()
searcher = self.getSearcher()
reader = searcher.getIndexReader()
pos = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
"field", BytesRef("1"))
pos.nextDoc()
# first token should be at position 0
self.assertEqual(0, pos.nextPosition())
pos = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
"field", BytesRef("2"))
pos.nextDoc()
# second token should be at position 2
self.assertEqual(2, pos.nextPosition())
q = PhraseQuery()
q.add(Term("field", "1"))
q.add(Term("field", "2"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# same as previous, just specify positions explicitely.
q = PhraseQuery()
q.add(Term("field", "1"), 0)
q.add(Term("field", "2"), 1)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# specifying correct positions should find the phrase.
q = PhraseQuery()
q.add(Term("field", "1"), 0)
q.add(Term("field", "2"), 2)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "3"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "3"))
q.add(Term("field", "4"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# phrase query would find it when correct positions are specified.
q = PhraseQuery()
q.add(Term("field", "3"), 0)
q.add(Term("field", "4"), 0)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
# phrase query should fail for non existing searched term
# even if there exist another searched terms in the same searched
# position.
q = PhraseQuery()
q.add(Term("field", "3"), 0)
q.add(Term("field", "9"), 0)
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
# multi-phrase query should succed for non existing searched term
# because there exist another searched terms in the same searched
# position.
mq = MultiPhraseQuery()
mq.add([Term("field", "3"), Term("field", "9")], 0)
hits = searcher.search(mq, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "4"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "3"))
q.add(Term("field", "5"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "4"))
q.add(Term("field", "5"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "5"))
hits = searcher.search(q, None, 1000).scoreDocs
self.assertEqual(0, len(hits))
def testPayloadsPos0(self):
writer = self.getWriter(analyzer=TestPayloadAnalyzer())
doc = Document()
doc.add(
Field("content", "a a b c d e a f g h i j a b k k",
TextField.TYPE_STORED))
writer.addDocument(doc)
reader = writer.getReader()
writer.close()
tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
"content", BytesRef("a"))
count = 0
self.assert_(tp.nextDoc() != tp.NO_MORE_DOCS)
# "a" occurs 4 times
self.assertEqual(4, tp.freq())
expected = 0
self.assertEqual(expected, tp.nextPosition())
self.assertEqual(1, tp.nextPosition())
self.assertEqual(3, tp.nextPosition())
self.assertEqual(6, tp.nextPosition())
# only one doc has "a"
self.assert_(tp.nextDoc() == tp.NO_MORE_DOCS)
searcher = self.getSearcher(reader=reader)
stq1 = SpanTermQuery(Term("content", "a"))
stq2 = SpanTermQuery(Term("content", "k"))
sqs = [stq1, stq2]
snq = SpanNearQuery(sqs, 30, False)
count = 0
sawZero = False
pspans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq)
while pspans.next():
payloads = pspans.getPayload()
sawZero |= pspans.start() == 0
it = payloads.iterator()
while it.hasNext():
count += 1
it.next()
self.assertEqual(5, count)
self.assert_(sawZero)
spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq)
count = 0
sawZero = False
while spans.next():
count += 1
sawZero |= spans.start() == 0
self.assertEqual(4, count)
self.assert_(sawZero)
sawZero = False
psu = PayloadSpanUtil(searcher.getTopReaderContext())
pls = psu.getPayloadsForQuery(snq)
count = pls.size()
it = pls.iterator()
while it.hasNext():
bytes = JArray('byte').cast_(it.next())
s = bytes.string_
sawZero |= s == "pos: 0"
self.assertEqual(5, count)
self.assert_(sawZero)
