def indexReader(self, indexWriter, reader, owner, attribute, version): doc = Document() doc.add(Field("owner", owner.str16(), True, False, False)) doc.add(Field("attribute", attribute, True, False, False)) doc.add(Field("version", str(version), True, False, False)) doc.add(Field.Text("contents", reader)) indexWriter.addDocument(doc)
def create_feed_document(self, feed): doc = Document() doc.add( Field('id', str(feed.id), Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add( Field('url', feed.xml_url, Field.Store.YES, Field.Index.UN_TOKENIZED)) if feed.channel_link: doc.add( Field('link', feed.channel_link, Field.Store.YES, Field.Index.UN_TOKENIZED)) if feed.title: doc.add( Field('title', feed.title, Field.Store.YES, Field.Index.TOKENIZED)) if feed.subtitle: doc.add( Field('subtitle', feed.subtitle, Field.Store.YES, Field.Index.TOKENIZED)) return doc
def create_entry_documents(self, feed): docs = [] for entry in feed.get_entries(): try: doc = Document() id = '%s:%s' % (feed.xml_url, entry.get('id', None)) doc.add( Field('id', id, Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add( Field('feed_url', feed.xml_url, Field.Store.YES, Field.Index.UN_TOKENIZED)) if entry.get('title', None): doc.add( Field('title', entry['title'], Field.Store.YES, Field.Index.TOKENIZED)) if entry.get('summary', None): doc.add( Field('summary', entry['summary'], Field.Store.YES, Field.Index.TOKENIZED)) if entry.get('link', None): doc.add( Field('link', entry['link'], Field.Store.YES, Field.Index.UN_TOKENIZED)) try: updated = parser.parse(entry.get('updated', None), ignoretz=True) doc.add( Field('updated', updated.isoformat(' '), Field.Store.YES, Field.Index.NO)) except: {} try: doc.add( Field('pickle', pickle.dumps(entry), Field.Store.YES, Field.Index.NO)) except Exception, e: logging.error('Unable to store pickled entry: %s' % e) docs.append(doc) except Exception, e: logging.error(e)
def indexArticle(pmid, text): res = p.split(text) i = 1 for r in res[1::2]: span_id = r span_text = res[i + 1] try: doc = Document() doc.add( Field("span_id", span_id, Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add( Field("pmid", pmid, Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add( Field("text", span_text, Field.Store.YES, Field.Index.TOKENIZED)) addAnnotations(doc, span_id) writer.addDocument(doc) except Exception, e: sys.stderr.write("error: %s pmid: %s span_id: %s\n" % (e, pmid, span_id)) i += 2