def tokenize_for_words(self, phrase):
"""Return list of words found in PHRASE. Note that the phrase is
split into groups depending on the alphanumeric characters and
punctuation characters definition present in the config file.
"""
words = {}
formulas = []
if self.remove_html_markup and phrase.find("</") > -1:
phrase = remove_html_markup(phrase)
if self.remove_latex_markup:
formulas = latex_formula_re.findall(phrase)
phrase = remove_latex_markup(phrase)
phrase = latex_formula_re.sub(' ', phrase)
phrase = wash_for_utf8(phrase)
phrase = lower_index_term(phrase)
# 1st split phrase into blocks according to whitespace
for block in strip_accents(phrase).split():
# 2nd remove leading/trailing punctuation and add block:
block = re_block_punctuation_begin.sub("", block)
block = re_block_punctuation_end.sub("", block)
if block:
stemmed_block = remove_stopwords(block, self.remove_stopwords)
stemmed_block = length_check(stemmed_block)
stemmed_block = apply_stemming(stemmed_block,
self.stemming_language)
if stemmed_block:
words[stemmed_block] = 1
if re_arxiv.match(block):
# special case for blocks like `arXiv:1007.5048' where
# we would like to index the part after the colon
# regardless of dot or other punctuation characters:
words[block.split(':', 1)[1]] = 1
# 3rd break each block into subblocks according to punctuation and add subblocks:
for subblock in re_punctuation.split(block):
stemmed_subblock = remove_stopwords(
subblock, self.remove_stopwords)
stemmed_subblock = length_check(stemmed_subblock)
stemmed_subblock = apply_stemming(stemmed_subblock,
self.stemming_language)
if stemmed_subblock:
words[stemmed_subblock] = 1
# 4th break each subblock into alphanumeric groups and add groups:
for alphanumeric_group in re_separators.split(subblock):
stemmed_alphanumeric_group = remove_stopwords(
alphanumeric_group, self.remove_stopwords)
stemmed_alphanumeric_group = length_check(
stemmed_alphanumeric_group)
stemmed_alphanumeric_group = apply_stemming(
stemmed_alphanumeric_group, self.stemming_language)
if stemmed_alphanumeric_group:
words[stemmed_alphanumeric_group] = 1
for block in formulas:
words[block] = 1
return words.keys()
def tokenize_for_words(self, phrase):
"""Return list of words found in PHRASE. Note that the phrase is
split into groups depending on the alphanumeric characters and
punctuation characters definition present in the config file.
"""
words = {}
formulas = []
if self.remove_html_markup and phrase.find("</") > -1:
phrase = remove_html_markup(phrase)
if self.remove_latex_markup:
formulas = latex_formula_re.findall(phrase)
phrase = remove_latex_markup(phrase)
phrase = latex_formula_re.sub(' ', phrase)
phrase = wash_for_utf8(phrase)
phrase = lower_index_term(phrase)
# 1st split phrase into blocks according to whitespace
for block in strip_accents(phrase).split():
# 2nd remove leading/trailing punctuation and add block:
block = re_block_punctuation_begin.sub("", block)
block = re_block_punctuation_end.sub("", block)
if block:
stemmed_block = remove_stopwords(block, self.remove_stopwords)
stemmed_block = length_check(stemmed_block)
stemmed_block = apply_stemming(stemmed_block, self.stemming_language)
if stemmed_block:
words[stemmed_block] = 1
if re_arxiv.match(block):
# special case for blocks like `arXiv:1007.5048' where
# we would like to index the part after the colon
# regardless of dot or other punctuation characters:
words[block.split(':', 1)[1]] = 1
# 3rd break each block into subblocks according to punctuation and add subblocks:
for subblock in re_punctuation.split(block):
stemmed_subblock = remove_stopwords(subblock, self.remove_stopwords)
stemmed_subblock = length_check(stemmed_subblock)
stemmed_subblock = apply_stemming(stemmed_subblock, self.stemming_language)
if stemmed_subblock:
words[stemmed_subblock] = 1
# 4th break each subblock into alphanumeric groups and add groups:
for alphanumeric_group in re_separators.split(subblock):
stemmed_alphanumeric_group = remove_stopwords(alphanumeric_group, self.remove_stopwords)
stemmed_alphanumeric_group = length_check(stemmed_alphanumeric_group)
stemmed_alphanumeric_group = apply_stemming(stemmed_alphanumeric_group, self.stemming_language)
if stemmed_alphanumeric_group:
words[stemmed_alphanumeric_group] = 1
for block in formulas:
words[block] = 1
return words.keys()
def tokenize_for_pairs(self, phrase):
"""Return list of words found in PHRASE. Note that the phrase is
split into groups depending on the alphanumeric characters and
punctuation characters definition present in the config file.
"""
words = {}
if self.remove_html_markup and phrase.find("</") > -1:
phrase = remove_html_markup(phrase)
if self.remove_latex_markup:
phrase = remove_latex_markup(phrase)
phrase = latex_formula_re.sub(' ', phrase)
phrase = wash_for_utf8(phrase)
phrase = lower_index_term(phrase)
# 1st split phrase into blocks according to whitespace
last_word = ''
for block in strip_accents(phrase).split():
# 2nd remove leading/trailing punctuation and add block:
block = re_block_punctuation_begin.sub("", block)
block = re_block_punctuation_end.sub("", block)
if block:
block = remove_stopwords(block, self.remove_stopwords)
block = length_check(block)
block = apply_stemming(block, self.stemming_language)
# 3rd break each block into subblocks according to punctuation and add subblocks:
for subblock in re_punctuation.split(block):
subblock = remove_stopwords(subblock,
self.remove_stopwords)
subblock = length_check(subblock)
subblock = apply_stemming(subblock, self.stemming_language)
if subblock:
# 4th break each subblock into alphanumeric groups and add groups:
for alphanumeric_group in re_separators.split(
subblock):
alphanumeric_group = remove_stopwords(
alphanumeric_group, self.remove_stopwords)
alphanumeric_group = length_check(
alphanumeric_group)
alphanumeric_group = apply_stemming(
alphanumeric_group, self.stemming_language)
if alphanumeric_group:
if last_word:
words['%s %s' %
(last_word, alphanumeric_group)] = 1
last_word = alphanumeric_group
return words.keys()
def tokenize_for_pairs(self, phrase):
"""Return list of words found in PHRASE. Note that the phrase is
split into groups depending on the alphanumeric characters and
punctuation characters definition present in the config file.
"""
words = {}
if self.remove_html_markup and phrase.find("</") > -1:
phrase = remove_html_markup(phrase)
if self.remove_latex_markup:
phrase = remove_latex_markup(phrase)
phrase = latex_formula_re.sub(' ', phrase)
phrase = wash_for_utf8(phrase)
phrase = lower_index_term(phrase)
# 1st split phrase into blocks according to whitespace
last_word = ''
for block in strip_accents(phrase).split():
# 2nd remove leading/trailing punctuation and add block:
block = re_block_punctuation_begin.sub("", block)
block = re_block_punctuation_end.sub("", block)
if block:
block = remove_stopwords(block, self.remove_stopwords)
block = length_check(block)
block = apply_stemming(block, self.stemming_language)
# 3rd break each block into subblocks according to punctuation and add subblocks:
for subblock in re_punctuation.split(block):
subblock = remove_stopwords(subblock, self.remove_stopwords)
subblock = length_check(subblock)
subblock = apply_stemming(subblock, self.stemming_language)
if subblock:
# 4th break each subblock into alphanumeric groups and add groups:
for alphanumeric_group in re_separators.split(subblock):
alphanumeric_group = remove_stopwords(alphanumeric_group, self.remove_stopwords)
alphanumeric_group = length_check(alphanumeric_group)
alphanumeric_group = apply_stemming(alphanumeric_group, self.stemming_language)
if alphanumeric_group:
if last_word:
words['%s %s' % (last_word, alphanumeric_group)] = 1
last_word = alphanumeric_group
return words.keys()