コード例 #1
0
 def test_get_additional_tokens(self):
     assert jautils.get_additional_tokens([u'ABC']) == set()
     assert jautils.get_additional_tokens(set([u'ABC'])) == set()
     assert jautils.get_additional_tokens([u'ABC', u'ひらがな']) == \
         set([u'HIRAGANA'])
     assert jautils.get_additional_tokens([u'やまだ', u'たろう']) == \
         set([u'YAMADA', u'TARO', u'やまだたろう', u'たろうやまだ'])
     assert jautils.get_additional_tokens([u'はい', u'やまだ', u'たろう']) == \
         set([u'HAI', u'YAMADA', u'TARO'])
コード例 #2
0
 def test_get_additional_tokens(self):
     assert jautils.get_additional_tokens([u'ABC']) == set()
     assert jautils.get_additional_tokens(set([u'ABC'])) == set()
     assert jautils.get_additional_tokens([u'ABC', u'ひらがな']) == \
         set([u'HIRAGANA'])
     assert jautils.get_additional_tokens([u'やまだ', u'たろう']) == \
         set([u'YAMADA', u'TARO', u'やまだたろう', u'たろうやまだ'])
     assert jautils.get_additional_tokens([u'はい', u'やまだ', u'たろう']) == \
         set([u'HAI', u'YAMADA', u'TARO'])
コード例 #3
0
ファイル: indexing.py プロジェクト: asethu5/personfinder
def get_alternate_name_tokens(person):
    """Returns alternate name tokens and their variations."""
    tokens = set(TextQuery(person.alternate_names).query_words)
    # This is no-op for non-Japanese.
    tokens |= set(jautils.get_additional_tokens(tokens))
    return tokens
コード例 #4
0
def get_alternate_name_tokens(person):
    """Returns alternate name tokens and their variations."""
    tokens = set(TextQuery(person.alternate_names).query_words)
    # This is no-op for non-Japanese.
    tokens |= set(jautils.get_additional_tokens(tokens))
    return tokens