def test_get_additional_tokens(self):
assert jautils.get_additional_tokens([u'ABC']) == set()
assert jautils.get_additional_tokens(set([u'ABC'])) == set()
assert jautils.get_additional_tokens([u'ABC', u'ひらがな']) == \
set([u'HIRAGANA'])
assert jautils.get_additional_tokens([u'やまだ', u'たろう']) == \
set([u'YAMADA', u'TARO', u'やまだたろう', u'たろうやまだ'])
assert jautils.get_additional_tokens([u'はい', u'やまだ', u'たろう']) == \
set([u'HAI', u'YAMADA', u'TARO'])
def test_get_additional_tokens(self):
assert jautils.get_additional_tokens([u'ABC']) == set()
assert jautils.get_additional_tokens(set([u'ABC'])) == set()
assert jautils.get_additional_tokens([u'ABC', u'ひらがな']) == \
set([u'HIRAGANA'])
assert jautils.get_additional_tokens([u'やまだ', u'たろう']) == \
set([u'YAMADA', u'TARO', u'やまだたろう', u'たろうやまだ'])
assert jautils.get_additional_tokens([u'はい', u'やまだ', u'たろう']) == \
set([u'HAI', u'YAMADA', u'TARO'])
def get_alternate_name_tokens(person):
"""Returns alternate name tokens and their variations."""
tokens = set(TextQuery(person.alternate_names).query_words)
# This is no-op for non-Japanese.
tokens |= set(jautils.get_additional_tokens(tokens))
return tokens
def get_alternate_name_tokens(person):
"""Returns alternate name tokens and their variations."""
tokens = set(TextQuery(person.alternate_names).query_words)
# This is no-op for non-Japanese.
tokens |= set(jautils.get_additional_tokens(tokens))
return tokens