def test_replace_prices_with_tags(self):
headline = '日経平均222円安、1500円割れ 円安一服で'
tokens = self.tokenizer.tokenize(headline)
tag_tokens = replace_prices_with_tags([t.surface for t in tokens])
expected = [
'日経平均', '<yen val="222"/>', '安', '、', '<yen val="1500"/>', '割れ',
IDEOGRAPHIC_SPACE, '円安', '一服', 'で'
]
self.assertEqual(tag_tokens, expected)
def test_replace_prices_with_tags():
tokenizer = Tokenizer('resources/user-dict.csv')
headline = '日経平均222円安、1500円割れ 円安一服で'
tokens = tokenizer.tokenize(headline)
result = replace_prices_with_tags([t.surface for t in tokens])
expected = [
'日経平均', '<yen val="222"/>', '安', '、', '<yen val="1500"/>', '割れ',
IDEOGRAPHIC_SPACE, '円安', '一服', 'で'
]
assert result == expected
def update_headlines(session: Session, user_dict: Path,
logger: Logger) -> None:
query_result = session \
.query(Headline) \
.filter(Headline.is_used.is_(None)) \
.all()
headlines = list(query_result)
if len(headlines) == 0:
return
tokenizer = Tokenizer(str(user_dict))
mappings = []
logger.info('start updating headlines')
for headline in tqdm(headlines):
h = simplify_headline(headline.headline)
is_about_di = headline.categories is not None and \
DOMESTIC_INDEX in headline.categories
# We stopped using `is_template` because the size of the dataset decreased and the result got worse.
# if is_template(h) or not is_interesting(h) or not is_about_di:
if not is_interesting(h) or not is_about_di:
mappings.append({
'article_id': headline.article_id,
'is_used': False
})
continue
tokens = kansuuzi2number(
[token.surface for token in tokenizer.tokenize(h)])
tag_tokens = replace_prices_with_tags(tokens)
mappings.append({
'article_id': headline.article_id,
'simple_headline': h,
'tokens': tokens,
'tag_tokens': tag_tokens,
'is_used': True,
})
session.bulk_update_mappings(Headline, mappings)
session.commit()
logger.info('end updating headlines')