def test_should_not_include_line_feed_in_tag_if_previous_token_has_different_tag(
self):
original_root = _tei(front_items=[TOKEN_1, '\n ' + TOKEN_2])
LOGGER.debug('original_root: %s', _to_xml(original_root))
doc = GrobidTrainingTeiStructuredDocument(original_root)
lines = _get_all_lines(doc)
line1_tokens = list(doc.get_all_tokens_of_line(lines[0]))
space_tokens = [t for t in line1_tokens if isinstance(t, TeiSpace)]
assert space_tokens
for token in space_tokens:
doc.set_tag(token, TAG_1)
doc.set_tag(line1_tokens[0], TAG_1)
doc.set_tag(line1_tokens[-1], TAG_2)
LOGGER.debug('line1_tokens: %s', line1_tokens)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><{tag1}>{token1}</{tag1}>'
'\n <{tag2}>{token2}</{tag2}></front>'.format(tag1=TAG_1,
tag2=TAG_2,
token1=TOKEN_1,
token2=TOKEN_2))
def test_should_set_empty_token_whitespace(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note('1.')]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [['1', '.']]
tokens = doc.get_all_tokens_of_line(lines[0])
assert tokens[0].whitespace == ''
def test_should_reverse_map_tags(self):
tag_to_tei_path_mapping = {TAG_1: 'docTitle/titlePart'}
original_tei_xml = _tei(front_items=[E.docTitle(E.titlePart(TOKEN_1))])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(
original_tei_xml,
tag_to_tei_path_mapping=tag_to_tei_path_mapping,
preserve_tags=True)
LOGGER.debug('doc: %s', doc)
assert [[
doc.get_tag_or_preserved_tag_value(t)
for t in doc.get_all_tokens_of_line(line)
] for line in _get_all_lines(doc)] == [[TAG_1]]