def test_should_not_create_person_name_for_note(self):
label_and_layout_line_list_list = [[
('O', get_next_layout_line_for_text('before')),
('<forename>', get_next_layout_line_for_text('John')),
('<surname>', get_next_layout_line_for_text('Smith')),
('<marker>', get_next_layout_line_for_text('1')),
('O', get_next_layout_line_for_text(',')),
('<forename>', get_next_layout_line_for_text('Maria')),
('<surname>', get_next_layout_line_for_text('Madison')),
('<marker>', get_next_layout_line_for_text('2')),
('O', get_next_layout_line_for_text('after')),
]]
labeled_model_data_list_list = get_labeled_model_data_list_list(
label_and_layout_line_list_list,
data_generator=get_data_generator())
training_data_generator = get_tei_training_data_generator()
xml_root = training_data_generator.get_training_tei_xml_for_multiple_model_data_iterables(
labeled_model_data_list_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
nodes = tei_xpath(xml_root, AUTHOR_XPATH)
assert len(nodes) == 2
assert get_tei_xpath_text_content_list(nodes[0],
'./tei:forename') == ['John']
assert get_tei_xpath_text_content_list(nodes[1],
'./tei:forename') == ['Maria']
def test_should_generate_tei_for_most_labels(self):
label_and_layout_line_list = [
('<figure_head>', get_next_layout_line_for_text('Figure Head 1')),
('<figDesc>', get_next_layout_line_for_text('Figure Desc 1')),
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/head') == ['Figure Head 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/figDesc') == ['Figure Desc 1']
def test_should_add_label_to_head_element_without_additional_text(self):
label_and_layout_line_list = [
('<label>', get_next_layout_line_for_text('Figure Label 1')),
('<figDesc>', get_next_layout_line_for_text('Figure Desc 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/head/label') == ['Figure Label 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/head') == ['Figure Label 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/figDesc') == ['Figure Desc 1']
def test_should_add_label_at_the_end_inside_head_element(self):
label_and_layout_line_list = [
('<figure_head>', get_next_layout_line_for_text('Figure Head 1')),
('<label>', get_next_layout_line_for_text('Figure Label 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/head/label') == ['Figure Label 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{FIGURE_XPATH}/head') == [
'\n'.join(['Figure Head 1', 'Figure Label 1'])
]
def test_should_not_join_separate_labels(self):
label_and_layout_line_list = [
('<section>', get_next_layout_line_for_text(TEXT_1)),
('<section>', get_next_layout_line_for_text(TEXT_2))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = FullTextTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/head')) == [TEXT_1, TEXT_2]
assert get_text_content_list(
xml_root.xpath('./text')) == [f'{TEXT_1}\n{TEXT_2}\n']
def test_should_generate_multiple_tei_bibl_elements_without_label(self):
label_and_layout_line_list = [
('<reference>', get_next_layout_line_for_text(TEXT_1)),
('<reference>', get_next_layout_line_for_text(TEXT_2))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = get_tei_training_data_generator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/listBibl/bibl')) == [TEXT_1, TEXT_2]
assert get_text_content_list(
xml_root.xpath('./text/listBibl')) == [f'{TEXT_1}\n{TEXT_2}\n']
def test_should_generate_tei_from_model_data_using_model_labels(self):
label_and_layout_line_list = [
('<header>', get_next_layout_line_for_text(TEXT_1)),
('<body>', get_next_layout_line_for_text(TEXT_2))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = SegmentationTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/front')) == [TEXT_1]
assert get_text_content_list(xml_root.xpath('./text/body')) == [TEXT_2]
assert get_text_content_list(
xml_root.xpath('./text')) == [f'{TEXT_1}\n{TEXT_2}\n']
def test_should_generate_tei_from_multiple_model_data_lists_using_model_labels(
self):
label_and_layout_line_list_list = [[
('<figure_head>', get_next_layout_line_for_text(TEXT_1))
], [('<figure_head>', get_next_layout_line_for_text(TEXT_2))]]
labeled_model_data_list_list = get_labeled_model_data_list_list(
label_and_layout_line_list_list,
data_generator=get_data_generator())
training_data_generator = get_tei_training_data_generator()
xml_root = training_data_generator.get_training_tei_xml_for_multiple_model_data_iterables(
labeled_model_data_list_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
nodes = tei_xpath(xml_root, FIGURE_XPATH)
assert len(nodes) == 2
assert get_tei_xpath_text_content_list(nodes[0], './head') == [TEXT_1]
assert get_tei_xpath_text_content_list(nodes[1], './head') == [TEXT_2]
def test_should_add_equation_label_at_the_end_inside_equation_element(
self):
label_and_layout_line_list = [
('<equation>', get_next_layout_line_for_text('Eq 1')),
('<equation_label>', get_next_layout_line_for_text('Eq Label 1')),
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = FullTextTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/formula/label')) == ['Eq Label 1']
assert get_text_content_list(
xml_root.xpath('./text/formula')) == ['Eq 1\nEq Label 1']
def test_should_generate_single_tei_bibl_element_with_label_at_the_beginning(
self):
label_and_layout_line_list = [
('<label>', get_next_layout_line_for_text('Label 1')),
('<reference>', get_next_layout_line_for_text('Ref 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = get_tei_training_data_generator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/listBibl/bibl/label')) == ['Label 1']
assert get_text_content_list(
xml_root.xpath('./text/listBibl/bibl')) == ['Label 1\nRef 1']
def test_should_not_join_separate_labels(self):
label_and_layout_line_list = [
('<institution>', get_next_layout_line_for_text(TEXT_1)),
('<institution>', get_next_layout_line_for_text(TEXT_2))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = AffiliationAddressTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
aff_nodes = tei_xpath(xml_root, AFFILIATION_XPATH)
assert len(aff_nodes) == 1
assert get_tei_xpath_text_content_list(
aff_nodes[0],
'./tei:orgName[@type="institution"]') == [TEXT_1, TEXT_2]
assert get_text_content(aff_nodes[0]) == f'{TEXT_1}\n{TEXT_2}\n'
def test_should_generate_tei_from_multiple_model_data_lists_using_model_labels(
self):
label_and_layout_line_list_list = [[
('<institution>', get_next_layout_line_for_text(TEXT_1))
], [('<institution>', get_next_layout_line_for_text(TEXT_2))]]
labeled_model_data_list_list = get_labeled_model_data_list_list(
label_and_layout_line_list_list,
data_generator=get_data_generator())
training_data_generator = AffiliationAddressTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_multiple_model_data_iterables(
labeled_model_data_list_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
aff_nodes = tei_xpath(xml_root, AFFILIATION_XPATH)
assert len(aff_nodes) == 2
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:orgName[@type="institution"]') == [TEXT_1]
assert get_tei_xpath_text_content_list(
aff_nodes[1], './tei:orgName[@type="institution"]') == [TEXT_2]
def test_should_generate_tei_from_model_data(self):
layout_document = LayoutDocument.for_blocks([
LayoutBlock(lines=[
get_next_layout_line_for_text(TEXT_1),
get_next_layout_line_for_text(TEXT_2)
])
])
data_generator = get_data_generator()
model_data_iterable = data_generator.iter_model_data_for_layout_document(
layout_document)
xml_root = get_training_tei_xml_for_model_data_iterable(
model_data_iterable)
nodes = tei_xpath(xml_root, AUTHOR_XPATH)
assert len(nodes) == 1
lb_nodes = tei_xpath(nodes[0], 'tei:lb')
assert len(lb_nodes) == 2
assert lb_nodes[0].getparent().text == TEXT_1
assert lb_nodes[0].tail == '\n' + TEXT_2
def test_should_map_unknown_label_to_note(self):
label_and_layout_line_list = [('<unknown>',
get_next_layout_line_for_text(TEXT_1))]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:note[@type="unknown"]') == [TEXT_1]
def test_should_generate_tei_from_model_data(self):
layout_document = LayoutDocument.for_blocks([
LayoutBlock(lines=[
get_next_layout_line_for_text(TEXT_1),
get_next_layout_line_for_text(TEXT_2)
])
])
data_generator = get_data_generator()
model_data_iterable = data_generator.iter_model_data_for_layout_document(
layout_document)
training_data_generator = AffiliationAddressTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
model_data_iterable)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
aff_nodes = tei_xpath(xml_root, AFFILIATION_XPATH)
assert len(aff_nodes) == 1
lb_nodes = tei_xpath(aff_nodes[0], 'tei:lb')
assert len(lb_nodes) == 2
assert lb_nodes[0].getparent().text == TEXT_1
assert lb_nodes[0].tail == '\n' + TEXT_2
def test_should_generate_tei_from_model_data(self):
layout_document = LayoutDocument.for_blocks([
LayoutBlock(lines=[
get_next_layout_line_for_text(TEXT_1),
get_next_layout_line_for_text(TEXT_2)
])
])
data_generator = get_data_generator()
model_data_iterable = data_generator.iter_model_data_for_layout_document(
layout_document)
training_data_generator = get_tei_training_data_generator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
model_data_iterable)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
text_nodes = xml_root.xpath('./text/listBibl')
assert len(text_nodes) == 1
lb_nodes = text_nodes[0].xpath('lb')
assert len(lb_nodes) == 2
assert lb_nodes[0].getparent().text == TEXT_1
assert lb_nodes[0].tail == '\n' + TEXT_2
def test_should_map_other_label_as_text_without_note(self):
label_and_layout_line_list = [('<other>',
get_next_layout_line_for_text(TEXT_1))]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = get_tei_training_data_generator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert not xml_root.xpath('./text/listBibl//note')
assert get_text_content_list(
xml_root.xpath('./text/listBibl')) == [f'{TEXT_1}\n']
def test_should_generate_tei_from_model_data(self):
layout_document = LayoutDocument.for_blocks([
LayoutBlock(lines=[
get_next_layout_line_for_text(TEXT_1),
get_next_layout_line_for_text(TEXT_2)
])
])
data_generator = SegmentationDataGenerator(
DEFAULT_DOCUMENT_FEATURES_CONTEXT, use_first_token_of_block=True)
model_data_iterable = data_generator.iter_model_data_for_layout_document(
layout_document)
training_data_generator = SegmentationTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
model_data_iterable)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
text_nodes = xml_root.xpath('./text')
assert len(text_nodes) == 1
lb_nodes = text_nodes[0].xpath('lb')
assert len(lb_nodes) == 2
assert lb_nodes[0].getparent().text == TEXT_1
assert lb_nodes[0].tail == '\n' + TEXT_2
def test_should_generate_tei_for_most_labels(self):
label_and_layout_line_list = [
('<marker>', get_next_layout_line_for_text('Marker 1')),
('<title>', get_next_layout_line_for_text('Title 1')),
('<forename>', get_next_layout_line_for_text('Forename 1')),
('<middlename>', get_next_layout_line_for_text('Middlename 1')),
('<surname>', get_next_layout_line_for_text('Surname 1')),
('<suffix>', get_next_layout_line_for_text('Suffix 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:marker') == ['Marker 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:roleName') == ['Title 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:forename') == ['Forename 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:middlename') == ['Middlename 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:surname') == ['Surname 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{AUTHOR_XPATH}/tei:suffix') == ['Suffix 1']
def test_should_map_unknown_label_to_note(self):
label_and_layout_line_list = [('<unknown>',
get_next_layout_line_for_text(TEXT_1))]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = FullTextTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/note[@type="unknown"]')) == [TEXT_1]
assert get_text_content_list(
xml_root.xpath('./text')) == [f'{TEXT_1}\n']
def test_should_map_unknown_label_to_note(self):
label_and_layout_line_list = [('<unknown>',
get_next_layout_line_for_text(TEXT_1))]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = AffiliationAddressTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list, )
LOGGER.debug('xml: %r', etree.tostring(xml_root))
aff_nodes = tei_xpath(xml_root, AFFILIATION_XPATH)
assert len(aff_nodes) == 1
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:note[@type="unknown"]') == [TEXT_1]
assert get_text_content(aff_nodes[0]) == f'{TEXT_1}\n'
def test_should_generate_tei_from_model_data_using_model_labels(self):
label_and_layout_line_list = [
('<title>', get_next_layout_line_for_text(TEXT_1)),
('<abstract>', get_next_layout_line_for_text(TEXT_2))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list,
data_generator=get_data_generator()
)
training_data_generator = HeaderTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list
)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/front/docTitle/titlePart')
) == [TEXT_1]
assert get_text_content_list(
xml_root.xpath('./text/front/div[@type="abstract"]')
) == [TEXT_2]
assert get_text_content_list(
xml_root.xpath('./text/front')
) == [f'{TEXT_1}\n{TEXT_2}\n']
def test_should_add_label_to_head_element_without_additional_text(self):
label_and_layout_line_list = [
('<label>', get_next_layout_line_for_text('Table Label 1')),
('<figDesc>', get_next_layout_line_for_text('Table Desc 1')),
('<content>', get_next_layout_line_for_text('Content 1')),
('<other>', get_next_layout_line_for_text('Other 1')),
('<note>', get_next_layout_line_for_text('Note 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{TABLE_XPATH}/head/label') == ['Table Label 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{TABLE_XPATH}/head') == ['Table Label 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{TABLE_XPATH}/figDesc') == ['Table Desc 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{TABLE_XPATH}/table') == ['Content 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{TABLE_XPATH}/other') == ['Other 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{TABLE_XPATH}/note') == ['Note 1']
def test_should_map_pubnum_to_idno_with_type_if_detected(
self, test_input: str, expected_type: Optional[str]):
label_and_layout_line_list = [
('<pubnum>', get_next_layout_line_for_text(test_input))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
if expected_type:
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:idno[@type="{expected_type}"]') == [
test_input
]
else:
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:idno[not(@type)]') == [test_input]
def test_should_generate_tei_for_most_labels(self):
label_and_layout_line_list = [
('<section>', get_next_layout_line_for_text('Section Title 1')),
('<paragraph>', get_next_layout_line_for_text('Paragraph 1')),
('<citation_marker>',
get_next_layout_line_for_text('See Citation 1')),
('<figure_marker>', get_next_layout_line_for_text('See Figure 1')),
('<table_marker>', get_next_layout_line_for_text('See Table 1')),
('<equation_marker>', get_next_layout_line_for_text('See Eq 1')),
('<section_marker>',
get_next_layout_line_for_text('See Section 1')),
('<figure>', get_next_layout_line_for_text('Figure 1')),
('<table>', get_next_layout_line_for_text('Table 1')),
('<equation>', get_next_layout_line_for_text('Eq 1')),
('<item>', get_next_layout_line_for_text('Item 1')),
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = FullTextTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
assert get_text_content_list(
xml_root.xpath('./text/head')) == ['Section Title 1']
assert get_text_content_list(
xml_root.xpath('./text/p/ref[@type="biblio"]')) == [
'See Citation 1'
]
assert get_text_content_list(
xml_root.xpath('./text/p/ref[@type="table"]')) == ['See Table 1']
assert get_text_content_list(
xml_root.xpath('./text/p/ref[@type="figure"]')) == [
'See Figure 1'
]
assert get_text_content_list(
xml_root.xpath('./text/p/ref[@type="formula"]')) == ['See Eq 1']
assert get_text_content_list(
xml_root.xpath('./text/p/ref[@type="section"]')) == [
'See Section 1'
]
assert get_text_content_list(
xml_root.xpath('./text/figure[not(@type="table")]')) == [
'Figure 1'
]
assert get_text_content_list(
xml_root.xpath('./text/figure[@type="table"]')) == ['Table 1']
assert get_text_content_list(
xml_root.xpath('./text/formula')) == ['Eq 1']
assert get_text_content_list(
xml_root.xpath('./text/item')) == ['Item 1']
def test_should_generate_tei_for_most_labels(self):
label_and_layout_line_list = [
('<marker>', get_next_layout_line_for_text('Marker 1')),
('<institution>', get_next_layout_line_for_text('Institution 1')),
('<department>', get_next_layout_line_for_text('Department 1')),
('<laboratory>', get_next_layout_line_for_text('Laboratory 1')),
('<addrLine>', get_next_layout_line_for_text('AddrLine 1')),
('O', get_next_layout_line_for_text(',')),
('<postCode>', get_next_layout_line_for_text('PostCode 1')),
('O', get_next_layout_line_for_text(',')),
('<postBox>', get_next_layout_line_for_text('PostBox 1')),
('O', get_next_layout_line_for_text(',')),
('<region>', get_next_layout_line_for_text('Region 1')),
('O', get_next_layout_line_for_text(',')),
('<settlement>', get_next_layout_line_for_text('Settlement 1')),
('O', get_next_layout_line_for_text(',')),
('<country>', get_next_layout_line_for_text('Country 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
training_data_generator = AffiliationAddressTeiTrainingDataGenerator()
xml_root = training_data_generator.get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
LOGGER.debug('xml: %r', etree.tostring(xml_root))
aff_nodes = tei_xpath(xml_root, AFFILIATION_XPATH)
assert len(aff_nodes) == 1
assert get_tei_xpath_text_content_list(aff_nodes[0],
'./tei:marker') == ['Marker 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0],
'./tei:orgName[@type="institution"]') == ['Institution 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0],
'./tei:orgName[@type="department"]') == ['Department 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0],
'./tei:orgName[@type="laboratory"]') == ['Laboratory 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:address/tei:addrLine') == ['AddrLine 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:address/tei:postCode') == ['PostCode 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:address/tei:postBox') == ['PostBox 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:address/tei:region') == ['Region 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:address/tei:settlement') == ['Settlement 1']
assert get_tei_xpath_text_content_list(
aff_nodes[0], './tei:address/tei:country') == ['Country 1']
assert get_tei_xpath_text_content_list(aff_nodes[0],
'./tei:address') == [
'\n,\n'.join([
'AddrLine 1',
'PostCode 1',
'PostBox 1', 'Region 1',
'Settlement 1',
'Country 1'
])
]
def test_should_generate_tei_for_most_labels(self):
label_and_layout_line_list = [
('<title>', get_next_layout_line_for_text('Title 1')),
('<author>', get_next_layout_line_for_text('Author 1')),
('<editor>', get_next_layout_line_for_text('Editor 1')),
('<institution>', get_next_layout_line_for_text('Institution 1')),
('<collaboration>',
get_next_layout_line_for_text('Collaboration 1')),
('<journal>', get_next_layout_line_for_text('Journal 1')),
('<series>', get_next_layout_line_for_text('Series 1')),
('<booktitle>', get_next_layout_line_for_text('Book Title 1')),
('<date>', get_next_layout_line_for_text('Date 1')),
('<volume>', get_next_layout_line_for_text('Volume 1')),
('<issue>', get_next_layout_line_for_text('Issue 1')),
('<pages>', get_next_layout_line_for_text('Pages 1')),
('<publisher>', get_next_layout_line_for_text('Publisher 1')),
('<location>', get_next_layout_line_for_text('Location 1')),
('<tech>', get_next_layout_line_for_text('Tech 1')),
('<pubnum>', get_next_layout_line_for_text('Pubnum 1')),
('<web>', get_next_layout_line_for_text('Web 1')),
('<note>', get_next_layout_line_for_text('Note 1'))
]
labeled_model_data_list = get_labeled_model_data_list(
label_and_layout_line_list, data_generator=get_data_generator())
xml_root = get_training_tei_xml_for_model_data_iterable(
labeled_model_data_list)
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:title[@level="a"]') == ['Title 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:author') == ['Author 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:editor') == ['Editor 1']
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:orgName[not(@type)]') == ['Institution 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:orgName[@type="collaboration"]') == [
'Collaboration 1'
]
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:title[@level="j"]') == ['Journal 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:title[@level="s"]') == ['Series 1']
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:title[@level="m"]') == ['Book Title 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:date') == ['Date 1']
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:biblScope[@unit="volume"]') == ['Volume 1']
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:biblScope[@unit="issue"]') == ['Issue 1']
assert get_tei_xpath_text_content_list(
xml_root,
f'{BIBL_XPATH}/tei:biblScope[@unit="page"]') == ['Pages 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:publisher') == ['Publisher 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:pubPlace') == ['Location 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:note[@type="report"]') == ['Tech 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:idno') == ['Pubnum 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:ptr[@type="web"]') == ['Web 1']
assert get_tei_xpath_text_content_list(
xml_root, f'{BIBL_XPATH}/tei:note[not(@type)]') == ['Note 1']
