class ManuscriptLineFactory(factory.Factory): class Meta: model = ManuscriptLine manuscript_id = factory.Sequence(lambda n: n) labels = ( SurfaceLabel.from_label(Surface.OBVERSE), ColumnLabel.from_label("iii", [Status.COLLATION, Status.CORRECTION]), ) line = factory.Sequence( lambda n: TextLine.of_iterable( LineNumber(n), ( Word.of( [ Reading.of_name("ku"), Joiner.hyphen(), BrokenAway.open(), Reading.of_name("nu"), Joiner.hyphen(), Reading.of_name("ši"), BrokenAway.close(), ] ), ), ) ) paratext = (NoteLine((StringPart("note"),)), RulingDollarLine(Ruling.SINGLE)) omitted_words = (1,)
def test_text_line_atf_erasure(erasure, expected: str) -> None: word = Word.of( [Reading.of_name("mu"), Joiner.hyphen(), Reading.of_name("mu")]) line = TextLine.of_iterable(LINE_NUMBER, [word, *erasure, word]) assert line.atf == f"{line.line_number.atf} {word.value} {expected} {word.value}"
def test_dump_line(): text = Text( ( TextLine.of_iterable( LineNumber(1), [ Word.of( parts=[ Reading.of_name("ha"), Joiner.hyphen(), Reading.of_name("am"), ] ) ], ), EmptyLine(), ControlLine("#", " comment"), ), "1.0.0", ) assert TextSchema().dump(text) == { "lines": OneOfLineSchema().dump(text.lines, many=True), "parser_version": text.parser_version, "numberOfLines": 1, }
def test_update_lemmatization() -> None: tokens = [list(line) for line in TEXT.lemmatization.tokens] tokens[0][0] = LemmatizationToken(tokens[0][0].value, (WordId("nu I"),)) lemmatization = Lemmatization(tokens) expected = Text( ( TextLine( LineNumber(1), ( Word.of( unique_lemma=(WordId("nu I"),), parts=[ Reading.of_name("ha"), Joiner.hyphen(), Reading.of_name("am"), ], ), ), ), RulingDollarLine(atf.Ruling.SINGLE), ), TEXT.parser_version, ) assert TEXT.update_lemmatization(lemmatization) == expected
def test_update_lemmatization_wrong_lenght() -> None: line = TextLine.of_iterable( LINE_NUMBER, [Word.of([Reading.of_name("bu")]), Word.of([Reading.of_name("bu")])], ) lemmatization = (LemmatizationToken("bu", (WordId("nu I"), )), ) with pytest.raises(LemmatizationError): line.update_lemmatization(lemmatization)
def test_extent_before_translation() -> None: with pytest.raises(ValueError): Text.of_iterable( [ TextLine.of_iterable(LineNumber(1), [Word.of([Reading.of_name("bu")])]), TextLine.of_iterable(LineNumber(2), [Word.of([Reading.of_name("bu")])]), TranslationLine(tuple(), "en", Extent(LineNumber(1))), ] )
def test_update_lemmatization() -> None: line = TextLine.of_iterable(LINE_NUMBER, [Word.of([Reading.of_name("bu")])]) lemmatization = (LemmatizationToken("bu", (WordId("nu I"), )), ) expected = TextLine.of_iterable( LINE_NUMBER, [Word.of([Reading.of_name("bu")], unique_lemma=(WordId("nu I"), ))]) assert line.update_lemmatization(lemmatization) == expected
def test_extent_overlapping_languages() -> None: Text.of_iterable( [ TextLine.of_iterable(LineNumber(1), [Word.of([Reading.of_name("bu")])]), TranslationLine(tuple(), "en", Extent(LineNumber(2))), TextLine.of_iterable(LineNumber(2), [Word.of([Reading.of_name("bu")])]), TranslationLine(tuple(), "de"), ] )
def test_exent_overlapping() -> None: with pytest.raises(ValueError): Text.of_iterable( [ TextLine.of_iterable(LineNumber(1), [Word.of([Reading.of_name("bu")])]), TranslationLine(tuple(), extent=Extent(LineNumber(2))), TextLine.of_iterable(LineNumber(2), [Word.of([Reading.of_name("bu")])]), TranslationLine(tuple()), ] )
def text_with_labels(): return Text.of_iterable([ TextLine.of_iterable(LineNumber(1), [Word.of([Reading.of_name("bu")])]), ColumnAtLine(ColumnLabel.from_int(1)), SurfaceAtLine(SurfaceLabel([], atf.Surface.SURFACE, "Stone wig")), ObjectAtLine(ObjectLabel([], atf.Object.OBJECT, "Stone wig")), TextLine.of_iterable(LineNumber(2), [Word.of([Reading.of_name("bu")])]), ])
def test_updating_alignment(client, bibliography, sign_repository, signs, text_repository): allow_signs(signs, sign_repository) chapter = ChapterFactory.build() allow_references(chapter, bibliography) text_repository.create_chapter(chapter) alignment = 0 omitted_words = (1, ) updated_chapter = attr.evolve( chapter, lines=(attr.evolve( chapter.lines[0], variants=(attr.evolve( chapter.lines[0].variants[0], manuscripts=(attr.evolve( chapter.lines[0].variants[0].manuscripts[0], line=TextLine.of_iterable( chapter.lines[0].variants[0].manuscripts[0].line. line_number, (Word.of( [ Reading.of_name("ku"), Joiner.hyphen(), BrokenAway.open(), Reading.of_name("nu"), Joiner.hyphen(), Reading.of_name("ši"), BrokenAway.close(), ], alignment=alignment, variant=Word.of( [Logogram.of_name("KU")], language=Language.SUMERIAN, ), ), ), ), omitted_words=omitted_words, ), ), ), ), ), ), ) expected_chapter = ApiChapterSchema().dump(updated_chapter) post_result = client.simulate_post(create_chapter_url( chapter, "/alignment"), body=json.dumps(DTO)) assert post_result.status == falcon.HTTP_OK assert post_result.json == expected_chapter get_result = client.simulate_get(create_chapter_url(chapter)) assert get_result.status == falcon.HTTP_OK assert get_result.json == expected_chapter
def test_text_line_atf_gloss() -> None: line = TextLine.of_iterable( LINE_NUMBER, [ DocumentOrientedGloss.open(), Word.of([Reading.of_name("mu")]), Word.of([Reading.of_name("bu")]), DocumentOrientedGloss.close(), ], ) assert line.atf == f"{line.line_number.atf} {{(mu bu)}}"
def test_updating_alignment(corpus, text_repository, bibliography, changelog, signs, sign_repository, user, when) -> None: aligmnet = 0 omitted_words = (1, ) updated_chapter = attr.evolve( CHAPTER, lines=(attr.evolve( CHAPTER.lines[0], variants=(attr.evolve( CHAPTER.lines[0].variants[0], manuscripts=(attr.evolve( CHAPTER.lines[0].variants[0].manuscripts[0], line=TextLine.of_iterable( CHAPTER.lines[0].variants[0].manuscripts[0].line. line_number, (Word.of( [ Reading.of_name("ku"), Joiner.hyphen(), BrokenAway.open(), Reading.of_name("nu"), Joiner.hyphen(), Reading.of_name("ši"), BrokenAway.close(), ], alignment=aligmnet, ), ), ), omitted_words=omitted_words, ), ), ), ), ), ), ) expect_find_and_update_chapter( bibliography, changelog, CHAPTER_WITHOUT_DOCUMENTS, updated_chapter, signs, sign_repository, text_repository, user, when, ) alignment = Alignment((((ManuscriptLineAlignment( (AlignmentToken("ku-[nu-ši]", aligmnet), ), omitted_words), ), ), )) assert corpus.update_alignment(CHAPTER.id_, alignment, user) == updated_chapter
def test_lemmatization() -> None: line = TextLine.of_iterable( LINE_NUMBER, [ Word.of([Reading.of_name("bu")], unique_lemma=(WordId("nu I"), )), UnknownNumberOfSigns.of(), Word.of([Reading.of_name("nu")]), ], ) assert line.lemmatization == ( LemmatizationToken("bu", (WordId("nu I"), )), LemmatizationToken("..."), LemmatizationToken("nu", tuple()), )
def test_parse_normalized_akkadain_shift() -> None: word = "ha" line = f"1. {word} %n {word} %sux {word}" expected = Text((TextLine.of_iterable( LineNumber(1), ( Word.of((Reading.of_name(word), ), DEFAULT_LANGUAGE), LanguageShift.normalized_akkadian(), AkkadianWord.of((ValueToken.of(word), )), LanguageShift.of("%sux"), Word.of((Reading.of_name(word), ), Language.SUMERIAN), ), ), )) assert parse_atf_lark(line).lines == expected.lines
def test_set_language(): parts = [Determinative.of([Reading.of_name("bu")])] language = Language.SUMERIAN lone_determinative = LoneDeterminative.of(parts, Language.AKKADIAN) expected = LoneDeterminative.of(parts, language) assert lone_determinative.set_language(language) == expected
def test_parse_atf_language_shifts(code: str, expected_language: Language) -> None: word = "ha-am" parts = [Reading.of_name("ha"), Joiner.hyphen(), Reading.of_name("am")] line = f"1. {word} {code} {word} %sb {word}" expected = Text((TextLine.of_iterable( LineNumber(1), ( Word.of(parts, DEFAULT_LANGUAGE), LanguageShift.of(code), Word.of(parts, expected_language), LanguageShift.of("%sb"), Word.of(parts, Language.AKKADIAN), ), ), )) assert parse_atf_lark(line).lines == expected.lines
def test_determinative(): parts = [Reading.of_name("kur"), Joiner.hyphen(), Reading.of_name("kur")] determinative = Determinative.of(parts) expected_value = f"{{{''.join(part.value for part in parts)}}}" expected_clean_value = f"{{{''.join(part.clean_value for part in parts)}}}" expected_parts = f"⟨{'⁚'.join(part.get_key() for part in parts)}⟩" assert determinative.value == expected_value assert determinative.clean_value == expected_clean_value assert determinative.get_key() == f"Determinative⁝{expected_value}{expected_parts}" assert determinative.parts == tuple(parts) assert determinative.lemmatizable is False serialized = { "type": "Determinative", "parts": OneOfTokenSchema().dump(parts, many=True), } assert_token_serialization(determinative, serialized)
def test_linguistic_gloss(): parts = [Reading.of_name("kur"), Joiner.hyphen(), Reading.of_name("kur")] gloss = LinguisticGloss.of(parts) expected_value = f"{{{{{''.join(part.value for part in parts)}}}}}" expected_clean_value = f"{{{{{''.join(part.clean_value for part in parts)}}}}}" expected_parts = f"⟨{'⁚'.join(part.get_key() for part in parts)}⟩" assert gloss.value == expected_value assert gloss.clean_value == expected_clean_value assert gloss.get_key() == f"LinguisticGloss⁝{expected_value}{expected_parts}" assert gloss.parts == tuple(parts) assert gloss.lemmatizable is False serialized = { "type": "LinguisticGloss", "parts": OneOfTokenSchema().dump(parts, many=True), } assert_token_serialization(gloss, serialized)
def test_updating_lines_edit(corpus, text_repository, bibliography, changelog, signs, sign_repository, user, when) -> None: updated_chapter = attr.evolve( CHAPTER, lines=(attr.evolve( CHAPTER.lines[0], number=LineNumber(1, True), variants=(attr.evolve( CHAPTER.lines[0].variants[0], manuscripts=(attr.evolve( CHAPTER.lines[0].variants[0].manuscripts[0], line=TextLine.of_iterable( LineNumber(1, True), (Word.of([ Reading.of_name("nu"), Joiner.hyphen(), BrokenAway.open(), Reading.of_name("ku"), Joiner.hyphen(), Reading.of_name("ši"), BrokenAway.close(), ]), ), ), ), ), ), ), ), ), signs=("ABZ075 KU ABZ207a\\u002F207b\\u0020X\nKU\nABZ075", ), parser_version=ATF_PARSER_VERSION, ) expect_find_and_update_chapter( bibliography, changelog, CHAPTER_WITHOUT_DOCUMENTS, updated_chapter, signs, sign_repository, text_repository, user, when, ) assert (corpus.update_lines( CHAPTER.id_, LinesUpdate([], set(), {0: updated_chapter.lines[0]}), user) == updated_chapter)
def test_statistics(database, fragment_repository): database[COLLECTION].insert_many( [ SCHEMA.dump( FragmentFactory.build( text=Text( ( TextLine( LineNumber(1), ( Word.of([Reading.of_name("first")]), Word.of([Reading.of_name("line")]), ), ), ControlLine("#", "ignore"), EmptyLine(), ) ) ) ), SCHEMA.dump( FragmentFactory.build( text=Text( ( ControlLine("#", "ignore"), TextLine( LineNumber(1), (Word.of([Reading.of_name("second")]),) ), TextLine( LineNumber(2), (Word.of([Reading.of_name("third")]),) ), ControlLine("#", "ignore"), TextLine( LineNumber(3), (Word.of([Reading.of_name("fourth")]),) ), ) ) ) ), SCHEMA.dump(FragmentFactory.build(text=Text())), ] ) assert fragment_repository.count_transliterated_fragments() == 2 assert fragment_repository.count_lines() == 4
def test_of_value(): value = "{bu}" parts = [Determinative.of([Reading.of_name("bu")])] lone_determinative = LoneDeterminative.of_value(parts) assert lone_determinative.value == value assert lone_determinative.lemmatizable is False assert lone_determinative.language == DEFAULT_LANGUAGE assert lone_determinative.normalized is False assert lone_determinative.unique_lemma == tuple() assert lone_determinative.erasure == ErasureState.NONE assert lone_determinative.alignment is None
def make_token(self, data, **kwargs): return ( Reading.of( data["name_parts"], data["sub_index"], data["modifiers"], data["flags"], data["sign"], ) .set_enclosure_type(frozenset(data["enclosure_type"])) .set_erasure(data["erasure"]) )
def expected_transliteration(language: Language) -> Sequence[Token]: return ( Word.of([Reading.of_name("bu")], language), LanguageShift.of("%es"), Word.of( [ BrokenAway.open(), Reading.of((ValueToken( frozenset({EnclosureType.BROKEN_AWAY}), ErasureState.NONE, "kur", ), )).set_enclosure_type(frozenset({EnclosureType.BROKEN_AWAY })), ], Language.EMESAL, ), UnknownNumberOfSigns(frozenset({EnclosureType.BROKEN_AWAY}), ErasureState.NONE), BrokenAway.close().set_enclosure_type( frozenset({EnclosureType.BROKEN_AWAY})), )
class ManuscriptFactory(factory.Factory): class Meta: model = Manuscript id = factory.Sequence(lambda n: n + 1) siglum_disambiguator = factory.Faker("word") museum_number = factory.Sequence( lambda n: MuseumNumber("M", str(n)) if pydash.is_odd(n) else None ) accession = factory.Sequence(lambda n: f"A.{n}" if pydash.is_even(n) else "") period_modifier = factory.fuzzy.FuzzyChoice(PeriodModifier) period = factory.fuzzy.FuzzyChoice(set(Period) - {Period.NONE}) provenance = factory.fuzzy.FuzzyChoice(set(Provenance) - {Provenance.STANDARD_TEXT}) type = factory.fuzzy.FuzzyChoice(set(ManuscriptType) - {ManuscriptType.NONE}) notes = factory.Faker("sentence") colophon = Transliteration.of_iterable( [TextLine.of_iterable(LineNumber(1, True), (Word.of([Reading.of_name("ku")]),))] ) unplaced_lines = Transliteration.of_iterable( [TextLine.of_iterable(LineNumber(1, True), (Word.of([Reading.of_name("nu")]),))] ) references = factory.List( [factory.SubFactory(ReferenceFactory, with_document=True)], TupleFactory )
def test_updating_manuscripts(corpus, text_repository, bibliography, changelog, signs, sign_repository, user, when) -> None: uncertain_fragments = (MuseumNumber.of("K.1"), ) updated_chapter = attr.evolve( CHAPTER, manuscripts=(attr.evolve( CHAPTER.manuscripts[0], colophon=Transliteration.of_iterable([ TextLine.of_iterable(LineNumber(1, True), (Word.of([Reading.of_name("ba")]), )) ]), unplaced_lines=Transliteration.of_iterable([ TextLine.of_iterable(LineNumber(1, True), (Word.of([Reading.of_name("ku")]), )) ]), notes="Updated manuscript.", ), ), uncertain_fragments=uncertain_fragments, signs=("KU ABZ075 ABZ207a\\u002F207b\\u0020X\nBA\nKU", ), ) expect_find_and_update_chapter( bibliography, changelog, CHAPTER_WITHOUT_DOCUMENTS, updated_chapter, signs, sign_repository, text_repository, user, when, ) manuscripts = (updated_chapter.manuscripts[0], ) assert (corpus.update_manuscripts(CHAPTER.id_, manuscripts, uncertain_fragments, user) == updated_chapter)
def test_lone_determinative(language): value = "{mu}" parts = [Determinative.of([Reading.of_name("mu")])] lone_determinative = LoneDeterminative.of(parts, language) equal = LoneDeterminative.of(parts, language) other_language = LoneDeterminative.of(parts, Language.UNKNOWN) other_parts = LoneDeterminative.of( [Determinative.of([Reading.of_name("bu")])], language) assert lone_determinative.value == value assert lone_determinative.lemmatizable is False assert lone_determinative.language == language assert lone_determinative.normalized is False assert lone_determinative.unique_lemma == tuple() serialized = { "type": "LoneDeterminative", "uniqueLemma": [], "normalized": False, "language": lone_determinative.language.name, "lemmatizable": lone_determinative.lemmatizable, "alignable": lone_determinative.lemmatizable, "erasure": ErasureState.NONE.name, "parts": OneOfTokenSchema().dump(parts, many=True), } assert_token_serialization(lone_determinative, serialized) assert lone_determinative == equal assert hash(lone_determinative) == hash(equal) for not_equal in [other_language, other_parts]: assert lone_determinative != not_equal assert hash(lone_determinative) != hash(not_equal) assert lone_determinative != ValueToken.of(value)
def test_text_line_of_iterable(code: str, language: Language) -> None: tokens = [ Word.of([Reading.of_name("first")]), LanguageShift.of(code), Word.of([Reading.of_name("second")]), LanguageShift.of("%sb"), LoneDeterminative.of([Determinative.of([Reading.of_name("third")])]), Word.of([BrokenAway.open(), Reading.of_name("fourth")]), UnknownNumberOfSigns.of(), BrokenAway.close(), ] expected_tokens = ( Word.of([Reading.of_name("first")], DEFAULT_LANGUAGE), LanguageShift.of(code), Word.of([Reading.of_name("second")], language), LanguageShift.of("%sb"), LoneDeterminative.of([Determinative.of([Reading.of_name("third")])], Language.AKKADIAN), Word.of( [ BrokenAway.open(), Reading.of((ValueToken( frozenset({EnclosureType.BROKEN_AWAY}), ErasureState.NONE, "fourth", ), )).set_enclosure_type(frozenset({EnclosureType.BROKEN_AWAY })), ], DEFAULT_LANGUAGE, ), UnknownNumberOfSigns(frozenset({EnclosureType.BROKEN_AWAY}), ErasureState.NONE), BrokenAway.close().set_enclosure_type( frozenset({EnclosureType.BROKEN_AWAY})), ) line = TextLine.of_iterable(LINE_NUMBER, tokens) assert line.line_number == LINE_NUMBER assert line.content == expected_tokens assert ( line.key == f"TextLine⁞{line.atf}⟨{'⁚'.join(token.get_key() for token in expected_tokens)}⟩" ) assert line.atf == f"1. first {code} second %sb {{third}} [fourth ...]"
def test_variant(): reading = Reading.of([ValueToken.of("sa"), BrokenAway.open(), ValueToken.of("l")]) divider = Divider.of(":") variant = Variant.of(reading, divider) expected_value = "sa[l/:" assert variant.value == expected_value assert variant.clean_value == "sal/:" assert variant.tokens == (reading, divider) assert variant.parts == variant.tokens assert ( variant.get_key() == f"Variant⁝{expected_value}⟨{'⁚'.join(token.get_key() for token in variant.tokens)}⟩" ) assert variant.lemmatizable is False serialized = { "type": "Variant", "tokens": OneOfTokenSchema().dump([reading, divider], many=True), } assert_token_serialization(variant, serialized)
def test_reading( name_parts, sub_index, modifiers, flags, sign, expected_value, expected_clean_value, expected_name, ) -> None: reading = Reading.of(name_parts, sub_index, modifiers, flags, sign) expected_parts = (*name_parts, sign) if sign else name_parts assert reading.value == expected_value assert reading.clean_value == expected_clean_value assert ( reading.get_key() == f"Reading⁝{expected_value}⟨{'⁚'.join(token.get_key() for token in expected_parts)}⟩" ) assert reading.name_parts == name_parts assert reading.name == expected_name assert reading.modifiers == tuple(modifiers) assert reading.flags == tuple(flags) assert reading.lemmatizable is False assert reading.sign == sign serialized = { "type": "Reading", "name": expected_name, "nameParts": OneOfTokenSchema().dump(name_parts, many=True), "subIndex": sub_index, "modifiers": modifiers, "flags": [flag.value for flag in flags], "sign": sign and OneOfTokenSchema().dump(sign), } assert_token_serialization(reading, serialized)