def test_set_unique_lemma_empty() -> None: word = AkkadianWord.of((ValueToken.of("bu"), ), unique_lemma=(WordId("nu I"), )) lemma = LemmatizationToken("bu", tuple()) expected = AkkadianWord.of((ValueToken.of("bu"), )) assert word.set_unique_lemma(lemma) == expected
def make_token(self, data, **kwargs): return AkkadianWord.of( tuple(data["parts"]), tuple(data["modifiers"]), tuple(data["unique_lemma"]), data["alignment"], data["variant"], ).set_enclosure_type(frozenset(data["enclosure_type"]))
def test_text_line_of_iterable_normalized() -> None: tokens = [ LanguageShift.normalized_akkadian(), AkkadianWord.of((ValueToken.of("kur"), )), ] expected_tokens = ( LanguageShift.normalized_akkadian(), AkkadianWord.of((ValueToken.of("kur"), )), ) line = TextLine.of_iterable(LINE_NUMBER, tokens) assert line.content == expected_tokens assert ( line.key == f"TextLine⁞{line.atf}⟨{'⁚'.join(token.get_key() for token in expected_tokens)}⟩" ) assert line.atf == "1. %n kur"
class LineVariantFactory(factory.Factory): class Meta: model = LineVariant class Params: manuscript_id = factory.Sequence(lambda n: n) manuscript = factory.SubFactory( ManuscriptLineFactory, manuscript_id=factory.SelfAttribute("..manuscript_id"), ) reconstruction = ( LanguageShift.normalized_akkadian(), AkkadianWord.of((ValueToken.of("buāru"),)), MetricalFootSeparator.uncertain(), BrokenAway.open(), UnknownNumberOfSigns.of(), Caesura.certain(), AkkadianWord.of( ( UnknownNumberOfSigns.of(), BrokenAway.close(), Joiner.hyphen(), ValueToken.of("buāru"), ), (Flag.DAMAGE,), ), ) note = factory.fuzzy.FuzzyChoice([None, NoteLine((StringPart("a note"),))]) manuscripts = factory.List([factory.SelfAttribute("..manuscript")], TupleFactory) intertext = factory.fuzzy.FuzzyChoice([tuple(), (StringPart("bar"),)]) parallel_lines = factory.List( [ factory.SubFactory(ParallelCompositionFactory), factory.SubFactory(ParallelTextFactory), factory.SubFactory(ParallelFragmentFactory), ], TupleFactory, )
def test_parse_normalized_akkadain_shift() -> None: word = "ha" line = f"1. {word} %n {word} %sux {word}" expected = Text((TextLine.of_iterable( LineNumber(1), ( Word.of((Reading.of_name(word), ), DEFAULT_LANGUAGE), LanguageShift.normalized_akkadian(), AkkadianWord.of((ValueToken.of(word), )), LanguageShift.of("%sux"), Word.of((Reading.of_name(word), ), Language.SUMERIAN), ), ), )) assert parse_atf_lark(line).lines == expected.lines
MANUSCRIPT_ID = 9001 SIGLUM_DISAMBIGUATOR = "1c" MUSEUM_NUMBER = MuseumNumber("BM", "x") ACCESSION = "" PERIOD_MODIFIER = PeriodModifier.LATE PERIOD = Period.OLD_BABYLONIAN PROVENANCE = Provenance.NINEVEH TYPE = ManuscriptType.LIBRARY NOTES = "some notes" COLOPHON = Transliteration.of_iterable( [TextLine(LineNumber(1, True), (Word.of([Reading.of_name("ku")]), ))]) UNPLACED_LINES = Transliteration.of_iterable( [TextLine(LineNumber(4, True), (Word.of([Reading.of_name("bu")]), ))]) REFERENCES = (ReferenceFactory.build(), ) LINE_NUMBER = LineNumber(1) LINE_RECONSTRUCTION = (AkkadianWord.of((ValueToken.of("buāru"), )), ) IS_SECOND_LINE_OF_PARALLELISM = True IS_BEGINNING_OF_SECTION = True LABELS = (SurfaceLabel.from_label(Surface.OBVERSE), ) PARATEXT = (NoteLine((StringPart("note"), )), RulingDollarLine(Ruling.SINGLE)) OMITTED_WORDS = (1, ) NOTE = None PARALLEL_LINES = (ParallelComposition(False, "a composition", LineNumber(7)), ) TRANSLATION = (TranslationLine((StringPart("foo"), ), "en", None), ) SIGNS = ("FOO BAR", ) MANUSCRIPT_TEXT_1 = TextLine(LineNumber(1), (Word.of([Reading.of([ValueToken.of("ku")])]), )) LINE_VARIANT_1 = LineVariant(
def test_akkadian_word_invalid_modifier() -> None: with pytest.raises(ValueError): AkkadianWord.of((ValueToken.of("ibnû"), ), (Flag.COLLATION, ))
BrokenAway, Emendation, PerhapsBrokenAway, ) from ebl.transliteration.domain.normalized_akkadian import ( AkkadianWord, Caesura, MetricalFootSeparator, ) from ebl.transliteration.domain.tokens import Joiner, UnknownNumberOfSigns, ValueToken @pytest.mark.parametrize( # pyre-ignore[56] "word,expected,lemmatizable", [ (AkkadianWord.of((ValueToken.of("ibnû"), )), "ibnû", True), ( AkkadianWord.of((ValueToken.of("ibnû"), ), (Flag.UNCERTAIN, Flag.DAMAGE, Flag.CORRECTION)), "ibnû?#!", True, ), (AkkadianWord.of( (BrokenAway.open(), ValueToken.of("ibnû"))), "[ibnû", True), ( AkkadianWord.of(( BrokenAway.open(), PerhapsBrokenAway.open(), ValueToken.of("ib"), PerhapsBrokenAway.close(), ValueToken.of("nû"),
def _make_sequence(values: List[str]) -> List[Token]: return [AkkadianWord.of((ValueToken.of(value), )) for value in values]
class LemmatizedFragmentFactory(TransliteratedFragmentFactory): text = Text(( TextLine.of_iterable( LineNumber(1, True), ( Word.of([UnidentifiedSign.of()]), Word.of([ Logogram.of_name( "BA", surrogate=[ Reading.of_name("ku"), Joiner.hyphen(), Reading.of_name("u", 4), ], ) ]), Column.of(), Tabulation.of(), Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), Joiner.hyphen(), Reading.of_name("ku"), BrokenAway.close(), Joiner.hyphen(), Reading.of_name("nu"), Joiner.hyphen(), Reading.of_name("ši"), ]), Variant.of(Divider.of(":"), Reading.of_name("ku")), Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), BrokenAway.close(), ]), Column.of(2), Divider.of(":", ("@v", ), (Flag.DAMAGE, )), CommentaryProtocol.of("!qt"), Word.of([Number.of_name("10", flags=[Flag.DAMAGE])]), ), ), TextLine.of_iterable( LineNumber(2, True), ( Word.of([BrokenAway.open(), UnknownNumberOfSigns.of()]), Word.of([Logogram.of_name("GI", 6)], unique_lemma=(WordId("ginâ I"), )), Word.of([Reading.of_name("ana")], unique_lemma=(WordId("ana I"), )), Word.of( [ Reading.of_name("u₄"), Joiner.hyphen(), Reading.of_name("š[u"), ], unique_lemma=(WordId("ūsu I"), ), ), Word.of([UnknownNumberOfSigns.of(), BrokenAway.close()]), ), ), TextLine.of_iterable( LineNumber(3, True), ( Word.of([BrokenAway.open(), UnknownNumberOfSigns.of()]), Word.of( unique_lemma=(WordId("kīdu I"), ), parts=[ Reading.of(( ValueToken.of("k"), BrokenAway.close(), ValueToken.of("i"), )), Joiner.hyphen(), Reading.of_name("du"), ], ), Word.of(unique_lemma=(WordId("u I"), ), parts=[Reading.of_name("u")]), Word.of( unique_lemma=(WordId("bamātu I"), ), parts=[ Reading.of_name("ba"), Joiner.hyphen(), Reading.of_name("ma"), Joiner.hyphen(), Reading.of(( ValueToken.of("t"), BrokenAway.open(), ValueToken.of("i"), )), ], ), Word.of([UnknownNumberOfSigns.of(), BrokenAway.close()]), ), ), TextLine.of_iterable( LineNumber(6, True), ( Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), BrokenAway.close(), ]), Word.of([UnclearSign.of([Flag.DAMAGE])]), Word.of(unique_lemma=(WordId("mu I"), ), parts=[Reading.of_name("mu")]), Word.of( unique_lemma=(WordId("tamalāku I"), ), parts=[ Reading.of_name("ta"), Joiner.hyphen(), Reading.of_name("ma"), InWordNewline.of(), Joiner.hyphen(), Reading.of_name("tu", 2), ], ), ), ), TextLine.of_iterable( LineNumber(7, True), ( Word.of([ Variant.of(Reading.of_name("šu"), CompoundGrapheme.of(["BI×IS"])) ]), LanguageShift.normalized_akkadian(), AkkadianWord.of([ValueToken.of("kur")], unique_lemma=(WordId("normalized I"), )), ), ), StateDollarLine( atf.Qualification.AT_LEAST, 1, ScopeContainer(atf.Surface.OBVERSE, ""), atf.State.MISSING, None, ), ImageDollarLine("1", None, "numbered diagram of triangle"), RulingDollarLine(atf.Ruling.SINGLE), LooseDollarLine("this is a loose line"), SealDollarLine(1), SealAtLine(1), HeadingAtLine(1), ColumnAtLine(ColumnLabel([atf.Status.COLLATION], 1)), SurfaceAtLine( SurfaceLabel([atf.Status.COLLATION], atf.Surface.SURFACE, "stone wig")), ObjectAtLine( ObjectLabel([atf.Status.COLLATION], atf.Object.OBJECT, "stone wig")), DiscourseAtLine(atf.Discourse.DATE), DivisionAtLine("paragraph", 5), CompositeAtLine(atf.Composite.DIV, "part", 1), NoteLine(( StringPart("a note "), EmphasisPart("italic"), LanguagePart.of_transliteration( Language.AKKADIAN, (Word.of([Reading.of_name("bu")]), )), )), ParallelComposition(False, "my name", LineNumber(1)), ParallelText( True, TextId(CorpusGenre.LITERATURE, 1, 1), ChapterName(Stage.OLD_BABYLONIAN, "", "my name"), LineNumber(1), False, ), ParallelFragment(False, MuseumNumber.of("K.1"), True, Labels(), LineNumber(1), False), ))
class TransliteratedFragmentFactory(FragmentFactory): text = Text(( TextLine.of_iterable( LineNumber(1, True), ( Word.of([UnidentifiedSign.of()]), Word.of([ Logogram.of_name( "BA", surrogate=[ Reading.of_name("ku"), Joiner.hyphen(), Reading.of_name("u", 4), ], ) ]), Column.of(), Tabulation.of(), Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), Joiner.hyphen(), Reading.of_name("ku"), BrokenAway.close(), Joiner.hyphen(), Reading.of_name("nu"), Joiner.hyphen(), Reading.of_name("ši"), ]), Variant.of(Divider.of(":"), Reading.of_name("ku")), Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), BrokenAway.close(), ]), Column.of(2), Divider.of(":", ("@v", ), (Flag.DAMAGE, )), CommentaryProtocol.of("!qt"), Word.of([Number.of_name("10", flags=[Flag.DAMAGE])]), ), ), TextLine.of_iterable( LineNumber(2, True), ( Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), BrokenAway.close(), ]), Word.of([Logogram.of_name("GI", 6)]), Word.of([Reading.of_name("ana")]), Word.of([ Reading.of_name("u", 4), Joiner.hyphen(), Reading.of(( ValueToken.of("š"), BrokenAway.open(), ValueToken.of("u"), )), ]), Word.of([UnknownNumberOfSigns.of(), BrokenAway.close()]), ), ), TextLine.of_iterable( LineNumber(3, True), ( Word.of([BrokenAway.open(), UnknownNumberOfSigns.of()]), Word.of([ Reading.of(( ValueToken.of("k"), BrokenAway.close(), ValueToken.of("i"), )), Joiner.hyphen(), Reading.of_name("du"), ]), Word.of([Reading.of_name("u")]), Word.of([ Reading.of_name("ba"), Joiner.hyphen(), Reading.of_name("ma"), Joiner.hyphen(), Reading.of(( ValueToken.of("t"), BrokenAway.open(), ValueToken.of("i"), )), ]), Word.of([UnknownNumberOfSigns.of(), BrokenAway.close()]), ), ), TextLine.of_iterable( LineNumber(6, True), ( Word.of([ BrokenAway.open(), UnknownNumberOfSigns.of(), BrokenAway.close(), ]), Word.of([UnclearSign.of([Flag.DAMAGE])]), Word.of([Reading.of_name("mu")]), Word.of([ Reading.of_name("ta"), Joiner.hyphen(), Reading.of_name("ma"), InWordNewline.of(), Joiner.hyphen(), Reading.of_name("tu", 2), ]), ), ), TextLine.of_iterable( LineNumber(7, True), ( Word.of([ Variant.of(Reading.of_name("šu"), CompoundGrapheme.of(["BI×IS"])) ]), LanguageShift.normalized_akkadian(), AkkadianWord.of([ValueToken.of("kur")]), ), ), StateDollarLine( atf.Qualification.AT_LEAST, 1, ScopeContainer(atf.Surface.OBVERSE, ""), atf.State.MISSING, None, ), ImageDollarLine("1", None, "numbered diagram of triangle"), RulingDollarLine(atf.Ruling.SINGLE), LooseDollarLine("this is a loose line"), SealDollarLine(1), SealAtLine(1), HeadingAtLine(1), ColumnAtLine(ColumnLabel([atf.Status.COLLATION], 1)), SurfaceAtLine( SurfaceLabel([atf.Status.COLLATION], atf.Surface.SURFACE, "stone wig")), ObjectAtLine( ObjectLabel([atf.Status.COLLATION], atf.Object.OBJECT, "stone wig")), DiscourseAtLine(atf.Discourse.DATE), DivisionAtLine("paragraph", 5), CompositeAtLine(atf.Composite.DIV, "part", 1), NoteLine(( StringPart("a note "), EmphasisPart("italic"), LanguagePart.of_transliteration( Language.AKKADIAN, (Word.of([Reading.of_name("bu")]), )), )), ParallelComposition(False, "my name", LineNumber(1)), ParallelText( True, TextId(CorpusGenre.LITERATURE, 1, 1), ChapterName(Stage.OLD_BABYLONIAN, "", "my name"), LineNumber(1), False, ), ParallelFragment(False, MuseumNumber.of("K.1"), True, Labels(), LineNumber(1), False), )) signs = ( "X BA KU ABZ075 ABZ207a\\u002F207b\\u0020X ABZ377n1/KU ABZ377n1 ABZ411\n" "MI DIŠ UD ŠU\n" "KI DU ABZ411 BA MA TI\n" "X MU TA MA UD\n" "ŠU/|BI×IS|") folios = Folios((Folio("WGL", "3"), Folio("XXX", "3"))) record = Record((RecordEntry("test", RecordType.TRANSLITERATION), )) line_to_vec = (( LineToVecEncoding.TEXT_LINE, LineToVecEncoding.TEXT_LINE, LineToVecEncoding.TEXT_LINE, LineToVecEncoding.TEXT_LINE, LineToVecEncoding.TEXT_LINE, LineToVecEncoding.SINGLE_RULING, ), )
def ebl_atf_text_line__akkadian_word( self, parts: Tree, modifiers: Sequence[Flag], closing_enclosures: Tree ) -> AkkadianWord: return AkkadianWord.of( tuple(parts.children + closing_enclosures.children), modifiers )
def test_set_alignment_empty() -> None: word = AkkadianWord.of((ValueToken.of("bu"), ), alignment=1) expected = AkkadianWord.of((ValueToken.of("bu"), )) assert word.set_alignment(None, None) == expected
Emendation.open(), UnknownNumberOfSigns(frozenset({EnclosureType.EMENDATION}), ErasureState.NONE), Emendation.close().set_enclosure_type( frozenset({EnclosureType.EMENDATION})), ), ), ( "%n kur-[kur ...]", ( LanguageShift.normalized_akkadian(), AkkadianWord.of(( ValueToken.of("kur"), Joiner.hyphen(), BrokenAway.open(), ValueToken( frozenset({EnclosureType.BROKEN_AWAY}), ErasureState.NONE, "kur", ), )), UnknownNumberOfSigns.of().set_enclosure_type( frozenset({EnclosureType.BROKEN_AWAY})), BrokenAway.close().set_enclosure_type( frozenset({EnclosureType.BROKEN_AWAY})), ), ), ( "%n <... | ...>", ( LanguageShift.normalized_akkadian(), Emendation.open(),
def test_word(text, expected) -> None: assert parse_reconstructed_word(text) == AkkadianWord.of( tuple(expected[:-1]), tuple(expected[-1]) )
@pytest.mark.parametrize("text,is_uncertain", [("|", False), ("(|)", True)]) def test_feet_separator(text, is_uncertain) -> None: assert ( parse_break(text) == MetricalFootSeparator.uncertain() if is_uncertain else MetricalFootSeparator.certain() ) @pytest.mark.parametrize("text", ["|||", "||||", "[|]", "[(|)]", "[||]", "[(||)]"]) def test_invalid_break(text) -> None: assert_parse_error(parse_break, text) WORD = AkkadianWord.of((ValueToken.of("ibnû"),)) # pyre-ignore[56] @pytest.mark.parametrize( "text,expected", [ ("ibnû", (WORD,)), ("... ibnû", (UnknownNumberOfSigns.of(), WORD)), ("ibnû ...", (WORD, UnknownNumberOfSigns.of())), ( "[...] ibnû", (BrokenAway.open(), UnknownNumberOfSigns.of(), BrokenAway.close(), WORD), ), ( "ibnû [...]",
"old,new,expected", [ (MANUSCRIPT_LINE, MANUSCRIPT_LINE, MANUSCRIPT_LINE), ( MANUSCRIPT_LINE, ManuscriptLine(NEW_MANUSCRIPT_ID, NEW_LABELS, NEW_TEXT_LINE), ManuscriptLine(NEW_MANUSCRIPT_ID, NEW_LABELS, TEXT_LINE.merge(NEW_TEXT_LINE)), ), ], ) def test_merge_manuscript_line(old, new, expected): assert old.merge(new) == expected RECONSTRUCTION = (AkkadianWord.of((ValueToken.of("buāru"), ), unique_lemma=(WordId("buāru I"), )), ) RECONSTRUCTION_WITHOUT_LEMMA = (AkkadianWord.of((ValueToken.of("buāru"), )), ) IS_SECOND_LINE_OF_PARALLELISM = True IS_BEGINNING_OF_SECTION = True NOTE = None LINE = Line( LineNumber(1), (LineVariant(RECONSTRUCTION, NOTE, (MANUSCRIPT_LINE, )), ), IS_SECOND_LINE_OF_PARALLELISM, IS_BEGINNING_OF_SECTION, ) @pytest.mark.parametrize( # pyre-ignore[56] "old,new,expected",
def test_alignment() -> None: assert AlignmentSchema().load({ "alignment": [[ [{ "alignment": [{ "value": "ku]-nu-ši", "alignment": 1, "variant": "kunusi", "type": "AkkadianWord", "language": "AKKADIAN", }], "omittedWords": [], }], [{ "alignment": [{ "value": "ku]-nu-ši", "alignment": 1, "variant": "kur", "type": "Word", "language": "SUMERIAN", }], "omittedWords": [1], }], [{ "alignment": [{ "value": "ku]-nu-ši", "alignment": 1, "variant": "", "type": "", "language": "", }], "omittedWords": [], }], [{ "alignment": [{ "value": "ku]-nu-ši", "alignment": 1, "variant": "β", "type": "GreekWord", "language": "AKKADIAN", }], "omittedWords": [], }], ]] }) == Alignment((( (ManuscriptLineAlignment((AlignmentToken( "ku]-nu-ši", 1, AkkadianWord.of([ValueToken.of("kunusi")]), ), )), ), (ManuscriptLineAlignment( (AlignmentToken( "ku]-nu-ši", 1, Word.of([Reading.of_name("kur")], language=Language.SUMERIAN), ), ), (1, ), ), ), (ManuscriptLineAlignment((AlignmentToken("ku]-nu-ši", 1, None), )), ), (ManuscriptLineAlignment((AlignmentToken( "ku]-nu-ši", 1, GreekWord.of([GreekLetter.of("β")], language=Language.AKKADIAN), ), )), ), ), ))
( Word.of( [Logogram.of_name("GI", 6)], unique_lemma=(WordId("ginâ I"),) ), Word.of([Reading.of_name("ana")], unique_lemma=(WordId("ana II"),)), Word.of([Reading.of_name("ana")], unique_lemma=(WordId("ana II"),)), Word.of( [ Reading.of_name("u", 4), Joiner.hyphen(), Reading.of_name("šu"), ], unique_lemma=(WordId("ūsu I"),), ), AkkadianWord.of( [ValueToken.of("ana")], unique_lemma=(WordId("normalized I"),) ), ), ), ) ), signs="MI DIŠ DIŠ UD ŠU", ) SCHEMA = FragmentSchema() def test_create(database, fragment_repository): fragment = LemmatizedFragmentFactory.build() fragment_id = fragment_repository.create(fragment)
def test_merging_lines(corpus, text_repository, bibliography, changelog, signs, sign_repository, user, when) -> None: reconstruction = ( LanguageShift.normalized_akkadian(), AkkadianWord.of((ValueToken.of("buāru"), )), ) is_second_line_of_parallelism = False is_beginning_of_section = False text_line = TextLine.of_iterable( LineNumber(1), ( Word.of([Reading.of_name("ku")], unique_lemma=(WordId("word1"), ), alignment=0), Word.of([Reading.of_name("nu")], unique_lemma=(WordId("word2"), ), alignment=1), ), ) manuscript_id = CHAPTER_WITHOUT_DOCUMENTS.manuscripts[0].id line = Line( LineNumber(1), (LineVariant( reconstruction, None, (ManuscriptLine(manuscript_id, tuple(), text_line), ), ), ), not is_second_line_of_parallelism, not is_beginning_of_section, ) new_text_line = TextLine.of_iterable( LineNumber(1), (Word.of([Reading.of_name("ku")]), Word.of([Reading.of_name("ba")])), ) new_line = Line( LineNumber(1), (LineVariant( reconstruction, None, (ManuscriptLine(manuscript_id, tuple(), text_line.merge(new_text_line)), ), ), ), is_second_line_of_parallelism, is_beginning_of_section, ) old_chapter = attr.evolve(CHAPTER_WITHOUT_DOCUMENTS, lines=(line, )) updated_chapter = attr.evolve( CHAPTER, lines=(new_line, ), signs=("KU BA\nKU\nABZ075", ), parser_version=ATF_PARSER_VERSION, ) expect_find_and_update_chapter( bibliography, changelog, old_chapter, updated_chapter, signs, sign_repository, text_repository, user, when, ) assert (corpus.update_lines( CHAPTER.id_, LinesUpdate( [], set(), { 0: Line( LineNumber(1), (LineVariant( reconstruction, None, (ManuscriptLine(manuscript_id, tuple(), new_text_line), ), ), ), is_second_line_of_parallelism, is_beginning_of_section, ) }, ), user, ) == updated_chapter)