class TransliterationUpdate:
text: Text = Text()
notes: str = ""
signs: str = attr.ib(default="")
@signs.validator
def _check_signs(self, _attribute, value) -> None:
questionable_lines = self._get_questionable_lines(value)
if questionable_lines:
raise TransliterationError(
[
{"description": "Invalid value", "lineNumber": line_number}
for line_number in questionable_lines
]
)
def _get_questionable_lines(self, value: str) -> List[int]:
lines = [line.atf for line in self.text.lines]
text_lines = [line.atf for line in self.text.text_lines]
signs = value.split("\n")
def get_line_number(text_line_number: int) -> int:
line = text_lines[text_line_number]
return lines.index(line) + 1
return [
get_line_number(index) for index, line in enumerate(signs) if "?" in line
]
def test_update_lemmatization() -> None:
tokens = [list(line) for line in TEXT.lemmatization.tokens]
tokens[0][0] = LemmatizationToken(tokens[0][0].value, (WordId("nu I"),))
lemmatization = Lemmatization(tokens)
expected = Text(
(
TextLine(
LineNumber(1),
(
Word.of(
unique_lemma=(WordId("nu I"),),
parts=[
Reading.of_name("ha"),
Joiner.hyphen(),
Reading.of_name("am"),
],
),
),
),
RulingDollarLine(atf.Ruling.SINGLE),
),
TEXT.parser_version,
)
assert TEXT.update_lemmatization(lemmatization) == expected
def parse_atf_lark(atf_):
def parse_line_(line: str, line_number: int):
try:
parsed_line = parse_line(line) if line else EmptyLine()
validate_line(parsed_line)
return parsed_line, None
except PARSE_ERRORS as ex:
return (None,
create_transliteration_error_data(ex, line, line_number))
def check_errors(pairs):
errors = [error for line, error in pairs if error is not None]
if any(errors):
raise TransliterationError(errors)
lines = atf_.split("\n")
lines = list(dropwhile(lambda line: line == "", reversed(lines)))
lines.reverse()
lines = [parse_line_(line, number) for number, line in enumerate(lines)]
check_errors(lines)
lines = tuple(pair[0] for pair in lines)
text = Text(lines, f"{atf.ATF_PARSER_VERSION}")
if pydash.duplicates(text.labels):
raise DataError("Duplicate labels.")
else:
return text
def test_dump_line():
text = Text(
(
TextLine.of_iterable(
LineNumber(1),
[
Word.of(
parts=[
Reading.of_name("ha"),
Joiner.hyphen(),
Reading.of_name("am"),
]
)
],
),
EmptyLine(),
ControlLine("#", " comment"),
),
"1.0.0",
)
assert TextSchema().dump(text) == {
"lines": OneOfLineSchema().dump(text.lines, many=True),
"parser_version": text.parser_version,
"numberOfLines": 1,
}
def test_statistics(database, fragment_repository):
database[COLLECTION].insert_many(
[
SCHEMA.dump(
FragmentFactory.build(
text=Text(
(
TextLine(
LineNumber(1),
(
Word.of([Reading.of_name("first")]),
Word.of([Reading.of_name("line")]),
),
),
ControlLine("#", "ignore"),
EmptyLine(),
)
)
)
),
SCHEMA.dump(
FragmentFactory.build(
text=Text(
(
ControlLine("#", "ignore"),
TextLine(
LineNumber(1), (Word.of([Reading.of_name("second")]),)
),
TextLine(
LineNumber(2), (Word.of([Reading.of_name("third")]),)
),
ControlLine("#", "ignore"),
TextLine(
LineNumber(3), (Word.of([Reading.of_name("fourth")]),)
),
)
)
)
),
SCHEMA.dump(FragmentFactory.build(text=Text())),
]
)
assert fragment_repository.count_transliterated_fragments() == 2
assert fragment_repository.count_lines() == 4
class InterestingFragmentFactory(FragmentFactory):
collection = "Kuyunjik" # pyre-ignore[15]
publication = "" # pyre-ignore[15]
joins: Sequence[str] = tuple()
text = Text()
notes = ""
uncurated_references = (
UncuratedReference("7(0)"),
UncuratedReference("CAD 51", (34, 56)),
UncuratedReference("7(1)"),
)
references = tuple()
class Manuscript:
id: int
siglum_disambiguator: str = ""
museum_number: Optional[MuseumNumber] = None
accession: str = attr.ib(default="")
period_modifier: PeriodModifier = PeriodModifier.NONE
period: Period = Period.NEO_ASSYRIAN
provenance: Provenance = attr.ib(default=Provenance.NINEVEH)
type: ManuscriptType = ManuscriptType.LIBRARY
notes: str = ""
colophon: Text = Text()
unplaced_lines: Text = Text()
references: Sequence[Reference] = tuple()
joins: Joins = Joins()
is_in_fragmentarium: bool = False
@accession.validator
def validate_accession(self, _, value) -> None:
if self.museum_number and value:
raise ValueError("Accession given when museum number present.")
@provenance.validator
def validate_provenance(self, _, value) -> None:
if is_invalid_standard_text(value, self.period, self.type):
raise ValueError(
"Manuscript must not have period and type when provenance is Standard Text."
)
elif is_invalid_non_standard_text(value, self.period, self.type):
raise ValueError(
"Manuscript must have period and type unless provenance is Standard Text."
)
@property
def text_lines(self) -> Sequence[TextLine]:
return [*self.colophon.text_lines, *self.unplaced_lines.text_lines]
@property
def siglum(self) -> Siglum:
return Siglum(self.provenance, self.period, self.type,
self.siglum_disambiguator)
def test_parse_normalized_akkadain_shift() -> None:
word = "ha"
line = f"1. {word} %n {word} %sux {word}"
expected = Text((TextLine.of_iterable(
LineNumber(1),
(
Word.of((Reading.of_name(word), ), DEFAULT_LANGUAGE),
LanguageShift.normalized_akkadian(),
AkkadianWord.of((ValueToken.of(word), )),
LanguageShift.of("%sux"),
Word.of((Reading.of_name(word), ), Language.SUMERIAN),
),
), ))
assert parse_atf_lark(line).lines == expected.lines
def test_parse_atf_language_shifts(code: str,
expected_language: Language) -> None:
word = "ha-am"
parts = [Reading.of_name("ha"), Joiner.hyphen(), Reading.of_name("am")]
line = f"1. {word} {code} {word} %sb {word}"
expected = Text((TextLine.of_iterable(
LineNumber(1),
(
Word.of(parts, DEFAULT_LANGUAGE),
LanguageShift.of(code),
Word.of(parts, expected_language),
LanguageShift.of("%sb"),
Word.of(parts, Language.AKKADIAN),
),
), ))
assert parse_atf_lark(line).lines == expected.lines
def test_get(client, text_repository):
chapter = ChapterFactory.build(
lines=tuple(),
manuscripts=(
ManuscriptFactory.build(references=tuple()),
ManuscriptFactory.build(colophon=Text(), references=tuple()),
ManuscriptFactory.build(references=tuple()),
),
)
text_repository.create_chapter(chapter)
result = client.simulate_get(create_chapter_url(chapter, "/colophons"))
assert result.status == falcon.HTTP_OK
assert result.json == [{
"siglum": str(manuscript.siglum),
"text": TextSchema().dump(manuscript.colophon),
} for manuscript in chapter.manuscripts
if not manuscript.colophon.is_empty]
def test_query_by_parallel_line_exists(database, fragment_repository):
parallel_number = MuseumNumber.of("K.1")
fragment = FragmentFactory.build(
text=Text(
(
ParallelFragment(
False, parallel_number, True, Labels(), LineNumber(1), True
),
)
)
)
parallel_fragment = FragmentFactory.build(number=parallel_number)
database[COLLECTION].insert_many(
[
FragmentSchema(exclude=["joins"]).dump(fragment),
FragmentSchema(exclude=["joins"]).dump(parallel_fragment),
]
)
assert fragment_repository.query_by_museum_number(fragment.number) == fragment
class Fragment:
number: MuseumNumber
accession: str = ""
cdli_number: str = ""
bm_id_number: str = ""
publication: str = ""
description: str = ""
collection: str = ""
script: str = ""
museum: str = ""
width: Measure = Measure()
length: Measure = Measure()
thickness: Measure = Measure()
joins: Joins = Joins()
record: Record = Record()
folios: Folios = Folios()
text: Text = Text()
signs: str = ""
notes: str = ""
references: Sequence[Reference] = tuple()
uncurated_references: Optional[Sequence[UncuratedReference]] = None
genres: Sequence[Genre] = tuple()
line_to_vec: Tuple[LineToVecEncodings, ...] = tuple()
@property
def is_lowest_join(self) -> bool:
return (self.joins.lowest or self.number) == self.number
def set_references(self, references: Sequence[Reference]) -> "Fragment":
return attr.evolve(self, references=references)
def set_text(self, text: Text) -> "Fragment":
return attr.evolve(self, text=text)
def update_lowest_join_transliteration(
self, transliteration: TransliterationUpdate,
user: User) -> "Fragment":
if transliteration.text.is_empty or self.is_lowest_join:
return self.update_transliteration(transliteration, user)
else:
raise NotLowestJoinError(
"Transliteration must be empty unless fragment is the lowest in join."
)
def update_transliteration(self, transliteration: TransliterationUpdate,
user: User) -> "Fragment":
record = self.record.add_entry(self.text.atf, transliteration.text.atf,
user)
text = self.text.merge(transliteration.text)
return attr.evolve(
self,
text=text,
notes=transliteration.notes,
signs=transliteration.signs,
record=record,
line_to_vec=create_line_to_vec(text.lines),
)
def set_genres(self, genres_new: Sequence[Genre]) -> "Fragment":
return attr.evolve(self, genres=tuple(genres_new))
def update_lemmatization(self, lemmatization: Lemmatization) -> "Fragment":
text = self.text.update_lemmatization(lemmatization)
return attr.evolve(self, text=text)
def get_matching_lines(self, query: TransliterationQuery) -> Lines:
line_numbers = query.match(self.signs)
lines = [line.atf for line in self.text.text_lines]
return tuple(
tuple(lines[numbers[0]:numbers[1] + 1])
for numbers, _ in groupby(line_numbers))
ANOTHER_LEMMATIZED_FRAGMENT = attr.evolve(
TransliteratedFragmentFactory.build(),
text=Text(
(
TextLine(
LineNumber(1),
(
Word.of(
[Logogram.of_name("GI", 6)], unique_lemma=(WordId("ginâ I"),)
),
Word.of([Reading.of_name("ana")], unique_lemma=(WordId("ana II"),)),
Word.of([Reading.of_name("ana")], unique_lemma=(WordId("ana II"),)),
Word.of(
[
Reading.of_name("u", 4),
Joiner.hyphen(),
Reading.of_name("šu"),
],
unique_lemma=(WordId("ūsu I"),),
),
AkkadianWord.of(
[ValueToken.of("ana")], unique_lemma=(WordId("normalized I"),)
),
),
),
)
),
signs="MI DIŠ DIŠ UD ŠU",
)
def test_set_text():
fragment = FragmentFactory.build()
text = Text((ParallelCompositionFactory.build(), ))
updated_fragment = fragment.set_text(text)
assert updated_fragment.text == text
def test_text():
fragment = FragmentFactory.build()
assert fragment.text == Text()
class TransliteratedFragmentFactory(FragmentFactory):
text = Text((
TextLine.of_iterable(
LineNumber(1, True),
(
Word.of([UnidentifiedSign.of()]),
Word.of([
Logogram.of_name(
"BA",
surrogate=[
Reading.of_name("ku"),
Joiner.hyphen(),
Reading.of_name("u", 4),
],
)
]),
Column.of(),
Tabulation.of(),
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
Joiner.hyphen(),
Reading.of_name("ku"),
BrokenAway.close(),
Joiner.hyphen(),
Reading.of_name("nu"),
Joiner.hyphen(),
Reading.of_name("ši"),
]),
Variant.of(Divider.of(":"), Reading.of_name("ku")),
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
BrokenAway.close(),
]),
Column.of(2),
Divider.of(":", ("@v", ), (Flag.DAMAGE, )),
CommentaryProtocol.of("!qt"),
Word.of([Number.of_name("10", flags=[Flag.DAMAGE])]),
),
),
TextLine.of_iterable(
LineNumber(2, True),
(
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
BrokenAway.close(),
]),
Word.of([Logogram.of_name("GI", 6)]),
Word.of([Reading.of_name("ana")]),
Word.of([
Reading.of_name("u", 4),
Joiner.hyphen(),
Reading.of((
ValueToken.of("š"),
BrokenAway.open(),
ValueToken.of("u"),
)),
]),
Word.of([UnknownNumberOfSigns.of(),
BrokenAway.close()]),
),
),
TextLine.of_iterable(
LineNumber(3, True),
(
Word.of([BrokenAway.open(),
UnknownNumberOfSigns.of()]),
Word.of([
Reading.of((
ValueToken.of("k"),
BrokenAway.close(),
ValueToken.of("i"),
)),
Joiner.hyphen(),
Reading.of_name("du"),
]),
Word.of([Reading.of_name("u")]),
Word.of([
Reading.of_name("ba"),
Joiner.hyphen(),
Reading.of_name("ma"),
Joiner.hyphen(),
Reading.of((
ValueToken.of("t"),
BrokenAway.open(),
ValueToken.of("i"),
)),
]),
Word.of([UnknownNumberOfSigns.of(),
BrokenAway.close()]),
),
),
TextLine.of_iterable(
LineNumber(6, True),
(
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
BrokenAway.close(),
]),
Word.of([UnclearSign.of([Flag.DAMAGE])]),
Word.of([Reading.of_name("mu")]),
Word.of([
Reading.of_name("ta"),
Joiner.hyphen(),
Reading.of_name("ma"),
InWordNewline.of(),
Joiner.hyphen(),
Reading.of_name("tu", 2),
]),
),
),
TextLine.of_iterable(
LineNumber(7, True),
(
Word.of([
Variant.of(Reading.of_name("šu"),
CompoundGrapheme.of(["BI×IS"]))
]),
LanguageShift.normalized_akkadian(),
AkkadianWord.of([ValueToken.of("kur")]),
),
),
StateDollarLine(
atf.Qualification.AT_LEAST,
1,
ScopeContainer(atf.Surface.OBVERSE, ""),
atf.State.MISSING,
None,
),
ImageDollarLine("1", None, "numbered diagram of triangle"),
RulingDollarLine(atf.Ruling.SINGLE),
LooseDollarLine("this is a loose line"),
SealDollarLine(1),
SealAtLine(1),
HeadingAtLine(1),
ColumnAtLine(ColumnLabel([atf.Status.COLLATION], 1)),
SurfaceAtLine(
SurfaceLabel([atf.Status.COLLATION], atf.Surface.SURFACE,
"stone wig")),
ObjectAtLine(
ObjectLabel([atf.Status.COLLATION], atf.Object.OBJECT,
"stone wig")),
DiscourseAtLine(atf.Discourse.DATE),
DivisionAtLine("paragraph", 5),
CompositeAtLine(atf.Composite.DIV, "part", 1),
NoteLine((
StringPart("a note "),
EmphasisPart("italic"),
LanguagePart.of_transliteration(
Language.AKKADIAN, (Word.of([Reading.of_name("bu")]), )),
)),
ParallelComposition(False, "my name", LineNumber(1)),
ParallelText(
True,
TextId(CorpusGenre.LITERATURE, 1, 1),
ChapterName(Stage.OLD_BABYLONIAN, "", "my name"),
LineNumber(1),
False,
),
ParallelFragment(False, MuseumNumber.of("K.1"), True, Labels(),
LineNumber(1), False),
))
signs = (
"X BA KU ABZ075 ABZ207a\\u002F207b\\u0020X ABZ377n1/KU ABZ377n1 ABZ411\n"
"MI DIŠ UD ŠU\n"
"KI DU ABZ411 BA MA TI\n"
"X MU TA MA UD\n"
"ŠU/|BI×IS|")
folios = Folios((Folio("WGL", "3"), Folio("XXX", "3")))
record = Record((RecordEntry("test", RecordType.TRANSLITERATION), ))
line_to_vec = ((
LineToVecEncoding.TEXT_LINE,
LineToVecEncoding.TEXT_LINE,
LineToVecEncoding.TEXT_LINE,
LineToVecEncoding.TEXT_LINE,
LineToVecEncoding.TEXT_LINE,
LineToVecEncoding.SINGLE_RULING,
), )
def test_of_iterable() -> None:
assert Text.of_iterable(LINES) == Text(LINES, atf.ATF_PARSER_VERSION)
from ebl.transliteration.domain.sign_tokens import Reading
from ebl.transliteration.domain.text import LineLabel, Text
from ebl.transliteration.domain.text_line import TextLine
from ebl.transliteration.domain.tokens import Joiner
from ebl.transliteration.domain.translation_line import Extent, TranslationLine
from ebl.transliteration.domain.word_tokens import Word
LINES: Sequence[Line] = (
TextLine(
LineNumber(1),
(Word.of([Reading.of_name("ha"), Joiner.hyphen(), Reading.of_name("am")]),),
),
RulingDollarLine(atf.Ruling.SINGLE),
)
PARSER_VERSION = "1.0.0"
TEXT: Text = Text(LINES, PARSER_VERSION)
def test_of_iterable() -> None:
assert Text.of_iterable(LINES) == Text(LINES, atf.ATF_PARSER_VERSION)
def test_lines() -> None:
assert TEXT.lines == LINES
def test_text_lines() -> None:
assert TEXT.text_lines == LINES[:1]
def test_number_of_lines() -> None:
class LemmatizedFragmentFactory(TransliteratedFragmentFactory):
text = Text((
TextLine.of_iterable(
LineNumber(1, True),
(
Word.of([UnidentifiedSign.of()]),
Word.of([
Logogram.of_name(
"BA",
surrogate=[
Reading.of_name("ku"),
Joiner.hyphen(),
Reading.of_name("u", 4),
],
)
]),
Column.of(),
Tabulation.of(),
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
Joiner.hyphen(),
Reading.of_name("ku"),
BrokenAway.close(),
Joiner.hyphen(),
Reading.of_name("nu"),
Joiner.hyphen(),
Reading.of_name("ši"),
]),
Variant.of(Divider.of(":"), Reading.of_name("ku")),
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
BrokenAway.close(),
]),
Column.of(2),
Divider.of(":", ("@v", ), (Flag.DAMAGE, )),
CommentaryProtocol.of("!qt"),
Word.of([Number.of_name("10", flags=[Flag.DAMAGE])]),
),
),
TextLine.of_iterable(
LineNumber(2, True),
(
Word.of([BrokenAway.open(),
UnknownNumberOfSigns.of()]),
Word.of([Logogram.of_name("GI", 6)],
unique_lemma=(WordId("ginâ I"), )),
Word.of([Reading.of_name("ana")],
unique_lemma=(WordId("ana I"), )),
Word.of(
[
Reading.of_name("u₄"),
Joiner.hyphen(),
Reading.of_name("š[u"),
],
unique_lemma=(WordId("ūsu I"), ),
),
Word.of([UnknownNumberOfSigns.of(),
BrokenAway.close()]),
),
),
TextLine.of_iterable(
LineNumber(3, True),
(
Word.of([BrokenAway.open(),
UnknownNumberOfSigns.of()]),
Word.of(
unique_lemma=(WordId("kīdu I"), ),
parts=[
Reading.of((
ValueToken.of("k"),
BrokenAway.close(),
ValueToken.of("i"),
)),
Joiner.hyphen(),
Reading.of_name("du"),
],
),
Word.of(unique_lemma=(WordId("u I"), ),
parts=[Reading.of_name("u")]),
Word.of(
unique_lemma=(WordId("bamātu I"), ),
parts=[
Reading.of_name("ba"),
Joiner.hyphen(),
Reading.of_name("ma"),
Joiner.hyphen(),
Reading.of((
ValueToken.of("t"),
BrokenAway.open(),
ValueToken.of("i"),
)),
],
),
Word.of([UnknownNumberOfSigns.of(),
BrokenAway.close()]),
),
),
TextLine.of_iterable(
LineNumber(6, True),
(
Word.of([
BrokenAway.open(),
UnknownNumberOfSigns.of(),
BrokenAway.close(),
]),
Word.of([UnclearSign.of([Flag.DAMAGE])]),
Word.of(unique_lemma=(WordId("mu I"), ),
parts=[Reading.of_name("mu")]),
Word.of(
unique_lemma=(WordId("tamalāku I"), ),
parts=[
Reading.of_name("ta"),
Joiner.hyphen(),
Reading.of_name("ma"),
InWordNewline.of(),
Joiner.hyphen(),
Reading.of_name("tu", 2),
],
),
),
),
TextLine.of_iterable(
LineNumber(7, True),
(
Word.of([
Variant.of(Reading.of_name("šu"),
CompoundGrapheme.of(["BI×IS"]))
]),
LanguageShift.normalized_akkadian(),
AkkadianWord.of([ValueToken.of("kur")],
unique_lemma=(WordId("normalized I"), )),
),
),
StateDollarLine(
atf.Qualification.AT_LEAST,
1,
ScopeContainer(atf.Surface.OBVERSE, ""),
atf.State.MISSING,
None,
),
ImageDollarLine("1", None, "numbered diagram of triangle"),
RulingDollarLine(atf.Ruling.SINGLE),
LooseDollarLine("this is a loose line"),
SealDollarLine(1),
SealAtLine(1),
HeadingAtLine(1),
ColumnAtLine(ColumnLabel([atf.Status.COLLATION], 1)),
SurfaceAtLine(
SurfaceLabel([atf.Status.COLLATION], atf.Surface.SURFACE,
"stone wig")),
ObjectAtLine(
ObjectLabel([atf.Status.COLLATION], atf.Object.OBJECT,
"stone wig")),
DiscourseAtLine(atf.Discourse.DATE),
DivisionAtLine("paragraph", 5),
CompositeAtLine(atf.Composite.DIV, "part", 1),
NoteLine((
StringPart("a note "),
EmphasisPart("italic"),
LanguagePart.of_transliteration(
Language.AKKADIAN, (Word.of([Reading.of_name("bu")]), )),
)),
ParallelComposition(False, "my name", LineNumber(1)),
ParallelText(
True,
TextId(CorpusGenre.LITERATURE, 1, 1),
ChapterName(Stage.OLD_BABYLONIAN, "", "my name"),
LineNumber(1),
False,
),
ParallelFragment(False, MuseumNumber.of("K.1"), True, Labels(),
LineNumber(1), False),
))