def test_set_mapping(self):
authors = Authors()
compare("William Abbott", authors._mapping["Abbott"])
self.assertFalse("New" in authors._mapping)
authors.set_mappings({"William Abbott": "Abbott_new", "New": "New"})
compare("Abbott_new", authors._mapping["William Abbott"])
compare("New", authors._mapping["New"])
def test_header_vg_nf(self):
copy_tst_data("I_1_base", "I_1")
i1 = VolumeRegister(Volumes()["I,1"], Authors())._get_header()
self.assertTrue("VG=" in i1)
self.assertTrue("NF=I,2" in i1)
copy_tst_data("I_1_base", "S I")
si = VolumeRegister(Volumes()["S I"], Authors())._get_header()
self.assertTrue("VG=X A" in si)
self.assertTrue("NF=S II" in si)
def setUp(self):
copy_tst_data("I_1_alpha", "I_1")
copy_tst_data("III_1_alpha", "III_1")
self.authors = Authors()
self.volumes = Volumes()
self.registers = OrderedDict()
self.registers["I,1"] = VolumeRegister(self.volumes["I,1"],
self.authors)
self.registers["III,1"] = VolumeRegister(self.volumes["III,1"],
self.authors)
def test_duplicate_lemmas_in_supplements(self):
copy_tst_data("S_I_no_dublicates", "S I")
register = VolumeRegister(Volumes()["S I"], Authors())
update_dict = {"lemma": "Abdymon", "previous": "Abd Hadad", "next": "Abeikta"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
compare(11, len(register.lemmas))
def test_get_lemma_by_id(self):
copy_tst_data("I_1_base", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
lemma = register[2]
compare("Aarassos", lemma["previous"])
with self.assertRaises(IndexError):
print(register[8])
def test_header_proof_read(self):
copy_tst_data("I_1_base", "I_1")
i1 = VolumeRegister(Volumes()["I,1"], Authors())._get_header().replace("\n", "")
compare(StringComparison(".*SUM=8.*"), i1)
compare(StringComparison(".*UNK=3.*"), i1)
compare(StringComparison(".*KOR=2.*"), i1)
compare(StringComparison(".*FER=3.*"), i1)
def test_persist(self):
copy_tst_data("I_1_two_entries", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
register._lemmas[0]._lemma_dict["previous"] = None
register._lemmas[0]._chapters[0]._dict["author"] = "ÄäÖöÜüß"
register.persist()
expect = """[
{
"lemma": "Aal",
"next": "Aarassos",
"proof_read": 3,
"short_description": "Ein Fisch",
"chapters": [
{
"start": 1,
"end": 4,
"author": "ÄäÖöÜüß"
}
]
},
{
"lemma": "Aarassos",
"previous": "Aal",
"proof_read": 2,
"chapters": [
{
"start": 4,
"end": 4,
"author": "Abert"
}
]
}
]"""
with open(DataRepo.get_data_path().joinpath("I_1.json"), mode="r", encoding="utf-8") as register_file:
compare(expect, register_file.read())
def test_update_pre_and_next_not_possible(self):
copy_tst_data("I_1_sorting2", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "B", "previous": "A", "next": "D"}
with self.assertRaisesRegex(RegisterException, "Diff between previous and next aren't 1 or 2"):
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
def test_prokleides_2(self):
copy_tst_data("prokleides_2_bug", "XXIII_1")
register = VolumeRegister(Volumes()["XXIII,1"], Authors())
update_dict = {"lemma": "Prokleides 2", "previous": "Prokleides", "next": "Prokles"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
compare(None, register.lemmas[0]["previous"])
compare(None, register.lemmas[2]["next"])
def test_update_lemma(self):
copy_tst_data("I_1_base", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "Aal", "redirect": True}
with Updater(register) as updater:
updater.update_lemma(update_dict, ["next"])
post_lemma = register.get_lemma_by_name("Aal")
self.assertTrue(post_lemma["redirect"])
self.assertIsNone(post_lemma["next"])
def test_iter(self):
authors = iter(Authors())
compare("Herman Abel", next(authors).name)
compare("Abel", next(authors).name)
compare("Abert", next(authors).name)
compare("William Abbott", next(authors).name)
with self.assertRaises(StopIteration):
# redirects doesn't count
next(authors)
def __init__(self,
wiki: pywikibot.Site,
logger: WikiLogger,
debug: bool = True):
super().__init__(wiki, logger, debug)
self.authors = Authors()
current_year = datetime.datetime.now().year
self.pd_todesjahr, self.pd_geburtsjahr = \
current_year - public_domain.YEARS_AFTER_DEATH, current_year - public_domain.YEARS_AFTER_BIRTH
def test_get_id_of_lemma(self):
copy_tst_data("I_1_self_append", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
compare(0, register.get_index_of_lemma("Aal"))
compare(2, register.get_index_of_lemma("Something"))
compare(3, register.get_index_of_lemma("Aal", self_supplement=True))
lemma = register.get_lemma_by_name("Aal", self_supplement=True)
compare(3, register.get_index_of_lemma(lemma))
compare(None, register.get_index_of_lemma("Lemma not there"))
def test_update_next_and_previous(self):
copy_tst_data("I_1_sorting2", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "O", "previous": "Ä", "next": "Ü"}
with Updater(register) as updater:
updater._try_update_next_and_previous(update_dict, register.get_lemma_by_name("O"))
post_lemma_previous = register.get_lemma_by_name("Ä")
compare("O", post_lemma_previous["next"])
post_lemma_next = register.get_lemma_by_name("Ü")
compare("O", post_lemma_next["previous"])
def test_persist(self):
authors = Authors()
authors.set_mappings({"Foo": "Bar"})
authors.set_author({"Foo Bar": {"birth": 1234}})
authors.persist()
base_path = Path(__file__).parent.joinpath("mock_data")
with open(str(base_path.joinpath("authors_mapping.json")),
mode="r",
encoding="utf8") as mapping:
compare(True, "\"Foo\": \"Bar\"" in mapping.read())
with open(str(base_path.joinpath("authors.json")),
mode="r",
encoding="utf8") as authors:
compare(
True, " \"Foo Bar\": {\n \"birth\": 1234\n }"
in authors.read())
def test_get_lemma_self_append(self):
copy_tst_data("I_1_self_append", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
lemma = register.get_lemma_by_name("Aal")
compare(None, lemma["previous"])
lemma = register.get_lemma_by_name("Aal", self_supplement=True)
compare("Something", lemma["previous"])
lemma = register.get_lemma_by_sort_key("AAL", self_supplement=True)
compare("Something", lemma["previous"])
lemma = register.get_lemma_by_name("Something", self_supplement=True)
compare(None, lemma)
def test_update_by_sortkey_raise_error(self):
copy_tst_data("I_1_update_previous_wrong", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "Äarassos", "previous": "Aal", "next": "Aba 1", "sort_key": "Aarassos"}
with self.assertRaisesRegex(RegisterException, "!= next lemma name \"Ab 1\""):
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
previous_lemma = register.get_lemma_by_name("Aal")
compare("Aarassos", previous_lemma["next"])
next_lemma = register.get_lemma_by_name("Ab 1")
compare("Aarassos", next_lemma["previous"])
def test_update_lemma_by_sortkey(self):
copy_tst_data("I_1_base", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "Äal", "redirect": True, "sort_key": "Aal", "next": "Aarassos"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
post_lemma = register.get_lemma_by_name("Äal")
compare(True, post_lemma["redirect"])
compare("Aal", post_lemma["sort_key"])
post_lemma_next = register.get_lemma_by_name("Aarassos")
compare("Äal", post_lemma_next["previous"])
def test_acutius_1a(self):
copy_tst_data("acutius_1a_bug", "S I")
register = VolumeRegister(Volumes()["S I"], Authors())
update_dict = {"lemma": "Acutius a", "previous": "Acronoma", "next": "Acutius 1a", "sort_key": "Acutius 0a"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
update_dict = {"lemma": "Acutius 1a", "previous": "Acutius a", "next": "Adaba", "redirect": True}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
# don't create a new lemma, because Acutius a != Acutius 0a
compare(5, len(register.lemmas))
def test_update_no_update_possible(self):
copy_tst_data("I_1_base", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "bubum",
"redirect": True,
"sort_key": "babam",
"previous": "rubbish",
"next": "something"}
with self.assertRaisesRegex(RegisterException, "No strategy available"):
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
def __init__(self, update_data=False):
self.repo = DataRepo()
if update_data:
self.repo.pull()
self._authors: Authors = Authors()
self._registers: Dict[str, VolumeRegister] = OrderedDict()
self._alphabetic_registers: Dict[str,
AlphabeticRegister] = OrderedDict()
for volume in Volumes().all_volumes:
with contextlib.suppress(FileNotFoundError):
self._registers[volume.name] = VolumeRegister(
volume, self._authors)
def test_set_author(self):
authors = Authors()
author = authors.get_author_by_mapping("Abel", "I,1")
compare("Herman Abel", author[0].name)
compare(1998, author[0].death)
compare(None, author[0].birth)
authors.set_author({"Herman Abel": {"birth": 1900, "death": 1990}})
author = authors.get_author_by_mapping("Abel", "I,1")
compare("Herman Abel", author[0].name)
compare(1990, author[0].death)
compare(1900, author[0].birth)
authors.set_author({"Abel2": {"birth": 1950}})
author = authors._authors["Abel2"]
compare("Abel2", author.name)
compare(1950, author.birth)
def test_update_by_insert_before_next_no_previous(self):
copy_tst_data("I_1_sorting2", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "B", "next": "Ö"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
post_lemma = register.get_lemma_by_name("B")
compare(None, post_lemma["previous"])
compare("Ö", post_lemma["next"])
post_lemma_previous = register.get_lemma_by_name("A")
compare(None, post_lemma_previous["next"])
post_lemma_next = register.get_lemma_by_name("Ö")
compare("B", post_lemma_next["previous"])
post_lemma_next_next = register.get_lemma_by_name("U")
compare("Ö", post_lemma_next_next["previous"])
def test_load_data(self):
authors = Authors()
author = authors.get_author_by_mapping("Abbott", "I,1")
compare("William Abbott", author[0].name)
compare(None, author[0].death)
author = authors.get_author_by_mapping("Abel", "I,1")
compare("Herman Abel", author[0].name)
compare(1998, author[0].death)
author = authors.get_author_by_mapping("Abel", "XVI,1")
compare("Abel", author[0].name)
compare(1987, author[0].death)
author = authors.get_author_by_mapping("redirect.", "XVI,1")
compare("Abert", author[0].name)
compare(1927, author[0].death)
author = authors.get_author_by_mapping("redirect_list", "XVI,1")
compare("Abert", author[0].name)
compare("Herman Abel", author[1].name)
compare(1927, author[0].death)
compare([], authors.get_author_by_mapping("Tada", "XVI,1"))
author = authors.get_author("Abert|")
compare("Abert", author.name)
def test_update_lemma_by_sortkey_pre_and_next_lemma_other_name(self):
copy_tst_data("I_1_sorting2", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "Ö", "sort_key": "O", "previous": "Ä", "next": "Ü"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
post_lemma = register.get_lemma_by_name("Ö")
compare("O", post_lemma["sort_key"])
post_lemma_previous = register.get_lemma_by_name("Ä")
compare("Ö", post_lemma_previous["next"])
post_lemma_next = register.get_lemma_by_name("Ü")
compare("Ö", post_lemma_next["previous"])
post_lemma_start = register.get_lemma_by_name("Vor A")
compare("Ä", post_lemma_start["next"])
post_lemma_end = register.get_lemma_by_name("D")
compare("Ü", post_lemma_end["previous"])
def test_update_by_replace(self):
copy_tst_data("I_1_sorting2", "I_1")
register = VolumeRegister(Volumes()["I,1"], Authors())
update_dict = {"lemma": "B", "previous": "Ä", "next": "Ü"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
post_lemma = register.get_lemma_by_name("B")
compare("Ä", post_lemma["previous"])
compare("Ü", post_lemma["next"])
post_lemma_previous = register.get_lemma_by_name("Ä")
compare("B", post_lemma_previous["next"])
post_lemma_previous_previous = register.get_lemma_by_name("Vor A")
compare("Ä", post_lemma_previous_previous["next"])
post_lemma_next = register.get_lemma_by_name("Ü")
compare("B", post_lemma_next["previous"])
post_lemma_next_next = register.get_lemma_by_name("D")
compare("Ü", post_lemma_next_next["previous"])
class ClaimFactory:
_authors = Authors()
_volumes = Volumes()
_IMPORTED_FROM_WIKISOURCE = SnakParameter(property_str="P143",
target_type="wikibase-item",
target="Q15522295")
def __init__(self, re_page: RePage, logger: WikiLogger):
self.wikisource: pywikibot.Site = pywikibot.Site(code="de", fam="wikisource", user="******")
self.wikipedia: pywikibot.Site = pywikibot.Site(code="de", fam="wikipedia", user="******")
self.wikidata: pywikibot.site.DataSite = self.wikisource.data_repository()
self.re_page = re_page
self.logger = logger
self._current_year = datetime.now().year
def _get_claim_json(self) -> List[JsonClaimDict]:
raise NotImplementedError
def execute_pre_action(self):
pass
def get_claims_to_update(self, data_item: pywikibot.ItemPage) -> ChangedClaimsDict:
"""
Every claim that is updated can possible add new claims, but can also remove existing claims at the item.
Which claims is removed or added depends of the specific implementation of the property factory. The standart
implementation will update all claims as expected by the factory (this include removing possible existing
claims).
:param: data_item: item where the specific property is going to be altered
:returns: A dictionary with claims to add and to remove is returned
"""
claim_list = [pywikibot.Claim.fromJSON(self.wikidata, claim_json)
for claim_json in self._get_claim_json()]
return self.get_diff_claims_for_replacement(claim_list, data_item)
@classmethod
def get_property_string(cls) -> str:
"""
Returns the property string for this class
"""
if regex_hit := re.search(r"^P\d{1,6}", cls.__name__):
return regex_hit.group(0)
return ""
def test_update_create_next_previous_supplement_by_sort_key(self):
copy_tst_data("I_1_sorting2", "S I")
register = VolumeRegister(Volumes()["S I"], Authors())
update_dict = {"lemma": "Ö", "previous": "blub", "next": "Ä"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
compare(8, len(register.lemmas))
update_dict = {"lemma": "Ä", "previous": "Ö", "next": "blab"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
compare(9, len(register.lemmas))
post_lemma = register.get_lemma_by_name("Ö")
compare("blub", post_lemma["previous"])
compare("Ä", post_lemma["next"])
post_lemma = register.get_lemma_by_name("Ä")
compare("Ö", post_lemma["previous"])
compare("blab", post_lemma["next"])
self.assertTrue(register.get_index_of_lemma("Ä") <
register.get_index_of_lemma("blab") <
register.get_index_of_lemma("blub") <
register.get_index_of_lemma("Ö"))
def test_update_create_next_previous_supplement_by_name_next_exists(self):
copy_tst_data("I_1_sorting2", "R")
register = VolumeRegister(Volumes()["R"], Authors())
update_dict = {"lemma": "O", "previous": "N", "next": "Ü"}
with Updater(register) as updater:
updater.update_lemma(update_dict, [])
post_lemma = register.get_lemma_by_name("O")
compare("N", post_lemma["previous"])
compare("Ü", post_lemma["next"])
post_lemma_previous = register.get_lemma_by_name("N")
compare("O", post_lemma_previous["next"])
post_lemma_previous_previous = register.get_lemma_by_name("A")
compare(None, post_lemma_previous_previous["next"])
post_lemma_next = register.get_lemma_by_name("Ü")
compare("O", post_lemma_next["previous"])
post_lemma_next_next = register.get_lemma_by_name("D")
compare("Ü", post_lemma_next_next["previous"])
self.assertTrue(register.get_index_of_lemma("A") <
register.get_index_of_lemma("N") <
register.get_index_of_lemma("O") <
register.get_index_of_lemma("Ü") <
register.get_index_of_lemma("D"))
def setUp(self):
self.authors = Authors()
self.volumes = Volumes()
self.basic_dict = {
"lemma":
"lemma",
"previous":
"previous",
"next":
"next",
"redirect":
True,
"chapters": [{
"start": 1,
"end": 1,
"author": "Herman Abel"
}, {
"start": 1,
"end": 2,
"author": "William Abbott"
}]
}