def test_get_passage_hypercontext_complex_xpath(self):
simple = self.text_complex.getTextualNode(Reference("pr.1-1.2"))
str_simple = simple.tostring(encoding=str)
text = CapitainsCtsText(resource=str_simple,
citation=self.text_complex.citation)
self.assertIn(
"Pervincis tandem",
text.getTextualNode(Reference("pr.1"),
simple=True).export(output=Mimetypes.PLAINTEXT,
exclude=["tei:note"
]).strip(),
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
text.getTextualNode(
Reference("1.2"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"lusimus quos in Suebae gratiam virgunculae,",
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
list(map(lambda x: str(x),
text.getValidReff(level=2))), ["pr.1", "1.1", "1.2"],
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
def onekgreek_tei_xml_to_text_capitains():
"""Use MyCapitains program to convert TEI to plaintext."""
file = os.path.expanduser(
'~/cltk_data/greek/text/greek_text_first1kgreek/data/tlg0627/tlg021/tlg0627.tlg021.1st1K-grc1.xml')
xml_dir = os.path.expanduser('~/cltk_data/greek/text/greek_text_first1kgreek/data/*/*/*.xml')
xml_paths = glob.glob(xml_dir)
if not len(xml_paths):
logger.error('1K Greek corpus not installed. Use CorpusInstaller to get `First1KGreek`.')
raise FileNotFoundError
xml_paths = [path for path in xml_paths if '__cts__' not in path]
# new dir
new_dir = os.path.expanduser('~/cltk_data/greek/text/greek_text_first1kgreek_plaintext/')
if not os.path.isdir(new_dir):
os.makedirs(new_dir)
for xml_path in xml_paths:
_, xml_name = os.path.split(xml_path)
xml_name = xml_name.rstrip('.xml')
xml_name += '.txt'
plain_text = ''
with open(xml_path) as file_open:
text = CapitainsCtsText(resource=file_open)
for ref in text.getReffs(level=len(text.citation)):
psg = text.getTextualNode(subreference=ref, simple=True)
text_line = psg.export(Mimetypes.PLAINTEXT, exclude=["tei:note"])
plain_text += text_line
new_plaintext_path = os.path.join(new_dir, xml_name)
with open(new_plaintext_path, 'w') as file_open:
file_open.write(plain_text)
def test_get_passage_hyper_context_double_slash_xpath(self):
simple = self.seneca.getTextualNode(Reference("1-10"))
str_simple = simple.export(output=Mimetypes.XML.Std)
text = CapitainsCtsText(resource=str_simple,
citation=self.seneca.citation)
self.assertEqual(
text.getTextualNode(Reference("1"),
simple=True).export(output=Mimetypes.PLAINTEXT,
exclude=["tei:note"
]).strip(),
"Di coniugales tuque genialis tori,",
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
text.getTextualNode(
Reference("10"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"aversa superis regna manesque impios",
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
list(map(lambda x: str(x), text.getValidReff(level=1))),
["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"],
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
simple = self.seneca.getTextualNode(Reference("1"))
str_simple = simple.tostring(encoding=str)
text = CapitainsCtsText(resource=str_simple,
citation=self.seneca.citation)
self.assertEqual(
text.getTextualNode(Reference("1"),
simple=True).export(output=Mimetypes.PLAINTEXT,
exclude=["tei:note"
]).strip(),
"Di coniugales tuque genialis tori,",
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
list(map(lambda x: str(x), text.getValidReff(level=1))), ["1"],
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
def test_passage_extraction_fail_when_reffs_are_found(self):
""" This issues is drawn from https://github.com/PerseusDL/canonical-latinLit/issues/226
"""
with open("tests/testing_data/texts/extraction_issue.xml") as text:
interactive_text = CapitainsCtsText(resource=etree.parse(text).getroot())
reffs = interactive_text.getReffs(level=len(interactive_text.citation))
passages = []
# The failing passage was 5.1
for reff in reffs:
try:
passages.append(interactive_text.getTextualNode(reff))
except IndexError:
raise Exception("Unable to extract %s " % reff)
plaintext = [r.export(Mimetypes.PLAINTEXT, exclude=["tei:note"]).strip() for r in passages]
self.assertIn(
"NUNC et praedictos et regni sorte sequentes", plaintext,
"The text of 5.1 should be in plaintext"
)
def build_texts(self, text):
interactive_text = CapitainsCtsText(
resource=etree.parse(text).getroot())
reffs = interactive_text.getReffs(level=len(interactive_text.citation))
passages = [
interactive_text.getTextualNode(passage) for passage in reffs
]
plaintext = [
r.export(Mimetypes.PLAINTEXT, exclude=["tei:note"]).strip()
for r in passages
]
if self.cites is True:
for i, t in enumerate(plaintext):
plaintext[i] = '#' + reffs[i] + '#\n' + t
with open('{}text/{}.txt'.format(
self.dest,
text.split('/')[-1].replace('.xml', '')),
mode='w') as f:
f.write('\n\n'.join(plaintext))
def onekgreek_tei_xml_to_text_capitains():
"""Use MyCapitains program to convert TEI to plaintext."""
file = os.path.expanduser(
get_cltk_data_dir() +
'/greek/text/greek_text_first1kgreek/data/tlg0627/tlg021/tlg0627.tlg021.1st1K-grc1.xml'
)
xml_dir = os.path.normpath(
get_cltk_data_dir() +
'/greek/text/greek_text_first1kgreek/data/*/*/*.xml')
xml_paths = glob.glob(xml_dir)
if not len(xml_paths):
logger.error(
'1K Greek corpus not installed. Use CorpusInstaller to get `First1KGreek`.'
)
raise FileNotFoundError
xml_paths = [path for path in xml_paths if '__cts__' not in path]
# new dir
new_dir = os.path.normpath(
get_cltk_data_dir() + '/greek/text/greek_text_first1kgreek_plaintext/')
if not os.path.isdir(new_dir):
os.makedirs(new_dir)
for xml_path in xml_paths:
_, xml_name = os.path.split(xml_path)
xml_name = xml_name.rstrip('.xml')
xml_name += '.txt'
plain_text = ''
with open(xml_path) as file_open:
text = CapitainsCtsText(resource=file_open)
for ref in text.getReffs(level=len(text.citation)):
psg = text.getTextualNode(subreference=ref, simple=True)
text_line = psg.export(Mimetypes.PLAINTEXT,
exclude=["tei:note"])
plain_text += text_line
new_plaintext_path = os.path.join(new_dir, xml_name)
with open(new_plaintext_path, 'w') as file_open:
file_open.write(plain_text)
def test_get_Passage_context_no_double_slash(self):
""" Check that get CapitainsCtsPassage contexts return right information """
simple = self.TEI.getTextualNode(Reference("1.pr.2"))
str_simple = simple.tostring(encoding=str)
text = CapitainsCtsText(resource=str_simple,
citation=self.TEI.citation)
self.assertEqual(
text.getTextualNode(
Reference("1.pr.2"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"tum, ut de illis queri non possit quisquis de se bene",
"Ensure passage finding with context is fully TEI / Capitains compliant (One reference CapitainsCtsPassage)"
)
simple = self.TEI.getTextualNode(Reference("1.pr.2-1.pr.7"))
str_simple = simple.tostring(encoding=str)
text = CapitainsCtsText(resource=str_simple,
citation=self.TEI.citation)
self.assertEqual(
text.getTextualNode(
Reference("1.pr.2"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"tum, ut de illis queri non possit quisquis de se bene",
"Ensure passage finding with context is fully TEI / Capitains compliant (Same level same "
"parent range CapitainsCtsPassage)")
self.assertEqual(
text.getTextualNode(
Reference("1.pr.3"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"senserit, cum salva infimarum quoque personarum re-",
"Ensure passage finding with context is fully TEI / Capitains compliant (Same level same "
"parent range CapitainsCtsPassage)")
self.assertEqual(
list(map(lambda x: str(x), text.getValidReff(level=3))),
["1.pr.2", "1.pr.3", "1.pr.4", "1.pr.5", "1.pr.6", "1.pr.7"],
"Ensure passage finding with context is fully TEI / Capitains compliant (Same level same "
"parent range CapitainsCtsPassage)")
simple = self.TEI.getTextualNode(Reference("1.pr.2-1.1.6"))
str_simple = simple.tostring(encoding=str)
text = CapitainsCtsText(resource=str_simple,
citation=self.TEI.citation)
self.assertEqual(
text.getTextualNode(
Reference("1.pr.2"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"tum, ut de illis queri non possit quisquis de se bene",
"Ensure passage finding with context is fully TEI / Capitains compliant (Same level range CapitainsCtsPassage)"
)
self.assertEqual(
text.getTextualNode(
Reference("1.1.6"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"Rari post cineres habent poetae.",
"Ensure passage finding with context is fully TEI / Capitains compliant (Same level range CapitainsCtsPassage)"
)
self.assertEqual(
list(map(lambda x: str(x), text.getValidReff(level=3))), [
"1.pr.2",
"1.pr.3",
"1.pr.4",
"1.pr.5",
"1.pr.6",
"1.pr.7",
"1.pr.8",
"1.pr.9",
"1.pr.10",
"1.pr.11",
"1.pr.12",
"1.pr.13",
"1.pr.14",
"1.pr.15",
"1.pr.16",
"1.pr.17",
"1.pr.18",
"1.pr.19",
"1.pr.20",
"1.pr.21",
"1.pr.22",
"1.1.1",
"1.1.2",
"1.1.3",
"1.1.4",
"1.1.5",
"1.1.6",
],
"Ensure passage finding with context is fully TEI / Capitains compliant (Same level range CapitainsCtsPassage)"
)
simple = self.TEI.getTextualNode(Reference("1.pr.2-1.2"))
str_simple = simple.tostring(encoding=str)
text = CapitainsCtsText(resource=str_simple,
citation=self.TEI.citation)
self.assertEqual(
text.getTextualNode(
Reference("1.pr.2"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"tum, ut de illis queri non possit quisquis de se bene",
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
text.getTextualNode(
Reference("1.1.6"),
simple=True).export(output=Mimetypes.PLAINTEXT).strip(),
"Rari post cineres habent poetae.",
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
self.assertEqual(
list(map(lambda x: str(x), text.getValidReff(level=3))), [
"1.pr.2", "1.pr.3", "1.pr.4", "1.pr.5", "1.pr.6", "1.pr.7",
"1.pr.8", "1.pr.9", "1.pr.10", "1.pr.11", "1.pr.12", "1.pr.13",
"1.pr.14", "1.pr.15", "1.pr.16", "1.pr.17", "1.pr.18",
"1.pr.19", "1.pr.20", "1.pr.21", "1.pr.22", "1.1.1", "1.1.2",
"1.1.3", "1.1.4", "1.1.5", "1.1.6", '1.2.1', '1.2.2', '1.2.3',
'1.2.4', '1.2.5', '1.2.6', '1.2.7', '1.2.8'
],
"Ensure passage finding with context is fully TEI / Capitains compliant (Different level range CapitainsCtsPassage)"
)
from lxml.etree import tostring
# We open a file
with open("./tests/testing_data/examples/text.martial.xml") as f:
# We initiate a Text object giving the IO instance to resource argument
text = CapitainsCtsText(resource=f)
# Text objects have a citation property
# len(Citation(...)) gives the depth of the citation scheme
# in the case of this sample, this would be 3 (Book, Poem, Line)
for ref in text.getReffs(level=len(text.citation)):
# We retrieve a Passage object for each reference that we find
# We can pass the reference many way, including in the form of a list of strings
# We use the _simple parameter to get a fairly simple object
# Simple makes a straight object that has only the targeted node inside of it
psg = text.getTextualNode(subreference=ref, simple=True)
# We print the passage from which we retrieve <note> nodes
print("\t".join([ref, psg.export(Mimetypes.PLAINTEXT, exclude=["tei:note"])]))
"""
You'll print something like the following :
1.pr.1 Spero me secutum in libellis meis tale temperamen-
1.pr.2 tum, ut de illis queri non possit quisquis de se bene
1.pr.3 senserit, cum salva infimarum quoque personarum re-
1.pr.4 verentia ludant; quae adeo antiquis auctoribus defuit, ut
1.pr.5 nominibus non tantum veris abusi sint, sed et magnis.
1.pr.6 Mihi fama vilius constet et probetur in me novissimum
"""