class PlaintextParser(AbstractParser):
def __init__(self, filename):
super(PlaintextParser, self).__init__(filename)
if not self.wants_this_file():
return
self._init_line_count_progress()
self.nlp_pipeline = NlpPipeline()
def _wanted_file_endings(self):
return (".txt", )
def parse(self):
text = Text()
with open(self.filename, "r") as file_:
for line_unenc in file_:
self._progress += 1
line = unicode(line_unenc.encode('utf8'))
if line.startswith(TEXT_SEPARATOR):
if (len(text.sentences) > 0):
yield text
text = Text()
continue
sentences = self.nlp_pipeline.sentence_segmentation(line)
for sentence in sentences:
s = Sentence()
s.set_sentence_text(sentence)
s.set_tokens(self.nlp_pipeline.parse_text(sentence))
text.add_sentence(s)
if (len(text.sentences) > 0):
yield text
def progress(self):
return self._line_count_progress()
class PlaintextParser(AbstractParser):
def __init__(self, filename):
super(PlaintextParser, self).__init__(filename)
if not self.wants_this_file():
return
self._init_line_count_progress()
self.nlp_pipeline = NlpPipeline()
def _wanted_file_endings(self):
return (".txt",)
def parse(self):
text = Text()
with open(self.filename, "r") as file_:
for line_unenc in file_:
self._progress += 1
line = unicode(line_unenc.encode('utf8'))
if line.startswith(TEXT_SEPARATOR):
if (len(text.sentences) > 0):
yield text
text = Text()
continue
sentences = self.nlp_pipeline.sentence_segmentation(line)
for sentence in sentences:
s = Sentence()
s.set_sentence_text(sentence)
s.set_tokens(self.nlp_pipeline.parse_text(sentence))
text.add_sentence(s)
if (len(text.sentences) > 0):
yield text
def progress(self):
return self._line_count_progress()
