def load(self, trim=None):
LOGGER.info('Loading data from %s and %s' %
(self.full_text_directory, self.summary_directory))
full_text_lang = Language(type='full_text')
summary_text_lang = Language(type='summary_text')
pairs = []
for doc, summary in self:
full_text_lang.add_text(doc)
summary_text_lang.add_text(summary)
pairs.append((doc, summary))
if trim and len(pairs) == trim:
break
LOGGER.info('Finished loading %i data samples' % len(pairs))
return full_text_lang, summary_text_lang, pairs
