def load(self, trim=None): LOGGER.info('Loading data from %s and %s' % (self.full_text_directory, self.summary_directory)) full_text_lang = Language(type='full_text') summary_text_lang = Language(type='summary_text') pairs = [] for doc, summary in self: full_text_lang.add_text(doc) summary_text_lang.add_text(summary) pairs.append((doc, summary)) if trim and len(pairs) == trim: break LOGGER.info('Finished loading %i data samples' % len(pairs)) return full_text_lang, summary_text_lang, pairs