def getBatchData(self): src_lang = self.src_lang tgt_lang = self.tgt_lang tgt_vocab_size = self.tgt_vocab_size ngram_size = self.ngram_size is_shuffle = self.is_shuffle chunk_size = self.chunk_size src_window = self.src_window opt = self.opt (data_x, data_y) = io_read_ngram.get_joint_ngrams(self.src_f, self.tgt_f, self.align_f, \ tgt_vocab_size, ngram_size, src_window, opt, num_read_lines=chunk_size) return (data_x, data_y)
def loadBatchData(self, isInitialLoad=False): src_lang = self.src_lang tgt_lang = self.tgt_lang tgt_vocab_size = self.tgt_vocab_size ngram_size = self.ngram_size chunk_size = self.chunk_size src_window = self.src_window opt = self.opt (self.data_x, self.data_y) = io_read_ngram.get_joint_ngrams(self.src_f, self.tgt_f, self.align_f, \ tgt_vocab_size, ngram_size, src_window, opt, num_read_lines=chunk_size) if isInitialLoad == False: assert (type(self.model) == model_nnlm.ModelNNLM) return self.model.updateTrainModelInput(self.data_x, self.data_y)
def loadBatchData(self, isInitialLoad=False): src_lang = self.src_lang tgt_lang = self.tgt_lang tgt_vocab_size = self.tgt_vocab_size ngram_size = self.ngram_size chunk_size = self.chunk_size src_window = self.src_window opt = self.opt (self.data_x, self.data_y) = io_read_ngram.get_joint_ngrams(self.src_f, self.tgt_f, self.align_f, \ tgt_vocab_size, ngram_size, src_window, opt, num_read_lines=chunk_size) if isInitialLoad == False: assert(type(self.model) == model_nnlm.ModelNNLM) return self.model.updateTrainModelInput(self.data_x, self.data_y)