def __call__(self): def cut_eol(words): for i, word in enumerate(words): if words[i] == '<eol>': return words[:i + 1] raise Exception("No end-of-line found") sample_idx = 0 while sample_idx < self.state['n_examples']: batch = self.train_iter.next(peek=True) xs, ys = batch['x'], batch['y'] if self.state['rolling_vocab']: small_xs = replace_array(xs, self.model.large2small_src) small_ys = replace_array(ys, self.model.large2small_trgt) for seq_idx in range(xs.shape[1]): if sample_idx == self.state['n_examples']: break x, y = xs[:, seq_idx], ys[:, seq_idx] if self.state['rolling_vocab']: small_x = small_xs[:, seq_idx] small_y = small_ys[:, seq_idx] x_words = cut_eol( map(lambda w_idx: self.model.large2word_src[w_idx], x)) y_words = cut_eol( map(lambda w_idx: self.model.large2word_trgt[w_idx], y)) #Alternatively x_words_alt = cut_eol( map(lambda w_idx: self.model.word_indxs_src[w_idx], small_x)) y_words_alt = cut_eol( map(lambda w_idx: self.model.word_indxs[w_idx], small_y)) if (x_words == x_words_alt) and (y_words == y_words_alt): logger.debug("OK. Small and large index2word match.") else: logger.error( "Small and large index2word DO NOT MATCH.") else: x_words = cut_eol( map(lambda w_idx: self.model.word_indxs_src[w_idx], x)) y_words = cut_eol( map(lambda w_idx: self.model.word_indxs[w_idx], y)) if len(x_words) == 0: continue print "Input: {}".format(" ".join(x_words)) print "Target: {}".format(" ".join(y_words)) if self.state['rolling_vocab']: self.model.get_samples(self.state['seqlen'] + 1, self.state['n_samples'], small_x[:len(x_words)]) else: self.model.get_samples(self.state['seqlen'] + 1, self.state['n_samples'], x[:len(x_words)]) sample_idx += 1
def __call__(self): batch = self.data.next() assert batch if self.state['rolling_vocab']: # Assumes batch is a dictionary batch['x'] = replace_array(batch['x'], self.model.large2small_src) batch['y'] = replace_array(batch['y'], self.model.large2small_trgt) # Perturb the data (! and the model) if isinstance(batch, dict): batch = self.model.perturb(**batch) else: batch = self.model.perturb(*batch) # Load the dataset into GPU # Note: not the most efficient approach in general, as it involves # each batch is copied individually on gpu if isinstance(batch, dict): for gdata in self.gdata: gdata.set_value(batch[gdata.name], borrow=True) else: for gdata, data in zip(self.gdata, batch): gdata.set_value(data, borrow=True) # Run the trianing function g_st = time.time() rvals = self.train_fn() for schedule in self.schedules: schedule(self, rvals[-1]) self.update_fn() g_ed = time.time() self.state['lr'] = float(self.lr) cost = rvals[-1] self.old_cost = cost whole_time = time.time() - self.step_timer if self.step % self.state['trainFreq'] == 0: msg = '.. iter %4d cost %.3f' vals = [self.step, cost] for dx, prop in enumerate(self.prop_names): msg += ' ' + prop + ' %.2e' vals += [float(numpy.array(rvals[dx]))] msg += ' step time %s whole time %s lr %.2e' vals += [ print_time(g_ed - g_st), print_time(time.time() - self.step_timer), float(self.lr) ] print msg % tuple(vals) self.step += 1 ret = dict([('cost', float(cost)), ('error', float(cost)), ('lr', float(self.lr)), ('time_step', float(g_ed - g_st)), ('whole_time', float(whole_time))] + zip(self.prop_names, rvals)) return ret
def __call__(self): batch = self.data.next() assert batch if self.state['rolling_vocab']: # Assumes batch is a dictionary batch['x'] = replace_array(batch['x'], self.model.large2small_src) batch['y'] = replace_array(batch['y'], self.model.large2small_trgt) # Perturb the data (! and the model) if isinstance(batch, dict): batch = self.model.perturb(**batch) else: batch = self.model.perturb(*batch) # Load the dataset into GPU # Note: not the most efficient approach in general, as it involves # each batch is copied individually on gpu if isinstance(batch, dict): for gdata in self.gdata: gdata.set_value(batch[gdata.name], borrow=True) else: for gdata, data in zip(self.gdata, batch): gdata.set_value(data, borrow=True) # Run the trianing function g_st = time.time() rvals = self.train_fn() for schedule in self.schedules: schedule(self, rvals[-1]) self.update_fn() g_ed = time.time() self.state['lr'] = float(self.lr) cost = rvals[-1] self.old_cost = cost whole_time = time.time() - self.step_timer if self.step % self.state['trainFreq'] == 0: msg = '.. iter %4d cost %.3f' vals = [self.step, cost] for dx, prop in enumerate(self.prop_names): msg += ' '+prop+' %.2e' vals += [float(numpy.array(rvals[dx]))] msg += ' step time %s whole time %s lr %.2e' vals += [print_time(g_ed - g_st), print_time(time.time() - self.step_timer), float(self.lr)] print msg % tuple(vals) self.step += 1 ret = dict([('cost', float(cost)), ('error', float(cost)), ('lr', float(self.lr)), ('time_step', float(g_ed - g_st)), ('whole_time', float(whole_time))]+zip(self.prop_names, rvals)) return ret
def __call__(self): def cut_eol(words): for i, word in enumerate(words): if words[i] == '<eol>': return words[:i + 1] raise Exception("No end-of-line found") sample_idx = 0 while sample_idx < self.state['n_examples']: batch = self.train_iter.next(peek=True) xs, ys = batch['x'], batch['y'] if self.state['rolling_vocab']: small_xs = replace_array(xs, self.model.large2small_src) small_ys = replace_array(ys, self.model.large2small_trgt) for seq_idx in range(xs.shape[1]): if sample_idx == self.state['n_examples']: break x, y = xs[:, seq_idx], ys[:, seq_idx] if self.state['rolling_vocab']: small_x = small_xs[:, seq_idx] small_y = small_ys[:, seq_idx] x_words = cut_eol(map(lambda w_idx : self.model.large2word_src[w_idx], x)) y_words = cut_eol(map(lambda w_idx : self.model.large2word_trgt[w_idx], y)) #Alternatively x_words_alt = cut_eol(map(lambda w_idx : self.model.word_indxs_src[w_idx], small_x)) y_words_alt = cut_eol(map(lambda w_idx : self.model.word_indxs[w_idx], small_y)) if (x_words == x_words_alt) and (y_words == y_words_alt): logger.debug("OK. Small and large index2word match.") else: logger.error("Small and large index2word DO NOT MATCH.") else: x_words = cut_eol(map(lambda w_idx : self.model.word_indxs_src[w_idx], x)) y_words = cut_eol(map(lambda w_idx : self.model.word_indxs[w_idx], y)) if len(x_words) == 0: continue print "Input: {}".format(" ".join(x_words)) print "Target: {}".format(" ".join(y_words)) if self.state['rolling_vocab']: self.model.get_samples(self.state['seqlen'] + 1, self.state['n_samples'], small_x[:len(x_words)]) else: self.model.get_samples(self.state['seqlen'] + 1, self.state['n_samples'], x[:len(x_words)]) sample_idx += 1
def __call__(self): batch = self.data.next() assert batch null_inputs = sum(batch["x"].flatten() == self.null_word) / float(len(batch["x"][0])) # replace occurrences of <null> with </s> (<null> should be last word in sentence) for i in range(1, len(batch["x"]) - 1): batch["x_mask"][i + 1][batch["x"][i] == self.null_word] = 0 batch["x"][batch["x"] == self.null_word] = 0 # if <null> was only word in sentence, add it back in to prevent empty input batch["x"][0][batch["x"][0] == 0] = self.null_word if self.state["rolling_vocab"]: # Assumes batch is a dictionary batch["x"] = replace_array(batch["x"], self.model.large2small_src) batch["y"] = replace_array(batch["y"], self.model.large2small_trgt) # Perturb the data (! and the model) if isinstance(batch, dict): batch = self.model.perturb(**batch) else: batch = self.model.perturb(*batch) # Load the dataset into GPU # Note: not the most efficient approach in general, as it involves # each batch is copied individually on gpu if isinstance(batch, dict): for gdata in self.gdata: gdata.set_value(batch[gdata.name], borrow=True) else: for gdata, data in zip(self.gdata, batch): gdata.set_value(data, borrow=True) # Run the trianing function g_st = time.time() rvals = self.train_fn() for schedule in self.schedules: schedule(self, rvals[-1]) if null_inputs > 0.5: self.update_fn_lm() else: self.update_fn() g_ed = time.time() self.state["lr"] = float(self.lr) cost = rvals[-1] self.old_cost = cost whole_time = time.time() - self.step_timer if self.step % self.state["trainFreq"] == 0: msg = ".. iter %4d cost %.3f" vals = [self.step, cost] for dx, prop in enumerate(self.prop_names): msg += " " + prop + " %.2e" vals += [float(numpy.array(rvals[dx]))] msg += " step time %s whole time %s lr %.2e" vals += [print_time(g_ed - g_st), print_time(time.time() - self.step_timer), float(self.lr)] print msg % tuple(vals) self.step += 1 ret = dict( [ ("cost", float(cost)), ("error", float(cost)), ("lr", float(self.lr)), ("time_step", float(g_ed - g_st)), ("whole_time", float(whole_time)), ] + zip(self.prop_names, rvals) ) return ret