Example #1
0
    def __call__(self):
        def cut_eol(words):
            for i, word in enumerate(words):
                if words[i] == '<eol>':
                    return words[:i + 1]
            raise Exception("No end-of-line found")

        sample_idx = 0
        while sample_idx < self.state['n_examples']:
            batch = self.train_iter.next(peek=True)
            xs, ys = batch['x'], batch['y']
            if self.state['rolling_vocab']:
                small_xs = replace_array(xs, self.model.large2small_src)
                small_ys = replace_array(ys, self.model.large2small_trgt)
            for seq_idx in range(xs.shape[1]):
                if sample_idx == self.state['n_examples']:
                    break

                x, y = xs[:, seq_idx], ys[:, seq_idx]
                if self.state['rolling_vocab']:
                    small_x = small_xs[:, seq_idx]
                    small_y = small_ys[:, seq_idx]
                    x_words = cut_eol(
                        map(lambda w_idx: self.model.large2word_src[w_idx], x))
                    y_words = cut_eol(
                        map(lambda w_idx: self.model.large2word_trgt[w_idx],
                            y))
                    #Alternatively
                    x_words_alt = cut_eol(
                        map(lambda w_idx: self.model.word_indxs_src[w_idx],
                            small_x))
                    y_words_alt = cut_eol(
                        map(lambda w_idx: self.model.word_indxs[w_idx],
                            small_y))
                    if (x_words == x_words_alt) and (y_words == y_words_alt):
                        logger.debug("OK. Small and large index2word match.")
                    else:
                        logger.error(
                            "Small and large index2word DO NOT MATCH.")
                else:
                    x_words = cut_eol(
                        map(lambda w_idx: self.model.word_indxs_src[w_idx], x))
                    y_words = cut_eol(
                        map(lambda w_idx: self.model.word_indxs[w_idx], y))
                if len(x_words) == 0:
                    continue

                print "Input: {}".format(" ".join(x_words))
                print "Target: {}".format(" ".join(y_words))
                if self.state['rolling_vocab']:
                    self.model.get_samples(self.state['seqlen'] + 1,
                                           self.state['n_samples'],
                                           small_x[:len(x_words)])
                else:
                    self.model.get_samples(self.state['seqlen'] + 1,
                                           self.state['n_samples'],
                                           x[:len(x_words)])
                sample_idx += 1
Example #2
0
    def __call__(self):
        batch = self.data.next()
        assert batch

        if self.state['rolling_vocab']:  # Assumes batch is a dictionary
            batch['x'] = replace_array(batch['x'], self.model.large2small_src)
            batch['y'] = replace_array(batch['y'], self.model.large2small_trgt)

        # Perturb the data (! and the model)
        if isinstance(batch, dict):
            batch = self.model.perturb(**batch)
        else:
            batch = self.model.perturb(*batch)
        # Load the dataset into GPU
        # Note: not the most efficient approach in general, as it involves
        # each batch is copied individually on gpu
        if isinstance(batch, dict):
            for gdata in self.gdata:
                gdata.set_value(batch[gdata.name], borrow=True)
        else:
            for gdata, data in zip(self.gdata, batch):
                gdata.set_value(data, borrow=True)
        # Run the trianing function
        g_st = time.time()
        rvals = self.train_fn()
        for schedule in self.schedules:
            schedule(self, rvals[-1])
        self.update_fn()
        g_ed = time.time()
        self.state['lr'] = float(self.lr)
        cost = rvals[-1]
        self.old_cost = cost
        whole_time = time.time() - self.step_timer
        if self.step % self.state['trainFreq'] == 0:
            msg = '.. iter %4d cost %.3f'
            vals = [self.step, cost]
            for dx, prop in enumerate(self.prop_names):
                msg += ' ' + prop + ' %.2e'
                vals += [float(numpy.array(rvals[dx]))]
            msg += ' step time %s whole time %s lr %.2e'
            vals += [
                print_time(g_ed - g_st),
                print_time(time.time() - self.step_timer),
                float(self.lr)
            ]
            print msg % tuple(vals)
        self.step += 1
        ret = dict([('cost', float(cost)), ('error', float(cost)),
                    ('lr', float(self.lr)), ('time_step', float(g_ed - g_st)),
                    ('whole_time', float(whole_time))] +
                   zip(self.prop_names, rvals))
        return ret
Example #3
0
    def __call__(self):
        batch = self.data.next()
        assert batch
        
        if self.state['rolling_vocab']: # Assumes batch is a dictionary
            batch['x'] = replace_array(batch['x'], self.model.large2small_src)
            batch['y'] = replace_array(batch['y'], self.model.large2small_trgt)

        # Perturb the data (! and the model)
        if isinstance(batch, dict):
            batch = self.model.perturb(**batch)
        else:
            batch = self.model.perturb(*batch)
        # Load the dataset into GPU
        # Note: not the most efficient approach in general, as it involves
        # each batch is copied individually on gpu
        if isinstance(batch, dict):
            for gdata in self.gdata:
                gdata.set_value(batch[gdata.name], borrow=True)
        else:
            for gdata, data in zip(self.gdata, batch):
                gdata.set_value(data, borrow=True)
        # Run the trianing function
        g_st = time.time()
        rvals = self.train_fn()
        for schedule in self.schedules:
            schedule(self, rvals[-1])
        self.update_fn()
        g_ed = time.time()
        self.state['lr'] = float(self.lr)
        cost = rvals[-1]
        self.old_cost = cost
        whole_time = time.time() - self.step_timer
        if self.step % self.state['trainFreq'] == 0:
            msg = '.. iter %4d cost %.3f'
            vals = [self.step, cost]
            for dx, prop in enumerate(self.prop_names):
                msg += ' '+prop+' %.2e'
                vals += [float(numpy.array(rvals[dx]))]
            msg += ' step time %s whole time %s lr %.2e'
            vals += [print_time(g_ed - g_st),
                     print_time(time.time() - self.step_timer),
                     float(self.lr)]
            print msg % tuple(vals)
        self.step += 1
        ret = dict([('cost', float(cost)),
                    ('error', float(cost)),
                       ('lr', float(self.lr)),
                       ('time_step', float(g_ed - g_st)),
                       ('whole_time', float(whole_time))]+zip(self.prop_names, rvals))
        return ret
Example #4
0
    def __call__(self):
        def cut_eol(words):
            for i, word in enumerate(words):
                if words[i] == '<eol>':
                    return words[:i + 1]
            raise Exception("No end-of-line found")

        sample_idx = 0
        while sample_idx < self.state['n_examples']:
            batch = self.train_iter.next(peek=True)
            xs, ys = batch['x'], batch['y']
            if self.state['rolling_vocab']:
                small_xs = replace_array(xs, self.model.large2small_src)
                small_ys = replace_array(ys, self.model.large2small_trgt)
            for seq_idx in range(xs.shape[1]):
                if sample_idx == self.state['n_examples']:
                    break

                x, y = xs[:, seq_idx], ys[:, seq_idx]
                if self.state['rolling_vocab']:
                    small_x = small_xs[:, seq_idx]
                    small_y = small_ys[:, seq_idx]
                    x_words = cut_eol(map(lambda w_idx : self.model.large2word_src[w_idx], x))
                    y_words = cut_eol(map(lambda w_idx : self.model.large2word_trgt[w_idx], y))
                    #Alternatively
                    x_words_alt = cut_eol(map(lambda w_idx : self.model.word_indxs_src[w_idx], small_x))
                    y_words_alt = cut_eol(map(lambda w_idx : self.model.word_indxs[w_idx], small_y))
                    if (x_words == x_words_alt) and (y_words == y_words_alt):
                        logger.debug("OK. Small and large index2word match.")
                    else:
                        logger.error("Small and large index2word DO NOT MATCH.")
                else:
                    x_words = cut_eol(map(lambda w_idx : self.model.word_indxs_src[w_idx], x))
                    y_words = cut_eol(map(lambda w_idx : self.model.word_indxs[w_idx], y))
                if len(x_words) == 0:
                    continue

                print "Input: {}".format(" ".join(x_words))
                print "Target: {}".format(" ".join(y_words))
                if self.state['rolling_vocab']:
                    self.model.get_samples(self.state['seqlen'] + 1, self.state['n_samples'], small_x[:len(x_words)])
                else:
                    self.model.get_samples(self.state['seqlen'] + 1, self.state['n_samples'], x[:len(x_words)])
                sample_idx += 1
Example #5
0
    def __call__(self):
        batch = self.data.next()
        assert batch

        null_inputs = sum(batch["x"].flatten() == self.null_word) / float(len(batch["x"][0]))

        # replace occurrences of <null> with </s> (<null> should be last word in sentence)
        for i in range(1, len(batch["x"]) - 1):
            batch["x_mask"][i + 1][batch["x"][i] == self.null_word] = 0
        batch["x"][batch["x"] == self.null_word] = 0

        # if <null> was only word in sentence, add it back in to prevent empty input
        batch["x"][0][batch["x"][0] == 0] = self.null_word

        if self.state["rolling_vocab"]:  # Assumes batch is a dictionary
            batch["x"] = replace_array(batch["x"], self.model.large2small_src)
            batch["y"] = replace_array(batch["y"], self.model.large2small_trgt)

        # Perturb the data (! and the model)
        if isinstance(batch, dict):
            batch = self.model.perturb(**batch)
        else:
            batch = self.model.perturb(*batch)
        # Load the dataset into GPU
        # Note: not the most efficient approach in general, as it involves
        # each batch is copied individually on gpu
        if isinstance(batch, dict):
            for gdata in self.gdata:
                gdata.set_value(batch[gdata.name], borrow=True)
        else:
            for gdata, data in zip(self.gdata, batch):
                gdata.set_value(data, borrow=True)
        # Run the trianing function
        g_st = time.time()
        rvals = self.train_fn()
        for schedule in self.schedules:
            schedule(self, rvals[-1])
        if null_inputs > 0.5:
            self.update_fn_lm()
        else:
            self.update_fn()
        g_ed = time.time()
        self.state["lr"] = float(self.lr)
        cost = rvals[-1]
        self.old_cost = cost
        whole_time = time.time() - self.step_timer
        if self.step % self.state["trainFreq"] == 0:
            msg = ".. iter %4d cost %.3f"
            vals = [self.step, cost]
            for dx, prop in enumerate(self.prop_names):
                msg += " " + prop + " %.2e"
                vals += [float(numpy.array(rvals[dx]))]
            msg += " step time %s whole time %s lr %.2e"
            vals += [print_time(g_ed - g_st), print_time(time.time() - self.step_timer), float(self.lr)]
            print msg % tuple(vals)
        self.step += 1
        ret = dict(
            [
                ("cost", float(cost)),
                ("error", float(cost)),
                ("lr", float(self.lr)),
                ("time_step", float(g_ed - g_st)),
                ("whole_time", float(whole_time)),
            ]
            + zip(self.prop_names, rvals)
        )
        return ret