class LanguageBitBilingualSpeakerLearner(BilingualSpeakerLearner): def get_extra_vars(self): id_tag = (self.id + '/') if self.id else '' return [T.ivector(id_tag + 'languages')] def _data_to_arrays(self, training_instances, init_vectorizer=False, test=False, inverted=False): xs, ys = super(LanguageBitBilingualSpeakerLearner, self)._data_to_arrays( training_instances=training_instances, init_vectorizer=init_vectorizer, test=test, inverted=inverted) langs = [inst.output.split(':', 1)[0] for inst in training_instances] if init_vectorizer: self.lang_vec = SymbolVectorizer(use_unk=False) self.lang_vec.add_all(langs) langs_vec = self.lang_vec.vectorize_all(langs) return xs[:-2] + [langs_vec] + xs[-2:], ys def modify_context(self, l_context_repr, extra_vars): language = extra_vars[0] id_tag = (self.id + '/') if self.id else '' print('l_context_repr: {}'.format(l_context_repr.output_shape)) l_lang_input = InputLayer(shape=(None, ), input_var=language, name=id_tag + 'lang_input') l_lang_embed = EmbeddingLayer( l_lang_input, input_size=len(self.lang_vec.tokens), output_size=self.options.bilingual_lang_embed_size, name=id_tag + 'lang_embed') print('l_lang_embed: {}'.format(l_lang_embed.output_shape)) l_modified_context = ConcatLayer([l_lang_embed, l_context_repr]) print('l_modified_context: {}'.format(l_modified_context.output_shape)) return (l_modified_context, [l_lang_input]) def __getstate__(self): super_state = super(LanguageBitBilingualSpeakerLearner, self).__getstate__() return super_state + (self.lang_vec, ) def unpickle(self, state, model_class=SimpleLasagneModel): if isinstance( state, dict) and 'quickpickle' in state and state['quickpickle']: super(LanguageBitBilingualSpeakerLearner, self).unpickle(state, model_class=model_class) else: self.lang_vec = state[-1] super(LanguageBitBilingualSpeakerLearner, self).unpickle(state[:-1], model_class=model_class)
class AtomicUniformPrior(object): ''' >>> p = AtomicUniformPrior() >>> p.train([instance.Instance('blue')]) >>> p.sample(3) # doctest: +ELLIPSIS [Instance('...', None), Instance('...', None), Instance('...', None)] ''' def __init__(self): self.vec = SymbolVectorizer() def train(self, training_instances, listener_data=True): self.vec.add_all([inst.input if listener_data else inst.output for inst in training_instances]) def apply(self, input_vars): c = input_vars[0] if c.ndim == 1: ones = T.ones_like(c) else: ones = T.ones_like(c[:, 0]) return -np.log(self.vec.num_types) * ones def sample(self, num_samples=1): indices = random.randint(0, self.vec.num_types, size=(num_samples,)) return [instance.Instance(c) for c in self.vec.unvectorize_all(indices)]
class AtomicUniformPrior(object): ''' >>> p = AtomicUniformPrior() >>> p.train([instance.Instance('blue')]) >>> p.sample(3) # doctest: +ELLIPSIS [Instance('...', None), Instance('...', None), Instance('...', None)] ''' def __init__(self): self.vec = SymbolVectorizer() def train(self, training_instances, listener_data=True): self.vec.add_all([ inst.input if listener_data else inst.output for inst in training_instances ]) def apply(self, input_vars): c = input_vars[0] if c.ndim == 1: ones = T.ones_like(c) else: ones = T.ones_like(c[:, 0]) return -np.log(self.vec.num_types) * ones def sample(self, num_samples=1): indices = random.randint(0, self.vec.num_types, size=(num_samples, )) return [ instance.Instance(c) for c in self.vec.unvectorize_all(indices) ]
def unpickle(self, state, model_class=SimpleLasagneModel): if isinstance(state, tuple) and isinstance(state[-1], SymbolVectorizer): self.lang_vec = state[-1] super(BilingualGaussianListenerLearner, self).unpickle(state[:-1], model_class=model_class) else: self.lang_vec = SymbolVectorizer(use_unk=False) self.lang_vec.add_all(['en', 'zh']) super(BilingualGaussianListenerLearner, self).unpickle(state, model_class=model_class)
def _data_to_arrays(self, training_instances, init_vectorizer=False, test=False, inverted=False): xs, ys = super(BilingualGaussianListenerLearner, self)._data_to_arrays( training_instances=training_instances, init_vectorizer=init_vectorizer, test=test, inverted=inverted) langs = [inst.input.split(':', 1)[0] for inst in training_instances] if init_vectorizer: self.lang_vec = SymbolVectorizer(use_unk=False) self.lang_vec.add_all(langs) langs_vec = self.lang_vec.vectorize_all(langs) return xs[:1] + [langs_vec] + xs[1:], ys
class BilingualGaussianListenerLearner(listener.GaussianContextListenerLearner ): def get_extra_vars(self): id_tag = (self.id + '/') if self.id else '' return [T.ivector(id_tag + 'languages')] def _data_to_arrays(self, training_instances, init_vectorizer=False, test=False, inverted=False): xs, ys = super(BilingualGaussianListenerLearner, self)._data_to_arrays( training_instances=training_instances, init_vectorizer=init_vectorizer, test=test, inverted=inverted) langs = [inst.input.split(':', 1)[0] for inst in training_instances] if init_vectorizer: self.lang_vec = SymbolVectorizer(use_unk=False) self.lang_vec.add_all(langs) langs_vec = self.lang_vec.vectorize_all(langs) return xs[:1] + [langs_vec] + xs[1:], ys def get_embedding_layer(self, l_in, extra_vars): language = extra_vars[0] context_vars = extra_vars[1:] id_tag = (self.id + '/') if self.id else '' l_lang = InputLayer(shape=(None, ), input_var=language, name=id_tag + 'lang_input') if self.options.bilingual_en_embed_file: en_embeddings = load_embeddings( self.options.bilingual_en_embed_file, self.seq_vec) en_embed_size = en_embeddings.shape[1] else: en_embeddings = Normal() en_embed_size = self.options.bilingual_embed_size if self.options.bilingual_zh_embed_file: zh_embeddings = load_embeddings( self.options.bilingual_zh_embed_file, self.seq_vec) zh_embed_size = zh_embeddings.shape[1] else: zh_embeddings = Normal() zh_embed_size = self.options.bilingual_embed_size l_en = EmbeddingLayer(l_in, input_size=len(self.seq_vec.tokens), output_size=en_embed_size, W=en_embeddings, name=id_tag + 'desc_embed_en') l_en_transformed = dimshuffle(l_en, (0, 2, 1)) l_en_transformed = NINLayer(l_en_transformed, num_units=self.options.listener_cell_size, nonlinearity=None, name=id_tag + 'desc_embed_en_transformed') l_en_transformed = dimshuffle(l_en_transformed, (0, 2, 1)) l_zh = EmbeddingLayer(l_in, input_size=len(self.seq_vec.tokens), output_size=zh_embed_size, W=zh_embeddings, name=id_tag + 'desc_embed_zh') l_zh_transformed = dimshuffle(l_zh, (0, 2, 1)) l_zh_transformed = NINLayer(l_zh_transformed, num_units=self.options.listener_cell_size, nonlinearity=None, name=id_tag + 'desc_embed_zh_transformed') l_zh_transformed = dimshuffle(l_zh_transformed, (0, 2, 1)) l_merged = SwitchLayer(l_lang, [l_en_transformed, l_zh_transformed], name=id_tag + 'desc_embed_switch') return (l_merged, context_vars) def unpickle(self, state, model_class=SimpleLasagneModel): if isinstance(state, tuple) and isinstance(state[-1], SymbolVectorizer): self.lang_vec = state[-1] super(BilingualGaussianListenerLearner, self).unpickle(state[:-1], model_class=model_class) else: self.lang_vec = SymbolVectorizer(use_unk=False) self.lang_vec.add_all(['en', 'zh']) super(BilingualGaussianListenerLearner, self).unpickle(state, model_class=model_class)
def __init__(self, id=None): super(AtomicSpeakerLearner, self).__init__(id=id) self.seq_vec = SymbolVectorizer() color_repr = COLOR_REPRS[self.options.speaker_color_repr] self.color_vec = color_repr(self.options.speaker_color_resolution, hsv=self.options.speaker_hsv)
class AtomicSpeakerLearner(NeuralLearner): ''' A speaker that learns to produce descriptions as indivisible symbols (as opposed to word-by-word sequences) given colors. ''' def __init__(self, id=None): super(AtomicSpeakerLearner, self).__init__(id=id) self.seq_vec = SymbolVectorizer() color_repr = COLOR_REPRS[self.options.speaker_color_repr] self.color_vec = color_repr(self.options.speaker_color_resolution, hsv=self.options.speaker_hsv) def predict_and_score(self, eval_instances, random=False, verbosity=0): predictions = [] scores = [] batches = iterators.iter_batches(eval_instances, self.options.speaker_eval_batch_size) num_batches = (len(eval_instances) - 1) // self.options.speaker_eval_batch_size + 1 if self.options.verbosity + verbosity >= 2: print('Testing') if self.options.verbosity + verbosity >= 1: progress.start_task('Eval batch', num_batches) for batch_num, batch in enumerate(batches): if self.options.verbosity + verbosity >= 1: progress.progress(batch_num) batch = list(batch) xs, (y, ) = self._data_to_arrays(batch, test=True) probs = self.model.predict(xs) if random: indices = sample(probs) else: indices = probs.argmax(axis=1) predictions.extend(self.seq_vec.unvectorize_all(indices)) scores_arr = np.log(probs[np.arange(len(batch)), y]) scores.extend(scores_arr.tolist()) if self.options.verbosity + verbosity >= 1: progress.end_task() if self.options.verbosity >= 9: print('%s %ss:') % (self.id, 'sample' if random else 'prediction') for inst, prediction in zip(eval_instances, predictions): print('%s -> %s' % (repr(inst.input), repr(prediction))) return predictions, scores def _data_to_arrays(self, training_instances, init_vectorizer=False, test=False, inverted=False): get_i, get_o = (lambda inst: inst.input), (lambda inst: inst.output) get_color, get_desc = (get_o, get_i) if inverted else (get_i, get_o) if init_vectorizer: self.seq_vec.add_all(get_desc(inst) for inst in training_instances) sentences = [] colors = [] if self.options.verbosity >= 9: print('%s _data_to_arrays:' % self.id) for i, inst in enumerate(training_instances): desc = get_desc(inst) if desc is None: assert test desc = '<unk>' color = get_color(inst) assert color if self.options.verbosity >= 9: print('%s -> %s' % (repr(desc), repr(color))) sentences.append(desc) colors.append(color) x = self.color_vec.vectorize_all(colors, hsv=True) if len(x.shape) == 1: x = x[:, np.newaxis] y = self.seq_vec.vectorize_all(sentences) if self.options.verbosity >= 9: print('%s x: %s' % (self.id, x)) print('%s y: %s' % (self.id, y)) return [x], [y] def _build_model(self, model_class=SimpleLasagneModel): id_tag = (self.id + '/') if self.id else '' input_vars = self.color_vec.get_input_vars(self.id) target_var = T.ivector(id_tag + 'targets') self.l_out, self.input_layers = self._get_l_out(input_vars) self.loss = categorical_crossentropy self.model = model_class(input_vars, [target_var], self.l_out, loss=self.loss, optimizer=rmsprop, id=self.id) def train_priors(self, training_instances, listener_data=False): prior_class = PRIORS[self.options.speaker_prior] self.prior_emp = prior_class() self.prior_smooth = prior_class() self.prior_emp.train(training_instances, listener_data=listener_data) self.prior_smooth.train(training_instances, listener_data=listener_data) def _get_l_out(self, input_vars): id_tag = (self.id + '/') if self.id else '' cell_size = self.options.speaker_cell_size or self.seq_vec.num_types l_color_repr, color_inputs = self.color_vec.get_input_layer( input_vars, recurrent_length=0, cell_size=cell_size, id=self.id) l_hidden_color = l_color_repr for i in range(1, self.options.speaker_hidden_color_layers + 1): l_hidden_color = NINLayer( l_hidden_color, num_units=cell_size, nonlinearity=NONLINEARITIES[self.options.speaker_nonlinearity], name=id_tag + 'hidden_color%d' % i) l_hidden_color = l_hidden_color if self.options.speaker_cell_size == 0: l_scores = l_color_repr # BiasLayer(l_color_repr, name=id_tag + 'bias') else: if self.options.speaker_dropout > 0.0: l_color_drop = DropoutLayer(l_hidden_color, p=self.options.speaker_dropout, name=id_tag + 'color_drop') else: l_color_drop = l_hidden_color l_hidden = DenseLayer( l_color_drop, num_units=self.options.speaker_cell_size, nonlinearity=NONLINEARITIES[self.options.speaker_nonlinearity], name=id_tag + 'hidden') if self.options.speaker_dropout > 0.0: l_hidden_drop = DropoutLayer(l_hidden, p=self.options.speaker_dropout, name=id_tag + 'hidden_drop') else: l_hidden_drop = l_hidden l_scores = DenseLayer(l_hidden_drop, num_units=self.seq_vec.num_types, nonlinearity=None, name=id_tag + 'scores') l_out = NonlinearityLayer(l_scores, nonlinearity=softmax, name=id_tag + 'softmax') return l_out, color_inputs def sample_prior_smooth(self, num_samples): return self.prior_smooth.sample(num_samples)
def __init__(self): self.vec = SymbolVectorizer()
def __init__(self, id=None): super(AtomicListenerLearner, self).__init__(id=id) self.seq_vec = SymbolVectorizer()
class AtomicListenerLearner(ListenerLearner): ''' An single-embedding listener (guesses colors from descriptions, where the descriptions are treated as indivisible symbols). ''' def __init__(self, id=None): super(AtomicListenerLearner, self).__init__(id=id) self.seq_vec = SymbolVectorizer() def _data_to_arrays(self, training_instances, init_vectorizer=False, test=False, inverted=False): get_i, get_o = (lambda inst: inst.input), (lambda inst: inst.output) get_desc, get_color = (get_o, get_i) if inverted else (get_i, get_o) if init_vectorizer: self.seq_vec.add_all(get_desc(inst) for inst in training_instances) sentences = [] colors = [] if self.options.verbosity >= 9: print('%s _data_to_arrays:' % self.id) for i, inst in enumerate(training_instances): self.word_counts.update([get_desc(inst)]) desc = get_desc(inst) color = get_color(inst) if not color: assert test color = (0.0, 0.0, 0.0) if self.options.verbosity >= 9: print('%s -> %s' % (repr(desc), repr(color))) sentences.append(desc) colors.append(color) x = np.zeros((len(sentences), ), dtype=np.int32) y = np.zeros((len(sentences), ), dtype=np.int32) for i, sentence in enumerate(sentences): x[i] = self.seq_vec.vectorize(sentence) y[i] = self.color_vec.vectorize(colors[i], hsv=True) return [x], [y] def _build_model(self, model_class=SimpleLasagneModel): id_tag = (self.id + '/') if self.id else '' input_var = T.ivector(id_tag + 'inputs') target_var = T.ivector(id_tag + 'targets') self.l_out, self.input_layers = self._get_l_out([input_var]) self.loss = categorical_crossentropy self.model = model_class([input_var], [target_var], self.l_out, loss=self.loss, optimizer=rmsprop, id=self.id) def train_priors(self, training_instances, listener_data=False): prior_class = PRIORS[self.options.listener_prior] self.prior_emp = prior_class( ) # TODO: accurate values for the empirical prior self.prior_smooth = prior_class() self.prior_emp.train(training_instances, listener_data=listener_data) self.prior_smooth.train(training_instances, listener_data=listener_data) def _get_l_out(self, input_vars): id_tag = (self.id + '/') if self.id else '' input_var = input_vars[0] l_in = InputLayer(shape=(None, ), input_var=input_var, name=id_tag + 'desc_input') embed_size = self.options.listener_cell_size or self.color_vec.num_types l_in_embed = EmbeddingLayer(l_in, input_size=len(self.seq_vec.tokens), output_size=embed_size, name=id_tag + 'desc_embed') if self.options.listener_cell_size == 0: l_scores = l_in_embed # BiasLayer(l_in_embed, name=id_tag + 'bias') else: l_hidden = DenseLayer(l_in_embed, num_units=self.options.listener_cell_size, nonlinearity=NONLINEARITIES[ self.options.listener_nonlinearity], name=id_tag + 'hidden') if self.options.listener_dropout > 0.0: l_hidden_drop = DropoutLayer(l_hidden, p=self.options.listener_dropout, name=id_tag + 'hidden_drop') else: l_hidden_drop = l_hidden l_scores = DenseLayer(l_hidden_drop, num_units=self.color_vec.num_types, nonlinearity=None, name=id_tag + 'scores') l_out = NonlinearityLayer(l_scores, nonlinearity=softmax, name=id_tag + 'out') return l_out, [l_in] def sample_prior_smooth(self, num_samples): return self.prior_smooth.sample(num_samples)
def __init__(self, id=None): super(AtomicListenerLearner, self).__init__(id=id) self.seq_vec = SymbolVectorizer()
class AtomicListenerLearner(ListenerLearner): ''' An single-embedding listener (guesses colors from descriptions, where the descriptions are treated as indivisible symbols). ''' def __init__(self, id=None): super(AtomicListenerLearner, self).__init__(id=id) self.seq_vec = SymbolVectorizer() def _data_to_arrays(self, training_instances, init_vectorizer=False, test=False, inverted=False): get_i, get_o = (lambda inst: inst.input), (lambda inst: inst.output) get_desc, get_color = (get_o, get_i) if inverted else (get_i, get_o) if init_vectorizer: self.seq_vec.add_all(get_desc(inst) for inst in training_instances) sentences = [] colors = [] if self.options.verbosity >= 9: print('%s _data_to_arrays:' % self.id) for i, inst in enumerate(training_instances): self.word_counts.update([get_desc(inst)]) desc = get_desc(inst) color = get_color(inst) if not color: assert test color = (0.0, 0.0, 0.0) if self.options.verbosity >= 9: print('%s -> %s' % (repr(desc), repr(color))) sentences.append(desc) colors.append(color) x = np.zeros((len(sentences),), dtype=np.int32) y = np.zeros((len(sentences),), dtype=np.int32) for i, sentence in enumerate(sentences): x[i] = self.seq_vec.vectorize(sentence) y[i] = self.color_vec.vectorize(colors[i], hsv=True) return [x], [y] def _build_model(self, model_class=SimpleLasagneModel): id_tag = (self.id + '/') if self.id else '' input_var = T.ivector(id_tag + 'inputs') target_var = T.ivector(id_tag + 'targets') self.l_out, self.input_layers = self._get_l_out([input_var]) self.loss = categorical_crossentropy self.model = model_class([input_var], [target_var], self.l_out, loss=self.loss, optimizer=rmsprop, id=self.id) def train_priors(self, training_instances, listener_data=False): prior_class = PRIORS[self.options.listener_prior] self.prior_emp = prior_class() # TODO: accurate values for the empirical prior self.prior_smooth = prior_class() self.prior_emp.train(training_instances, listener_data=listener_data) self.prior_smooth.train(training_instances, listener_data=listener_data) def _get_l_out(self, input_vars): id_tag = (self.id + '/') if self.id else '' input_var = input_vars[0] l_in = InputLayer(shape=(None,), input_var=input_var, name=id_tag + 'desc_input') embed_size = self.options.listener_cell_size or self.color_vec.num_types l_in_embed = EmbeddingLayer(l_in, input_size=len(self.seq_vec.tokens), output_size=embed_size, name=id_tag + 'desc_embed') if self.options.listener_cell_size == 0: l_scores = l_in_embed # BiasLayer(l_in_embed, name=id_tag + 'bias') else: l_hidden = DenseLayer(l_in_embed, num_units=self.options.listener_cell_size, nonlinearity=NONLINEARITIES[self.options.listener_nonlinearity], name=id_tag + 'hidden') if self.options.listener_dropout > 0.0: l_hidden_drop = DropoutLayer(l_hidden, p=self.options.listener_dropout, name=id_tag + 'hidden_drop') else: l_hidden_drop = l_hidden l_scores = DenseLayer(l_hidden_drop, num_units=self.color_vec.num_types, nonlinearity=None, name=id_tag + 'scores') l_out = NonlinearityLayer(l_scores, nonlinearity=softmax, name=id_tag + 'out') return l_out, [l_in] def sample_prior_smooth(self, num_samples): return self.prior_smooth.sample(num_samples)
def __init__(self): self.vec = SymbolVectorizer()