class LanguageBitBilingualSpeakerLearner(BilingualSpeakerLearner):
def get_extra_vars(self):
id_tag = (self.id + '/') if self.id else ''
return [T.ivector(id_tag + 'languages')]
def _data_to_arrays(self,
training_instances,
init_vectorizer=False,
test=False,
inverted=False):
xs, ys = super(LanguageBitBilingualSpeakerLearner,
self)._data_to_arrays(
training_instances=training_instances,
init_vectorizer=init_vectorizer,
test=test,
inverted=inverted)
langs = [inst.output.split(':', 1)[0] for inst in training_instances]
if init_vectorizer:
self.lang_vec = SymbolVectorizer(use_unk=False)
self.lang_vec.add_all(langs)
langs_vec = self.lang_vec.vectorize_all(langs)
return xs[:-2] + [langs_vec] + xs[-2:], ys
def modify_context(self, l_context_repr, extra_vars):
language = extra_vars[0]
id_tag = (self.id + '/') if self.id else ''
print('l_context_repr: {}'.format(l_context_repr.output_shape))
l_lang_input = InputLayer(shape=(None, ),
input_var=language,
name=id_tag + 'lang_input')
l_lang_embed = EmbeddingLayer(
l_lang_input,
input_size=len(self.lang_vec.tokens),
output_size=self.options.bilingual_lang_embed_size,
name=id_tag + 'lang_embed')
print('l_lang_embed: {}'.format(l_lang_embed.output_shape))
l_modified_context = ConcatLayer([l_lang_embed, l_context_repr])
print('l_modified_context: {}'.format(l_modified_context.output_shape))
return (l_modified_context, [l_lang_input])
def __getstate__(self):
super_state = super(LanguageBitBilingualSpeakerLearner,
self).__getstate__()
return super_state + (self.lang_vec, )
def unpickle(self, state, model_class=SimpleLasagneModel):
if isinstance(
state,
dict) and 'quickpickle' in state and state['quickpickle']:
super(LanguageBitBilingualSpeakerLearner,
self).unpickle(state, model_class=model_class)
else:
self.lang_vec = state[-1]
super(LanguageBitBilingualSpeakerLearner,
self).unpickle(state[:-1], model_class=model_class)
class AtomicUniformPrior(object):
'''
>>> p = AtomicUniformPrior()
>>> p.train([instance.Instance('blue')])
>>> p.sample(3) # doctest: +ELLIPSIS
[Instance('...', None), Instance('...', None), Instance('...', None)]
'''
def __init__(self):
self.vec = SymbolVectorizer()
def train(self, training_instances, listener_data=True):
self.vec.add_all([inst.input if listener_data else inst.output
for inst in training_instances])
def apply(self, input_vars):
c = input_vars[0]
if c.ndim == 1:
ones = T.ones_like(c)
else:
ones = T.ones_like(c[:, 0])
return -np.log(self.vec.num_types) * ones
def sample(self, num_samples=1):
indices = random.randint(0, self.vec.num_types, size=(num_samples,))
return [instance.Instance(c) for c in self.vec.unvectorize_all(indices)]
class AtomicUniformPrior(object):
'''
>>> p = AtomicUniformPrior()
>>> p.train([instance.Instance('blue')])
>>> p.sample(3) # doctest: +ELLIPSIS
[Instance('...', None), Instance('...', None), Instance('...', None)]
'''
def __init__(self):
self.vec = SymbolVectorizer()
def train(self, training_instances, listener_data=True):
self.vec.add_all([
inst.input if listener_data else inst.output
for inst in training_instances
])
def apply(self, input_vars):
c = input_vars[0]
if c.ndim == 1:
ones = T.ones_like(c)
else:
ones = T.ones_like(c[:, 0])
return -np.log(self.vec.num_types) * ones
def sample(self, num_samples=1):
indices = random.randint(0, self.vec.num_types, size=(num_samples, ))
return [
instance.Instance(c) for c in self.vec.unvectorize_all(indices)
]
def unpickle(self, state, model_class=SimpleLasagneModel):
if isinstance(state, tuple) and isinstance(state[-1],
SymbolVectorizer):
self.lang_vec = state[-1]
super(BilingualGaussianListenerLearner,
self).unpickle(state[:-1], model_class=model_class)
else:
self.lang_vec = SymbolVectorizer(use_unk=False)
self.lang_vec.add_all(['en', 'zh'])
super(BilingualGaussianListenerLearner,
self).unpickle(state, model_class=model_class)
def _data_to_arrays(self,
training_instances,
init_vectorizer=False,
test=False,
inverted=False):
xs, ys = super(BilingualGaussianListenerLearner, self)._data_to_arrays(
training_instances=training_instances,
init_vectorizer=init_vectorizer,
test=test,
inverted=inverted)
langs = [inst.input.split(':', 1)[0] for inst in training_instances]
if init_vectorizer:
self.lang_vec = SymbolVectorizer(use_unk=False)
self.lang_vec.add_all(langs)
langs_vec = self.lang_vec.vectorize_all(langs)
return xs[:1] + [langs_vec] + xs[1:], ys
class BilingualGaussianListenerLearner(listener.GaussianContextListenerLearner
):
def get_extra_vars(self):
id_tag = (self.id + '/') if self.id else ''
return [T.ivector(id_tag + 'languages')]
def _data_to_arrays(self,
training_instances,
init_vectorizer=False,
test=False,
inverted=False):
xs, ys = super(BilingualGaussianListenerLearner, self)._data_to_arrays(
training_instances=training_instances,
init_vectorizer=init_vectorizer,
test=test,
inverted=inverted)
langs = [inst.input.split(':', 1)[0] for inst in training_instances]
if init_vectorizer:
self.lang_vec = SymbolVectorizer(use_unk=False)
self.lang_vec.add_all(langs)
langs_vec = self.lang_vec.vectorize_all(langs)
return xs[:1] + [langs_vec] + xs[1:], ys
def get_embedding_layer(self, l_in, extra_vars):
language = extra_vars[0]
context_vars = extra_vars[1:]
id_tag = (self.id + '/') if self.id else ''
l_lang = InputLayer(shape=(None, ),
input_var=language,
name=id_tag + 'lang_input')
if self.options.bilingual_en_embed_file:
en_embeddings = load_embeddings(
self.options.bilingual_en_embed_file, self.seq_vec)
en_embed_size = en_embeddings.shape[1]
else:
en_embeddings = Normal()
en_embed_size = self.options.bilingual_embed_size
if self.options.bilingual_zh_embed_file:
zh_embeddings = load_embeddings(
self.options.bilingual_zh_embed_file, self.seq_vec)
zh_embed_size = zh_embeddings.shape[1]
else:
zh_embeddings = Normal()
zh_embed_size = self.options.bilingual_embed_size
l_en = EmbeddingLayer(l_in,
input_size=len(self.seq_vec.tokens),
output_size=en_embed_size,
W=en_embeddings,
name=id_tag + 'desc_embed_en')
l_en_transformed = dimshuffle(l_en, (0, 2, 1))
l_en_transformed = NINLayer(l_en_transformed,
num_units=self.options.listener_cell_size,
nonlinearity=None,
name=id_tag + 'desc_embed_en_transformed')
l_en_transformed = dimshuffle(l_en_transformed, (0, 2, 1))
l_zh = EmbeddingLayer(l_in,
input_size=len(self.seq_vec.tokens),
output_size=zh_embed_size,
W=zh_embeddings,
name=id_tag + 'desc_embed_zh')
l_zh_transformed = dimshuffle(l_zh, (0, 2, 1))
l_zh_transformed = NINLayer(l_zh_transformed,
num_units=self.options.listener_cell_size,
nonlinearity=None,
name=id_tag + 'desc_embed_zh_transformed')
l_zh_transformed = dimshuffle(l_zh_transformed, (0, 2, 1))
l_merged = SwitchLayer(l_lang, [l_en_transformed, l_zh_transformed],
name=id_tag + 'desc_embed_switch')
return (l_merged, context_vars)
def unpickle(self, state, model_class=SimpleLasagneModel):
if isinstance(state, tuple) and isinstance(state[-1],
SymbolVectorizer):
self.lang_vec = state[-1]
super(BilingualGaussianListenerLearner,
self).unpickle(state[:-1], model_class=model_class)
else:
self.lang_vec = SymbolVectorizer(use_unk=False)
self.lang_vec.add_all(['en', 'zh'])
super(BilingualGaussianListenerLearner,
self).unpickle(state, model_class=model_class)
def __init__(self, id=None):
super(AtomicSpeakerLearner, self).__init__(id=id)
self.seq_vec = SymbolVectorizer()
color_repr = COLOR_REPRS[self.options.speaker_color_repr]
self.color_vec = color_repr(self.options.speaker_color_resolution,
hsv=self.options.speaker_hsv)
class AtomicSpeakerLearner(NeuralLearner):
'''
A speaker that learns to produce descriptions as indivisible symbols (as
opposed to word-by-word sequences) given colors.
'''
def __init__(self, id=None):
super(AtomicSpeakerLearner, self).__init__(id=id)
self.seq_vec = SymbolVectorizer()
color_repr = COLOR_REPRS[self.options.speaker_color_repr]
self.color_vec = color_repr(self.options.speaker_color_resolution,
hsv=self.options.speaker_hsv)
def predict_and_score(self, eval_instances, random=False, verbosity=0):
predictions = []
scores = []
batches = iterators.iter_batches(eval_instances,
self.options.speaker_eval_batch_size)
num_batches = (len(eval_instances) -
1) // self.options.speaker_eval_batch_size + 1
if self.options.verbosity + verbosity >= 2:
print('Testing')
if self.options.verbosity + verbosity >= 1:
progress.start_task('Eval batch', num_batches)
for batch_num, batch in enumerate(batches):
if self.options.verbosity + verbosity >= 1:
progress.progress(batch_num)
batch = list(batch)
xs, (y, ) = self._data_to_arrays(batch, test=True)
probs = self.model.predict(xs)
if random:
indices = sample(probs)
else:
indices = probs.argmax(axis=1)
predictions.extend(self.seq_vec.unvectorize_all(indices))
scores_arr = np.log(probs[np.arange(len(batch)), y])
scores.extend(scores_arr.tolist())
if self.options.verbosity + verbosity >= 1:
progress.end_task()
if self.options.verbosity >= 9:
print('%s %ss:') % (self.id, 'sample' if random else 'prediction')
for inst, prediction in zip(eval_instances, predictions):
print('%s -> %s' % (repr(inst.input), repr(prediction)))
return predictions, scores
def _data_to_arrays(self,
training_instances,
init_vectorizer=False,
test=False,
inverted=False):
get_i, get_o = (lambda inst: inst.input), (lambda inst: inst.output)
get_color, get_desc = (get_o, get_i) if inverted else (get_i, get_o)
if init_vectorizer:
self.seq_vec.add_all(get_desc(inst) for inst in training_instances)
sentences = []
colors = []
if self.options.verbosity >= 9:
print('%s _data_to_arrays:' % self.id)
for i, inst in enumerate(training_instances):
desc = get_desc(inst)
if desc is None:
assert test
desc = '<unk>'
color = get_color(inst)
assert color
if self.options.verbosity >= 9:
print('%s -> %s' % (repr(desc), repr(color)))
sentences.append(desc)
colors.append(color)
x = self.color_vec.vectorize_all(colors, hsv=True)
if len(x.shape) == 1:
x = x[:, np.newaxis]
y = self.seq_vec.vectorize_all(sentences)
if self.options.verbosity >= 9:
print('%s x: %s' % (self.id, x))
print('%s y: %s' % (self.id, y))
return [x], [y]
def _build_model(self, model_class=SimpleLasagneModel):
id_tag = (self.id + '/') if self.id else ''
input_vars = self.color_vec.get_input_vars(self.id)
target_var = T.ivector(id_tag + 'targets')
self.l_out, self.input_layers = self._get_l_out(input_vars)
self.loss = categorical_crossentropy
self.model = model_class(input_vars, [target_var],
self.l_out,
loss=self.loss,
optimizer=rmsprop,
id=self.id)
def train_priors(self, training_instances, listener_data=False):
prior_class = PRIORS[self.options.speaker_prior]
self.prior_emp = prior_class()
self.prior_smooth = prior_class()
self.prior_emp.train(training_instances, listener_data=listener_data)
self.prior_smooth.train(training_instances,
listener_data=listener_data)
def _get_l_out(self, input_vars):
id_tag = (self.id + '/') if self.id else ''
cell_size = self.options.speaker_cell_size or self.seq_vec.num_types
l_color_repr, color_inputs = self.color_vec.get_input_layer(
input_vars, recurrent_length=0, cell_size=cell_size, id=self.id)
l_hidden_color = l_color_repr
for i in range(1, self.options.speaker_hidden_color_layers + 1):
l_hidden_color = NINLayer(
l_hidden_color,
num_units=cell_size,
nonlinearity=NONLINEARITIES[self.options.speaker_nonlinearity],
name=id_tag + 'hidden_color%d' % i)
l_hidden_color = l_hidden_color
if self.options.speaker_cell_size == 0:
l_scores = l_color_repr # BiasLayer(l_color_repr, name=id_tag + 'bias')
else:
if self.options.speaker_dropout > 0.0:
l_color_drop = DropoutLayer(l_hidden_color,
p=self.options.speaker_dropout,
name=id_tag + 'color_drop')
else:
l_color_drop = l_hidden_color
l_hidden = DenseLayer(
l_color_drop,
num_units=self.options.speaker_cell_size,
nonlinearity=NONLINEARITIES[self.options.speaker_nonlinearity],
name=id_tag + 'hidden')
if self.options.speaker_dropout > 0.0:
l_hidden_drop = DropoutLayer(l_hidden,
p=self.options.speaker_dropout,
name=id_tag + 'hidden_drop')
else:
l_hidden_drop = l_hidden
l_scores = DenseLayer(l_hidden_drop,
num_units=self.seq_vec.num_types,
nonlinearity=None,
name=id_tag + 'scores')
l_out = NonlinearityLayer(l_scores,
nonlinearity=softmax,
name=id_tag + 'softmax')
return l_out, color_inputs
def sample_prior_smooth(self, num_samples):
return self.prior_smooth.sample(num_samples)
def __init__(self):
self.vec = SymbolVectorizer()
def __init__(self, id=None):
super(AtomicListenerLearner, self).__init__(id=id)
self.seq_vec = SymbolVectorizer()
class AtomicListenerLearner(ListenerLearner):
'''
An single-embedding listener (guesses colors from descriptions, where
the descriptions are treated as indivisible symbols).
'''
def __init__(self, id=None):
super(AtomicListenerLearner, self).__init__(id=id)
self.seq_vec = SymbolVectorizer()
def _data_to_arrays(self,
training_instances,
init_vectorizer=False,
test=False,
inverted=False):
get_i, get_o = (lambda inst: inst.input), (lambda inst: inst.output)
get_desc, get_color = (get_o, get_i) if inverted else (get_i, get_o)
if init_vectorizer:
self.seq_vec.add_all(get_desc(inst) for inst in training_instances)
sentences = []
colors = []
if self.options.verbosity >= 9:
print('%s _data_to_arrays:' % self.id)
for i, inst in enumerate(training_instances):
self.word_counts.update([get_desc(inst)])
desc = get_desc(inst)
color = get_color(inst)
if not color:
assert test
color = (0.0, 0.0, 0.0)
if self.options.verbosity >= 9:
print('%s -> %s' % (repr(desc), repr(color)))
sentences.append(desc)
colors.append(color)
x = np.zeros((len(sentences), ), dtype=np.int32)
y = np.zeros((len(sentences), ), dtype=np.int32)
for i, sentence in enumerate(sentences):
x[i] = self.seq_vec.vectorize(sentence)
y[i] = self.color_vec.vectorize(colors[i], hsv=True)
return [x], [y]
def _build_model(self, model_class=SimpleLasagneModel):
id_tag = (self.id + '/') if self.id else ''
input_var = T.ivector(id_tag + 'inputs')
target_var = T.ivector(id_tag + 'targets')
self.l_out, self.input_layers = self._get_l_out([input_var])
self.loss = categorical_crossentropy
self.model = model_class([input_var], [target_var],
self.l_out,
loss=self.loss,
optimizer=rmsprop,
id=self.id)
def train_priors(self, training_instances, listener_data=False):
prior_class = PRIORS[self.options.listener_prior]
self.prior_emp = prior_class(
) # TODO: accurate values for the empirical prior
self.prior_smooth = prior_class()
self.prior_emp.train(training_instances, listener_data=listener_data)
self.prior_smooth.train(training_instances,
listener_data=listener_data)
def _get_l_out(self, input_vars):
id_tag = (self.id + '/') if self.id else ''
input_var = input_vars[0]
l_in = InputLayer(shape=(None, ),
input_var=input_var,
name=id_tag + 'desc_input')
embed_size = self.options.listener_cell_size or self.color_vec.num_types
l_in_embed = EmbeddingLayer(l_in,
input_size=len(self.seq_vec.tokens),
output_size=embed_size,
name=id_tag + 'desc_embed')
if self.options.listener_cell_size == 0:
l_scores = l_in_embed # BiasLayer(l_in_embed, name=id_tag + 'bias')
else:
l_hidden = DenseLayer(l_in_embed,
num_units=self.options.listener_cell_size,
nonlinearity=NONLINEARITIES[
self.options.listener_nonlinearity],
name=id_tag + 'hidden')
if self.options.listener_dropout > 0.0:
l_hidden_drop = DropoutLayer(l_hidden,
p=self.options.listener_dropout,
name=id_tag + 'hidden_drop')
else:
l_hidden_drop = l_hidden
l_scores = DenseLayer(l_hidden_drop,
num_units=self.color_vec.num_types,
nonlinearity=None,
name=id_tag + 'scores')
l_out = NonlinearityLayer(l_scores,
nonlinearity=softmax,
name=id_tag + 'out')
return l_out, [l_in]
def sample_prior_smooth(self, num_samples):
return self.prior_smooth.sample(num_samples)
def __init__(self, id=None):
super(AtomicListenerLearner, self).__init__(id=id)
self.seq_vec = SymbolVectorizer()
class AtomicListenerLearner(ListenerLearner):
'''
An single-embedding listener (guesses colors from descriptions, where
the descriptions are treated as indivisible symbols).
'''
def __init__(self, id=None):
super(AtomicListenerLearner, self).__init__(id=id)
self.seq_vec = SymbolVectorizer()
def _data_to_arrays(self, training_instances,
init_vectorizer=False, test=False, inverted=False):
get_i, get_o = (lambda inst: inst.input), (lambda inst: inst.output)
get_desc, get_color = (get_o, get_i) if inverted else (get_i, get_o)
if init_vectorizer:
self.seq_vec.add_all(get_desc(inst) for inst in training_instances)
sentences = []
colors = []
if self.options.verbosity >= 9:
print('%s _data_to_arrays:' % self.id)
for i, inst in enumerate(training_instances):
self.word_counts.update([get_desc(inst)])
desc = get_desc(inst)
color = get_color(inst)
if not color:
assert test
color = (0.0, 0.0, 0.0)
if self.options.verbosity >= 9:
print('%s -> %s' % (repr(desc), repr(color)))
sentences.append(desc)
colors.append(color)
x = np.zeros((len(sentences),), dtype=np.int32)
y = np.zeros((len(sentences),), dtype=np.int32)
for i, sentence in enumerate(sentences):
x[i] = self.seq_vec.vectorize(sentence)
y[i] = self.color_vec.vectorize(colors[i], hsv=True)
return [x], [y]
def _build_model(self, model_class=SimpleLasagneModel):
id_tag = (self.id + '/') if self.id else ''
input_var = T.ivector(id_tag + 'inputs')
target_var = T.ivector(id_tag + 'targets')
self.l_out, self.input_layers = self._get_l_out([input_var])
self.loss = categorical_crossentropy
self.model = model_class([input_var], [target_var], self.l_out,
loss=self.loss, optimizer=rmsprop, id=self.id)
def train_priors(self, training_instances, listener_data=False):
prior_class = PRIORS[self.options.listener_prior]
self.prior_emp = prior_class() # TODO: accurate values for the empirical prior
self.prior_smooth = prior_class()
self.prior_emp.train(training_instances, listener_data=listener_data)
self.prior_smooth.train(training_instances, listener_data=listener_data)
def _get_l_out(self, input_vars):
id_tag = (self.id + '/') if self.id else ''
input_var = input_vars[0]
l_in = InputLayer(shape=(None,), input_var=input_var,
name=id_tag + 'desc_input')
embed_size = self.options.listener_cell_size or self.color_vec.num_types
l_in_embed = EmbeddingLayer(l_in, input_size=len(self.seq_vec.tokens),
output_size=embed_size,
name=id_tag + 'desc_embed')
if self.options.listener_cell_size == 0:
l_scores = l_in_embed # BiasLayer(l_in_embed, name=id_tag + 'bias')
else:
l_hidden = DenseLayer(l_in_embed, num_units=self.options.listener_cell_size,
nonlinearity=NONLINEARITIES[self.options.listener_nonlinearity],
name=id_tag + 'hidden')
if self.options.listener_dropout > 0.0:
l_hidden_drop = DropoutLayer(l_hidden, p=self.options.listener_dropout,
name=id_tag + 'hidden_drop')
else:
l_hidden_drop = l_hidden
l_scores = DenseLayer(l_hidden_drop, num_units=self.color_vec.num_types,
nonlinearity=None, name=id_tag + 'scores')
l_out = NonlinearityLayer(l_scores, nonlinearity=softmax, name=id_tag + 'out')
return l_out, [l_in]
def sample_prior_smooth(self, num_samples):
return self.prior_smooth.sample(num_samples)
def __init__(self):
self.vec = SymbolVectorizer()