def build_grid(self, batch):
options = self.options
# for inst in batch:
# for i in range(len(inst.context)):
# for utt in sample_utts(inst.context, i):
# (utt -> inst.context, i)
sampler_inputs = [
instance.Instance(i, None, alt_inputs=inst.alt_outputs)
for inst in batch for i in range(len(inst.alt_outputs))
for _ in range(options.ac_num_samples)
]
context_len = len(batch[0].alt_outputs)
assert len(sampler_inputs) == (len(batch) *
context_len *
options.ac_num_samples), \
'Building grid: inconsistent context length %s' % \
(len(sampler_inputs), len(batch),
context_len, options.ac_num_samples)
outputs = self.sampler.sample(sampler_inputs)
if self.options.verbosity >= 8:
print('S0 samples:')
for inst, output in zip(sampler_inputs, outputs):
print("%s -> %s" % (inst.alt_inputs[inst.input], repr(output)))
outputs = (np.array(outputs).reshape(
len(batch), context_len * options.ac_num_samples).tolist())
return [
instance.Instance(utt, 0, alt_outputs=[(0, 0, 0)] * context_len)
for inst, samples in zip(batch, outputs)
for utt in [inst.input] + samples
]
def build_grid(self, batch, all_utts):
# for inst in batch:
# for j in range(num_sample_sets):
# for i in range(len(inst.context)):
# for utt in sample_utts(inst.context, i):
# (utt -> inst.context, i)
options = self.get_options()
if options.exhaustive_num_samples > 0:
sampler_inputs = [
instance.Instance(i, None, alt_inputs=inst.alt_outputs)
for inst in batch
for _ in range(options.exhaustive_num_sample_sets)
for i in range(len(inst.alt_outputs))
for _ in range(options.exhaustive_num_samples)
]
context_len = len(batch[0].alt_outputs)
assert len(sampler_inputs) == (len(batch) *
options.exhaustive_num_sample_sets *
context_len *
options.exhaustive_num_samples), \
'Building grid: inconsistent context length %s' % \
(len(sampler_inputs), len(batch), options.exhaustive_num_sample_sets,
context_len, options.exhaustive_num_samples)
outputs = self.sampler.sample(sampler_inputs)
if options.exhaustive_reject_duplicates > 0:
dupes = None
for _ in range(options.exhaustive_reject_duplicates):
dupes = self.get_duplicate_indices(
outputs, batch, options.exhaustive_num_sample_sets,
dupes)
if len(dupes) == 0:
break
resample_inputs = np.array(sampler_inputs)[dupes].tolist()
new_outputs = self.sampler.sample(resample_inputs)
for i, output in zip(dupes, new_outputs):
outputs[i] = output
if len(dupes) != 0 and options.verbosity >= 7:
print >> sys.stderr, "Unable to suppress duplicates for instances:"
for i in dupes:
print >> sys.stderr, "{}[{}] -> '{}'".format(
sampler_inputs[i].alt_inputs,
sampler_inputs[i].input, outputs[i])
outputs = (np.array(outputs).reshape(
len(batch), options.exhaustive_num_sample_sets,
context_len * options.exhaustive_num_samples).tolist())
return [
instance.Instance(utt, j, alt_outputs=inst.alt_outputs)
for inst, sample_sets in zip(batch, outputs)
for samples in sample_sets
for j in range(len(inst.alt_outputs))
for utt in [inst.input] + samples
]
else:
return [
instance.Instance(utt, j, alt_outputs=inst.alt_outputs)
for inst in batch for j in range(len(inst.alt_outputs))
for utt in [inst.input] + all_utts
]
def evaluate_l1_eval():
options = config.options(read=True)
grids_path = os.path.join(options.run_dir, 's0_grids.0.jsons.gz')
with gzip.open(grids_path, 'rb') as infile:
grids = [json.loads(line.strip()) for line in infile]
data_path = os.path.join(options.run_dir, 'data.eval.jsons')
with open(data_path, 'r') as infile:
insts = [
instance.Instance(**json.loads(line.strip())) for line in infile
]
assert len(grids) == len(insts), '{} != {}'.format(len(grids), len(insts))
gold_outputs = np.array([inst.output for inst in insts])
s0 = np.array([[np.array(ss['S0']).T for ss in grid['sets']]
for grid in grids])
l1 = compute_l1(s0, alpha=options.alpha)
l1_scores = l1[np.arange(l1.shape[0]), gold_outputs].tolist()
l1_preds = np.argmax(l1, axis=1).tolist()
m = [
metrics.log_likelihood, metrics.log_likelihood_bits,
metrics.perplexity, metrics.accuracy
]
learner = DummyLearner(l1_preds, l1_scores)
results = evaluate.evaluate(learner,
insts,
metrics=m,
split_id='l1_eval',
write_data=False)
output.output_results(results, 'l1_eval')
def run_listener(self,
listener_class=ListenerLearner,
cell='LSTM',
tensorboard=True):
sys.argv = []
options = config.options()
options.train_iters = 2
options.train_epochs = 3
options.listener_cell = cell
options.listener = True
mo = MockOpen(TEST_DIR)
mgfp = mock_get_file_path(TEST_DIR)
with mock.patch('stanza.monitoring.summary.open', mo), \
mock.patch('stanza.monitoring.summary.SummaryWriter', MockSummaryWriter), \
mock.patch('stanza.research.config.open', mo), \
mock.patch('stanza.research.config.get_file_path', mgfp):
listener = listener_class()
train_data = [instance.Instance('green', (0, 255, 0))]
listener.train(train_data)
predictions, scores = listener.predict_and_score(train_data)
# predictions = [(123, 45, 67)]
self.assertIsInstance(predictions, list)
self.assertEqual(len(predictions), 1)
self.assertEqual(len(predictions[0]), 3)
self.assertIsInstance(predictions[0][0], Number)
# scores = [123.456]
self.assertIsInstance(scores, list)
self.assertEqual(len(scores), 1)
self.assertIsInstance(scores[0], float)
if tensorboard:
self.check_tensorboard(mo, mgfp, images=True)
def sample(self, num_samples):
'''
:return: a list of `num_samples` colors sampled uniformly in RGB space,
but expressed as HSV triples.
'''
colors = self.sampler.unvectorize_all(np.zeros(num_samples, dtype=np.int32),
random=True, hsv=True)
return [instance.Instance(c) for c in colors]
def sample(self, num_samples=1):
descs = super(AtomicUniformContextPrior,
self).sample(num_samples=num_samples)
colors = self.uniform_colors.sample(num_samples)
insts = [
instance.Instance(d.input, c.input) for d, c in zip(descs, colors)
]
return color_instances.reference_game(insts,
color_instances.uniform,
listener=True)
def sample(self, num_samples=1):
indices = np.array([[
sample(self.counts.get_value() * 1.0 / self.total.get_value())
for _t in range(self.vec.max_len)
] for _s in range(num_samples)],
dtype=np.int32)
return [
instance.Instance(' '.join(strip_invalid_tokens(s)))
for s in self.vec.unvectorize_all(indices)
]
def on_iter_end(self, step, writer):
most_common = [desc for desc, count in self.word_counts.most_common(10)]
insts = [instance.Instance(input=desc) for desc in most_common]
xs, (y,) = self._data_to_arrays(insts, test=True)
probs = self.model.predict(xs)
for i, desc in enumerate(most_common):
dist = probs[i, :]
for image, channel in zip(self.color_vec.visualize_distribution(dist), '012'):
writer.log_image(step, '%s/%s/%s' % (self.id, desc, channel), image)
super(ListenerLearner, self).on_iter_end(step, writer)
def write_metric_for_split(output, run_dir, split, metric_name):
filename = '%s.%s.jsons' % (metric_name, split)
learner = Learner()
learner.num_params = output.results['%s.num_params' % split]
metric_func = metrics.METRICS[metric_name]
if output.data[0].keys() == ['error']:
data_insts = []
else:
data_insts = (instance.Instance(**d) for d in output.data)
metric_scores = metric_func(data_insts, output.predictions, output.scores,
learner)
config.dump(metric_scores, filename, lines=True)
def get_context_speaker_utts(self, base_samples):
from fields import get_context
contexts = [get_context(inst, self.options.listener) for inst in base_samples]
context_insts = [instance.Instance(c, None)
for context in contexts
for c in context]
utts = self.speaker_model.predict(context_insts, random=True)
grouped_utts = []
utt_iter = iter(utts)
for context in contexts:
grouped_utts.append([next(utt_iter) for _ in range(len(context))])
return grouped_utts
def get_trial_data(dir_output, size, tag):
try:
use_source = 'alt_inputs' in dir_output.data[0]['source']
except (ValueError, KeyError):
use_source = False
src = lambda inst: (inst['source'] if use_source else inst)
return [
instance.Instance(pred,
src(inst)['input'],
alt_outputs=src(inst)['alt_inputs'],
source=tag) for inst, pred in
zip(dir_output.data[:size], dir_output.predictions[:size])
]
def build_insts(orig_inst, grid):
# num_sample_sets * context_len * num_alt_utts
num_sample_sets = len(grid['sets'])
context_len = len(orig_inst['alt_outputs'])
num_alt_utts = len(grid['sets'][0]['utts'])
new_insts = []
for s, ss in enumerate(grid['sets']):
for t in range(len(orig_inst['alt_outputs'])):
for u, utt in enumerate(ss['utts']):
new_insts.append(
instance.Instance(input=t,
alt_inputs=orig_inst['alt_outputs'],
output=utt))
return new_insts, (num_sample_sets, context_len, num_alt_utts)
def run_speaker(self,
speaker_class,
cell='LSTM',
color_repr='buckets',
tensorboard=True,
images=False):
sys.argv = []
options = config.options()
options.train_iters = 2
options.train_epochs = 3
options.speaker_cell = cell
options.speaker_color_repr = color_repr
options.listener = False
mo = MockOpen(TEST_DIR)
mgfp = mock_get_file_path(TEST_DIR)
with mock.patch('stanza.monitoring.summary.open', mo), \
mock.patch('stanza.monitoring.summary.SummaryWriter', MockSummaryWriter), \
mock.patch('stanza.research.config.open', mo), \
mock.patch('stanza.research.config.get_file_path', mgfp):
speaker = speaker_class()
train_data = [instance.Instance((0, 255, 0), 'green')]
speaker.train(train_data)
predictions, scores = speaker.predict_and_score(train_data)
# predictions = ['somestring']
self.assertIsInstance(predictions, list)
self.assertEqual(len(predictions), 1)
self.assertIsInstance(predictions[0], basestring)
# scores = [123.456]
self.assertIsInstance(scores, list)
self.assertEqual(len(scores), 1)
self.assertIsInstance(scores[0], float)
if tensorboard:
self.check_tensorboard(mo, mgfp, images=images)
def evaluate_ak_blending():
options = config.options(read=True)
grids_path = os.path.join(options.run_dir, 's0_grids.0.jsons.gz')
with gzip.open(grids_path, 'rb') as infile:
grids = [json.loads(line.strip()) for line in infile]
data_path = os.path.join(options.run_dir, 'data.eval.jsons')
with open(data_path, 'r') as infile:
insts = [
instance.Instance(**json.loads(line.strip())) for line in infile
]
assert len(grids) == len(insts), '{} != {}'.format(len(grids), len(insts))
gold_outputs = np.array([inst.output for inst in insts])
l0 = np.array([[np.array(ss['L0']).T for ss in grid['sets']]
for grid in grids])
s0 = np.array([[np.array(ss['S0']).T for ss in grid['sets']]
for grid in grids])
if options.additive:
ak = compute_additive(l0,
s0,
bw=options.base_weight,
sw=options.speaker_weight,
alpha_s1=options.alpha,
alpha_l1=options.alpha_l1)
else:
ak = compute_ak(l0,
s0,
bw=options.base_weight,
sw=options.speaker_weight,
alpha=options.alpha,
gamma=options.gamma)
ak_scores = ak[np.arange(ak.shape[0]), gold_outputs].tolist()
ak_preds = np.argmax(ak, axis=1).tolist()
m = [
metrics.log_likelihood, metrics.log_likelihood_bits,
metrics.perplexity, metrics.accuracy
]
learner = DummyLearner(ak_preds,
ak_scores,
params={
'base_weight': options.base_weight,
'speaker_weight': options.speaker_weight,
'alpha': options.alpha,
'alpha_l1': options.alpha_l1,
'gamma': options.gamma,
'additive': options.additive,
})
split_id = '{}_eval'.format(options.blend_name)
results = evaluate.evaluate(learner,
insts,
metrics=m,
split_id=split_id,
write_data=False)
output.output_results(results, split_id)
options_dump = vars(options)
del options_dump['overwrite']
del options_dump['config']
config.dump_pretty(options_dump, split_id + '_config.json')
def get_reprs(self, utts):
insts = [instance.Instance(utt, 0, alt_outputs=[(0, 0, 0)] * 3)
for utt in utts]
xs, (_,) = self._data_to_arrays(insts, test=True)
return self.repr_fn(*xs)
def predict_and_score(self, eval_instances, random=False, verbosity=0):
options = config.options()
predictions = []
scores = []
all_utts = self.base.seq_vec.tokens
sym_vec = vectorizers.SymbolVectorizer()
sym_vec.add_all(all_utts)
prior_scores = self.prior_scores(all_utts)
base_is_listener = (type(self.base) in listener.LISTENERS.values())
true_batch_size = options.listener_eval_batch_size / len(all_utts)
batches = iterators.iter_batches(eval_instances, true_batch_size)
num_batches = (len(eval_instances) - 1) // true_batch_size + 1
if options.verbosity + verbosity >= 2:
print('Testing')
progress.start_task('Eval batch', num_batches)
for batch_num, batch in enumerate(batches):
progress.progress(batch_num)
batch = list(batch)
context = len(
batch[0].alt_inputs) if batch[0].alt_inputs is not None else 0
if context:
output_grid = [
(instance.Instance(utt, color)
if base_is_listener else instance.Instance(color, utt))
for inst in batch for color in inst.alt_inputs
for utt in sym_vec.tokens
]
assert len(output_grid) == context * len(batch) * len(all_utts), \
'Context must be the same number of colors for all examples'
true_indices = np.array([inst.input for inst in batch])
else:
output_grid = [
(instance.Instance(utt, inst.input) if base_is_listener
else instance.Instance(inst.input, utt)) for inst in batch
for utt in sym_vec.tokens
]
true_indices = sym_vec.vectorize_all(
[inst.input for inst in batch])
if len(true_indices.shape) == 2:
# Sequence vectorizer; we're only using single tokens for now.
true_indices = true_indices[:, 0]
scores = self.base.score(output_grid, verbosity=verbosity)
if context:
log_probs = np.array(scores).reshape(
(len(batch), context, len(all_utts)))
orig_log_probs = log_probs[np.arange(len(batch)),
true_indices, :]
# Renormalize over only the context colors, and extract the score of
# the true color.
log_probs -= logsumexp(log_probs, axis=1)[:, np.newaxis, :]
log_probs = log_probs[np.arange(len(batch)), true_indices, :]
else:
log_probs = np.array(scores).reshape(
(len(batch), len(all_utts)))
orig_log_probs = log_probs
assert log_probs.shape == (len(batch), len(all_utts))
# Add in the prior scores, if used (S1 \propto L0 * P)
if prior_scores is not None:
log_probs = log_probs + 0.5 * prior_scores
if options.exhaustive_base_weight:
w = options.exhaustive_base_weight
log_probs = w * orig_log_probs + (1.0 - w) * log_probs
# Normalize across utterances. Note that the listener returns probability
# densities over colors.
log_probs -= logsumexp(log_probs, axis=1)[:, np.newaxis]
if random:
pred_indices = sample(np.exp(log_probs))
else:
pred_indices = np.argmax(log_probs, axis=1)
predictions.extend(sym_vec.unvectorize_all(pred_indices))
scores.extend(log_probs[np.arange(len(batch)),
true_indices].tolist())
progress.end_task()
return predictions, scores
def get_gaussian_params(self, utt):
inst = instance.Instance(utt, 0, alt_outputs=[(0, 0, 0)] * 3)
xs, (_,) = self._data_to_arrays([inst], test=True)
mean, covar, points, scores = self.gaussian_fn(*xs)
return mean[0], covar[0]
def get_trial_data(dir_output, size, tag):
return [
instance.Instance(pred, inst['input'], source=tag) for inst, pred in
zip(dir_output.data[:size], dir_output.predictions[:size])
]
def build_instance(utt, target, context, listener):
if listener:
return instance.Instance(utt, target, alt_outputs=context)
else:
return instance.Instance(target, utt, alt_inputs=context)
def sample(self, num_samples=1):
indices = random.randint(0, self.vec.num_types, size=(num_samples, ))
return [
instance.Instance(c) for c in self.vec.unvectorize_all(indices)
]
def sample(self, num_samples=1):
colors = super(UniformContextPrior, self).sample(num_samples)
insts = [instance.Instance(c.input) for c in colors]
return color_instances.reference_game(insts,
color_instances.uniform_int,
listener=False)
