def _build(self, img, z_tm1, temporal_state, prior_state, sample_from_prior=False, do_generate=False):
"""
:param img: `Tensor` of shape `[B, H, W, C]` representing images.
:param z_tm1: 4-tuple of [what, where, presence, presence_logit] at the previous time-step.
:param temporal_state: Hidden state of the temporal RNN.
:param prior_state: Hidden state of the prior RNN.
:param sample_from_prior: see Discovery class.
:param do_generate: see Discovery class.
:return: AttrDict of results.
"""
presence_tm1 = z_tm1[2]
prior_stats, prior_state = self._prior(z_tm1, prior_state)
hidden_outputs, num_steps, delta_what, delta_where = self._ssm(img, z_tm1, temporal_state)
hidden_outputs, log_probs = self._compute_log_probs(presence_tm1, hidden_outputs, prior_stats, delta_what,
delta_where, sample_from_prior = sample_from_prior,
do_generate = do_generate)
outputs = orderedattrdict.AttrDict(
prior_stats = prior_stats,
prior_state = prior_state,
hidden_outputs = hidden_outputs,
num_steps = num_steps,
)
outputs.update(hidden_outputs)
outputs.update(log_probs)
return outputs
def _build(self, img, n_present_obj, conditioning_from_prop=None, time_step=0,
prior_conditioning=None, sample_from_prior=False, do_generate=False):
"""
:param img: `Tensor` of shape `[B, H, W, C]` of images.
:param n_present_obj: `Tensor` of integer numbers (but dtype=tf.float32) of shape `[B]` representing
number of already present object for every data example in the batch.
:param conditioning_from_prop: `Tensor` of shape `[B, n]` representing summury of propagated latent variables.
:param time_step: Scalar tensor.
:param prior_conditioning: `Tensor` of shape `[B, m]`, additional conditioning passed to prior distributions.
:param sample_from_prior: Boolean; if True samples from the prior instead of the inference network.
:param do_generate: if True, replaces sample from the posterior with a sample from the prior. Useful for
conditional generation from a few observations (i.e. prediction).
:return: AttrDict of results.
"""
max_disc_steps = self._n_steps - n_present_obj
hidden_outputs, num_steps = self._discover(img, max_disc_steps, conditioning_from_prop, time_step)
hidden_outputs, log_probs = self._compute_log_probs(hidden_outputs, num_steps, time_step,
conditioning_from_prop, prior_conditioning,
sample_from_prior, do_generate)
outputs = orderedattrdict.AttrDict(
hidden_outputs=hidden_outputs,
num_steps=num_steps,
max_disc_steps=max_disc_steps
)
outputs.update(hidden_outputs)
outputs.update(log_probs)
return outputs
def _compute_log_probs(self, hidden_outputs, num_steps, time_step,
conditioning_from_prop, prior_conditioning,
sample_from_prior, do_generate):
"""Computes log probabilities of latent variables from discovery under both q and p.
"""
where_conditioning = tf.concat(
(conditioning_from_prop, prior_conditioning), -1)
priors = self._make_priors(time_step, prior_conditioning)
if sample_from_prior:
what = priors[0].sample(hidden_outputs.what.shape)
where = priors[1].sample(hidden_outputs.where.shape[:-1],
conditioning=where_conditioning)
pres_sample = priors[2].sample()
pres_sample = tf.sequence_mask(pres_sample,
maxlen=self._n_steps,
dtype=tf.float32)
pres_sample = tf.expand_dims(pres_sample, -1) * 0.
dg = tf.to_float(do_generate)
ndg = 1. - dg
hidden_outputs.what = dg * what + ndg * hidden_outputs.what
hidden_outputs.where = dg * where + ndg * hidden_outputs.where
hidden_outputs.presence = dg * pres_sample + ndg * hidden_outputs.presence
squeezed_presence = tf.squeeze(hidden_outputs.presence, -1)
# outputs; short name due to frequent usage
o = orderedattrdict.AttrDict()
posteriors = self._make_posteriors(hidden_outputs)
samples = [hidden_outputs.what, hidden_outputs.where, num_steps]
posterior_log_probs = [
distrib.log_prob(sample)
for (distrib, sample) in zip(posteriors, samples)
]
kwargs = [dict(), {'conditioning': where_conditioning}, dict()]
prior_log_probs = [
distrib.log_prob(sample, **kw)
for (distrib, sample, kw) in zip(priors, samples, kwargs)
]
for probs in (posterior_log_probs, prior_log_probs):
for i in xrange(2):
probs[i] = tf.reduce_sum(probs[i], -1) * squeezed_presence
o.q_z_given_x = self._reduce_prob(posterior_log_probs)
o.p_z = self._reduce_prob(prior_log_probs)
for i, k in enumerate('what where num_step'.split()):
o['{}_log_prob'.format(k)] = posterior_log_probs[i]
o['{}_prior_log_prob'.format(k)] = prior_log_probs[i]
o.num_steps_prob = posteriors[-1].probs
return hidden_outputs, o
def _compute_log_probs(self, presence_tm1, hidden_outputs, prior_stats, delta_what,
delta_where, sample_from_prior=False, do_generate=False):
"""Computes log probabilities, see Discovery class.
"""
presence = tf.squeeze(hidden_outputs.presence, -1)
presence_tm1 = tf.squeeze(presence_tm1, -1)
o = orderedattrdict.AttrDict()
posteriors = self._make_posteriors(hidden_outputs)
priors = self._prior.make_distribs(prior_stats)
samples = [delta_what, delta_where, presence]
#if we have already trained the VAE, given the data we can sample from prior to generate the sequences
if sample_from_prior:
samples = [p.sample() for p in priors]
dg = tf.to_float(do_generate)
ndg = 1. - dg
hidden_outputs.what = dg * samples[0] + ndg * hidden_outputs.what
hidden_outputs.where = dg * samples[1] + ndg * hidden_outputs.where
pres = tf.to_float(tf.expand_dims(samples[2], -1))
hidden_outputs.presence = dg * pres + ndg * hidden_outputs.presence
#computing log probabilities for the posterior distribution of what where and presence latent variables
posterior_log_probs = [distrib.log_prob(sample) for (distrib, sample) in zip(posteriors, samples)]
samples = [hidden_outputs.what, hidden_outputs.where, presence]
#computing prior log probabilities for where what and presence samples we got from the inference part
prior_log_probs = [distrib.log_prob(sample) for (distrib, sample) in zip(priors, samples)]
#getting propogation probabilities by taking the probabilities only for the objects that were present at the
#previous timestep
o.prop_prob = tf.exp(posterior_log_probs[-1]) * presence_tm1
#summing log probabilities along dimensions
for probs in (posterior_log_probs, prior_log_probs):
for i in xrange(2):
if probs[i].shape.ndims == 3:
probs[i] = tf.reduce_sum(probs[i], -1)
probs[i] = probs[i] * presence_tm1 * presence
probs[-1] = tf.reduce_sum(probs[-1] * presence_tm1, -1)
#getting final q(approximate posterior)and p(prior) probabilities
o.q_z_given_x = self._reduce_prob(posterior_log_probs)
o.p_z = self._reduce_prob(prior_log_probs)
for i, k in enumerate('what where prop'.split()):
o['{}_log_prob'.format(k)] = posterior_log_probs[i]
o['{}_prior_log_prob'.format(k)] = prior_log_probs[i]
return hidden_outputs, o
def _compute_log_probs(self,
presence_tm1,
hidden_outputs,
prior_stats,
delta_what,
delta_where,
sample_from_prior=False,
do_generate=False):
"""Computes log probabilities, see Discovery class.
"""
presence = tf.squeeze(hidden_outputs.presence, -1)
presence_tm1 = tf.squeeze(presence_tm1, -1)
o = orderedattrdict.AttrDict()
posteriors = self._make_posteriors(hidden_outputs)
priors = self._prior.make_distribs(prior_stats)
samples = [delta_what, delta_where, presence]
if sample_from_prior:
samples = [p.sample() for p in priors]
dg = tf.to_float(do_generate)
ndg = 1. - dg
hidden_outputs.what = dg * samples[0] + ndg * hidden_outputs.what
hidden_outputs.where = dg * samples[1] + ndg * hidden_outputs.where
pres = tf.to_float(tf.expand_dims(samples[2], -1))
hidden_outputs.presence = dg * pres + ndg * hidden_outputs.presence
posterior_log_probs = [
distrib.log_prob(sample)
for (distrib, sample) in zip(posteriors, samples)
]
samples = [hidden_outputs.what, hidden_outputs.where, presence]
prior_log_probs = [
distrib.log_prob(sample)
for (distrib, sample) in zip(priors, samples)
]
o.prop_prob = tf.exp(posterior_log_probs[-1]) * presence_tm1
for probs in (posterior_log_probs, prior_log_probs):
for i in xrange(2):
if probs[i].shape.ndims == 3:
probs[i] = tf.reduce_sum(probs[i], -1)
probs[i] = probs[i] * presence_tm1 * presence
probs[-1] = tf.reduce_sum(probs[-1] * presence_tm1, -1)
o.q_z_given_x = self._reduce_prob(posterior_log_probs)
o.p_z = self._reduce_prob(prior_log_probs)
for i, k in enumerate('what where prop'.split()):
o['{}_log_prob'.format(k)] = posterior_log_probs[i]
o['{}_prior_log_prob'.format(k)] = prior_log_probs[i]
return hidden_outputs, o
def outputs_by_name(cls, hidden_outputs, stack=True):
if stack:
hidden_outputs = nested.stack(hidden_outputs, axis=1)
d = orderedattrdict.AttrDict()
for n, o in zip(cls._output_names, hidden_outputs):
d[n] = o
return d
def parse_response(message):
result = orderedattrdict.AttrDict()
remainder = message
next_unknown_field_num = 0
for name, format_code in fields:
if not name:
name = 'unknown_%02d_%s' % (next_unknown_field_num, format_code)
next_unknown_field_num += 1
result[name], remainder = get_field(remainder, format_code)
print name, len(remainder), result[name]
return result
def _build(self, img, z_tm1, temporal_hidden_state, prop_prior_state, highest_used_ids, prev_ids,
time_step=0, sample_from_prior=False, do_generate=False):
"""
:param img: `Tensor` of size `[B, H, W, C]` representing images.
:param z_tm1: 4-tuple of [what, where, presence, presence_logit] from previous time-step.
:param temporal_hidden_state: Hidden state of the time_cell.
:param prop_prior_state: Hidden state of the propagation prior.
:param highest_used_ids: Integer `Tensor` of size `[B]`, where each entry represent the highest used object
ID for the corresponding data example in the batch.
:param prev_ids: Integer `Tensor` of size `[B, n_steps]`, with each entry representing object ID of the
corresponding object at the previous time-step.
:param time_step: Integer.
:param sample_from_prior: Boolean; if True samples from the prior instead of the inference network.
:param do_generate: if True, replaces sample from the posterior with a sample from the prior. Useful for
conditional generation from a few observations (i.e. prediction).
:return: AttrDict of results.
"""
#get batch size
batch_size = int(img.shape[0])
#call propogate and discover
propogate_outputs, discover_outputs = self._propagate_and_discover(img, z_tm1, temporal_hidden_state, prop_prior_state,
time_step, sample_from_prior, do_generate)
#call choose_latents
latents = self._choose_latents(batch_size, propogate_output, discover_output, highest_used_ids, prev_ids)
#store into orderedattrdict.AttrDict as "outputs"
outputs = orderedattrdict.AttrDict(
hidden_outputs=hidden_outputs,
obj_ids=obj_ids,
z_t=z_t,
prop_prior_state=prop_prior_state,
ids=obj_ids,
highest_used_ids=highest_used_ids,
prop=prop_output,
disc=disc_output,
temporal_hidden_state=temporal_hidden_state,
presence_log_prob=prop_output.prop_log_prob + disc_output.num_step_log_prob,
p_z=disc_output.p_z + prop_output.p_z,
q_z_given_x=disc_output.q_z_given_x + prop_output.q_z_given_x
)
#update the outputs with hidden outputs
outputs.update(hidden_outputs)
#update number of steps
outputs.num_steps = tf.reduce_sum(tf.squeeze(outputs.presence, -1), -1)
#return outputs
return outputs
def _build(self,
z_tm1,
temporal_hidden_state,
prop_prior_state,
highest_used_ids,
prev_ids,
timestep=0,
sample_from_prior=False):
"""
:param z_tm1: 4-tuple of [what, where, presence, presence_logit] from previous time-step.
:param temporal_hidden_state: Hidden state of the time_cell.
:param prop_prior_state: Hidden state of the propagation prior.
:param highest_used_ids: Integer `Tensor` of size `[B]`, where each entry represent the highest used object
ID for the corresponding data example in the batch.
:param prev_ids: Integer `Tensor` of size `[B, n_steps]`, with each entry representing object ID of the
corresponding object at the previous time-step.
:param timestep: Integer.
:param sample_from_prior: Boolean; if True samples from the prior instead of the inference network.
:return: AttrDict of results.
"""
batch_size = self._discover._cell.batch_size
prop_output, disc_output = \
self._propagate_and_discover(z_tm1, temporal_hidden_state, prop_prior_state,
timestep, sample_from_prior)
hidden_outputs, z_t, obj_ids, prop_prior_state, temporal_hidden_state, highest_used_ids = \
self._choose_latents(batch_size, prop_output, disc_output, highest_used_ids, prev_ids)
outputs = orderedattrdict.AttrDict(
hidden_outputs=hidden_outputs,
obj_ids=obj_ids,
z_t=z_t,
prop_prior_state=prop_prior_state,
ids=obj_ids,
highest_used_ids=highest_used_ids,
prop=prop_output,
disc=disc_output,
temporal_hidden_state=temporal_hidden_state,
presence_log_prob=prop_output.prop_log_prob +
disc_output.num_step_log_prob,
p_z=disc_output.p_z + prop_output.p_z,
q_z_given_x=disc_output.q_z_given_x + prop_output.q_z_given_x)
outputs.update(hidden_outputs)
outputs.num_steps = tf.reduce_sum(tf.squeeze(outputs.presence, -1), -1)
return outputs
def _build(self,
timestep,
z_tm1,
temporal_state,
prior_state,
sample_from_prior=False):
"""
:param z_tm1: 4-tuple of [what, where, presence, presence_logit] at the previous time-step.
:param temporal_state: Hidden state of the temporal RNN.
:param prior_state: Hidden state of the prior RNN.
:param sample_from_prior: see Discovery class.
:return: AttrDict of results.
"""
presence_tm1 = z_tm1[2]
prior_stats, prior_state = self._prior(z_tm1, prior_state)
hidden_outputs, num_steps, delta_what, delta_where = self._propagate(
timestep, z_tm1, temporal_state)
hidden_outputs, log_probs = self._compute_log_probs(
presence_tm1,
hidden_outputs,
prior_stats,
delta_what,
delta_where,
sample_from_prior=sample_from_prior)
outputs = orderedattrdict.AttrDict(
prior_stats=prior_stats,
prior_state=prior_state,
hidden_outputs=hidden_outputs,
num_steps=num_steps,
)
outputs.update(hidden_outputs)
outputs.update(log_probs)
return outputs
