def _build(self, img, z_tm1, temporal_state, prior_state, sample_from_prior=False, do_generate=False): """ :param img: `Tensor` of shape `[B, H, W, C]` representing images. :param z_tm1: 4-tuple of [what, where, presence, presence_logit] at the previous time-step. :param temporal_state: Hidden state of the temporal RNN. :param prior_state: Hidden state of the prior RNN. :param sample_from_prior: see Discovery class. :param do_generate: see Discovery class. :return: AttrDict of results. """ presence_tm1 = z_tm1[2] prior_stats, prior_state = self._prior(z_tm1, prior_state) hidden_outputs, num_steps, delta_what, delta_where = self._ssm(img, z_tm1, temporal_state) hidden_outputs, log_probs = self._compute_log_probs(presence_tm1, hidden_outputs, prior_stats, delta_what, delta_where, sample_from_prior = sample_from_prior, do_generate = do_generate) outputs = orderedattrdict.AttrDict( prior_stats = prior_stats, prior_state = prior_state, hidden_outputs = hidden_outputs, num_steps = num_steps, ) outputs.update(hidden_outputs) outputs.update(log_probs) return outputs
def _build(self, img, n_present_obj, conditioning_from_prop=None, time_step=0, prior_conditioning=None, sample_from_prior=False, do_generate=False): """ :param img: `Tensor` of shape `[B, H, W, C]` of images. :param n_present_obj: `Tensor` of integer numbers (but dtype=tf.float32) of shape `[B]` representing number of already present object for every data example in the batch. :param conditioning_from_prop: `Tensor` of shape `[B, n]` representing summury of propagated latent variables. :param time_step: Scalar tensor. :param prior_conditioning: `Tensor` of shape `[B, m]`, additional conditioning passed to prior distributions. :param sample_from_prior: Boolean; if True samples from the prior instead of the inference network. :param do_generate: if True, replaces sample from the posterior with a sample from the prior. Useful for conditional generation from a few observations (i.e. prediction). :return: AttrDict of results. """ max_disc_steps = self._n_steps - n_present_obj hidden_outputs, num_steps = self._discover(img, max_disc_steps, conditioning_from_prop, time_step) hidden_outputs, log_probs = self._compute_log_probs(hidden_outputs, num_steps, time_step, conditioning_from_prop, prior_conditioning, sample_from_prior, do_generate) outputs = orderedattrdict.AttrDict( hidden_outputs=hidden_outputs, num_steps=num_steps, max_disc_steps=max_disc_steps ) outputs.update(hidden_outputs) outputs.update(log_probs) return outputs
def _compute_log_probs(self, hidden_outputs, num_steps, time_step, conditioning_from_prop, prior_conditioning, sample_from_prior, do_generate): """Computes log probabilities of latent variables from discovery under both q and p. """ where_conditioning = tf.concat( (conditioning_from_prop, prior_conditioning), -1) priors = self._make_priors(time_step, prior_conditioning) if sample_from_prior: what = priors[0].sample(hidden_outputs.what.shape) where = priors[1].sample(hidden_outputs.where.shape[:-1], conditioning=where_conditioning) pres_sample = priors[2].sample() pres_sample = tf.sequence_mask(pres_sample, maxlen=self._n_steps, dtype=tf.float32) pres_sample = tf.expand_dims(pres_sample, -1) * 0. dg = tf.to_float(do_generate) ndg = 1. - dg hidden_outputs.what = dg * what + ndg * hidden_outputs.what hidden_outputs.where = dg * where + ndg * hidden_outputs.where hidden_outputs.presence = dg * pres_sample + ndg * hidden_outputs.presence squeezed_presence = tf.squeeze(hidden_outputs.presence, -1) # outputs; short name due to frequent usage o = orderedattrdict.AttrDict() posteriors = self._make_posteriors(hidden_outputs) samples = [hidden_outputs.what, hidden_outputs.where, num_steps] posterior_log_probs = [ distrib.log_prob(sample) for (distrib, sample) in zip(posteriors, samples) ] kwargs = [dict(), {'conditioning': where_conditioning}, dict()] prior_log_probs = [ distrib.log_prob(sample, **kw) for (distrib, sample, kw) in zip(priors, samples, kwargs) ] for probs in (posterior_log_probs, prior_log_probs): for i in xrange(2): probs[i] = tf.reduce_sum(probs[i], -1) * squeezed_presence o.q_z_given_x = self._reduce_prob(posterior_log_probs) o.p_z = self._reduce_prob(prior_log_probs) for i, k in enumerate('what where num_step'.split()): o['{}_log_prob'.format(k)] = posterior_log_probs[i] o['{}_prior_log_prob'.format(k)] = prior_log_probs[i] o.num_steps_prob = posteriors[-1].probs return hidden_outputs, o
def _compute_log_probs(self, presence_tm1, hidden_outputs, prior_stats, delta_what, delta_where, sample_from_prior=False, do_generate=False): """Computes log probabilities, see Discovery class. """ presence = tf.squeeze(hidden_outputs.presence, -1) presence_tm1 = tf.squeeze(presence_tm1, -1) o = orderedattrdict.AttrDict() posteriors = self._make_posteriors(hidden_outputs) priors = self._prior.make_distribs(prior_stats) samples = [delta_what, delta_where, presence] #if we have already trained the VAE, given the data we can sample from prior to generate the sequences if sample_from_prior: samples = [p.sample() for p in priors] dg = tf.to_float(do_generate) ndg = 1. - dg hidden_outputs.what = dg * samples[0] + ndg * hidden_outputs.what hidden_outputs.where = dg * samples[1] + ndg * hidden_outputs.where pres = tf.to_float(tf.expand_dims(samples[2], -1)) hidden_outputs.presence = dg * pres + ndg * hidden_outputs.presence #computing log probabilities for the posterior distribution of what where and presence latent variables posterior_log_probs = [distrib.log_prob(sample) for (distrib, sample) in zip(posteriors, samples)] samples = [hidden_outputs.what, hidden_outputs.where, presence] #computing prior log probabilities for where what and presence samples we got from the inference part prior_log_probs = [distrib.log_prob(sample) for (distrib, sample) in zip(priors, samples)] #getting propogation probabilities by taking the probabilities only for the objects that were present at the #previous timestep o.prop_prob = tf.exp(posterior_log_probs[-1]) * presence_tm1 #summing log probabilities along dimensions for probs in (posterior_log_probs, prior_log_probs): for i in xrange(2): if probs[i].shape.ndims == 3: probs[i] = tf.reduce_sum(probs[i], -1) probs[i] = probs[i] * presence_tm1 * presence probs[-1] = tf.reduce_sum(probs[-1] * presence_tm1, -1) #getting final q(approximate posterior)and p(prior) probabilities o.q_z_given_x = self._reduce_prob(posterior_log_probs) o.p_z = self._reduce_prob(prior_log_probs) for i, k in enumerate('what where prop'.split()): o['{}_log_prob'.format(k)] = posterior_log_probs[i] o['{}_prior_log_prob'.format(k)] = prior_log_probs[i] return hidden_outputs, o
def _compute_log_probs(self, presence_tm1, hidden_outputs, prior_stats, delta_what, delta_where, sample_from_prior=False, do_generate=False): """Computes log probabilities, see Discovery class. """ presence = tf.squeeze(hidden_outputs.presence, -1) presence_tm1 = tf.squeeze(presence_tm1, -1) o = orderedattrdict.AttrDict() posteriors = self._make_posteriors(hidden_outputs) priors = self._prior.make_distribs(prior_stats) samples = [delta_what, delta_where, presence] if sample_from_prior: samples = [p.sample() for p in priors] dg = tf.to_float(do_generate) ndg = 1. - dg hidden_outputs.what = dg * samples[0] + ndg * hidden_outputs.what hidden_outputs.where = dg * samples[1] + ndg * hidden_outputs.where pres = tf.to_float(tf.expand_dims(samples[2], -1)) hidden_outputs.presence = dg * pres + ndg * hidden_outputs.presence posterior_log_probs = [ distrib.log_prob(sample) for (distrib, sample) in zip(posteriors, samples) ] samples = [hidden_outputs.what, hidden_outputs.where, presence] prior_log_probs = [ distrib.log_prob(sample) for (distrib, sample) in zip(priors, samples) ] o.prop_prob = tf.exp(posterior_log_probs[-1]) * presence_tm1 for probs in (posterior_log_probs, prior_log_probs): for i in xrange(2): if probs[i].shape.ndims == 3: probs[i] = tf.reduce_sum(probs[i], -1) probs[i] = probs[i] * presence_tm1 * presence probs[-1] = tf.reduce_sum(probs[-1] * presence_tm1, -1) o.q_z_given_x = self._reduce_prob(posterior_log_probs) o.p_z = self._reduce_prob(prior_log_probs) for i, k in enumerate('what where prop'.split()): o['{}_log_prob'.format(k)] = posterior_log_probs[i] o['{}_prior_log_prob'.format(k)] = prior_log_probs[i] return hidden_outputs, o
def outputs_by_name(cls, hidden_outputs, stack=True): if stack: hidden_outputs = nested.stack(hidden_outputs, axis=1) d = orderedattrdict.AttrDict() for n, o in zip(cls._output_names, hidden_outputs): d[n] = o return d
def parse_response(message): result = orderedattrdict.AttrDict() remainder = message next_unknown_field_num = 0 for name, format_code in fields: if not name: name = 'unknown_%02d_%s' % (next_unknown_field_num, format_code) next_unknown_field_num += 1 result[name], remainder = get_field(remainder, format_code) print name, len(remainder), result[name] return result
def _build(self, img, z_tm1, temporal_hidden_state, prop_prior_state, highest_used_ids, prev_ids, time_step=0, sample_from_prior=False, do_generate=False): """ :param img: `Tensor` of size `[B, H, W, C]` representing images. :param z_tm1: 4-tuple of [what, where, presence, presence_logit] from previous time-step. :param temporal_hidden_state: Hidden state of the time_cell. :param prop_prior_state: Hidden state of the propagation prior. :param highest_used_ids: Integer `Tensor` of size `[B]`, where each entry represent the highest used object ID for the corresponding data example in the batch. :param prev_ids: Integer `Tensor` of size `[B, n_steps]`, with each entry representing object ID of the corresponding object at the previous time-step. :param time_step: Integer. :param sample_from_prior: Boolean; if True samples from the prior instead of the inference network. :param do_generate: if True, replaces sample from the posterior with a sample from the prior. Useful for conditional generation from a few observations (i.e. prediction). :return: AttrDict of results. """ #get batch size batch_size = int(img.shape[0]) #call propogate and discover propogate_outputs, discover_outputs = self._propagate_and_discover(img, z_tm1, temporal_hidden_state, prop_prior_state, time_step, sample_from_prior, do_generate) #call choose_latents latents = self._choose_latents(batch_size, propogate_output, discover_output, highest_used_ids, prev_ids) #store into orderedattrdict.AttrDict as "outputs" outputs = orderedattrdict.AttrDict( hidden_outputs=hidden_outputs, obj_ids=obj_ids, z_t=z_t, prop_prior_state=prop_prior_state, ids=obj_ids, highest_used_ids=highest_used_ids, prop=prop_output, disc=disc_output, temporal_hidden_state=temporal_hidden_state, presence_log_prob=prop_output.prop_log_prob + disc_output.num_step_log_prob, p_z=disc_output.p_z + prop_output.p_z, q_z_given_x=disc_output.q_z_given_x + prop_output.q_z_given_x ) #update the outputs with hidden outputs outputs.update(hidden_outputs) #update number of steps outputs.num_steps = tf.reduce_sum(tf.squeeze(outputs.presence, -1), -1) #return outputs return outputs
def _build(self, z_tm1, temporal_hidden_state, prop_prior_state, highest_used_ids, prev_ids, timestep=0, sample_from_prior=False): """ :param z_tm1: 4-tuple of [what, where, presence, presence_logit] from previous time-step. :param temporal_hidden_state: Hidden state of the time_cell. :param prop_prior_state: Hidden state of the propagation prior. :param highest_used_ids: Integer `Tensor` of size `[B]`, where each entry represent the highest used object ID for the corresponding data example in the batch. :param prev_ids: Integer `Tensor` of size `[B, n_steps]`, with each entry representing object ID of the corresponding object at the previous time-step. :param timestep: Integer. :param sample_from_prior: Boolean; if True samples from the prior instead of the inference network. :return: AttrDict of results. """ batch_size = self._discover._cell.batch_size prop_output, disc_output = \ self._propagate_and_discover(z_tm1, temporal_hidden_state, prop_prior_state, timestep, sample_from_prior) hidden_outputs, z_t, obj_ids, prop_prior_state, temporal_hidden_state, highest_used_ids = \ self._choose_latents(batch_size, prop_output, disc_output, highest_used_ids, prev_ids) outputs = orderedattrdict.AttrDict( hidden_outputs=hidden_outputs, obj_ids=obj_ids, z_t=z_t, prop_prior_state=prop_prior_state, ids=obj_ids, highest_used_ids=highest_used_ids, prop=prop_output, disc=disc_output, temporal_hidden_state=temporal_hidden_state, presence_log_prob=prop_output.prop_log_prob + disc_output.num_step_log_prob, p_z=disc_output.p_z + prop_output.p_z, q_z_given_x=disc_output.q_z_given_x + prop_output.q_z_given_x) outputs.update(hidden_outputs) outputs.num_steps = tf.reduce_sum(tf.squeeze(outputs.presence, -1), -1) return outputs
def _build(self, timestep, z_tm1, temporal_state, prior_state, sample_from_prior=False): """ :param z_tm1: 4-tuple of [what, where, presence, presence_logit] at the previous time-step. :param temporal_state: Hidden state of the temporal RNN. :param prior_state: Hidden state of the prior RNN. :param sample_from_prior: see Discovery class. :return: AttrDict of results. """ presence_tm1 = z_tm1[2] prior_stats, prior_state = self._prior(z_tm1, prior_state) hidden_outputs, num_steps, delta_what, delta_where = self._propagate( timestep, z_tm1, temporal_state) hidden_outputs, log_probs = self._compute_log_probs( presence_tm1, hidden_outputs, prior_stats, delta_what, delta_where, sample_from_prior=sample_from_prior) outputs = orderedattrdict.AttrDict( prior_stats=prior_stats, prior_state=prior_state, hidden_outputs=hidden_outputs, num_steps=num_steps, ) outputs.update(hidden_outputs) outputs.update(log_probs) return outputs