def test_subsample_gibbs_partitioning(kernel_cls, num_blocks):
def model(obs):
with plate('N', obs.shape[0], subsample_size=100) as idx:
numpyro.sample('x', dist.Normal(0, 1), obs=obs[idx])
obs = random.normal(random.PRNGKey(0), (10000, )) / 100
kernel = HMCECS(kernel_cls(model), num_blocks=num_blocks)
state = kernel.init(random.PRNGKey(1),
10,
None,
model_args=(obs, ),
model_kwargs=None)
gibbs_sites = {'N': jnp.arange(100)}
def potential_fn(z_gibbs, z_hmc):
return kernel.inner_kernel._potential_fn_gen(
obs, _gibbs_sites=z_gibbs)(z_hmc)
gibbs_fn = numpyro.infer.hmc_gibbs._subsample_gibbs_fn(
potential_fn, kernel._plate_sizes, num_blocks)
new_gibbs_sites, _ = gibbs_fn(
random.PRNGKey(2), gibbs_sites, state.hmc_state.z,
state.hmc_state.potential_energy) # accept_prob > .999
block_size = 100 // num_blocks
for name in gibbs_sites:
assert block_size == jnp.not_equal(gibbs_sites[name],
new_gibbs_sites[name]).sum()
def weighted_misclassifications(logits, one_hot_targets, weights=None):
"""Compute weighted error rate over the given batch.
This computes the error rate over a single, potentially padded
minibatch. If the minibatch is padded (that is it contains null examples) it
is assumed that weights is a binary mask where 0 indicates that the example
is null. We assume the trainer will aggregate and divide by number of samples.
Args:
logits: [batch, ..., num_classes] float array.
one_hot_targets: one hot vector of shape [batch, ..., num_classes].
weights: None or array of shape [batch x ...] (rank of one_hot_targets -1).
Returns:
Binary vector indicated which examples are misclassified ([batch, ...]).
"""
if logits.ndim != one_hot_targets.ndim:
raise ValueError(
'Incorrect shapes. Got shape %s logits and %s one_hot_targets' %
(str(logits.shape), str(one_hot_targets.shape)))
preds = jnp.argmax(logits, axis=-1)
targets = jnp.argmax(one_hot_targets, axis=-1)
loss = jnp.not_equal(preds, targets)
if weights is not None:
if weights.ndim != one_hot_targets.ndim - 1:
raise ValueError(
'Incorrect shapes. Got shape %s weights and %s one_hot_targets'
% (str(weights.shape), str(one_hot_targets.shape)))
loss = loss * weights
return loss
def update(i, opt_state, batch, acc_grad, nsamples, base_rng):
params = get_params(opt_state)
bsz = int(math.ceil(batch[0].shape[0] / acc_grad))
first_batch = (batch[0][:bsz], batch[1][:bsz])
rngs = jax.random.split(base_rng, nsamples)
grads = loss_grad_fn(params, first_batch, rngs,
args.kl_coef / train_size)
grad_std = tree_map(lambda bg: jnp.std(bg, 0), grads)
avg_std = jnp.nanmean(ravel_pytree(grad_std)[0])
grads = tree_map(lambda bg: jnp.mean(bg, 0), grads)
grad_snr = tree_multimap(lambda m, sd: jnp.abs(m / sd), grads,
grad_std)
avg_snr = jnp.nanmean(ravel_pytree(grad_snr)[0])
for i in range(1, acc_grad):
batch_i = (batch[0][(i - 1) * bsz:i * bsz],
batch[1][(i - 1) * bsz:i * bsz])
grads_i = loss_grad_fn(params, batch_i, rngs,
args.kl_coef / train_size)
grads_i = tree_map(lambda bg: jnp.mean(bg, 0), grads_i)
grads = tree_multimap(lambda g, g_new: (g * i + g_new) / (i + 1),
grads, grads_i)
pre_update = get_params(opt_state)
post_update = jit(opt_update)(i, grads, opt_state)
assert jnp.not_equal(
ravel_pytree(pre_update)[0],
ravel_pytree(get_params(post_update))[0]).any()
return post_update, avg_std, avg_snr
def _randaugment_inner_for_loop(_, in_args):
"""
Loop body for for randougment.
Args:
i: loop iteration
in_args: loop body arguments
Returns:
updated loop arguments
"""
(image, geometric_transforms, random_key, available_ops, op_probs,
magnitude, cutout_const, translate_const, join_transforms,
default_replace_value) = in_args
random_keys = random.split(random_key, num=8)
random_key = random_keys[0] # keep for next iteration
op_to_select = random.choice(random_keys[1], available_ops, p=op_probs)
mask_value = jnp.where(default_replace_value > 0,
jnp.ones([image.shape[-1]]) * default_replace_value,
random.randint(random_keys[2],
[image.shape[-1]],
minval=-1, maxval=256))
random_magnitude = random.uniform(random_keys[3], [], minval=0.,
maxval=magnitude)
cutout_mask = color_transforms.get_random_cutout_mask(
random_keys[4],
image.shape,
cutout_const)
translate_vals = (random.uniform(random_keys[5], [], minval=0.0,
maxval=1.0) * translate_const,
random.uniform(random_keys[6], [], minval=0.0,
maxval=1.0) * translate_const)
negate = random.randint(random_keys[7], [], minval=0,
maxval=2).astype('bool')
args = level_to_arg(cutout_mask, translate_vals, negate,
random_magnitude, mask_value)
if DEBUG:
print(op_to_select, args[op_to_select])
image, geometric_transform = _apply_ops(image, args, op_to_select)
image, geometric_transform = jax.lax.cond(
jnp.logical_or(join_transforms, jnp.all(
jnp.not_equal(geometric_transform, jnp.identity(4)))),
lambda op: (op[0], op[1]),
lambda op: (transforms.apply_transform(op[0],
op[1],
mask_value=mask_value),
jnp.identity(4)),
(image, geometric_transform)
)
geometric_transforms = jnp.matmul(geometric_transforms, geometric_transform)
return(image, geometric_transforms, random_key, available_ops, op_probs,
magnitude, cutout_const, translate_const, join_transforms,
default_replace_value)
def _compute_xent_loss_helper(
predictions: NestedMap, input_batch: NestedMap,
return_predictions: bool) -> Tuple[Metrics, Dict[str, Any]]:
"""Helper for computing the xent loss for Language model and Sequence model.
Args:
predictions: A `.NestedMap` containing the keys `per_example_argmax`,
`total_loss`, `avg_xent`, `aux_loss`, `total_weight` which corresponds to
the output of the Softmax layer.
input_batch: A `.NestedMap` object containing input tensors which contains
the keys `labels` and `weights` which corresponds to the labels and the
`weights` for each token in the sequence.
return_predictions: Whether to return predictions, which can be more
expensive.
Returns:
- A dict or NestedMap containing str keys and (metric, weight) pairs as
values, where one of the entries is expected to correspond to the loss.
- A dict containing arbitrary tensors describing something about each
training example, where the first dimension of each tensor is the batch
index. The base class just returns an empty dict.
"""
if 'tgt' in input_batch:
labels = input_batch.tgt.labels
if 'paddings' in input_batch.tgt:
weights = 1.0 - input_batch.tgt.paddings
else:
weights = jnp.not_equal(input_batch.tgt.segment_ids, 0)
weights = weights.astype(labels.dtype)
else:
labels = input_batch.labels
weights = input_batch.weights
predicted_labels = predictions.per_example_argmax.astype(labels.dtype)
num_preds = predictions.total_weight
mean_acc = jnp.sum(
(labels == predicted_labels) * weights) / jnp.maximum(num_preds, 1)
metric_weight = jnp.array(num_preds, predictions.avg_xent.dtype)
if hasattr(predictions, 'avg_xent_weight'):
avg_xent_weight = predictions.avg_xent_weight
else:
avg_xent_weight = metric_weight
metrics = NestedMap(
total_loss=(predictions.total_loss, metric_weight),
avg_xent=(predictions.avg_xent, avg_xent_weight),
aux_loss=(predictions.aux_loss, jnp.array(1.0,
predictions.aux_loss.dtype)),
log_pplx=(predictions.avg_xent, avg_xent_weight),
fraction_of_correct_next_step_preds=(mean_acc, metric_weight),
num_predictions=(num_preds, jnp.array(1.0, num_preds.dtype)),
)
per_example_output = NestedMap()
if return_predictions:
per_example_output = predictions
return metrics, per_example_output
def _loss(
self,
theta: hk.Params,
trajectories: util.Transition,
) -> Tuple[jnp.ndarray, Dict[Text, jnp.ndarray]]:
"""Compute vtrace-based actor-critic loss."""
initial_state = jax.tree_map(lambda t: t[0], trajectories.agent_state)
learner_outputs = self._agent.unroll(theta, trajectories.timestep,
initial_state)
v_t = learner_outputs.values[1:]
# Remove bootstrap timestep from non-timesteps.
_, actor_out, _ = jax.tree_map(lambda t: t[:-1], trajectories)
learner_outputs = jax.tree_map(lambda t: t[:-1], learner_outputs)
v_tm1 = learner_outputs.values
# Get the discount, reward, step_type from the *next* timestep.
timestep = jax.tree_map(lambda t: t[1:], trajectories.timestep)
discounts = timestep.discount * self._discount_factor
rewards = timestep.reward
if self._max_abs_reward > 0:
rewards = jnp.clip(rewards, -self._max_abs_reward,
self._max_abs_reward)
# The step is uninteresting if we transitioned LAST -> FIRST.
# timestep corresponds to the *next* time step, so we filter for FIRST.
mask = jnp.not_equal(timestep.step_type, int(dm_env.StepType.FIRST))
mask = mask.astype(jnp.float32)
rhos = rlax.categorical_importance_sampling_ratios(
learner_outputs.policy_logits, actor_out.policy_logits,
actor_out.action)
# vmap vtrace_td_error_and_advantage to take/return [T, B, ...].
vtrace_td_error_and_advantage = jax.vmap(
rlax.vtrace_td_error_and_advantage, in_axes=1, out_axes=1)
vtrace_returns = vtrace_td_error_and_advantage(v_tm1, v_t, rewards,
discounts, rhos)
pg_advs = vtrace_returns.pg_advantage
pg_loss = policy_gradient_loss(learner_outputs.policy_logits,
actor_out.action, pg_advs, mask)
baseline_loss = 0.5 * jnp.sum(jnp.square(vtrace_returns.errors) * mask)
ent_loss = entropy_loss(learner_outputs.policy_logits, mask)
total_loss = pg_loss
total_loss += 0.5 * baseline_loss
total_loss += 0.01 * ent_loss
logs = {}
logs['PG_loss'] = pg_loss
logs['baseline_loss'] = baseline_loss
logs['entropy_loss'] = ent_loss
logs['total_loss'] = total_loss
return total_loss, logs
def loss(self, params: hk.Params, trajs: Transition) -> jnp.ndarray:
"""Computes a loss of trajs wrt params."""
# Re-run the agent over the trajectories.
# Due to https://github.com/google/jax/issues/1459, we use hk.BatchApply
# instead of vmap.
# BatchApply turns the input tensors from [T, B, ...] into [T*B, ...].
# We `functools.partial` params in so it does not get transformed.
net_curried = hk.BatchApply(functools.partial(self._net, params))
learner_logits, baseline_with_bootstrap = net_curried(trajs.timestep)
# Separate the bootstrap from the value estimates.
baseline = baseline_with_bootstrap[:-1]
baseline_tp1 = baseline_with_bootstrap[1:]
# Remove bootstrap timestep from non-observations.
_, actions, behavior_logits = jax.tree_map(lambda t: t[:-1], trajs)
learner_logits = learner_logits[:-1]
# Shift step_type/reward/discount back by one, so that actions match the
# timesteps caused by the action.
timestep = jax.tree_map(lambda t: t[1:], trajs.timestep)
discount = timestep.discount * self._discount
# The step is uninteresting if we transitioned LAST -> FIRST.
mask = jnp.not_equal(timestep.step_type, int(dm_env.StepType.FIRST))
mask = mask.astype(jnp.float32)
# Compute v-trace returns.
vtrace_td_error_and_advantage = jax.vmap(
rlax.vtrace_td_error_and_advantage, in_axes=1, out_axes=1)
rhos = rlax.categorical_importance_sampling_ratios(
learner_logits, behavior_logits, actions)
vtrace_returns = vtrace_td_error_and_advantage(baseline, baseline_tp1,
timestep.reward,
discount, rhos)
# Note that we use mean here, rather than sum as in canonical IMPALA.
# Compute policy gradient loss.
pg_advantage = jax.lax.stop_gradient(vtrace_returns.pg_advantage)
tb_pg_loss_fn = jax.vmap(rlax.policy_gradient_loss,
in_axes=1,
out_axes=0)
pg_loss = tb_pg_loss_fn(learner_logits, actions, pg_advantage, mask)
pg_loss = jnp.mean(pg_loss)
# Baseline loss.
bl_loss = 0.5 * jnp.mean(jnp.square(vtrace_returns.errors) * mask)
# Entropy regularization.
ent_loss_fn = jax.vmap(rlax.entropy_loss, in_axes=1, out_axes=0)
ent_loss = ent_loss_fn(learner_logits, mask)
ent_loss = jnp.mean(ent_loss)
total_loss = pg_loss + 0.5 * bl_loss + 0.01 * ent_loss
return total_loss
def apply(
server_state: mime.ServerState,
clients: Sequence[Tuple[federated_data.ClientId,
client_datasets.ClientDataset, PRNGKey]]
) -> Tuple[mime.ServerState, Mapping[federated_data.ClientId, Any]]:
# Training across clients using fixed optimizer state.
client_diagnostics = {}
client_num_examples = {cid: len(cds) for cid, cds, _ in clients}
batch_clients = [(cid, cds.shuffle_repeat_batch(client_batch_hparams),
crng) for cid, cds, crng in clients]
shared_input = {
'params': server_state.params,
'opt_state': server_state.opt_state
}
# Running weighted mean of client updates.
delta_params_sum = tree_util.tree_zeros_like(server_state.params)
num_examples_sum = 0.
for client_id, delta_params in train_for_each_client(
shared_input, batch_clients):
num_examples = client_num_examples[client_id]
client_diagnostics[client_id] = {
'delta_l2_norm': tree_util.tree_l2_norm(delta_params)
}
if client_delta_clip_norm is not None:
delta_params = tree_util.tree_clip_by_global_norm(
delta_params, client_delta_clip_norm)
client_diagnostics[client_id]['clipped_delta_l2_norm'] = (
tree_util.tree_l2_norm(delta_params))
client_diagnostics[client_id]['clipped'] = jnp.not_equal(
client_diagnostics[client_id]['delta_l2_norm'],
client_diagnostics[client_id]['clipped_delta_l2_norm'])
delta_params_sum = tree_util.tree_add(
delta_params_sum,
tree_util.tree_weight(delta_params, num_examples))
num_examples_sum += num_examples
mean_delta_params = tree_util.tree_inverse_weight(
delta_params_sum, num_examples_sum)
# Compute full-batch gradient at server params on train data.
grads_batch_clients = [(cid, cds.padded_batch(grads_batch_hparams),
crng) for cid, cds, crng in clients]
grads_sum_total, num_sum_total = tree_util.tree_sum(
(co for _, co in grads_for_each_client(server_state.params,
grads_batch_clients)))
server_grads = tree_util.tree_inverse_weight(grads_sum_total,
num_sum_total)
server_state = server_update(server_state, server_grads,
mean_delta_params)
return server_state, client_diagnostics
def test_block_update_partitioning(num_blocks):
plate_size = 10000, 100
plate_sizes = {'N': plate_size}
gibbs_sites = {'N': jnp.arange(plate_size[1])}
gibbs_state = {}
new_gibbs_sites, new_gibbs_state = numpyro.infer.hmc_gibbs._block_update(
plate_sizes, num_blocks, random.PRNGKey(2), gibbs_sites, gibbs_state)
block_size = 100 // num_blocks
for name in gibbs_sites:
assert block_size == jnp.not_equal(gibbs_sites[name],
new_gibbs_sites[name]).sum()
assert gibbs_state == new_gibbs_state
def test_subsample_gibbs_partitioning(kernel_cls, num_blocks):
def model(obs):
with plate('N', obs.shape[0], subsample_size=100) as idx:
numpyro.sample('x', dist.Normal(0, 1), obs=obs[idx])
obs = random.normal(random.PRNGKey(0), (10000,)) / 100
kernel = HMCECS(kernel_cls(model), num_blocks=num_blocks)
hmc_state = kernel.init(random.PRNGKey(1), 10, None, model_args=(obs,), model_kwargs=None)
gibbs_sites = {'N': jnp.arange(100)}
gibbs_fn = kernel._gibbs_fn
new_gibbs_sites = gibbs_fn(random.PRNGKey(2), gibbs_sites, hmc_state.z) # accept_prob > .999
block_size = 100 // num_blocks
for name in gibbs_sites:
assert block_size == jnp.not_equal(gibbs_sites[name], new_gibbs_sites[name]).sum()
def _update_controller(args):
counter = args[0]
jax.lax.cond(
np.logical_and(
np.logical_and(
np.greater(counter, 0),
np.equal(counter % print_rate, 0)),
np.not_equal(counter, max_iterations - remainder)),
lambda _: id_tap(
_update_pbar, (print_rate,) + args),
lambda _: (print_rate,) + args,
operand=None)
jax.lax.cond(
np.equal(counter, max_iterations - remainder),
lambda _: id_tap(
_update_pbar, (remainder,) + args),
lambda _: (remainder,) + args,
operand=None)
def _loss(self, params, actions, timesteps):
"""Calculates Q-lambda loss given parameters, actions and timesteps."""
network_apply_sequence = jax.vmap(self._network.apply,
in_axes=(None, 0))
q = network_apply_sequence(params, timesteps.observation)
# Use a mask since the sequence could cross episode boundaries.
mask = jnp.not_equal(timesteps.step_type, int(dm_env.StepType.LAST))
a_tm1 = actions[1:]
r_t = timesteps.reward[1:]
# Discount ought to be zero on a LAST timestep, use the mask to ensure this.
discount_t = timesteps.discount[1:] * mask[1:]
q_tm1 = q[:-1]
q_t = q[1:]
mask_tm1 = mask[:-1]
# Mask out TD errors for the last state in an episode.
td_error_tm1 = mask_tm1 * rlax.q_lambda(
q_tm1, a_tm1, r_t, discount_t, q_t, lambda_=self._lambda)
return jnp.sum(rlax.l2_loss(td_error_tm1)) / jnp.sum(mask_tm1)
def compute_loss(online_params, target_params, transitions, rng_key):
rng_key, apply_key = jax.random.split(rng_key)
outputs_tm1, outputs_t, outputs_target_t = network_outputs(
apply_key, online_params, target_params, transitions)
_, loss_key_active, loss_key_passive = jax.random.split(rng_key, 3)
loss_active = loss.active(sg_passive(outputs_tm1),
sg_passive(outputs_t), outputs_target_t,
transitions, loss_key_active)
loss_passive = loss.passive(sg_active(outputs_tm1),
sg_active(outputs_t), outputs_target_t,
transitions, loss_key_passive)
# Logging stuff.
a_tm1 = transitions.a_tm1
mc_return_tm1 = transitions.mc_return_tm1
q_values = TandemTuple(active=outputs_tm1.active.q_values,
passive=outputs_tm1.passive.q_values)
mc_error = jax.tree_map(
lambda q: losses.batch_mc_learning(q, a_tm1, mc_return_tm1),
q_values)
mc_error_abs = jax.tree_map(jnp.abs, mc_error)
q_argmax = jax.tree_map(lambda q: jnp.argmax(q, axis=-1), q_values)
argmax_diff = jnp.not_equal(q_argmax.active, q_argmax.passive)
batch_mean = lambda x: jnp.mean(x, axis=0)
logs = {'loss_active': loss_active, 'loss_passive': loss_passive}
logs.update(
jax.tree_map(
batch_mean, {
'frac_diff_argmax': argmax_diff,
'mc_error_active': mc_error.active,
'mc_error_passive': mc_error.passive,
'mc_error_abs_active': mc_error_abs.active,
'mc_error_abs_passive': mc_error_abs.passive,
}))
return loss_active + loss_passive, logs
def collapse_and_remove_blanks(labels: jnp.ndarray,
seq_length: jnp.ndarray,
blank_id: int = 0):
"""Merge repeated labels into single labels and remove the designated blank symbol.
Args:
labels: Array of shape (batch, seq_length)
seq_length: Arrray of shape (batch), sequence length of each batch element.
blank_id: Optional id of the blank symbol
Returns:
tuple of tf.SparseTensor of shape (batch, seq_length) with repeated labels
collapsed, eg: [[A, A, B, B, A],
[A, B, C, D, E]] => [[A, B, A],
[A, B, C, D, E]]
and int tensor of shape [batch] with new sequence lengths.
"""
b, t = labels.shape
# Zap out blank
blank_mask = 1 - jnp.equal(labels, blank_id)
labels = (labels * blank_mask).astype(labels.dtype)
# Mask labels that don't equal previous label.
label_mask = jnp.concatenate([
jnp.ones_like(labels[:, :1], dtype=jnp.int32),
jnp.not_equal(labels[:, 1:], labels[:, :-1])
],
axis=1)
# Filter labels that aren't in the original sequence.
maxlen = labels.shape[1]
seq_mask = sequence_mask(seq_length, maxlen=maxlen)
label_mask = label_mask * seq_mask
# remove repetitions from the labels
ulabels = label_mask * labels
# Count masks for new sequence lengths.
label_mask = jnp.not_equal(ulabels, 0).astype(labels.dtype)
new_seq_len = jnp.sum(label_mask, axis=1)
# Mask indexes based on sequence length mask.
new_maxlen = maxlen
idx_mask = sequence_mask(new_seq_len, maxlen=new_maxlen)
# Flatten everything and mask out labels to keep and sparse indices.
flat_labels = jnp.reshape(ulabels, [-1])
flat_idx_mask = jnp.reshape(idx_mask, [-1])
indices = jnp.nonzero(flat_idx_mask, size=b * t)[0]
values = jnp.nonzero(flat_labels, size=b * t)[0]
updates = jnp.take_along_axis(flat_labels, values, axis=-1)
# Scatter to flat shape.
flat = jnp.zeros(flat_idx_mask.shape).astype(labels.dtype)
flat = flat.at[indices].set(updates)
# 0'th position in the flat array gets clobbered by later padded updates,
# so reset it here to its original value
flat = flat.at[0].set(updates[0])
# Reshape back to square batch.
batch_size = labels.shape[0]
new_shape = [batch_size, new_maxlen]
return (jnp.reshape(flat, new_shape).astype(labels.dtype),
new_seq_len.astype(seq_length.dtype))
multiply_no_nan = utils.copy_docstring(
tf.math.multiply_no_nan,
lambda x, y, name=None: np.where( # pylint: disable=g-long-lambda
onp.broadcast_to(np.equal(y, 0.),
np.array(x).shape), np.zeros_like(np.multiply(x, y)),
np.multiply(x, y)))
negative = utils.copy_docstring(tf.math.negative,
lambda x, name=None: np.negative(x))
# nextafter = utils.copy_docstring(
# tf.math.nextafter,
# lambda x1, x2, name=None: np.nextafter)
not_equal = utils.copy_docstring(tf.math.not_equal,
lambda x, y, name=None: np.not_equal(x, y))
polygamma = utils.copy_docstring(
tf.math.polygamma, lambda a, x, name=None: scipy_special.polygamma(a, x))
polyval = utils.copy_docstring(
tf.math.polyval, lambda coeffs, x, name=None: np.polyval(coeffs, x))
pow = utils.copy_docstring( # pylint: disable=redefined-builtin
tf.math.pow,
lambda x, y, name=None: np.power(x, y))
real = utils.copy_docstring(tf.math.real,
lambda input, name=None: np.real(input))
reciprocal = utils.copy_docstring(tf.math.reciprocal,
def not_equal(x1, x2): if isinstance(x1, JaxArray): x1 = x1.value if isinstance(x2, JaxArray): x2 = x2.value return JaxArray(jnp.not_equal(x1, x2))
def ne(a: Numeric, b: Numeric):
return jnp.not_equal(a, b)
def error(inputs, targets):
prediction = net(inputs)
return np.mean(np.not_equal(np.argmax(targets, 2), np.argmax(prediction, 2)))
