def pdf(x): logpdf = reg.logpdf(tf.convert_to_tensor(x, dtype=tf.float32)) logpdf = float(logpdf) return math.exp(logpdf)
def _generate_boxes(self):
"""Generates multiscale anchor boxes."""
boxes = _generate_anchor_boxes(self.image_size, self.anchor_scale,
self.config)
boxes = tf.convert_to_tensor(boxes, dtype=tf.float32)
return boxes
def _define_collect(batch_env, ppo_hparams, scope, frame_stack_size, eval_phase,
sampling_temp, force_beginning_resets,
distributional_size=1):
"""Collect trajectories.
Args:
batch_env: Batch environment.
ppo_hparams: PPO hparams, defined in tensor2tensor.models.research.rl.
scope: var scope.
frame_stack_size: Number of last observations to feed into the policy.
eval_phase: TODO(koz4k): Write docstring.
sampling_temp: Sampling temperature for the policy.
force_beginning_resets: Whether to reset at the beginning of each episode.
distributional_size: optional, number of buckets in distributional RL.
Returns:
Returns memory (observations, rewards, dones, actions,
pdfs, values_functions)
containing a rollout of environment from nested wrapped structure.
"""
epoch_length = ppo_hparams.epoch_length
to_initialize = []
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
num_agents = batch_env.batch_size
to_initialize.append(batch_env)
wrappers = [(StackWrapper, {
"history": frame_stack_size
}), (_MemoryWrapper, {})]
rollout_metadata = None
speculum = None
for w in wrappers:
tf.logging.info("Applying wrapper %s(%s) to env %s." % (str(
w[0]), str(w[1]), str(batch_env)))
batch_env = w[0](batch_env, **w[1])
to_initialize.append(batch_env)
rollout_metadata = _rollout_metadata(batch_env, distributional_size)
speculum = batch_env.speculum
def initialization_lambda(sess):
for batch_env in to_initialize:
batch_env.initialize(sess)
memory = [
tf.get_variable( # pylint: disable=g-complex-comprehension
"collect_memory_%d_%s" % (epoch_length, name),
shape=[epoch_length] + shape,
dtype=dtype,
initializer=tf.zeros_initializer(),
trainable=False) for (shape, dtype, name) in rollout_metadata
]
cumulative_rewards = tf.get_variable(
"cumulative_rewards", len(batch_env), trainable=False)
eval_phase_t = tf.convert_to_tensor(eval_phase)
should_reset_var = tf.Variable(True, trainable=False)
zeros_tensor = tf.zeros(len(batch_env))
force_beginning_resets = tf.convert_to_tensor(force_beginning_resets)
def reset_ops_group():
return tf.group(
batch_env.reset(tf.range(len(batch_env))),
tf.assign(cumulative_rewards, zeros_tensor))
reset_op = tf.cond(
tf.logical_or(should_reset_var.read_value(), force_beginning_resets),
reset_ops_group, tf.no_op)
with tf.control_dependencies([reset_op]):
reset_once_op = tf.assign(should_reset_var, False)
with tf.control_dependencies([reset_once_op]):
def step(index, scores_sum, scores_num):
"""Single step."""
index %= epoch_length # Only needed in eval runs.
# Note - the only way to ensure making a copy of tensor is to run simple
# operation. We are waiting for tf.copy:
# https://github.com/tensorflow/tensorflow/issues/11186
obs_copy = batch_env.observ + 0
value_fun_shape = (num_agents,)
if distributional_size > 1:
value_fun_shape = (num_agents, distributional_size)
def env_step(arg1, arg2, arg3): # pylint: disable=unused-argument
"""Step of the environment."""
(logits, value_function) = get_policy(
obs_copy, ppo_hparams, batch_env.action_space, distributional_size
)
action = common_layers.sample_with_temperature(logits, sampling_temp)
action = tf.cast(action, tf.int32)
action = tf.reshape(action, shape=(num_agents,))
reward, done = batch_env.simulate(action)
pdf = tfp.distributions.Categorical(logits=logits).prob(action)
pdf = tf.reshape(pdf, shape=(num_agents,))
value_function = tf.reshape(value_function, shape=value_fun_shape)
done = tf.reshape(done, shape=(num_agents,))
with tf.control_dependencies([reward, done]):
return tf.identity(pdf), tf.identity(value_function), \
tf.identity(done)
# TODO(piotrmilos): while_body is executed at most once,
# thus should be replaced with tf.cond
pdf, value_function, top_level_done = tf.while_loop(
lambda _1, _2, _3: tf.equal(speculum.size(), 0),
env_step,
[
tf.constant(0.0, shape=(num_agents,)),
tf.constant(0.0, shape=value_fun_shape),
tf.constant(False, shape=(num_agents,))
],
parallel_iterations=1,
back_prop=False,
)
with tf.control_dependencies([pdf, value_function]):
obs, reward, done, action = speculum.dequeue()
to_save = [obs, reward, done, action, pdf, value_function]
save_ops = [
tf.scatter_update(memory_slot, index, value)
for memory_slot, value in zip(memory, to_save)
]
cumulate_rewards_op = cumulative_rewards.assign_add(reward)
agent_indices_to_reset = tf.where(top_level_done)[:, 0]
with tf.control_dependencies([cumulate_rewards_op]):
# TODO(piotrmilos): possibly we need cumulative_rewards.read_value()
scores_sum_delta = tf.reduce_sum(
tf.gather(cumulative_rewards.read_value(), agent_indices_to_reset))
scores_num_delta = tf.count_nonzero(done, dtype=tf.int32)
with tf.control_dependencies(save_ops +
[scores_sum_delta, scores_num_delta]):
reset_env_op = batch_env.reset(agent_indices_to_reset)
reset_cumulative_rewards_op = tf.scatter_update(
cumulative_rewards, agent_indices_to_reset,
tf.gather(zeros_tensor, agent_indices_to_reset))
with tf.control_dependencies([reset_env_op, reset_cumulative_rewards_op]):
return [
index + 1, scores_sum + scores_sum_delta,
scores_num + scores_num_delta
]
def stop_condition(i, _, resets):
return tf.cond(eval_phase_t, lambda: resets < num_agents,
lambda: i < epoch_length)
init = [tf.constant(0), tf.constant(0.0), tf.constant(0)]
index, scores_sum, scores_num = tf.while_loop(
stop_condition, step, init, parallel_iterations=1, back_prop=False)
# We handle force_beginning_resets differently. We assume that all envs are
# reseted at the end of episod (though it happens at the beginning of the
# next one
scores_num = tf.cond(force_beginning_resets,
lambda: scores_num + len(batch_env), lambda: scores_num)
with tf.control_dependencies([scores_sum]):
scores_sum = tf.cond(
force_beginning_resets,
lambda: scores_sum + tf.reduce_sum(cumulative_rewards.read_value()),
lambda: scores_sum)
mean_score = tf.cond(
tf.greater(scores_num, 0),
lambda: scores_sum / tf.cast(scores_num, tf.float32), lambda: 0.)
printing = tf.Print(0, [mean_score, scores_sum, scores_num], "mean_score: ")
with tf.control_dependencies([index, printing]):
memory = [mem.read_value() for mem in memory]
# When generating real data together with PPO training we must use single
# agent. For PPO to work we reshape the history, as if it was generated
# by real_ppo_effective_num_agents.
if ppo_hparams.effective_num_agents is not None and not eval_phase:
new_memory = []
effective_num_agents = ppo_hparams.effective_num_agents
assert epoch_length % ppo_hparams.effective_num_agents == 0, (
"The rollout of ppo_hparams.epoch_length will be distributed amongst"
"effective_num_agents of agents")
new_epoch_length = int(epoch_length / effective_num_agents)
for mem, info in zip(memory, rollout_metadata):
shape, _, name = info
new_shape = [effective_num_agents, new_epoch_length] + shape[1:]
perm = list(range(len(shape) + 1))
perm[0] = 1
perm[1] = 0
mem = tf.transpose(mem, perm=perm)
mem = tf.reshape(mem, shape=new_shape)
mem = tf.transpose(
mem,
perm=perm,
name="collect_memory_%d_%s" % (new_epoch_length, name))
new_memory.append(mem)
memory = new_memory
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
mean_score_summary = tf.cond(
tf.greater(scores_num, 0),
lambda: tf.summary.scalar("mean_score_this_iter", mean_score), str)
summaries = tf.summary.merge([
mean_score_summary,
tf.summary.scalar("episodes_finished_this_iter", scores_num)
])
return memory, summaries, initialization_lambda
def iou_loss(pred_boxes: FloatType,
target_boxes: FloatType,
iou_type: Text = 'iou') -> tf.Tensor:
"""A unified interface for computing various IoU losses.
Let B and B_gt denotes the pred_box and B_gt is the target box (ground truth):
IoU = |B & B_gt| / |B | B_gt|
GIoU = IoU - |C - B U B_gt| / C, where C is the smallest box covering B and
B_gt.
DIoU = IoU - E(B, B_gt)^2 / c^2, E is the Euclidean distance of the center
points of B and B_gt, and c is the diagonal length of the smallest box
covering the two boxes
CIoU = IoU - DIoU - a * v, where a is a positive trade-off parameter, and
v measures the consistency of aspect ratio:
v = (arctan(w_gt / h_gt) - arctan(w / h)) * 4 / pi^2
where (w_gt, h_gt) and (w, h) are the width and height of the target and
predicted box respectively.
The returned loss is computed as 1 - one of {IoU, GIoU, DIoU, CIoU}.
Args:
pred_boxes: predicted boxes, with coordinate [y_min, x_min, y_max, x_max]*.
It can be multiple anchors, with each anchor box has four coordinates.
target_boxes: target boxes, with coordinate [y_min, x_min, y_max, x_max]*.
It can be multiple anchors, with each anchor box has four coordinates.
iou_type: one of ['iou', 'ciou', 'diou', 'giou'].
Returns:
IoU loss float `Tensor`.
"""
if iou_type not in ('iou', 'ciou', 'diou', 'giou'):
raise ValueError(
'Unknown loss_type {}, not iou/ciou/diou/giou'.format(iou_type))
pred_boxes = tf.convert_to_tensor(pred_boxes, tf.float32)
target_boxes = tf.cast(target_boxes, pred_boxes.dtype)
# t_ denotes target boxes and p_ denotes predicted boxes: (y, x, y_max, x_max)
pred_boxes_list = tf.unstack(pred_boxes, None, axis=-1)
target_boxes_list = tf.unstack(target_boxes, None, axis=-1)
assert len(pred_boxes_list) == len(target_boxes_list)
assert len(pred_boxes_list) % 4 == 0
iou_loss_list = []
for i in range(0, len(pred_boxes_list), 4):
pred_boxes = pred_boxes_list[i:i + 4]
target_boxes = target_boxes_list[i:i + 4]
# Compute mask.
t_ymin, t_xmin, t_ymax, t_xmax = target_boxes
mask = tf.not_equal((t_ymax - t_ymin) * (t_xmax - t_xmin), 0)
mask = tf.cast(mask, t_ymin.dtype)
# Loss should be mask * (1 - iou) = mask - masked_iou.
pred_boxes = [b * mask for b in pred_boxes]
iou_loss_list.append(
mask -
tf.squeeze(_iou_per_anchor(pred_boxes, target_boxes, iou_type)))
if len(iou_loss_list) == 1:
return iou_loss_list[0]
return tf.reduce_sum(tf.stack(iou_loss_list), 0)
def func():
assert self._current_iter_id is not None, "Bridge not started"
x = self.receive(self._current_iter_id, name)
return tf.convert_to_tensor(x, dtype=dtype)
def _tensor(x):
return tf.to_float(tf.convert_to_tensor(x))
def test_make_global_local_transformer_side_inputs(self, use_hard_g2l_mask,
use_hard_l2g_mask):
tf.logging.set_verbosity(tf.logging.INFO)
# Example input:
# Q_ID is token corresponding to question. P* represents paragraph tokens
# and S* represents sentence level tokens.
#
# A total of 14 global tokens as follows:
#. 0. 1. 2. 3. 4. 5. 6 7 8 9. 10 11 12 13
# [CLS] Q_ID Q_ID Q_ID Q_ID P1 T1 S1 S2 P2 T2 S1 S2 --padding--
#
# Long Input:
# T*, W* represent (Title, Words) WordPieces in HotpotQA context.
# For example, (T1, W1) correspond to (Title1, Title1Words) and belong
# to the same sentence in the long input. Hence, there is only one
# corresponding global token for both of them.
# Q* represent the question WordPieces.
#
# S1, S2 are sentences (each with one WordPiece) of T1 and S3, S4 are
# sentences (each with one WordPiece) of T2
# Q1 Q2 Q3 Q4 T1 W1 S1 S2 T2 W2 S3 S4
#
# A total of 15 long tokens.
#
# Padding with 0s
long_paragraph_breakpoints = tf.convert_to_tensor(
[[0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0]])
# Note the difference here - padding with -1s instead.
long_sentence_ids = tf.convert_to_tensor(
[[1, 2, 3, 4, 6, 6, 7, 8, 10, 10, 11, 12, -1, -1, -1]])
# Note the difference here - padding with -1s instead.
long_paragraph_ids = tf.convert_to_tensor(
[[-1, -1, -1, -1, 5, 5, 5, 5, 9, 9, 9, 9, -1, -1, -1]])
# Let's say we want to link 0-th, 4-th, 5-th, 6-th long tokens to the 1st
# 2nd global tokens.
l2g_linked_ids = tf.convert_to_tensor(
[[1, -1, -1, -1, 1, 1, 2, -1, -1, -1, -1, -1, -1, -1, -1]])
# Padding with 0s
global_paragraph_breakpoints = tf.convert_to_tensor(
[[1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0]])
# Let's say we want the first and the third global tokens to attend to
# everything in the long (except padding) even if `hard_g2l` is enabled.
# Note that this tensor will be used / applicable only when
# `use_hard_g2l_mask` is enabled.
ignore_hard_g2l_mask = tf.convert_to_tensor(
[[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
# Let's say we want the first long token to attend to everything in the
# global (except padding) even if `hard_l2g` is enabled. Note that this
# tensor will be used / applicable only when `use_hard_l2g_mask` is
# enabled.
ignore_hard_l2g_mask = tf.convert_to_tensor(
[[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
side_inputs = (
multihop_utils.make_global_local_transformer_side_inputs(
long_paragraph_breakpoints=long_paragraph_breakpoints,
long_paragraph_ids=long_paragraph_ids,
long_sentence_ids=long_sentence_ids,
global_paragraph_breakpoints=global_paragraph_breakpoints,
local_radius=4,
relative_pos_max_distance=2,
use_hard_g2l_mask=use_hard_g2l_mask,
ignore_hard_g2l_mask=ignore_hard_g2l_mask,
use_hard_l2g_mask=use_hard_l2g_mask,
ignore_hard_l2g_mask=ignore_hard_l2g_mask,
l2g_linked_ids=l2g_linked_ids))
self._compare_side_inputs(
side_inputs,
use_hard_g2l_mask=use_hard_g2l_mask,
use_hard_l2g_mask=use_hard_l2g_mask)
def _construct_images(batch_size):
image_shape = (224, 224, 3)
images = tf.convert_to_tensor(
np.random.randn(batch_size, *image_shape), dtype=tf.float32)
return images
def conditional_instance_norm(inputs,
labels,
num_categories,
center=True,
scale=True,
activation_fn=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
"""Conditional instance normalization from TODO(vdumoulin): add link.
"A Learned Representation for Artistic Style"
Vincent Dumoulin, Jon Shlens, Manjunath Kudlur
Can be used as a normalizer function for conv2d.
Args:
inputs: a tensor with 4 dimensions. The normalization occurs over height
and width.
labels: tensor, style labels to condition on.
num_categories: int, total number of styles being modeled.
center: If True, subtract `beta`. If False, `beta` is ignored.
scale: If True, multiply by `gamma`. If False, `gamma` is
not used. When the next layer is linear (also e.g. `nn.relu`), this can be
disabled since the scaling can be done by the next layer.
activation_fn: Optional activation function.
reuse: whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
variables_collections: optional collections for the variables.
outputs_collections: collections to add the outputs.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
scope: Optional scope for `variable_scope`.
Returns:
A `Tensor` representing the output of the operation.
Raises:
ValueError: if rank or last dimension of `inputs` is undefined, or if the
input doesn't have 4 dimensions.
"""
with tf.variable_scope(scope, 'InstanceNorm', [inputs], reuse=reuse) as sc:
inputs = tf.convert_to_tensor(inputs)
inputs_shape = inputs.get_shape()
inputs_rank = inputs_shape.ndims
if inputs_rank is None:
raise ValueError('Inputs %s has undefined rank.' % inputs.name)
if inputs_rank != 4:
raise ValueError('Inputs %s is not a 4D tensor.' % inputs.name)
dtype = inputs.dtype.base_dtype
axis = [1, 2]
params_shape = inputs_shape[-1:]
if not params_shape.is_fully_defined():
raise ValueError('Inputs %s has undefined last dimension %s.' %
(inputs.name, params_shape))
def _label_conditioned_variable(name, initializer, labels,
num_categories):
"""Label conditioning."""
shape = tf.TensorShape([num_categories]).concatenate(params_shape)
var_collections = slim.utils.get_variable_collections(
variables_collections, name)
var = slim.model_variable(name,
shape=shape,
dtype=dtype,
initializer=initializer,
collections=var_collections,
trainable=trainable)
conditioned_var = tf.gather(var, labels)
conditioned_var = tf.expand_dims(
tf.expand_dims(conditioned_var, 1), 1)
return conditioned_var
# Allocate parameters for the beta and gamma of the normalization.
beta, gamma = None, None
if center:
beta = _label_conditioned_variable('beta', tf.zeros_initializer(),
labels, num_categories)
if scale:
gamma = _label_conditioned_variable('gamma', tf.ones_initializer(),
labels, num_categories)
# Calculate the moments on the last axis (instance activations).
mean, variance = tf.nn.moments(inputs, axis, keep_dims=True)
# Compute layer normalization using the batch_normalization function.
variance_epsilon = 1E-5
outputs = tf.nn.batch_normalization(inputs, mean, variance, beta,
gamma, variance_epsilon)
outputs.set_shape(inputs_shape)
if activation_fn:
outputs = activation_fn(outputs)
return slim.utils.collect_named_outputs(outputs_collections,
sc.original_name_scope,
outputs)
def bootstrap_results(self, state):
"""Trains the bijector and creates initial `previous_kernel_results`.
The supplied `state` is only used to determine the number of chains to run
in parallel_iterations
Args:
state: `Tensor` or Python `list` of `Tensor`s representing the initial
state(s) of the Markov chain(s). The first `r` dimensions index
independent chains, `r = tf.rank(target_log_prob_fn(*state))`.
Returns:
kernel_results: Instance of
`UncalibratedHamiltonianMonteCarloKernelResults` inside
`MetropolisHastingsResults` inside `TransformedTransitionKernelResults`
inside `SimpleStepSizeAdaptationResults`.
"""
def loss():
q = self._flattened_variational_distribution()
# TODO(siege): How to seed this?
samples = q.sample(self.train_batch_size)
return tf.reduce_mean(input_tensor=q.log_prob(samples) -
self._flattened_target_log_prob(samples),
axis=-1)
lr = tf.convert_to_tensor(value=self.learning_rate, dtype=self._dtype)
dtype = lr.dtype
learning_rate = tf.compat.v2.optimizers.schedules.PiecewiseConstantDecay(
list(self.num_train_steps *
np.array([0.2, 0.8]).astype(dtype.as_numpy_dtype)),
[lr, lr * 0.1, lr * 0.01])
opt = tf.compat.v2.optimizers.Adam(learning_rate)
@tf.function(autograph=False)
def train_step():
with tf.GradientTape() as tape:
loss_val = loss()
vals = tape.watched_variables()
grads = tape.gradient(loss_val, vals)
grads_and_vals = list(zip(grads, vals))
opt.apply_gradients(grads_and_vals)
return loss_val
for step in range(self.num_train_steps):
loss_val = train_step()
tf.debugging.assert_all_finite(
loss_val, 'NeuTra loss is NaN at step {}'.format(step))
if self.train_debug_fn:
# pylint: disable=not-callable
self.train_debug_fn(self, step, loss_val)
state_parts = tf.nest.flatten(state)
flat_state_shapes = tf.nest.flatten(self.state_shape)
batch_shape = tf.shape(
input=state_parts[0])[:-flat_state_shapes[0].ndims]
return self._kernel.bootstrap_results(
self._flattened_variational_distribution().sample(batch_shape,
seed=self.seed))
def apply_gradients(self, grads_and_vars, global_step, name=None):
"""Perform an update with the parameters."""
# we meta-train with 10k steps. When applying to longer problems we want to
# have a reasonable schedule so we rescale.
rescale_global_step = float(
10000) / self._training_steps * tf.to_float(global_step)
beta1_power = tf.get_variable(dtype=tf.float32,
name="beta1_power",
initializer=self._beta1)
beta2_power = tf.get_variable(dtype=tf.float32,
name="beta2_power",
initializer=self._beta2)
exp_factor = tf.exp(-self._exponential_decay *
tf.to_float(rescale_global_step))
# lr reduction per step.
linear_factor = tf.maximum(
1 - self._linear_decay * tf.to_float(rescale_global_step), 0.0)
lr = exp_factor * linear_factor * self._learning_rate
assignments = []
for (grad, param) in grads_and_vars:
if grad is None or param is None:
continue
# sparse to dense conversion
grad = tf.convert_to_tensor(grad)
param_name = self._get_variable_name(param.name)
m = tf.get_variable(name=param_name + "/adam_m",
shape=param.shape.as_list(),
dtype=tf.float32,
trainable=False,
initializer=tf.zeros_initializer())
v = tf.get_variable(name=param_name + "/adam_v",
shape=param.shape.as_list(),
dtype=tf.float32,
trainable=False,
initializer=tf.zeros_initializer())
next_m = (self._beta1 * m + (1.0 - self._beta1) * grad)
next_v = (self._beta2 * v + (1.0 - self._beta2) * tf.square(grad))
next_m_hat = next_m / (1 - beta1_power)
next_v_hat = next_v / (1 - beta2_power)
update = next_m_hat / (tf.sqrt(next_v_hat + 1e-10) + self._epsilon)
next_param = param - lr * update
assignments.extend(
[param.assign(next_param),
m.assign(next_m),
v.assign(next_v)])
# Do this after all other assignments are done to prevent a race condition.
with tf.control_dependencies(assignments):
assignments.extend([
beta1_power.assign(beta1_power * self._beta1),
beta2_power.assign(beta2_power * self._beta2),
global_step.assign_add(1),
])
return tf.group(*assignments, name=name)
def _num_leapfrog_steps(self, step_size):
step_size = tf.convert_to_tensor(value=step_size)
trajectory_length = np.float32(self._total_event_size)**0.25
return tf.cast(tf.math.ceil(trajectory_length / step_size),
dtype=tf.int32)
def compress(self, inputs):
"""Compress inputs and store their binary representations into strings.
Arguments:
inputs: `Tensor` with values to be compressed.
Returns:
compressed: String `Tensor` vector containing the compressed
representation of each batch element of `inputs`.
Raises:
ValueError: if `inputs` has an integral or inconsistent `DType`, or
inconsistent number of channels.
"""
with tf.name_scope(self._name_scope()):
inputs = tf.convert_to_tensor(inputs, dtype=self.dtype)
if not self.built:
# Check input assumptions set before layer building, e.g. input rank.
input_spec.assert_input_compatibility(self.input_spec, inputs,
self.name)
if self.dtype is None:
self._dtype = inputs.dtype.base_dtype.name
self.build(inputs.shape)
# Check input assumptions set after layer building, e.g. input shape.
if not tf.executing_eagerly():
input_spec.assert_input_compatibility(self.input_spec, inputs,
self.name)
if inputs.dtype.is_integer:
raise ValueError("{} can't take integer inputs.".format(
type(self).__name__))
symbols = self._quantize(inputs, "symbols")
assert symbols.dtype == tf.int32
ndim = self.input_spec.ndim
indexes = self._prepare_indexes(shape=tf.shape(symbols)[1:])
broadcast_indexes = (indexes.shape.ndims != ndim)
if broadcast_indexes:
# We can't currently broadcast over anything else but the batch axis.
assert indexes.shape.ndims == ndim - 1
args = (symbols, )
else:
args = (symbols, indexes)
def loop_body(args):
string = range_coding_ops.unbounded_index_range_encode(
args[0],
indexes if broadcast_indexes else args[1],
self._quantized_cdf,
self._cdf_length,
self._offset,
precision=self.range_coder_precision,
overflow_width=4,
debug_level=0)
return string
strings = tf.map_fn(loop_body,
args,
dtype=tf.string,
back_prop=False,
name="compress")
if not tf.executing_eagerly():
strings.set_shape(inputs.shape[:1])
return strings
def to_tensor(self, x): return tf.convert_to_tensor(x, dtype=tf.float32)
def initial_alpha(self):
alpha = self.initial_alpha_np()
return tf.convert_to_tensor(alpha, dtype=tf.float32)
def dynamic_decode(decoder,
output_time_major=False,
impute_finished=False,
maximum_iterations=None,
parallel_iterations=32,
swap_memory=False,
scope=None,
**kwargs):
"""Perform dynamic decoding with `decoder`.
Calls initialize() once and step() repeatedly on the Decoder object.
Args:
decoder: A `Decoder` instance.
output_time_major: Python boolean. Default: `False` (batch major). If
`True`, outputs are returned as time major tensors (this mode is faster).
Otherwise, outputs are returned as batch major tensors (this adds extra
time to the computation).
impute_finished: Python boolean. If `True`, then states for batch
entries which are marked as finished get copied through and the
corresponding outputs get zeroed out. This causes some slowdown at
each time step, but ensures that the final state and outputs have
the correct values and that backprop ignores time steps that were
marked as finished.
maximum_iterations: `int32` scalar, maximum allowed number of decoding
steps. Default is `None` (decode until the decoder is fully done).
parallel_iterations: Argument passed to `tf.while_loop`.
swap_memory: Argument passed to `tf.while_loop`.
scope: Optional variable scope to use.
**kwargs: dict, other keyword arguments for dynamic_decode. It might contain
arguments for `BaseDecoder` to initialize, which takes all tensor inputs
during call().
Returns:
`(final_outputs, final_state, final_sequence_lengths)`.
Raises:
TypeError: if `decoder` is not an instance of `Decoder`.
ValueError: if `maximum_iterations` is provided but is not a scalar.
"""
if not isinstance(decoder, (Decoder, BaseDecoder)):
raise TypeError("Expected decoder to be type Decoder, but saw: %s" %
type(decoder))
with tf.variable_scope(scope, "decoder") as varscope:
# Enable variable caching if it is safe to do so.
if _should_cache_variables():
if varscope.caching_device is None:
varscope.set_caching_device(lambda op: op.device)
if maximum_iterations is not None:
maximum_iterations = tf.convert_to_tensor(
maximum_iterations, dtype=tf.int32, name="maximum_iterations")
if maximum_iterations.get_shape().ndims != 0:
raise ValueError("maximum_iterations must be a scalar")
if isinstance(decoder, Decoder):
initial_finished, initial_inputs, initial_state = decoder.initialize()
else:
# For BaseDecoder that takes tensor inputs during call.
decoder_init_input = kwargs.pop("decoder_init_input", None)
decoder_init_kwargs = kwargs.pop("decoder_init_kwargs", {})
initial_finished, initial_inputs, initial_state = decoder.initialize(
decoder_init_input, **decoder_init_kwargs)
zero_outputs = _create_zero_outputs(decoder.output_size,
decoder.output_dtype,
decoder.batch_size)
# If we are in an XLA context, we set maximum_iterations on the while loop
# and set a fixed size for TensorArrays.
is_xla = control_flow_util.GraphOrParentsInXlaContext(
tf.get_default_graph())
if is_xla and maximum_iterations is None:
raise ValueError("maximum_iterations is required for XLA compilation.")
if maximum_iterations is not None:
initial_finished = tf.logical_or(
initial_finished, 0 >= maximum_iterations)
initial_sequence_lengths = tf.zeros_like(
initial_finished, dtype=tf.int32)
initial_time = tf.constant(0, dtype=tf.int32)
def _shape(batch_size, from_shape):
if (not isinstance(from_shape, tf.TensorShape) or
from_shape.ndims == 0):
return None
else:
batch_size = tensor_util.constant_value(
tf.convert_to_tensor(batch_size, name="batch_size"))
return tf.TensorShape([batch_size]).concatenate(from_shape)
dynamic_size = maximum_iterations is None or not is_xla
def _create_ta(s, d):
return tf.TensorArray(
dtype=d,
size=0 if dynamic_size else maximum_iterations,
dynamic_size=dynamic_size,
element_shape=_shape(decoder.batch_size, s))
initial_outputs_ta = tf.nest.map_structure(_create_ta, decoder.output_size,
decoder.output_dtype)
def condition(unused_time, unused_outputs_ta, unused_state, unused_inputs,
finished, unused_sequence_lengths):
return tf.logical_not(tf.reduce_all(finished))
def body(time, outputs_ta, state, inputs, finished, sequence_lengths):
"""Internal while_loop body.
Args:
time: scalar int32 tensor.
outputs_ta: structure of TensorArray.
state: (structure of) state tensors and TensorArrays.
inputs: (structure of) input tensors.
finished: bool tensor (keeping track of what's finished).
sequence_lengths: int32 tensor (keeping track of time of finish).
Returns:
`(time + 1, outputs_ta, next_state, next_inputs, next_finished,
next_sequence_lengths)`.
```
"""
(next_outputs, decoder_state, next_inputs,
decoder_finished) = decoder.step(time, inputs, state)
decoder_state_sequence_lengths = False
if decoder.tracks_own_finished:
next_finished = decoder_finished
lengths = getattr(decoder_state, "lengths", None)
if lengths is not None:
# sequence lengths are provided by decoder_state.lengths; overwrite
# our sequence lengths.
decoder_state_sequence_lengths = True
sequence_lengths = tf.cast(lengths, tf.int32)
else:
next_finished = tf.logical_or(decoder_finished, finished)
if decoder_state_sequence_lengths:
# Just pass something through the loop; at the next iteration we'll pull
# the sequence lengths from the decoder_state again.
next_sequence_lengths = sequence_lengths
else:
next_sequence_lengths = tf.where(
tf.logical_not(finished),
tf.fill(tf.shape(sequence_lengths), time + 1),
sequence_lengths)
tf.nest.assert_same_structure(state, decoder_state)
tf.nest.assert_same_structure(outputs_ta, next_outputs)
tf.nest.assert_same_structure(inputs, next_inputs)
# Zero out output values past finish
if impute_finished:
emit = tf.nest.map_structure(
lambda out, zero: tf.where(finished, zero, out),
next_outputs,
zero_outputs)
else:
emit = next_outputs
# Copy through states past finish
def _maybe_copy_state(new, cur):
# TensorArrays and scalar states get passed through.
if isinstance(cur, tf.TensorArray):
pass_through = True
else:
new.set_shape(cur.shape)
pass_through = (new.shape.ndims == 0)
return new if pass_through else tf.where(finished, cur, new)
if impute_finished:
next_state = tf.nest.map_structure(
_maybe_copy_state, decoder_state, state)
else:
next_state = decoder_state
outputs_ta = tf.nest.map_structure(lambda ta, out: ta.write(time, out),
outputs_ta, emit)
return (time + 1, outputs_ta, next_state, next_inputs, next_finished,
next_sequence_lengths)
res = tf.while_loop(
condition,
body,
loop_vars=(
initial_time,
initial_outputs_ta,
initial_state,
initial_inputs,
initial_finished,
initial_sequence_lengths,
),
parallel_iterations=parallel_iterations,
maximum_iterations=maximum_iterations,
swap_memory=swap_memory)
final_outputs_ta = res[1]
final_state = res[2]
final_sequence_lengths = res[5]
final_outputs = tf.nest.map_structure(
lambda ta: ta.stack(), final_outputs_ta)
try:
final_outputs, final_state = decoder.finalize(
final_outputs, final_state, final_sequence_lengths)
except NotImplementedError:
pass
if not output_time_major:
final_outputs = tf.nest.map_structure(
_transpose_batch_time, final_outputs)
return final_outputs, final_state, final_sequence_lengths
def __init__(self,
vocab_size=None,
embed_dim=None,
existing_vocab=None,
densify_gradients=False,
initializers=None,
partitioners=None,
regularizers=None,
trainable=True,
custom_getter=None,
name="embed"):
"""Constructs an Embed module.
Args:
vocab_size: int. Number of unique tokens to embed. If not provided, an
existing vocabulary matrix from which vocab_size can be inferred must
be provided as existing_vocab.
embed_dim: int or None. Number of dimensions to assign to each embedding.
If not specified, a sensible default is chosen based on `vocab_size`. If
an existing vocabulary matrix initializes the module, this should not be
provided as it will be inferred.
existing_vocab: a [vocab_size, embed_dim] vocabulary matrix. Will be
converted to a tf.float32 tensor. If provided, neither or vocab_size or
embed_dim should be provided as they are inferred.
densify_gradients: if True, we convert the embedding gradient from an
indexed-slices to a regular tensor before sending it back to the
parameter server. This avoids excess computation on the parameter
server. Use this option for moderately sized embeddings, e.g.,
a vocabulary size on the order of up to thousands. For embeddings larger
than these, e.g. a vocabulary size on the order of tens or hundreds of
thousands, set this to False.
initializers: Optional dict containing initializers for embeddings (with
key 'embeddings'). As a default, embeddings are initialized via a
truncated normal distribution.
partitioners: Optional dict containing partitioners for embeddings (with
key 'embeddings'). As a default, no partitioners are used.
regularizers: Optional dict containing regularizers for embeddings (with
key 'embeddings'). As a default, no regularizers are used. A regularizer
should be a function that takes a single `Tensor` as an input and
returns a scalar `Tensor` output, e.g. the L1 and L2 regularizers
in `tf.contrib.layers`.
trainable: if True, the embeddings will be updated during training. If
False, they are fixed to their initial values. If `trainable=False` and
a regularizer is given, the resulting loss stays constant.
custom_getter: Callable or dictionary of callables to use as
custom getters inside the module. If a dictionary, the keys
correspond to regexes to match variable names. See the `tf.get_variable`
documentation for information about the custom_getter API.
name: string. Name for this module.
Raises:
ValueError: if neither one of vocab_size or existing_vocab is provided, or
if existing_vocab is provided along with vocab_size, embedding_dim,
initializers, partitioners or regularizers (as these should
be inferred).
"""
if vocab_size is None and existing_vocab is None:
raise ValueError(
"Must provide on of vocab_size or existing_vocab.")
if existing_vocab is not None and not all(
x is None
for x in [vocab_size, embed_dim, initializers, partitioners]):
raise ValueError(
"If existing_vocab is provided, none of vocab_size, "
"embedding_dim, initializers, or partitioners is "
"needed.")
super(Embed, self).__init__(custom_getter=custom_getter, name=name)
self._existing_vocab = None
if existing_vocab is None:
self._vocab_size = vocab_size
self._embed_dim = embed_dim or _embedding_dim(self._vocab_size)
else:
self._existing_vocab = tf.convert_to_tensor(existing_vocab,
dtype=tf.float32)
existing_vocab_shape = self._existing_vocab.get_shape().with_rank(
2)
existing_vocab_shape.assert_is_fully_defined()
self._vocab_size, self._embed_dim = existing_vocab_shape.as_list()
self._initializers = util.check_initializers(
initializers, self.POSSIBLE_INITIALIZER_KEYS)
self._partitioners = util.check_partitioners(
partitioners, self.POSSIBLE_INITIALIZER_KEYS)
self._regularizers = util.check_regularizers(
regularizers, self.POSSIBLE_INITIALIZER_KEYS)
self._trainable = trainable
self._densify_gradients = densify_gradients
def graph_fn():
tf_interp_outputs = calibration_builder._tf_linear_interp1d(
tf.convert_to_tensor(new_x, dtype=tf.float32),
tf.convert_to_tensor(x, dtype=tf.float32),
tf.convert_to_tensor(y, dtype=tf.float32))
return tf_interp_outputs
def __init__(self,
usernum,
itemnum,
vocabsize,
use_last_query,
maxseqlen,
maxquerylen,
hidden_units,
l2_emb,
dropout_rate,
lr,
num_self_attn_heads,
num_query_attn_heads,
num_self_attn_layers,
num_query_attn_layers,
num_final_layers,
query_item_attention,
query_item_combine,
query_layer_norm,
query_residual,
time_exp_base,
overlapping_chunks,
reuse=None):
del usernum
# Variables.
self.is_training = tf.placeholder(tf.bool, shape=())
self.u = tf.placeholder(tf.int32, shape=(None))
self.item_seq = tf.placeholder(tf.int32, shape=(None, maxseqlen))
self.query_seq = tf.placeholder(tf.int32, shape=(None, maxseqlen))
self.query_words_seq = tf.placeholder(tf.int32,
shape=(None, maxseqlen,
maxquerylen))
self.time_seq = tf.placeholder(tf.int32, shape=(None, maxseqlen))
self.pos = tf.placeholder(tf.int32, shape=(None, maxseqlen))
self.neg = tf.placeholder(tf.int32, shape=(None, maxseqlen))
self.query_residual = query_residual
item_seq = self.item_seq
pos = self.pos
neg = self.neg
mask = tf.expand_dims(tf.to_float(tf.not_equal(self.item_seq, 0)), -1)
q_words_mask = tf.expand_dims(
tf.to_float(tf.not_equal(self.query_words_seq, 0)), -1)
batch_size = tf.shape(self.item_seq)[0]
with tf.variable_scope("MultiResolutionRec", reuse=reuse):
# Sequence embedding, item embedding table.
self.item_seq_emb, item_emb_table = modules.embedding(
self.item_seq,
vocab_size=itemnum + 1,
num_units=hidden_units,
zero_pad=True,
scale=True,
l2_reg=l2_emb,
scope="item_embeddings",
with_t=True,
reuse=reuse)
# Query embedding.
if use_last_query == "query_bow":
self.query_words_seq_emb = modules.embedding(
self.query_words_seq,
vocab_size=vocabsize + 1,
num_units=hidden_units,
zero_pad=True,
scale=True,
l2_reg=l2_emb,
scope="query_bow_embeddings",
with_t=False,
reuse=reuse)
# Masking padded query tokens.
self.query_words_seq_emb *= q_words_mask
# Computing the mean of query token embeddings (BOW). Shape changes from
# [None, maxseqlen, maxquerylen] to [None, maxseqlen].
self.query_seq_emb = tf.reduce_mean(self.query_words_seq_emb,
2)
# Masking padded items.
self.query_seq_emb *= mask
# Positional Encoding.
t, _ = modules.embedding(tf.tile(
tf.expand_dims(tf.range(tf.shape(self.item_seq)[1]), 0),
[tf.shape(self.item_seq)[0], 1]),
vocab_size=maxseqlen,
num_units=hidden_units,
zero_pad=False,
scale=False,
l2_reg=l2_emb,
scope="dec_pos",
reuse=reuse,
with_t=True)
self.item_seq_emb += t
# Dropout.
self.item_seq_emb = tf.layers.dropout(
self.item_seq_emb,
rate=dropout_rate,
training=tf.convert_to_tensor(self.is_training))
self.item_seq_emb *= mask
# Build self-attention layers.
for i in range(num_self_attn_layers):
with tf.variable_scope("num_self_attention_layers_%d" % i):
# Self-attention.
self.item_seq_emb = modules.multihead_attention(
queries=modules.normalize(self.item_seq_emb),
keys=self.item_seq_emb,
num_units=hidden_units,
num_heads=num_self_attn_heads,
dropout_rate=dropout_rate,
is_training=self.is_training,
causality=True,
scope="self_attention",
residual=True)
# Feed forward.
self.item_seq_emb = modules.feedforward(
modules.normalize(self.item_seq_emb),
num_units=[hidden_units, hidden_units],
dropout_rate=dropout_rate,
is_training=self.is_training)
self.item_seq_emb *= mask
self.item_seq_emb = modules.normalize(self.item_seq_emb)
# Query layer.
if use_last_query == "query_bow":
# Check whether to compute attention.
if query_item_combine != "only_query" and query_item_attention != "none":
# Compute query-attended history embeddings.
query_attended_seq_emb = self.compute_query_to_item_attention(
query_item_attention, self.query_seq_emb,
self.item_seq_emb, self.time_seq, num_query_attn_heads,
num_query_attn_layers, hidden_units, dropout_rate,
self.is_training, time_exp_base, overlapping_chunks)
self.item_seq_emb = tf.concat(
[
self.
item_seq_emb, # Keep non-attented history embedding.
query_attended_seq_emb
], # Attended history embedding.
axis=-1)
self.item_seq_emb *= mask
# Combine strategy can't be 'sum' due to last dimension mismatch.
query_item_combine = "concat"
# Combine history and query embeddings.
self.item_query_seq_emb = self.combine_item_query_embeddings(
query_item_combine, self.item_seq_emb, self.query_seq_emb)
# Feed-forward layer.
self.item_query_seq_emb = modules.query_feedforward(
self.item_query_seq_emb,
num_units=[hidden_units] * num_final_layers,
dropout_rate=dropout_rate,
is_training=self.is_training,
residual=self.query_residual)
self.item_query_seq_emb *= mask
if query_layer_norm:
self.item_query_seq_emb = modules.normalize(
self.item_query_seq_emb)
seq_emb = tf.reshape(self.item_query_seq_emb,
[batch_size * maxseqlen, hidden_units])
else:
seq_emb = tf.reshape(self.item_seq_emb,
[batch_size * maxseqlen, hidden_units])
# Position-wise positives/negatives.
item_seq = tf.reshape(item_seq, [batch_size * maxseqlen])
pos = tf.reshape(pos, [batch_size * maxseqlen])
neg = tf.reshape(neg, [batch_size * maxseqlen])
pos_emb = tf.nn.embedding_lookup(item_emb_table, pos)
neg_emb = tf.nn.embedding_lookup(item_emb_table, neg)
self.test_item = tf.placeholder(tf.int32, shape=(101))
test_item_emb = tf.nn.embedding_lookup(item_emb_table, self.test_item)
self.test_logits = tf.matmul(seq_emb, tf.transpose(test_item_emb))
self.test_logits = tf.reshape(self.test_logits,
[batch_size, maxseqlen, 101])
self.test_logits = self.test_logits[:, -1, :]
# Prediction layer.
self.pos_logits = tf.reduce_sum(pos_emb * seq_emb, -1)
self.neg_logits = tf.reduce_sum(neg_emb * seq_emb, -1)
# Ignore the first label item regardless of whether the query is used or not
# for consistency.
istarget = tf.reshape(tf.to_float(tf.not_equal(item_seq, 0)),
[batch_size * maxseqlen])
# Note: positives and negatives are present for exactly the same positions.
self.loss = tf.reduce_sum(
-tf.log(tf.sigmoid(self.pos_logits) + 1e-24) * istarget -
tf.log(1 - tf.sigmoid(self.neg_logits) + 1e-24) *
istarget) / tf.reduce_sum(istarget)
reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
self.loss += sum(reg_losses)
tf.summary.scalar("loss", self.loss)
if reuse is None:
self.global_step = tf.Variable(0,
name="global_step",
trainable=False)
self.optimizer = tf.train.AdamOptimizer(learning_rate=lr,
beta2=0.98)
self.train_op = self.optimizer.minimize(
self.loss, global_step=self.global_step)
def filter_for_label(features, target_label):
"""A filter for dataset to get seqs with a specific label."""
### TODO(jjren) not working
return tf.equal(features['y'],
tf.convert_to_tensor(target_label, dtype=tf.int32))
y.append(m)
return y
def innerproduct_layer(x, w, b):
return tf.matmul(x, tf.Variable(w)) + tf.Variable(b)
if __name__ == "__main__":
batch_size = 2
slot_num = 3
max_nnz_per_slot = 4
vocabulary_size = 10
embedding_vec_size = 5
keys = np.ones(shape=(batch_size, slot_num, max_nnz_per_slot),
dtype=np.int64) * -1
print(keys)
keys = tf.convert_to_tensor(keys)
init_values = np.reshape(np.array(
[i for i in range(0, vocabulary_size * embedding_vec_size)],
dtype=np.float32),
newshape=(vocabulary_size, embedding_vec_size))
embedding_feature = embedding_layer(keys, init_values)
print(embedding_feature)
def weight_variable_partial(shape, vari_name):
mask = np.zeros(shape)
#print (mask.shape)
mask[0, 0, :, 0:32, :] = 1
mask[3, 1, :, 0:32, :] = 1
mask[6, 2, :, 0:32, :] = 1
mask[2, 3, :, 0:32, :] = 1
mask[5, 4, :, 0:32, :] = 1
mask[1, 5, :, 0:32, :] = 1
mask[4, 6, :, 0:32, :] = 1
mask[0, 7, :, 0:32, :] = 1
mask[7, 0, :, 0:32, :] = 1
mask[7, 7, :, 0:32, :] = 1
mask[1, 1, :, 32:64, :] = 1
mask[4, 2, :, 32:64, :] = 1
mask[7, 3, :, 32:64, :] = 1
mask[3, 4, :, 32:64, :] = 1
mask[6, 5, :, 32:64, :] = 1
mask[2, 6, :, 32:64, :] = 1
mask[5, 7, :, 32:64, :] = 1
mask[5, 0, :, 32:64, :] = 1
mask[0, 3, :, 32:64, :] = 1
mask[2, 2, :, 64:96, :] = 1
mask[5, 3, :, 64:96, :] = 1
mask[4, 5, :, 64:96, :] = 1
mask[7, 6, :, 64:96, :] = 1
mask[3, 7, :, 64:96, :] = 1
mask[6, 1, :, 64:96, :] = 1
mask[1, 4, :, 64:96, :] = 1
mask[3, 0, :, 64:96, :] = 1
mask[0, 6, :, 64:96, :] = 1
mask[0, 0, :, 32 * 3:32 * 4, :] = 1
mask[3, 1, :, 32 * 3:32 * 4, :] = 1
mask[6, 2, :, 32 * 3:32 * 4, :] = 1
mask[2, 3, :, 32 * 3:32 * 4, :] = 1
mask[5, 4, :, 32 * 3:32 * 4, :] = 1
mask[1, 5, :, 32 * 3:32 * 4, :] = 1
mask[4, 6, :, 32 * 3:32 * 4, :] = 1
mask[0, 7, :, 32 * 3:32 * 4, :] = 1
mask[7, 0, :, 32 * 3:32 * 4, :] = 1
mask[7, 7, :, 32 * 3:32 * 4, :] = 1
mask[1, 1, :, 32 * 4:32 * 5, :] = 1
mask[4, 2, :, 32 * 4:32 * 5, :] = 1
mask[7, 3, :, 32 * 4:32 * 5, :] = 1
mask[3, 4, :, 32 * 4:32 * 5, :] = 1
mask[6, 5, :, 32 * 4:32 * 5, :] = 1
mask[2, 6, :, 32 * 4:32 * 5, :] = 1
mask[5, 7, :, 32 * 4:32 * 5, :] = 1
mask[5, 0, :, 32 * 4:32 * 5, :] = 1
mask[0, 3, :, 32 * 4:32 * 5, :] = 1
mask[2, 2, :, 32 * 5:32 * 6, :] = 1
mask[5, 3, :, 32 * 5:32 * 6, :] = 1
mask[4, 5, :, 32 * 5:32 * 6, :] = 1
mask[7, 6, :, 32 * 5:32 * 6, :] = 1
mask[3, 7, :, 32 * 5:32 * 6, :] = 1
mask[6, 1, :, 32 * 5:32 * 6, :] = 1
mask[1, 4, :, 32 * 5:32 * 6, :] = 1
mask[3, 0, :, 32 * 5:32 * 6, :] = 1
mask[0, 6, :, 32 * 5:32 * 6, :] = 1
'''
mask[0, 0, ...] = 1
mask[0, 7, ...] = 1
mask[7, 0, ...] = 1
mask[7, 7, ...] = 1
mask[3, 1, ...] = 1
mask[6, 2, ...] = 1
mask[2, 3, ...] = 1
mask[5, 4, ...] = 1
mask[1, 5, ...] = 1
mask[4, 6, ...] = 1
'''
'''
mask[0, 0, :,0:16,:] = 1
mask[3, 1, :,0:16,:] = 1
mask[6, 2, :,0:16,:] = 1
mask[2, 3, :,0:16,:] = 1
mask[5, 4, :,0:16,:] = 1
mask[1, 5, :,0:16,:] = 1
mask[4, 6, :,0:16,:] = 1
mask[0, 7, :,0:16,:] = 1
mask[7, 0, :,0:16,:] = 1
mask[7, 7, :,0:16,:] = 1
mask[0, 0, :,16:32,:] = 1
mask[3, 1, :,16:32,:] = 1
mask[6, 2, :,16:32,:] = 1
mask[2, 3, :,16:32,:] = 1
mask[5, 4, :,16:32,:] = 1
mask[1, 5, :,16:32,:] = 1
mask[4, 6, :,16:32,:] = 1
mask[0, 7, :,16:32,:] = 1
mask[7, 0, :,16:32,:] = 1
mask[7, 7, :,16:32,:] = 1
'''
'''
mask[0, 0, :,0:16,:] = 1
mask[3, 1, :,0:16,:] = 1
mask[6, 2, :,0:16,:] = 1
mask[2, 3, :,0:16,:] = 1
mask[5, 4, :,0:16,:] = 1
mask[1, 5, :,0:16,:] = 1
mask[4, 6, :,0:16,:] = 1
mask[0, 7, :,0:16,:] = 1
mask[7, 0, :,0:16,:] = 1
mask[7, 7, :,0:16,:] = 1
mask[1, 1, :,16:32,:] = 1
mask[4, 2, :,16:32,:] = 1
mask[7, 3, :,16:32,:] = 1
mask[3, 4, :,16:32,:] = 1
mask[6, 5, :,16:32,:] = 1
mask[2, 6, :,16:32,:] = 1
mask[5, 7, :,16:32,:] = 1
mask[5, 0, :,16:32,:] = 1
mask[0, 3, :,16:32,:] = 1
mask[2, 2, :,32:48,:] = 1
mask[5, 3, :,32:48,:] = 1
mask[4, 5, :,32:48,:] = 1
mask[7, 6, :,32:48,:] = 1
mask[3, 7, :,32:48,:] = 1
mask[6, 1, :,32:48,:] = 1
mask[1, 4, :,32:48,:] = 1
mask[3, 0, :,32:48,:] = 1
mask[0, 6, :,32:48,:] = 1
mask[0, 0, :,48:64,:] = 1
mask[3, 1, :,48:64,:] = 1
mask[6, 2, :,48:64,:] = 1
mask[2, 3, :,48:64,:] = 1
mask[5, 4, :,48:64,:] = 1
mask[1, 5, :,48:64,:] = 1
mask[4, 6, :,48:64,:] = 1
mask[0, 7, :,48:64,:] = 1
mask[7, 0, :,48:64,:] = 1
mask[7, 7, :,48:64,:] = 1
mask[1, 1, :,64:80,:] = 1
mask[4, 2, :,64:80,:] = 1
mask[7, 3, :,64:80,:] = 1
mask[3, 4, :,64:80,:] = 1
mask[6, 5, :,64:80,:] = 1
mask[2, 6, :,64:80,:] = 1
mask[5, 7, :,64:80,:] = 1
mask[5, 0, :,64:80,:] = 1
mask[0, 3, :,64:80,:] = 1
mask[2, 2, :,80:96,:] = 1
mask[5, 3, :,80:96,:] = 1
mask[4, 5, :,80:96,:] = 1
mask[7, 6, :,80:96,:] = 1
mask[3, 7, :,80:96,:] = 1
mask[6, 1, :,80:96,:] = 1
mask[1, 4, :,80:96,:] = 1
mask[3, 0, :,80:96,:] = 1
mask[0, 6, :,80:96,:] = 1
'''
mask = tf.convert_to_tensor(mask, dtype=tf.bool)
initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
w = tf.Variable(initial, name=vari_name)
w0 = tf.Variable(tf.zeros(shape))
#return tf.Variable(initial,name = vari_name)
return tf.where(mask, w, tf.stop_gradient(w0))
def _iou_per_anchor(pred_boxes: FloatType,
target_boxes: FloatType,
iou_type: Text = 'iou') -> tf.Tensor:
"""Computing the IoU for a single anchor.
Args:
pred_boxes: predicted boxes, with coordinate [y_min, x_min, y_max, x_max].
target_boxes: target boxes, with coordinate [y_min, x_min, y_max, x_max].
iou_type: one of ['iou', 'ciou', 'diou', 'giou'].
Returns:
IoU loss float `Tensor`.
"""
# t_ denotes target boxes and p_ denotes predicted boxes.
t_ymin, t_xmin, t_ymax, t_xmax = target_boxes
p_ymin, p_xmin, p_ymax, p_xmax = pred_boxes
zero = tf.convert_to_tensor(0.0, t_ymin.dtype)
p_width = tf.maximum(zero, p_xmax - p_xmin)
p_height = tf.maximum(zero, p_ymax - p_ymin)
t_width = tf.maximum(zero, t_xmax - t_xmin)
t_height = tf.maximum(zero, t_ymax - t_ymin)
p_area = p_width * p_height
t_area = t_width * t_height
intersect_ymin = tf.maximum(p_ymin, t_ymin)
intersect_xmin = tf.maximum(p_xmin, t_xmin)
intersect_ymax = tf.minimum(p_ymax, t_ymax)
intersect_xmax = tf.minimum(p_xmax, t_xmax)
intersect_width = tf.maximum(zero, intersect_xmax - intersect_xmin)
intersect_height = tf.maximum(zero, intersect_ymax - intersect_ymin)
intersect_area = intersect_width * intersect_height
union_area = p_area + t_area - intersect_area
iou_v = tf.math.divide_no_nan(intersect_area, union_area)
if iou_type == 'iou':
return iou_v # iou is the simplest form.
enclose_ymin = tf.minimum(p_ymin, t_ymin)
enclose_xmin = tf.minimum(p_xmin, t_xmin)
enclose_ymax = tf.maximum(p_ymax, t_ymax)
enclose_xmax = tf.maximum(p_xmax, t_xmax)
assert iou_type in ('giou', 'diou', 'ciou')
if iou_type == 'giou': # giou is the generalized iou.
enclose_width = tf.maximum(zero, enclose_xmax - enclose_xmin)
enclose_height = tf.maximum(zero, enclose_ymax - enclose_ymin)
enclose_area = enclose_width * enclose_height
giou_v = iou_v - tf.math.divide_no_nan(
(enclose_area - union_area), enclose_area)
return giou_v
assert iou_type in ('diou', 'ciou')
p_center = tf.stack([(p_ymin + p_ymax) / 2, (p_xmin + p_xmax) / 2])
t_center = tf.stack([(t_ymin + t_ymax) / 2, (t_xmin + t_xmax) / 2])
euclidean = tf.linalg.norm(t_center - p_center)
diag_length = tf.linalg.norm(
[enclose_ymax - enclose_ymin, enclose_xmax - enclose_xmin])
diou_v = iou_v - tf.math.divide_no_nan(euclidean**2, diag_length**2)
if iou_type == 'diou': # diou is the distance iou.
return diou_v
assert iou_type == 'ciou'
v = _get_v(p_height, p_width, t_height, t_width)
alpha = tf.math.divide_no_nan(v, ((1 - iou_v) + v))
return diou_v - alpha * v # the last one is ciou.
def Score(source, target, ckpt_prefix, hparams):
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
tf.reset_default_graph()
g = tf.Graph()
session = tf.Session(graph=g)
with g.as_default(), session.as_default():
A = nx.adjacency_matrix(target, weight=None)
x = tf.one_hot(list(target.nodes()),
target.number_of_nodes(),
dtype=tf.float64)
y = tf.convert_to_tensor(A.todense(), dtype=tf.float64)
with tf.variable_scope('attention'):
attention = tf.layers.dense(x,
source.number_of_nodes(),
use_bias=False)
source_node_prob = tf.nn.softmax(attention)
layer = tf.layers.dense(source_node_prob,
hparams.embedding_size,
use_bias=False)
for _ in range(hparams.num_dnn_layers):
layer = tf.layers.dense(layer,
hparams.embedding_size * 4,
activation=tf.nn.tanh)
logits = tf.layers.dense(layer,
source.number_of_nodes(),
activation=tf.nn.tanh)
with tf.variable_scope('attention_reverse'):
attention_reverse = tf.layers.dense(logits,
target.number_of_nodes())
target_neighbors_pred = tf.nn.sigmoid(attention_reverse)
target_neighbors_prob = ProbFromCounts(target_neighbors_pred)
loss = AdjMatrixLoss(attention_reverse, y)
if hparams.get('node_label_loss_coefficient', None):
label_loss = NodeLabelLoss(source, source_node_prob, target,
hparams.num_node_labels)
label_loss += NeighborNodesLabelLoss(target_neighbors_prob, target,
hparams.num_node_labels)
loss += label_loss * hparams.node_label_loss_coefficient
if hparams.get('incident_label_loss_coefficient', None):
edge_loss = EdgeLabelLoss(source, source_node_prob, target,
hparams.num_edge_labels)
edge_loss += NeighborEdgesLabelsLoss(target_neighbors_prob, target,
hparams.num_edge_labels)
loss += edge_loss * hparams.incident_label_loss_coefficient
vars_to_restore = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='(?!attention)')
vars_to_train = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='attention')
train_op = contrib_training.create_train_op(
loss,
tf.train.AdamOptimizer(hparams.learning_rate),
variables_to_train=vars_to_train,
summarize_gradients=False)
session.run(tf.global_variables_initializer())
tf.train.Saver(vars_to_restore).restore(session, ckpt_prefix)
losses = []
for _ in range(hparams.score_num_epochs):
losses.append(session.run([train_op, loss])[1])
return losses[-hparams.score_window:]
def preprocess_example(self, example, unused_mode, unused_model_hparams):
sep = tf.convert_to_tensor([self.QUESTION_SEPARATOR_ID],
dtype=example["inputs"].dtype)
example["inputs"] = tf.concat(
[example["inputs"], sep, example["context"]], 0)
return example
def call(self, inputs):
inputs = tf.convert_to_tensor(inputs)
outputs = inputs
# Not for all possible combinations of (`kernel_support`, `corr`,
# `strides_up`, `strides_down`) TF ops exist. We implement some additional
# combinations by manipulating the kernels and toggling `corr`.
kernel = self.kernel
corr = self.corr
# If a convolution with no upsampling is desired, we flip the kernels and
# use cross correlation to implement it, provided the kernels are odd-length
# in every dimension (with even-length kernels, the boundary handling
# would have to change).
if (not corr and all(s == 1 for s in self.strides_up)
and all(s % 2 == 1 for s in self.kernel_support)):
corr = True
slices = self._rank * (slice(None, None,
-1), ) + 2 * (slice(None), )
kernel = kernel[slices]
# Similarly, we can implement a cross correlation using convolutions.
# However, we do this only if upsampling is requested, as we are potentially
# wasting computation in the boundaries whenever we call the transpose ops.
elif (corr and any(s != 1 for s in self.strides_up)
and all(s % 2 == 1 for s in self.kernel_support)):
corr = False
slices = self._rank * (slice(None, None,
-1), ) + 2 * (slice(None), )
kernel = kernel[slices]
# Compute amount of necessary padding, and determine whether to use built-in
# padding or to pre-pad with a separate op.
if self.padding == "valid":
padding = prepadding = self._rank * ((0, 0), )
else: # same_*
padding = padding_ops.same_padding_for_kernel(
self.kernel_support, corr, self.strides_up)
if (self.padding == "same_zeros" and not self.channel_separable
and 1 <= self._rank <= 2 and self.use_explicit):
# Don't pre-pad and use built-in EXPLICIT mode.
prepadding = self._rank * ((0, 0), )
else:
# Pre-pad and then use built-in valid padding mode.
outputs = tf.pad(outputs, self._padded_tuple(padding, (0, 0)),
self._pad_mode)
prepadding = padding
padding = self._rank * ((0, 0), )
# Compute the convolution/correlation. Prefer EXPLICIT padding ops where
# possible, but don't use them to implement VALID padding.
if (corr and all(s == 1 for s in self.strides_up)
and not self.channel_separable and 1 <= self._rank <= 2
and not all(p[0] == p[1] == 0 for p in padding)):
outputs = self._correlate_down_explicit(outputs, kernel, padding)
elif (corr and all(s == 1 for s in self.strides_up)
and all(p[0] == p[1] == 0 for p in padding)):
outputs = self._correlate_down_valid(outputs, kernel)
elif (not corr and not self.channel_separable and 1 <= self._rank <= 2
and self.use_explicit):
outputs = self._up_convolve_transpose_explicit(
outputs, kernel, prepadding)
elif not corr:
outputs = self._up_convolve_transpose_valid(
outputs, kernel, prepadding)
else:
self._raise_notimplemented()
# Now, add bias if requested.
if self.use_bias:
bias = self.bias
if self.data_format == "channels_first":
# As of Mar 2017, direct addition is significantly slower than
# bias_add when computing gradients.
if self._rank == 1:
# tf.nn.bias_add does not accept a 1D input tensor.
outputs = tf.expand_dims(outputs, 2)
outputs = tf.nn.bias_add(outputs, bias, data_format="NCHW")
outputs = tf.squeeze(outputs, [2])
elif self._rank == 2:
outputs = tf.nn.bias_add(outputs, bias, data_format="NCHW")
elif self._rank >= 3:
shape = tf.shape(outputs)
outputs = tf.reshape(outputs,
tf.concat([shape[:3], [-1]], axis=0))
outputs = tf.nn.bias_add(outputs, bias, data_format="NCHW")
outputs = tf.reshape(outputs, shape)
else:
outputs = tf.nn.bias_add(outputs, bias)
# Finally, pass through activation function if requested.
if self.activation is not None:
outputs = self.activation(outputs) # pylint:disable=not-callable
# Aid shape inference, for some reason shape info is not always available.
if not tf.executing_eagerly():
outputs.set_shape(self.compute_output_shape(inputs.shape))
return outputs
def recon_model(mesh,
data,
R0,
x0,
nc=FLAGS.nc,
bs=FLAGS.box_size,
batch_size=FLAGS.batch_size,
a0=FLAGS.a0,
a=FLAGS.af,
nsteps=FLAGS.nsteps,
dtype=tf.float32):
"""
Prototype of function computing LPT deplacement.
Returns output tensorflow and mesh tensorflow tensors
"""
if dtype == tf.float32:
npdtype = "float32"
cdtype = tf.complex64
elif dtype == tf.float64:
npdtype = "float64"
cdtype = tf.complex128
print(dtype, npdtype)
# Compute a few things first, using simple tensorflow
stages = np.linspace(a0, a, nsteps, endpoint=True)
# Define the named dimensions
# Parameters of the small scales decomposition
n_block_x = FLAGS.nx
n_block_y = FLAGS.ny
n_block_z = 1
halo_size = FLAGS.hsize
if halo_size >= 0.5 * min(nc // n_block_x, nc // n_block_y,
nc // n_block_z):
new_size = int(0.5 *
min(nc // n_block_x, nc // n_block_y, nc // n_block_z))
print('WARNING: REDUCING HALO SIZE from %d to %d' %
(halo_size, new_size))
halo_size = new_size
# Parameters of the large scales decomposition
fx_dim = mtf.Dimension("nx", nc)
fy_dim = mtf.Dimension("ny", nc)
fz_dim = mtf.Dimension("nz", nc)
tfx_dim = mtf.Dimension("tx", nc)
tfy_dim = mtf.Dimension("ty", nc)
tfz_dim = mtf.Dimension("tz", nc)
tx_dim = mtf.Dimension("tx_lr", nc)
ty_dim = mtf.Dimension("ty_lr", nc)
tz_dim = mtf.Dimension("tz_lr", nc)
nx_dim = mtf.Dimension('nx_block', n_block_x)
ny_dim = mtf.Dimension('ny_block', n_block_y)
nz_dim = mtf.Dimension('nz_block', n_block_z)
sx_dim = mtf.Dimension('sx_block', nc // n_block_x)
sy_dim = mtf.Dimension('sy_block', nc // n_block_y)
sz_dim = mtf.Dimension('sz_block', nc // n_block_z)
k_dims = [tx_dim, ty_dim, tz_dim]
batch_dim = mtf.Dimension("batch", batch_size)
klin = np.loadtxt('../data/Planck15_a1p00.txt').T[0]
plin = np.loadtxt('../data/Planck15_a1p00.txt').T[1]
ipklin = iuspline(klin, plin)
pk_dim = mtf.Dimension("npk", len(plin))
pk = mtf.import_tf_tensor(mesh, plin.astype(npdtype), shape=[pk_dim])
# Compute necessary Fourier kernels
kvec = flowpm.kernels.fftk((nc, nc, nc), symmetric=False)
kx = mtf.import_tf_tensor(mesh,
kvec[0].squeeze().astype('float32'),
shape=[tfx_dim])
ky = mtf.import_tf_tensor(mesh,
kvec[1].squeeze().astype('float32'),
shape=[tfy_dim])
kz = mtf.import_tf_tensor(mesh,
kvec[2].squeeze().astype('float32'),
shape=[tfz_dim])
kv = [ky, kz, kx]
# kvec for low resolution grid
kvec_lr = flowpm.kernels.fftk([nc, nc, nc], symmetric=False)
kx_lr = mtf.import_tf_tensor(mesh,
kvec_lr[0].squeeze().astype('float32'),
shape=[tx_dim])
ky_lr = mtf.import_tf_tensor(mesh,
kvec_lr[1].squeeze().astype('float32'),
shape=[ty_dim])
kz_lr = mtf.import_tf_tensor(mesh,
kvec_lr[2].squeeze().astype('float32'),
shape=[tz_dim])
kv_lr = [ky_lr, kz_lr, kx_lr]
shape = [batch_dim, fx_dim, fy_dim, fz_dim]
lr_shape = [batch_dim, fx_dim, fy_dim, fz_dim]
hr_shape = [batch_dim, nx_dim, ny_dim, nz_dim, sx_dim, sy_dim, sz_dim]
part_shape = [batch_dim, fx_dim, fy_dim, fz_dim]
# Begin simulation
if x0 is None:
fieldvar = mtf.get_variable(mesh,
'linear',
part_shape,
initializer=tf.random_normal_initializer(
mean=0.0, stddev=1, seed=None))
else:
fieldvar = mtf.get_variable(mesh,
'linear',
part_shape,
initializer=tf.constant_initializer(x0))
print("\nfieldvar : \n", fieldvar)
# Here we can run our nbody
if FLAGS.nbody:
state = mtfpm.lpt_init_single(
fieldvar,
a0,
kv_lr,
halo_size,
lr_shape,
hr_shape,
part_shape[1:],
antialias=True,
)
# Here we can run our nbody
final_state = mtfpm.nbody_single(state, stages, lr_shape, hr_shape,
kv_lr, halo_size)
else:
final_state = mtfpm.lpt_init_single(
fieldvar,
stages[-1],
kv_lr,
halo_size,
lr_shape,
hr_shape,
part_shape[1:],
antialias=True,
)
# paint the field
final_field = mtf.zeros(mesh, shape=hr_shape)
for block_size_dim in hr_shape[-3:]:
final_field = mtf.pad(final_field, [halo_size, halo_size],
block_size_dim.name)
final_field = mesh_utils.cic_paint(final_field, final_state[0], halo_size)
# Halo exchange
for blocks_dim, block_size_dim in zip(hr_shape[1:4],
final_field.shape[-3:]):
final_field = mpm.halo_reduce(final_field, blocks_dim, block_size_dim,
halo_size)
# Remove borders
for block_size_dim in hr_shape[-3:]:
final_field = mtf.slice(final_field, halo_size, block_size_dim.size,
block_size_dim.name)
final_field = mtf.slicewise(
lambda x: x[:, 0, 0, 0], [final_field],
output_dtype=dtype,
output_shape=[batch_dim, fx_dim, fy_dim, fz_dim],
name='my_dumb_reshape',
splittable_dims=part_shape[:-1] + hr_shape[:4])
mtfdata = mtf.import_tf_tensor(mesh,
tf.convert_to_tensor(data),
shape=shape)
# Get prior
k_dims_pr = [d.shape[0] for d in kv]
k_dims_pr = [k_dims_pr[2], k_dims_pr[0], k_dims_pr[1]]
cfield = mesh_utils.r2c3d(fieldvar, k_dims_pr, dtype=cdtype)
def _cwise_prior(kfield, pk, kx, ky, kz):
kx = tf.reshape(kx, [-1, 1, 1])
ky = tf.reshape(ky, [1, -1, 1])
kz = tf.reshape(kz, [1, 1, -1])
kk = tf.sqrt((kx / bs * nc)**2 + (ky / bs * nc)**2 + (kz / bs * nc)**2)
kshape = kk.shape
kk = tf.reshape(kk, [-1])
pkmesh = tfp.math.interp_regular_1d_grid(
x=kk,
x_ref_min=1e-05,
x_ref_max=1000.0,
y_ref=pk,
grid_regularizing_transform=tf.log)
priormesh = tf.reshape(pkmesh, kshape)
return tf.abs(kfield) / priormesh**0.5
cpfield = mtf.cwise(_cwise_prior, [cfield, pk] + kv,
output_dtype=tf.float32)
prior = mtf.reduce_sum(mtf.square(cpfield)) * bs**3 #*nc**3
# Total loss
diff = (final_field - mtfdata)
R0 = tf.constant(R0)
print("R0 in the recon_model : ", R0)
def _cwise_smooth(kfield, kx, ky, kz):
kx = tf.reshape(kx, [-1, 1, 1])
ky = tf.reshape(ky, [1, -1, 1])
kz = tf.reshape(kz, [1, 1, -1])
kk = (kx / bs * nc)**2 + (ky / bs * nc)**2 + (kz / bs * nc)**2
wts = tf.cast(tf.exp(-kk * (R0 * bs / nc)**2), kfield.dtype)
return kfield * wts
# Element-wise function that applies a Fourier kernel
plambda = FLAGS.plambda
def _cwise_logprob(finalfield, data):
galmean = tfp.distributions.Poisson(rate=plambda * (1 + finalfield))
logprob = galmean.log_prob(data)
return -1 * logprob
cfield = mesh_utils.r2c3d(final_field, k_dims_pr, dtype=cdtype)
cfield = mtf.cwise(_cwise_smooth, [cfield] + kv, output_dtype=cdtype)
final_fieldsm = mesh_utils.c2r3d(cfield, diff.shape[-3:], dtype=dtype)
chisq = mtf.cwise(_cwise_logprob, [final_fieldsm, mtfdata],
output_dtype=tf.float32) #
chisq = mtf.reduce_sum(chisq)
## #
loss = chisq + prior
def _cwise_sample(finalfield, data):
galmean = tfp.distributions.Poisson(rate=plambda * (1 + finalfield))
sample = galmean.sample()
return sample
sample = mtf.cwise(_cwise_sample, [final_fieldsm, mtfdata],
output_dtype=tf.float32) #
fields = [fieldvar, sample]
metrics = [chisq, prior, loss]
return fields, metrics, kv
def _construct_images(batch_size):
image_shape = (50, 50, 3)
images = tf.convert_to_tensor(np.random.randn(*((batch_size, ) +
image_shape)),
dtype=tf.float32)
return images
def construct_network(self):
self.word_ids = tf.placeholder(tf.int32, [None, None], name="word_ids")
self.char_ids = tf.placeholder(tf.int32, [None, None, None],
name="char_ids")
self.sentence_lengths = tf.placeholder(tf.int32, [None],
name="sentence_lengths")
self.word_lengths = tf.placeholder(tf.int32, [None, None],
name="word_lengths")
self.sentence_labels = tf.placeholder(tf.float32, [None],
name="sentence_labels")
self.word_labels = tf.placeholder(tf.float32, [None, None],
name="word_labels")
self.word_objective_weights = tf.placeholder(
tf.float32, [None, None], name="word_objective_weights")
self.sentence_objective_weights = tf.placeholder(
tf.float32, [None], name="sentence_objective_weights")
self.learningrate = tf.placeholder(tf.float32, name="learningrate")
self.is_training = tf.placeholder(tf.int32, name="is_training")
self.loss = 0.0
input_tensor = None
input_vector_size = 0
self.initializer = None
if self.config["initializer"] == "normal":
self.initializer = tf.random_normal_initializer(mean=0.0,
stddev=0.1)
elif self.config["initializer"] == "glorot":
self.initializer = tf.glorot_uniform_initializer()
elif self.config["initializer"] == "xavier":
self.initializer = tf.glorot_normal_initializer()
zeros_initializer = tf.zeros_initializer()
self.word_embeddings = tf.get_variable(
"word_embeddings",
shape=[len(self.word2id), self.config["word_embedding_size"]],
initializer=(zeros_initializer if self.config["emb_initial_zero"]
== True else self.initializer),
trainable=(True
if self.config["train_embeddings"] == True else False))
input_tensor = tf.nn.embedding_lookup(self.word_embeddings,
self.word_ids)
input_vector_size = self.config["word_embedding_size"]
if self.config["char_embedding_size"] > 0 and self.config[
"char_recurrent_size"] > 0:
with tf.variable_scope("chars"), tf.control_dependencies([
tf.assert_equal(tf.shape(self.char_ids)[2],
tf.reduce_max(self.word_lengths),
message="Char dimensions don't match")
]):
self.char_embeddings = tf.get_variable(
"char_embeddings",
shape=[
len(self.char2id), self.config["char_embedding_size"]
],
initializer=self.initializer,
trainable=True)
char_input_tensor = tf.nn.embedding_lookup(
self.char_embeddings, self.char_ids)
s = tf.shape(char_input_tensor)
char_input_tensor = tf.reshape(
char_input_tensor,
shape=[
s[0] * s[1], s[2], self.config["char_embedding_size"]
])
_word_lengths = tf.reshape(self.word_lengths,
shape=[s[0] * s[1]])
char_lstm_cell_fw = tf.nn.rnn_cell.LSTMCell(
self.config["char_recurrent_size"],
use_peepholes=self.config["lstm_use_peepholes"],
state_is_tuple=True,
initializer=self.initializer,
reuse=False)
char_lstm_cell_bw = tf.nn.rnn_cell.LSTMCell(
self.config["char_recurrent_size"],
use_peepholes=self.config["lstm_use_peepholes"],
state_is_tuple=True,
initializer=self.initializer,
reuse=False)
char_lstm_outputs = tf.nn.bidirectional_dynamic_rnn(
char_lstm_cell_fw,
char_lstm_cell_bw,
char_input_tensor,
sequence_length=_word_lengths,
dtype=tf.float32,
time_major=False)
_, ((_, char_output_fw), (_,
char_output_bw)) = char_lstm_outputs
char_output_tensor = tf.concat(
[char_output_fw, char_output_bw], axis=-1)
char_output_tensor = tf.reshape(
char_output_tensor,
shape=[s[0], s[1], 2 * self.config["char_recurrent_size"]])
char_output_vector_size = 2 * self.config["char_recurrent_size"]
if self.config["lmcost_char_gamma"] > 0.0:
self.loss += self.config[
"lmcost_char_gamma"] * self.construct_lmcost(
char_output_tensor, char_output_tensor,
self.sentence_lengths, self.word_ids, "separate",
"lmcost_char_separate")
if self.config["lmcost_joint_char_gamma"] > 0.0:
self.loss += self.config[
"lmcost_joint_char_gamma"] * self.construct_lmcost(
char_output_tensor, char_output_tensor,
self.sentence_lengths, self.word_ids, "joint",
"lmcost_char_joint")
if self.config["char_hidden_layer_size"] > 0:
char_output_tensor = tf.layers.dense(
char_output_tensor,
self.config["char_hidden_layer_size"],
activation=tf.tanh,
kernel_initializer=self.initializer)
char_output_vector_size = self.config[
"char_hidden_layer_size"]
if self.config["char_integration_method"] == "concat":
input_tensor = tf.concat(
[input_tensor, char_output_tensor], axis=-1)
input_vector_size += char_output_vector_size
elif self.config["char_integration_method"] == "none":
input_tensor = input_tensor
else:
raise ValueError("Unknown char integration method")
self.word_representations = input_tensor
dropout_input = self.config["dropout_input"] * tf.cast(
self.is_training, tf.float32) + (
1.0 - tf.cast(self.is_training, tf.float32))
input_tensor = tf.nn.dropout(input_tensor,
dropout_input,
name="dropout_word")
word_lstm_cell_fw = tf.nn.rnn_cell.LSTMCell(
self.config["word_recurrent_size"],
use_peepholes=self.config["lstm_use_peepholes"],
state_is_tuple=True,
initializer=self.initializer,
reuse=False)
word_lstm_cell_bw = tf.nn.rnn_cell.LSTMCell(
self.config["word_recurrent_size"],
use_peepholes=self.config["lstm_use_peepholes"],
state_is_tuple=True,
initializer=self.initializer,
reuse=False)
with tf.control_dependencies([
tf.assert_equal(tf.shape(self.word_ids)[1],
tf.reduce_max(self.sentence_lengths),
message="Sentence dimensions don't match")
]):
(lstm_outputs_fw, lstm_outputs_bw), ((_, lstm_output_fw), (
_, lstm_output_bw)) = tf.nn.bidirectional_dynamic_rnn(
word_lstm_cell_fw,
word_lstm_cell_bw,
input_tensor,
sequence_length=self.sentence_lengths,
dtype=tf.float32,
time_major=False)
dropout_word_lstm = self.config["dropout_word_lstm"] * tf.cast(
self.is_training, tf.float32) + (
1.0 - tf.cast(self.is_training, tf.float32))
lstm_outputs_fw = tf.nn.dropout(
lstm_outputs_fw,
dropout_word_lstm,
noise_shape=tf.convert_to_tensor([
tf.shape(self.word_ids)[0], 1,
self.config["word_recurrent_size"]
],
dtype=tf.int32))
lstm_outputs_bw = tf.nn.dropout(
lstm_outputs_bw,
dropout_word_lstm,
noise_shape=tf.convert_to_tensor([
tf.shape(self.word_ids)[0], 1,
self.config["word_recurrent_size"]
],
dtype=tf.int32))
lstm_outputs = tf.concat([lstm_outputs_fw, lstm_outputs_bw], -1)
if self.config["whidden_layer_size"] > 0:
lstm_outputs = tf.layers.dense(lstm_outputs,
self.config["whidden_layer_size"],
activation=tf.tanh,
kernel_initializer=self.initializer)
self.lstm_outputs = lstm_outputs
lstm_output = tf.concat([lstm_output_fw, lstm_output_bw], -1)
lstm_output = tf.nn.dropout(lstm_output, dropout_word_lstm)
if self.config["sentence_composition"] == "last":
processed_tensor = lstm_output
self.attention_weights_unnormalised = tf.zeros_like(
self.word_ids, dtype=tf.float32)
elif self.config["sentence_composition"] == "attention":
with tf.variable_scope("attention"):
attention_evidence = tf.layers.dense(
lstm_outputs,
self.config["attention_evidence_size"],
activation=tf.tanh,
kernel_initializer=self.initializer)
attention_weights = tf.layers.dense(
attention_evidence,
1,
activation=None,
kernel_initializer=self.initializer)
attention_weights = tf.reshape(attention_weights,
shape=tf.shape(self.word_ids))
if self.config["attention_activation"] == "sharp":
attention_weights = tf.exp(attention_weights)
elif self.config["attention_activation"] == "soft":
attention_weights = tf.sigmoid(attention_weights)
elif self.config["attention_activation"] == "linear":
pass
else:
raise ValueError("Unknown activation for attention: " +
str(self.config["attention_activation"]))
word_objective_loss = tf.square(attention_weights -
self.word_labels)
word_objective_loss = tf.where(
tf.sequence_mask(self.sentence_lengths),
word_objective_loss, tf.zeros_like(word_objective_loss))
self.loss += self.config[
"word_objective_weight"] * tf.reduce_sum(
self.word_objective_weights * word_objective_loss)
self.attention_weights_unnormalised = attention_weights
attention_weights = tf.where(
tf.sequence_mask(self.sentence_lengths), attention_weights,
tf.zeros_like(attention_weights))
attention_weights = attention_weights / tf.reduce_sum(
attention_weights, 1, keep_dims=True)
processed_tensor = tf.reduce_sum(
lstm_outputs * attention_weights[:, :, numpy.newaxis], 1)
if self.config["hidden_layer_size"] > 0:
processed_tensor = tf.layers.dense(
processed_tensor,
self.config["hidden_layer_size"],
activation=tf.tanh,
kernel_initializer=self.initializer)
self.sentence_scores = tf.layers.dense(
processed_tensor,
1,
activation=tf.sigmoid,
kernel_initializer=self.initializer,
name="output_ff")
self.sentence_scores = tf.reshape(
self.sentence_scores, shape=[tf.shape(processed_tensor)[0]])
self.loss += self.config["sentence_objective_weight"] * tf.reduce_sum(
self.sentence_objective_weights *
tf.square(self.sentence_scores - self.sentence_labels))
if self.config["attention_objective_weight"] > 0.0:
self.loss += self.config["attention_objective_weight"] * \
(tf.reduce_sum(
self.sentence_objective_weights *tf.square(
tf.reduce_max(
tf.where(
tf.sequence_mask(self.sentence_lengths),
self.attention_weights_unnormalised,
tf.zeros_like(self.attention_weights_unnormalised) - 1e6),
axis=-1) - self.sentence_labels))
+
tf.reduce_sum(
self.sentence_objective_weights * tf.square(
tf.reduce_min(
tf.where(
tf.sequence_mask(self.sentence_lengths),
self.attention_weights_unnormalised,
tf.zeros_like(self.attention_weights_unnormalised) + 1e6),
axis=-1) - 0.0)))
self.token_scores = [
tf.where(tf.sequence_mask(self.sentence_lengths),
self.attention_weights_unnormalised,
tf.zeros_like(self.attention_weights_unnormalised) - 1e6)
]
if self.config["lmcost_lstm_gamma"] > 0.0:
self.loss += self.config[
"lmcost_lstm_gamma"] * self.construct_lmcost(
lstm_outputs_fw, lstm_outputs_bw, self.sentence_lengths,
self.word_ids, "separate", "lmcost_lstm_separate")
if self.config["lmcost_joint_lstm_gamma"] > 0.0:
self.loss += self.config[
"lmcost_joint_lstm_gamma"] * self.construct_lmcost(
lstm_outputs_fw, lstm_outputs_bw, self.sentence_lengths,
self.word_ids, "joint", "lmcost_lstm_joint")
self.train_op = self.construct_optimizer(self.config["opt_strategy"],
self.loss, self.learningrate,
self.config["clip"])
def tf_np_load(filepath):
return tf_v1.convert_to_tensor(np.load(filepath).astype(np.float32))
