def get_shape_list(x: tf.Tensor) -> List[Union[int, tf.Tensor]]:
"""Return list of dims, statically where possible.
Compute the static shape of a tensor. Where the dimension is not static
(e.g. batch or time dimension), symbolic Tensor is returned.
Based on tensor2tensor.
Arguments:
x: The ``Tensor`` to process.
Returns:
A list of integers and Tensors.
"""
x = tf.convert_to_tensor(x)
# If unknown rank, return dynamic shape
if x.get_shape().dims is None:
return tf.shape(x)
static = x.get_shape().as_list()
shape = tf.shape(x)
ret = []
for i, dim in enumerate(static):
if dim is None:
dim = shape[i]
ret.append(dim)
return ret
def layer_norm(x: tf.Tensor, epsilon: float = 1e-6) -> tf.Tensor:
"""Layer normalize the tensor x, averaging over the last dimension.
Implementation based on tensor2tensor.
Arguments:
x: The ``Tensor`` to normalize.
epsilon: The smoothing parameter of the normalization.
Returns:
The normalized tensor.
"""
with tf.variable_scope("LayerNorm"):
gamma = get_variable(
name="gamma",
shape=[x.get_shape()[-1]],
dtype=tf.float32,
initializer=tf.ones_initializer())
beta = get_variable(
name="beta",
shape=[x.get_shape()[-1]],
dtype=tf.float32,
initializer=tf.zeros_initializer())
mean = tf.reduce_mean(x, axis=[-1], keepdims=True)
variance = tf.reduce_mean(
tf.square(x - mean),
axis=[-1],
keepdims=True)
norm_x = (x - mean) * tf.rsqrt(variance + epsilon)
return norm_x * gamma + beta
def restore_shape(t0: tf.Tensor, t1: tf.Tensor):
shape = t0.get_shape()
if shape == tensor_shape.unknown_shape():
t1.set_shape([None, None, None])
else:
t1.set_shape(shape)
return t1
def __extract_vq_dimensions(x: tf.Tensor,
num_splits: int) -> Tuple[List[int], int]:
"""
Extracts and validates the shape and resulting quantization vector size used in a vq-layer. The parameter definition
is analogous to the parameter definition indicated in the vector_quantization function.
:return: A tuple of the input shape (as list of ints) and the quantization vector size (as int)
"""
in_shape = x.get_shape().as_list()
if not len(in_shape) == 3:
raise ValueError(
"Parameter 'x' must be a tensor of shape [batch, a, q]. Got {}.".
format(in_shape))
in_shape[0] = in_shape[0] if in_shape[
0] is not None else -1 # allow for variable-sized batch dimension
if num_splits <= 0:
raise ValueError(
"Parameter 'num_splits' must be greater than 0. Got '{}'.".format(
num_splits))
if not in_shape[2] % num_splits == 0:
raise ValueError(
"Parameter 'num_splits' must be a divisor of the third axis of 'x'. Got {} and {}."
.format(num_splits, in_shape[2]))
vec_size = in_shape[2] // num_splits
return in_shape, vec_size
def conv2d(x: tf.Tensor,
filters: int,
kernels: int = 4,
strides: int = 2,
stddev: float = 0.02,
use_bias: bool = False,
padding: str = "SAME"):
"""
卷积层
:param x: Tensor
:param filters: filter数量
:param kernels: kernel大小（正方形kernel）
:param strides: 步长
:param stddev: 初始化标准差
:param use_bias: 是否使用偏置
:param padding: 补零方式（VALID、SAME）
:return: Tensor
"""
w = tf.get_variable(
"w",
shape=[kernels, kernels, x.get_shape()[-1], filters],
initializer=tf.truncated_normal_initializer(stddev=stddev))
conv = tf.nn.conv2d(x,
w,
strides=[1, strides, strides, 1],
padding=padding)
if use_bias:
b = tf.get_variable("b",
shape=[filters],
initializer=tf.constant_initializer(0.))
conv = tf.nn.bias_add(conv, b)
return conv
def build(self, wavefunction: wavefunctions.Wavefunction,
inputs: tf.Tensor) -> Tuple[tf.Tensor, ...]:
"""Connects operations evaluating Sz element and Sxy term to the graph.
This operation computes matrix element <R|S_{i}^{z}S_{j}^{z}|R> and the
off-diagonal term <R|S_{i}^{x}S_{j}^{x}|psi>.
Args:
wavefunction: Wavefunction |psi>.
inputs: Input <R| on which terms are evaluated.
Returns:
Tuple containing SzSz matrix element and SxySxy term.
"""
batch_size = inputs.get_shape().as_list()[0]
i, j = self._bond
i_spins = tf.squeeze(tf.slice(inputs, [0, i], [-1, 1]))
j_spins = tf.squeeze(tf.slice(inputs, [0, j], [-1, 1]))
i_spin_index = tf.stack(
[np.arange(0, batch_size), i * np.ones(batch_size).astype(int)], 1)
j_spin_index = tf.stack(
[np.arange(0, batch_size), j * np.ones(batch_size).astype(int)], 1)
spin_i_update = tf.scatter_nd(i_spin_index, j_spins - i_spins,
inputs.shape)
spin_j_update = tf.scatter_nd(j_spin_index, i_spins - j_spins,
inputs.shape)
updated_config = tf.add_n([inputs, spin_i_update, spin_j_update])
sz_matrix_element = tf.multiply(i_spins, j_spins)
mask = tf.less(sz_matrix_element, np.zeros(batch_size))
mask = tf.cast(mask, sz_matrix_element.dtype)
s_perp_term = 2. * tf.multiply(mask, wavefunction(updated_config))
return 0.25 * self._j_z * sz_matrix_element, 0.25 * self._j_x * s_perp_term
def multilinear_grad(emb: tf.Tensor, tuples: tf.Tensor, score=False) -> tf.Tensor:
tuple_shape = [d.value for d in tuples.get_shape()]
# if len(tuple_shape) > 2:
# n = np.prod(tuple_shape[:-1])
# tuples = tf.reshape(tuples, (n, -1))
# n = tuples.get_shape()[0].value
order = tuples.get_shape()[2].value
rank = emb.get_shape()[-1].value
if order == 2:
if score:
emb_sel = tf.gather(emb, tuples)
grad_score = tf.reshape(tf.reverse(emb_sel, [False, False, True, False]), tuple_shape[:-1] + [2, rank])
prod = tf.reduce_prod(emb_sel, 2)
preds = tf.reshape(tf.reduce_sum(prod, 2), tuple_shape[:-1])
return grad_score, preds
raise NotImplementedError('Todo')
def compute_total_variation_loss(generated_image: tf.Tensor,
scope: str = None) -> tf.Tensor:
"""
Computes total variance of given image.
:param generated_image: input image.
:param scope: name of loss scope.
"""
batch_size, height, width, channels = [
item.value for item in generated_image.get_shape()
]
with tf.name_scope(scope):
vertical_total_variance = tf.nn.l2_loss(generated_image[:, 1:, ...] -
generated_image[:, :(height -
1), ...])
horizontal_total_variance = tf.nn.l2_loss(
generated_image[..., 1:, :] -
generated_image[..., :(width - 1), :])
total_variance_loss = 2. * (horizontal_total_variance / (
(height - 1) * width * channels) + vertical_total_variance /
(height * (width - 1) * channels))
total_variance_loss /= batch_size
total_variance_loss = tf.cast(total_variance_loss, tf.float32)
return total_variance_loss
def Normalize(x: tf.Tensor,
name: str,
epsilon: float = 1e-8,
trainable: bool = True,
dtype=tf.float32):
"""
attention is all you need 模型当中的normal
:param x:
:param name:
:param epsilon:
:param trainable:
:return:
"""
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
inputs_shape = x.get_shape()
params_shape = inputs_shape[-1:]
mean, variance = tf.nn.moments(x, [-1], keep_dims=True)
beta = tf.get_variable("beta",
params_shape,
initializer=tf.zeros_initializer(),
trainable=trainable,
dtype=dtype)
gamma = tf.get_variable("gamma",
params_shape,
initializer=tf.ones_initializer(),
trainable=trainable,
dtype=dtype)
normalized = (x - mean) / ((variance + epsilon)**(.5))
outputs = gamma * normalized + beta
return outputs
def _sentence_rnn(self, per_sentence_states: tf.Tensor) -> tf.Tensor:
assert len(per_sentence_states.get_shape()) == 3
assert per_sentence_states.get_shape()[1] == self.TOTAL_SENTENCES - 1
cell_fw = self._create_cell(self.rnn_cell_dim, name='sentence_cell_fw')
# cell_bw = self._create_cell(self.rnn_cell_dim, name='sentence_cell_bw')
cell_bw = cell_fw
inputs = per_sentence_states
_, (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw, cell_bw=cell_bw, inputs=inputs, dtype=tf.float32)
if self.rnn_cell == "LSTM":
state_fw = state_fw[0] # c_state
state_bw = state_bw[0] # c_state
per_story_states = tf.concat([state_bw, state_fw], axis=1)
print("per_story_states", per_story_states.get_shape())
return per_story_states
def resnet_shortcut(l: tf.Tensor,
n_out: int,
stride: int,
isDownsampling: bool,
activation=tf.identity):
data_format = get_arg_scope()["Conv2D"]["data_format"]
n_in = l.get_shape().as_list()[1 if data_format in
["NCHW", "channels_first"] else 3]
if n_in != n_out or stride != 1: # change dimension when channel is not the same
if isDownsampling:
return Conv2D("convshortcut",
l,
n_out,
1,
strides=stride,
activation=activation)
else:
return Conv2DTranspose("convshortcut",
l,
n_out,
1,
strides=stride,
activation=activation)
else:
return l
def fit(cls, parameters: Dict[str, Tensor],
data: tf.Tensor) -> Dict[str, Tensor]:
"""Optimal ML update using the EM algorithm."""
# regularized multiplicative
M, N = data.get_shape().as_list()
tau0, tau1 = parameters["tau0"], parameters["tau1"]
# hyperparameter optimization
alpha0, beta0 = cls.fitGamma(tau0)
alpha1, beta1 = cls.fitGamma(tau1)
# sampling taus
alphaPost0 = alpha0 + N / 2
betaPost0 = beta0 + tf.matmul(data**2, tau1[..., None])[..., 0] / 2
tau0 = tf.distributions.Gamma(concentration=alphaPost0,
rate=betaPost0).sample(1)[0]
tau0 = tf.where(tau0 < 1e-6, tf.ones_like(tau0) * 1e-6, tau0)
alphaPost1 = alpha1 + M / 2
betaPost1 = beta1 + tf.matmul(tau0[None, ...], data**2)[0, ...] / 2
tau1 = tf.distributions.Gamma(concentration=alphaPost1,
rate=betaPost1).sample(1)[0]
tau1 = tf.where(tau1 < 1e-6, tf.ones_like(tau1) * 1e-6, tau1)
# rescaling taus
normTau0 = tf.norm(tau0)
normTau1 = tf.norm(tau1)
normPerFactor = tf.sqrt(normTau0 * normTau1)
tau0 = tau0 / normTau0 * normPerFactor
tau1 = tau1 / normTau1 * normPerFactor
updatedParameters = {"tau0": tau0, "tau1": tau1}
return (updatedParameters)
def correct_pad(inputs: tf.Tensor, kernel_size):
"""Returns a tuple for zero-padding for 2D convolution with downsampling.
Args:
inputs: An integer or tuple/list of 2 integers.
kernel_size: An integer or tuple/list of 2 integers.
Returns:
A tuple.
"""
input_size = inputs.get_shape().as_list()[1:3]
if isinstance(kernel_size, int):
kernel_size = (kernel_size, kernel_size)
if input_size[0] is None:
adjust = (1, 1)
else:
adjust = (1 - input_size[0] % 2, 1 - input_size[1] % 2)
correct = (kernel_size[0] // 2, kernel_size[1] // 2)
return ((correct[0] - adjust[0], correct[0]), (correct[1] - adjust[1],
correct[1]))
def residual_block(self, x: tf.Tensor, filters: int, strides: int = 1) -> tf.Tensor:
downsample = x.get_shape().as_list()[-1] != filters
if downsample:
residual = tf.keras.layers.AvgPool2D(pool_size=2, strides=2)(x)
residual = tf.keras.layers.Conv2D(
filters,
kernel_size=1,
use_bias=False,
kernel_initializer="glorot_normal",
)(residual)
residual = tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5)(
residual
)
else:
residual = x
x = lq.layers.QuantConv2D(
filters,
kernel_size=3,
strides=strides,
padding="same",
input_quantizer=self.input_quantizer,
kernel_quantizer=self.kernel_quantizer,
kernel_constraint=self.kernel_constraint,
kernel_initializer="glorot_normal",
use_bias=False,
)(x)
x = tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5)(x)
return tf.keras.layers.add([x, residual])
def assert_shape(tensor: tf.Tensor,
expected_shape: List[Optional[int]]) -> None:
"""Check shape of a tensor.
Args:
tensor: Tensor to be chcecked.
expected_shape: Expected shape where `None` means the same as in TF and
`-1` means not checking the dimension.
"""
shape_list = tensor.get_shape().as_list()
if len(shape_list) != len(expected_shape):
raise CheckingException(
"Tensor '{}' with shape {} should have {} dimensions.".format(
tensor.name, shape_list, len(expected_shape)))
mismatching_dims = []
for i, (real, expected) in enumerate(zip(shape_list, expected_shape)):
if expected not in (real, -1):
mismatching_dims.append(i)
if mismatching_dims:
expected_str = ", ".join(
"?" if x == -1 else str(x) for x in expected_shape)
raise CheckingException(
("Shape mismatch of {} in dimensions: {}. "
"Shape was {}, but should be [{}]").format(
tensor.name,
", ".join(str(d) for d in mismatching_dims),
shape_list, expected_str))
def fully_connected(input_tensor: tf.Tensor,
num_outputs: int,
dtype: Optional[Any] = tf.float16,
name: Optional[str] = 'fc',
*_) -> tf.Tensor:
"""Applies fully connected layer to `input_tensor`.
Args:
input_tensor: 2D Tensor of dimensions [batch, in_units]
num_outputs: Number of output units
dtype: Data type of parameters
name: Optional name for this operation
Returns:
A 2-D Tensor computing matmul(x, weights) + biases, dimensions [batch, num_outputs]
"""
num_inputs = input_tensor.get_shape()[1]
w_init = tf.contrib.layers.xavier_initializer(dtype=dtype)
b_init = tf.constant_initializer(0.0, dtype=dtype)
with tf.variable_scope(name):
weights = tf.get_variable('kernel',
shape=[num_inputs, num_outputs],
initializer=w_init,
dtype=dtype)
biases = tf.get_variable('bias',
shape=[num_outputs],
initializer=b_init,
dtype=dtype)
return tf.nn.xw_plus_b(input_tensor, weights, biases, name=name)
def create_skip_connection_maps(self, auxiliary_feature_map: tf.Tensor,
input_feature_map: tf.Tensor) -> tf.Tensor:
"""
Apply convolutions to auxiliary_feature_map before combining it with
inputs
Parameters
----------
auxiliary_feature_map
auxiliary feature maps for skip connections
input_feature_map
input feature maps for skip connections
Returns
-------
skip_connection_maps
skip connection maps
"""
layer_name = self.get_current_layer_full_name("skip_connection")
num_channels_inputs = input_feature_map.get_shape().as_list()[-1]
skip_connection_layer = self.add_keras_layer(
tf.keras.layers.Conv2D(num_channels_inputs, kernel_size=3,
padding='same',
activation=self.activation,
bias_initializer=self.initializer,
kernel_initializer=self.initializer,
name=layer_name))
skip_connection_maps = skip_connection_layer(auxiliary_feature_map)
return skip_connection_maps
def _compute_critic_loss(sampled_q_t_all: tf.Tensor, q_tm1_all: tf.Tensor,
r_t_all: tf.Tensor, d_t: tf.Tensor, discount: float,
num_samples: int, num_critic_heads: int):
"""Compute loss and sampled Q-values for (non-distributional) critics."""
# Reshape Q-value samples back to original batch dimensions and average
# them to compute the TD-learning bootstrap target.
batch_size = r_t_all.get_shape()[0]
sampled_q_t = tf.reshape(
sampled_q_t_all,
(num_samples, batch_size, num_critic_heads)) # [N,B,C]
q_t = tf.reduce_mean(sampled_q_t, axis=0) # [B, C]
# Flatten q_t and q_tm1; necessary for trfl.td_learning
q_t = tf.reshape(q_t, [-1]) # [B*C]
q_tm1 = tf.reshape(q_tm1_all, [-1]) # [B*C]
# Flatten r_t_all; necessary for trfl.td_learning
r_t_all = tf.reshape(r_t_all, [-1]) # [B*C]
# Broadcast and then flatten d_t, to match shape of q_t and q_tm1
d_t = tf.tile(d_t, [num_critic_heads]) # [B*C]
# Cast the additional discount to match the environment discount dtype.
discount = tf.cast(discount, dtype=d_t.dtype)
# Critic loss.
critic_loss = trfl.td_learning(q_tm1, r_t_all, discount * d_t, q_t).loss
critic_loss = tf.reduce_mean(critic_loss)
return critic_loss, sampled_q_t
def assert_shape(tensor: tf.Tensor,
expected_shape: List[Optional[int]]) -> None:
"""Check shape of a tensor.
Args:
tensor: Tensor to be chcecked.
expected_shape: Expected shape where `None` means the same as in TF and
`-1` means not checking the dimension.
"""
shape_list = tensor.get_shape().as_list()
if len(shape_list) != len(expected_shape):
raise CheckingException(
"Tensor '{}' with shape {} should have {} dimensions.".format(
tensor.name, shape_list, len(expected_shape)))
mismatching_dims = []
for i, (real, expected) in enumerate(zip(shape_list, expected_shape)):
if expected != -1 and real != expected:
mismatching_dims.append(i)
if mismatching_dims:
expected_str = ", ".join("?" if x == -1 else str(x)
for x in expected_shape)
raise CheckingException(
("Shape mismatch of {} in dimensions: {}. "
"Shape was {}, but should be [{}]").format(
tensor.name, ", ".join(str(d) for d in mismatching_dims),
shape_list, expected_str))
def data_augmentation_fn(input_image: tf.Tensor, label_image: tf.Tensor, flip_lr: bool=True,
flip_ud: bool=True, color: bool=True) -> (tf.Tensor, tf.Tensor):
"""Applies data augmentation to both images and label images.
Includes left-right flip, up-down flip and color change.
:param input_image: images to be augmented [B, H, W, C]
:param label_image: corresponding label images [B, H, W, C]
:param flip_lr: option to flip image in left-right direction
:param flip_ud: option to flip image in up-down direction
:param color: option to change color of images
:return: the tuple (augmented images, augmented label images) [B, H, W, C]
"""
with tf.name_scope('DataAugmentation'):
if flip_lr:
with tf.name_scope('random_flip_lr'):
sample = tf.random_uniform([], 0, 1)
label_image = tf.cond(sample > 0.5, lambda: tf.image.flip_left_right(label_image), lambda: label_image)
input_image = tf.cond(sample > 0.5, lambda: tf.image.flip_left_right(input_image), lambda: input_image)
if flip_ud:
with tf.name_scope('random_flip_ud'):
sample = tf.random_uniform([], 0, 1)
label_image = tf.cond(sample > 0.5, lambda: tf.image.flip_up_down(label_image), lambda: label_image)
input_image = tf.cond(sample > 0.5, lambda: tf.image.flip_up_down(input_image), lambda: input_image)
chanels = input_image.get_shape()[-1]
if color:
input_image = tf.image.random_contrast(input_image, lower=0.8, upper=1.0)
if chanels == 3:
input_image = tf.image.random_hue(input_image, max_delta=0.1)
input_image = tf.image.random_saturation(input_image, lower=0.8, upper=1.2)
return input_image, label_image
def get_shape_static_or_dynamic(tensor: tf.Tensor):
shape = tensor.get_shape().as_list()
if None in shape:
shape_dynamic = tf.shape(tensor)
shape = [x if x is not None else shape_dynamic[i]
for i, x in enumerate(shape)]
return shape
def model_fn(self, features: tf.Tensor,
mode: tf.estimator.ModeKeys) -> tf.Tensor:
"""
alexnet image classification convolutional network
:param features: input of the network
:param mode: standard names for Estimator model modes
:return: output of the network (except last FC that evaluates the logits)
"""
assert_input_op = tf.debugging.assert_equal(
features.get_shape().as_list()[1:], self.INPUT_SHAPE[1:])
with tf.control_dependencies([assert_input_op]):
pool5 = self._cnn(features, mode)
flatten = Flatten()(pool5)
drop8 = self._fcn(flatten, mode)
assert_output_op = tf.debugging.assert_equal(
drop8.get_shape().as_list()[1:], self.OUTPUT_SHAPE[1:])
with tf.control_dependencies([assert_output_op]):
drop8 = tf.identity(drop8)
return drop8
def merge_fn(
self,
c_embeds: tf.Tensor, # 3D: [batch, temp_size, embed_dim]
c_tags: tf.Tensor, # 2D: [batch, temp_size]
p_tags: tf.Tensor # 1D: [batch]
) -> tf.Tensor: # 2D: [batch, embed_dim]
if not hasattr(self, 'embed_dim'):
self.temp_size = c_embeds.get_shape()[1]
self.embed_dim = c_embeds.get_shape()[2]
with tf.variable_scope('merge', reuse=tf.AUTO_REUSE):
tag_weights = tf.get_variable(
'tag_weight',
shape=(self.tags_num + 1, self.embed_dim), # +1 for padding
dtype=tf.float32,
initializer=tf.initializers.constant(1))
#initializer=tf.initializers.truncated_normal(1))
# shape: [tags, embed_dim]
tags = tf.count_nonzero(c_tags, 1)
# shape: [batch]
mask = tf.sequence_mask(tags, self.temp_size, dtype=tf.float32)
# shape: [batch, temp_size]
mask = tf.expand_dims(mask, -1)
# shape: [batch, temp_size, 1]
# Forward attention with POS tag
t_t = tf.gather(tag_weights, c_tags) * mask
# shape: [batch, temp_size, embed_dim]
att = tf.nn.softmax(c_embeds * t_t, axis=1)
# shape: [batch, temp_size, embed_dim]
p_t = tf.gather(tag_weights, p_tags)
# shape: [batch, embed_dim]
return self.attent(att, c_embeds) * p_t
def merge_fn(
self,
c_embeds: tf.Tensor, # 3D: [batch, temp_size, embed_dim]
c_tags: tf.Tensor, # 2D: [batch, temp_size]
p_tags: tf.Tensor # 1D: [batch]
) -> tf.Tensor: # 2D: [batch, embed_dim]
if not hasattr(self, 'embed_dim'):
self.embed_dim = c_embeds.get_shape()[-1]
with tf.variable_scope('merge', reuse=tf.AUTO_REUSE):
tag_weights = tf.get_variable(
'tag_weight',
shape=(self.tags_num + 1, self.embed_dim), # +1 for padding
dtype=tf.float32,
#initializer=tf.initializers.constant(1))
initializer=tf.initializers.truncated_normal(1))
# shape: [tags, embed_dim]
# Forward attention with POS tag
e_t = self.forward(c_embeds)
# shape: [batch, temp_size, embed_dim]
t_t = tf.gather(tag_weights, c_tags)
#t_t = tf_Print(t_t, [c_tags, t_t[:,:,:1]], message='t_t=')
#t_t = tf_Print(t_t, [c_tags, e_t[:,:,:5]], message='e_t=')
#t_t = tf_Print(t_t, [c_tags, c_embeds[:,:,1]], message='c_embeds=')
# shape: [batch, temp_size, embed_dim]
att = tf.nn.softmax(e_t * t_t, axis=1)
#att = tf_Print(att, [c_tags, att[:,:,1]], message='c_embeds=')
# shape: [batch, temp_size, embed_dim]
return self.attent(att, c_embeds)
def extract_patches_fn(image: tf.Tensor, patch_shape: Tuple[int, int],
offsets: Tuple[int, int]) -> tf.Tensor:
"""Will cut a given image into patches.
:param image: tf.Tensor
:param patch_shape: shape of the extracted patches [h, w]
:param offsets: offset to add to the origin of first patch top-right coordinate, useful during data augmentation \
to have slighlty different patches each time. This value will be multiplied by [h/2, w/2] (range values [0,1])
:return: patches [batch_patches, h, w, c]
"""
with tf.name_scope('patch_extraction'):
h, w = patch_shape
c = image.get_shape()[-1]
offset_h = tf.cast(tf.round(offsets[0] * h // 2), dtype=tf.int32)
offset_w = tf.cast(tf.round(offsets[1] * w // 2), dtype=tf.int32)
offset_img = image[offset_h:, offset_w:, :]
offset_img = offset_img[None, :, :, :]
patches = tf.extract_image_patches(offset_img,
ksizes=[1, h, w, 1],
strides=[1, h // 2, w // 2, 1],
rates=[1, 1, 1, 1],
padding='VALID')
patches_shape = tf.shape(patches)
return tf.reshape(patches,
[tf.reduce_prod(patches_shape[:3]), h, w,
int(c)])
def _select_top_k_scores(scores_in: tf.Tensor, pre_nms_num_detections: int):
"""Selects top_k scores and indices for each class.
Args:
scores_in: A `tf.Tensor` with shape `[batch_size, N, num_classes]`, which
stacks class logit outputs on all feature levels. The N is the number of
total anchors on all levels. The num_classes is the number of classes
predicted by the model.
pre_nms_num_detections: Number of candidates before NMS.
Returns:
scores and indices: A `tf.Tensor` with shape
`[batch_size, pre_nms_num_detections, num_classes]`.
"""
batch_size, num_anchors, num_class = scores_in.get_shape().as_list()
if batch_size is None:
batch_size = tf.shape(scores_in)[0]
scores_trans = tf.transpose(scores_in, perm=[0, 2, 1])
scores_trans = tf.reshape(scores_trans, [-1, num_anchors])
top_k_scores, top_k_indices = tf.nn.top_k(scores_trans,
k=pre_nms_num_detections,
sorted=True)
top_k_scores = tf.reshape(top_k_scores,
[batch_size, num_class, pre_nms_num_detections])
top_k_indices = tf.reshape(top_k_indices,
[batch_size, num_class, pre_nms_num_detections])
return tf.transpose(top_k_scores,
[0, 2, 1]), tf.transpose(top_k_indices, [0, 2, 1])
def rescale(image: tf.Tensor, scale: float, dtype=tf.float32, **kwargs) -> tf.Tensor:
assert image.get_shape().ndims in (3, 4), "The tensor must be of dimension 3 or 4"
image = tf.cast(image, tf.float32)
rescale_size = scale_shape(image, scale)
rescaled_image = tf.image.resize(image, size=rescale_size, **kwargs)
return tf.cast(rescaled_image, dtype)
def upsample(self, inputs: tf.Tensor, *,
kernel_size: Union[list, int], strides: Union[list, int],
name: Optional[str] = None, **additional_params) -> tf.Tensor:
"""
Upsample inputs in case of decoder
Parameters
----------
inputs
input tensor
kernel_size
kernel size for upsampling
strides
strides for upsampling
name
name of the operation
Returns
-------
upsampled_tensor
upsampled result
"""
layer_name = name or self.get_current_layer_full_name("upsample")
num_filters = inputs.get_shape().as_list()[-1]
upsample_layer = self.add_keras_layer(
tf.keras.layers.Conv2DTranspose(
num_filters, kernel_size=kernel_size,
strides=strides,
bias_initializer=self.initializer,
kernel_initializer=self.initializer,
activation=self.activation, name=layer_name,
**additional_params))
out = upsample_layer(inputs)
return out
def data_augmentation_fn(input_image: tf.Tensor,
label_image: tf.Tensor) -> (tf.Tensor, tf.Tensor):
with tf.name_scope('DataAugmentation'):
with tf.name_scope('random_flip_lr'):
sample = tf.random_uniform([], 0, 1)
label_image = tf.cond(
sample > 0.5, lambda: tf.image.flip_left_right(label_image),
lambda: label_image)
input_image = tf.cond(
sample > 0.5, lambda: tf.image.flip_left_right(input_image),
lambda: input_image)
with tf.name_scope('random_flip_ud'):
sample = tf.random_uniform([], 0, 1)
label_image = tf.cond(sample > 0.5,
lambda: tf.image.flip_up_down(label_image),
lambda: label_image)
input_image = tf.cond(sample > 0.5,
lambda: tf.image.flip_up_down(input_image),
lambda: input_image)
chanels = input_image.get_shape()[-1]
input_image = tf.image.random_contrast(input_image,
lower=0.8,
upper=1.0)
if chanels == 3:
input_image = tf.image.random_hue(input_image, max_delta=0.1)
input_image = tf.image.random_saturation(input_image,
lower=0.8,
upper=1.2)
return input_image, label_image
def multilabel_image_to_class(label_image: tf.Tensor, classes_file: str) -> tf.Tensor:
classes_color_values, colors_labels = get_classes_color_from_file_multilabel(classes_file)
# Convert label_image [H,W,3] to the classes [H,W,C],int32 according to the classes [C,3]
with tf.name_scope('LabelAssign'):
if len(label_image.get_shape()) == 3:
diff = tf.cast(label_image[:, :, None, :], tf.float32) - tf.constant(classes_color_values[None, None, :, :]) # [H,W,C,3]
elif len(label_image.get_shape()) == 4:
diff = tf.cast(label_image[:, :, :, None, :], tf.float32) - tf.constant(
classes_color_values[None, None, None, :, :]) # [B,H,W,C,3]
else:
raise NotImplementedError('Length is : {}'.format(len(label_image.get_shape())))
pixel_class_diff = tf.reduce_sum(tf.square(diff), axis=-1) # [H,W,C] or [B,H,W,C]
class_label = tf.argmin(pixel_class_diff, axis=-1) # [H,W] or [B,H,W]
return tf.gather(colors_labels, class_label) > 0
def compute_style_loss(style_image_features: tf.Tensor,
generate_image_features: tf.Tensor,
scope: str = None) -> tf.Tensor:
"""
Computes style loss for given inputs.
:param style_image_features: extracted features from style image.
:param generate_image_features extracted features from generated image.
:param scope: name of loss scope.
:return: computed style loss for give inputs.
"""
batch_size, _, _, channels = [
item.value for item in style_image_features.get_shape()
]
with tf.name_scope(scope):
style_image_gram_matrix = compute_gram_matrix(
inputs=style_image_features, scope='style_image_gram_matrix')
generate_image_gram_matrix = compute_gram_matrix(
inputs=generate_image_features,
scope='generated_image_gram_matrix')
style_loss = 2 * tf.nn.l2_loss(generate_image_gram_matrix -
style_image_gram_matrix)
style_loss /= (batch_size * channels * channels)
return style_loss
def fulllayer(X: tf.Tensor,
nOut: int,
activation=tf.nn.sigmoid,
bias: bool = True,
name="Fulllayer"):
with tf.variable_scope(name):
nIn = X.get_shape().as_list()[-1]
nAvg = (nOut + nIn) / 2
W = tf.get_variable(
'W', [nIn, nOut],
tf.float32,
initializer=tf.random_normal_initializer(stddev=1.0 / nAvg))
## Xavier intialization
res = cMatMul(X, W)
if (bias):
biasOffset = 0.0
if activation == tf.nn.relu:
biasOffset = 1.0 / nOut
b = tf.get_variable('b', [nOut],
tf.float32,
initializer=tf.constant_initializer(
biasOffset, tf.float32))
res += b
if activation is not None:
res = activation(res)
return res
def global_pool(
x: tf.Tensor, data_format: str = "channels_last", name: str = None
) -> tf.Tensor:
"""Global average 2D pooling and flattening.
Alternative to existing Keras implementation of GlobalAveragePooling2D.
AveragePooling2D is much faster than GlobalAveragePooling2D on Larq Compute Engine.
If the width or height of the input is dynamic, the function falls back to
GlobalAveragePooling2D.
# Arguments
x: 4D TensorFlow tensor.
data_format: A string, one of channels_last (default) or
channels_first. The ordering of the dimensions in the inputs. channels_last
corresponds to inputs with shape (batch, height, width, channels) while
channels_first corresponds to inputs with shape (batch, channels, height,
width). It defaults to "channels_last".
name: String name of the layer
# Returns
2D TensorFlow tensor.
# Raises
ValueError: if tensor is not 4D or data_format is not recognized.
"""
if len(x.get_shape()) != 4:
raise ValueError("Tensor is not 4D.")
if data_format not in ["channels_last", "channels_first"]:
raise ValueError("data_format not recognized.")
try:
input_shape = x.get_shape().as_list()
pool_size = (
input_shape[1:3] if data_format == "channels_last" else input_shape[2:4]
)
x = tf.keras.layers.AveragePooling2D(
pool_size=pool_size,
data_format=data_format,
name=f"{name}_pool" if name else None,
)(x)
x = tf.keras.layers.Flatten(name=f"{name}_flatten" if name else None)(x)
except ValueError:
x = tf.keras.layers.GlobalAveragePooling2D(data_format=data_format, name=name)(
x
)
return x
def multi_conv2d_modified2(inputs,
filters: tf.Tensor,
bias=None,
stride=list([1, 1, 1, 1]),
padding='SAME',
basis_rate=list([1, 3, 5]),
to_batch_norm=False,
batch_norm_decay=0.997,
is_training=True,
activation_fn=None):
_number_of_basis = len(basis_rate)
# _filter_shape = tf.shape(filters)
# _filter_center = tf.slice(filters, [1, 1, 0, 0], [1, 1, _filter_shape[2], _filter_shape[3]])
if _number_of_basis < 2:
raise ValueError('Number of basis_rate must be larger or equal than 2')
input_shape = inputs.get_shape()
output_channel = filters.get_shape()[-1]
global_average_pooling = global_avg_pooling_layer(inputs, upsample=False)
depth = 256
selection_weights1 = kernels([input_shape[-1], depth],
regularizer=slim.l2_regularizer(0.0001),
name='rate_selection_weights1')
selection_weights2 = kernels([depth, _number_of_basis],
regularizer=slim.l2_regularizer(0.0001),
name='rate_selection_weights2')
global_avg_pooling_squeezed = tf.squeeze(global_average_pooling,
axis=[1, 2])
selection = tf.matmul(global_avg_pooling_squeezed, selection_weights1)
selection = batch_norm(selection, is_training, batch_norm_decay)
selection = tf.nn.relu(selection)
selection = tf.matmul(selection, selection_weights2)
selection = batch_norm(selection, is_training, batch_norm_decay)
selection = tf.nn.relu(selection)
selection = tf.transpose(selection, [1, 0])
output = list()
for idx in range(8):
input_selection = tf.expand_dims(inputs[idx], axis=0)
_result = atrous_conv2d(input_selection, filters, basis_rate[0], bias,
padding, stride)
for r in basis_rate[1:]:
if idx == 0:
output = atrous_conv2d(input_selection, filters, r, bias,
padding, stride)
output += atrous_conv2d(input_selection, filters, r, bias, padding,
stride)
if to_batch_norm:
output = batch_norm(output, is_training, batch_norm_decay)
if activation_fn is not None:
output = activation_fn(output)
return output
def build_cnn(input: tf.Tensor, n_grams, n_features) -> tf.Tensor:
max_seq_len = int(input.get_shape()[1])
word_d = int(input.get_shape()[2])
tran_input = tf.reshape(input, [-1, 1, max_seq_len, word_d])
results = list()
for n_gram, n_feature in zip(n_grams, n_features):
with tf.name_scope("conv-maxpool-%s" % n_gram):
filter = tf.to_float(
tf.Variable(tf.random_uniform([1, n_gram, word_d, n_feature], 0.05, 0.05, dtype=tf.float32)))
f_result = tf.nn.conv2d(tran_input, filter, strides=[1, 1, 1, 1], padding='VALID')
# batch_size * (valid_height)(1) * valid_length * n_feature
b = tf.Variable(tf.constant(0.1, shape=[n_feature]), name="b")
f_result = tf.nn.relu(tf.nn.bias_add(f_result, b))
p_result = tf.reshape(
tf.nn.max_pool(f_result, ksize=[1, 1, max_seq_len - n_gram + 1, 1], strides=[1, 1, 1, 1],
padding='VALID'), [-1, n_feature])
results.append(p_result)
return tf.concat(concat_dim=1, values=results)
def split_by_factor(
tensor_3d: tf.Tensor, batch_size: tf.Tensor, factor: int) -> tf.Tensor:
max_time = tf.shape(tensor_3d)[1]
state_dim = tensor_3d.get_shape()[2].value
if state_dim % factor != 0:
raise ValueError((
"Dimension of the tensor ({}) must be dividable by the given "
"factor ({}).").format(state_dim, factor))
return tf.reshape(
tensor_3d, [batch_size, max_time * factor, state_dim // factor])
def __input(self, node_input, tensor: tf.Tensor):
if type(node_input) == list:
concatenated_input = np.concatenate(
[self.nodes[node_input_].output
for node_input_ in node_input])
expanded_input = np.lib.pad(
concatenated_input,
(0, tensor.get_shape()[-1].value -
concatenated_input.shape[0]),
'constant')
return expanded_input
else:
return self.nodes[node_input].output
def assert_same_shape(tensor_a: tf.Tensor, tensor_b: tf.Tensor) -> None:
"""Check if two tensors have the same shape."""
shape_a = tensor_a.get_shape().as_list()
shape_b = tensor_b.get_shape().as_list()
if len(shape_a) != len(shape_b):
raise CheckingException(
("Tensor '{}' has {} dimensions and tensor '{}' has {} "
"dimension, but should have the same shape.").format(
tensor_a.name, len(shape_a), tensor_b.name, len(shape_b)))
mismatching_dims = []
for i, (size_a, size_b) in enumerate(zip(shape_a, shape_b)):
if size_a != size_b:
mismatching_dims.append(i)
if mismatching_dims:
raise CheckingException(
("Shape mismatch of '{}' and '{}' in dimensions: {}. "
"Shapes were {} and {}").format(
tensor_a.name, tensor_b.name,
", ".join(str(d) for d in mismatching_dims),
shape_a, shape_b))
def attention(self,
query: tf.Tensor,
decoder_prev_state: tf.Tensor,
decoder_input: tf.Tensor,
loop_state: AttentionLoopState) -> Tuple[
tf.Tensor, AttentionLoopState]:
self.query_state_size = query.get_shape()[-1].value
y = tf.matmul(query, self.query_projection_matrix)
y = y + self.projection_bias_vector
y = tf.reshape(y, [-1, 1, 1, self.state_size])
energies = self.get_energies(y, loop_state.weights)
if self.attention_mask is None:
weights = tf.nn.softmax(energies)
else:
weights_all = tf.nn.softmax(energies) * self.attention_mask
norm = tf.reduce_sum(weights_all, 1, keepdims=True) + 1e-8
weights = weights_all / norm
# condition = tf.equal(self.attention_mask, 1)
# masked_logits = tf.where(
# tf.tile(condition, [tf.shape(energies)[0], 1]),
# energies, -np.inf * tf.ones_like(energies))
# weights = tf.nn.softmax(masked_logits)
# Now calculate the attention-weighted vector d.
context = tf.reduce_sum(
tf.expand_dims(tf.expand_dims(weights, -1), -1)
* self._att_states_reshaped, [1, 2])
context = tf.reshape(context, [-1, self.context_vector_size])
next_contexts = tf.concat(
[loop_state.contexts, tf.expand_dims(context, 0)], 0)
next_weights = tf.concat(
[loop_state.weights, tf.expand_dims(weights, 0)], 0)
next_loop_state = AttentionLoopState(
contexts=next_contexts,
weights=next_weights)
return context, next_loop_state
def label_smoothing(inputs: tf.Tensor, epsilon: float=0.1):
last_dim = inputs.get_shape().as_list()[-1]
return ((1 - epsilon) * inputs) + (epsilon / last_dim)
