def call(self, inputs, **kwargs):
inputs_shape = K.shape(inputs)
mask = K.cast(K.squeeze(K.any(K.not_equal(inputs, 0.),
axis=(-2, -1),
keepdims=True),
axis=-1),
dtype=inputs.dtype)
inputs_to_lstm = K.reshape(inputs,
(-1, inputs.shape[-2], inputs.shape[-1]))
inputs_embed = super(InferenceSpeakerEmbedding,
self).call(inputs_to_lstm)
inputs_embed = K.reshape(
inputs_embed,
(inputs_shape[0], inputs_shape[1], inputs_embed.shape[-1]))
inputs_embed = inputs_embed * mask
n = K.sum(mask, axis=1)
inputs_embed = K.sum(inputs_embed, axis=1) / n
return inputs_embed
def call(self, inputs):
boolean_mask = K.any(
math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
outputs = inputs * math_ops.cast(boolean_mask, inputs.dtype)
# Compute the mask and outputs simultaneously.
outputs._keras_mask = array_ops.squeeze(boolean_mask, axis=-1) # pylint: disable=protected-access
return outputs
def generate_val_set(self):
"""
Generates the actual dataset. It uses all the functions defined above to read images from disk and create croppings.
:return: K.data.Dataset
"""
parse_path_func = lambda x, y: self.parse_path(x, y)
process_label_func = lambda x, y: self.process_label(x, y)
resize_func = lambda x, y: self.resize_and_norm(x, y)
crops_func = lambda x, y: self.crop_img_and_serve(x, y)
filter_func = lambda x, y: K.equal(K.any(y), False)
batch_size = self.batch_size
n_el = len(list(self.val_id_ep_dict.keys()))
ids = []
labels = []
for k, v in self.val_id_ep_dict.items():
ids.append(os.path.join(self.train_images_folder, k))
labels.append(v)
id_tensor = K.constant(ids, dtype=tf.string, shape=([n_el]))
label_tensor = K.constant(labels, dtype=tf.string, shape=(n_el, 4))
return (tf.data.Dataset.from_tensor_slices((id_tensor, label_tensor))
.shuffle(buffer_size=n_el)
.map(parse_path_func, num_parallel_calls=AUTOTUNE)
.map(process_label_func, num_parallel_calls=AUTOTUNE) # create actual one_crop
.map(resize_func, num_parallel_calls=AUTOTUNE) # create actual one_crop
.map(crops_func, num_parallel_calls=AUTOTUNE) # create crops of image to enlarge output
.flat_map(
lambda x, y: tf.data.Dataset.from_tensor_slices((x, y))) # serve crops as new dataset to flat_map array
.filter(filter_func)
.batch(batch_size) # defined batch_size
.prefetch(AUTOTUNE) # number of batches to be prefetch.
.repeat() # repeats the dataset when it is finished
)
def call(self, inputs):
boolean_mask = K.any(
math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
outputs = inputs * math_ops.cast(boolean_mask, inputs.dtype)
# Compute the mask and outputs simultaneously.
outputs._keras_mask = array_ops.squeeze(boolean_mask, axis=-1) # pylint: disable=protected-access
return outputs
def loss_function(target_subtoken, y_pred):
# prediction is a probability, log probability for speed and smoothness
print("Model objective: y_pred.shape: {}".format(y_pred.shape))
# I_C = vector of a target subtoken exist in the input token - TODO probably not ok, debug using TF eager
I_C = K.expand_dims(
K.cast(K.any(K.equal(input_code_subtoken,
K.cast(target_subtoken, 'int32')),
axis=-1),
dtype='float32'), -1)
print("Model objective: I_C.shape: {}".format(I_C.shape))
# I_C shape = [batch_size, token, max_char_len, 1]
# TODO should I add a penality if there is no subtokens appearing in the model ? Yes
probability_correct_copy = K.log(copy_probability) + K.log(
K.sum(I_C * copy_weights) + mu)
print("Model objective: probability_correct_copy.shape: {}".format(
probability_correct_copy.shape))
# penalise the model when cnn-attention predicts unknown
# but the value can be predicted from the copy mechanism.
mask_unknown = K.cast(K.equal(target_subtoken, unknown_id),
dtype='float32') * mu
probability_target_token = K.sum(
K.log(1 - copy_probability) + K.log(y_pred) + mask_unknown, -1,
True)
print("Model objective: probability_target_token.shape: {}".format(
probability_target_token.shape))
loss = K.logsumexp(
[probability_correct_copy, probability_target_token])
return K.mean(loss)
def _fd_conditional(y_true, y_pred):
# if there are no masks annotations, return 0; else, compute fdl loss
return tf.cond(
K.any(K.equal(K.shape(y_true), 0)),
lambda: K.cast_to_floatx(0.0),
lambda: _fd_batch(y_true, y_pred,
iou_threshold=self.fdl_iou_threshold,
parallel_iterations=self.parallel_iterations))
def diaggreater3(x):
# x = K.print_tensor(x, 'input: ')
x1 = diaggreater3_on_axis(x, 0, 1, 2)
x2 = diaggreater3_on_axis(x, 1, 0, 2)
x3 = diaggreater3_on_axis(x, 2, 0, 1)
x = K.stack([x1, x2, x3])
# x = tf.Print(x, [x], summarize=64, message='diagonals: ')
return K.any(x)
def greater3(x):
# x = K.print_tensor(x, 'input: ')
x1 = greater3_on_axis(x, 1)
x2 = greater3_on_axis(x, 2)
x3 = greater3_on_axis(x, 3)
x = K.stack([x1, x2, x3])
# x = K.print_tensor(x, 'output: ')
return K.any(x)
def call(self, inputs, **kwargs):
pair1, pair2 = inputs
pair1_shape, pair2_shape = K.shape(pair1), K.shape(pair2)
pair1_mask = K.cast(K.squeeze(K.any(K.not_equal(pair1, 0.),
axis=(-2, -1),
keepdims=True),
axis=-1),
dtype=pair1.dtype)
pair2_mask = K.cast(K.squeeze(K.any(K.not_equal(pair2, 0.),
axis=(-2, -1),
keepdims=True),
axis=-1),
dtype=pair2.dtype)
pair1_to_lstm = K.reshape(pair1,
(-1, pair1.shape[-2], pair1.shape[-1]))
pair2_to_lstm = K.reshape(pair2,
(-1, pair2.shape[-2], pair2.shape[-1]))
batch = K.concatenate([pair1_to_lstm, pair2_to_lstm], axis=0)
embedded = super(TestSpeakerEmbedding, self).call(batch)
pair1_embed = embedded[:K.shape(pair1_to_lstm)[0]]
pair2_embed = embedded[K.shape(pair1_to_lstm)[0]:]
pair1_embed = K.reshape(pair1_embed,
(pair1_shape[0], pair1_shape[1], -1))
pair2_embed = K.reshape(pair2_embed,
(pair2_shape[0], pair2_shape[1], -1))
pair1_embed = pair1_embed * pair1_mask
pair2_embed = pair2_embed * pair2_mask
pair1_n = K.sum(pair1_mask, axis=1)
pair2_n = K.sum(pair2_mask, axis=1)
pair1_embed = K.sum(pair1_embed, axis=1) / pair1_n
pair2_embed = K.sum(pair2_embed, axis=1) / pair2_n
return pair1_embed, pair2_embed
def antidiaggreater3(x):
x1 = x * tf.constant(ANTIDIAG23)
x1 = K.sum(x1, axis=2)
x1 = K.sum(x1, axis=2)
x1 = K.greater(x1, 3)
x1 = K.any(x1)
x2 = x * tf.constant(ANTIDIAG13)
x2 = K.sum(x2, axis=2)
x2 = K.sum(x2, axis=2)
x2 = K.greater(x2, 3)
x2 = K.any(x2)
x3 = x * tf.constant(ANTIDIAG12)
x3 = K.sum(x3, axis=2)
x3 = K.sum(x3, axis=2)
x3 = K.greater(x3, 3)
x3 = K.any(x3)
x = K.stack([x1, x2, x3])
return K.any(x)
def fulldiaggreater3(x):
xs = []
for filt in [FULLDIAG1, FULLDIAG2, FULLDIAG3, FULLDIAG4]:
x1 = x * tf.constant(filt)
x1 = K.sum(x1, axis=1)
x1 = K.sum(x1, axis=1)
x1 = K.sum(x1, axis=1)
x1 = K.greater(x1, 3)
xs.append(x1)
x = K.stack(xs)
return K.any(x)
def diaggreater3_on_axis(x, i, j, k):
assert j < k
# x = tf.Print(x, [x], summarize=64, message='initial x: ')
diag = tf.diag(np.array([1, 1, 1, 1], dtype=np.int32))
# x = tf.Print(x, [diag], summarize=64, message='diagnonal: ')
diag = K.stack([diag, diag, diag, diag], axis=i)
# x = tf.Print(x, [diag], summarize=64, message='diagnonal: ')
x = x * diag
# x = tf.Print(x, [x], summarize=64, message='x * diag: ')
x = K.sum(x, axis=k + 1)
# x = tf.Print(x, [x], summarize=64, message='x after first sum: ')
x = K.sum(x, axis=j + 1)
# x = tf.Print(x, [x], summarize=64, message='x after second sum: ')
x = K.greater(x, 3)
x = K.any(x)
return x
def call(self, x, mask=None):
'''mask是上一层的'''
'''# using 'mask' you can access the mask passed from the previous layer'''
# x [batch_size, seq_len, embedding_size]
if self.supports_masking:
# mask [batch_size, seq_len]
if mask is None:
# 先判断是否非零，然后执行OR运算，计算每个序列的有效长度
mask = K.any(K.not_equal(x, 0), -1) # [batch_size, seq_len]
mask = K.cast(mask, K.floatx())
return K.sum(x, axis=1) / K.sum(mask, axis=1, keepdims=True)
if mask is not None:
mask = K.cast(mask, K.floatx())
# [batch_size, embedding_size, seq_len]
mask = K.repeat(mask, x.shape[-1].value)
# [batch_size, seq_len, embedding_size]
mask = tf.transpose(mask, [0, 2, 1])
x = x * mask
return K.sum(x, axis=1) / K.sum(mask, axis=1)
def compute_mask(self, inputs, mask=None):
# cases need to call the layer.compute_mask when input_mask is None:
# Masking layer and Embedding layer with mask_zero
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, we currently do not handle mask explicitly
return mask
inner_mask = mask
if inner_mask is not None:
inner_mask_shape = self._get_shape_tuple((-1, ), mask, 2)
inner_mask = K.reshape(inner_mask, inner_mask_shape)
inner_input_shape = self._get_shape_tuple((-1, ), inputs, 2)
inner_inputs = array_ops.reshape(inputs, inner_input_shape)
output_mask = self.layer.compute_mask(inner_inputs, inner_mask)
if output_mask is None:
if mask is None:
return None
# input_mask is not None, and output_mask is None:
# we should return a not-None mask
output_mask = mask
for _ in range(2, len(K.int_shape(mask))):
output_mask = K.any(output_mask, axis=-1)
else:
# output_mask is not None. We need to reshape it
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
output_mask_int_shape = K.int_shape(output_mask)
if output_mask_int_shape is None:
# if the output_mask does not have a static shape,
# its shape must be the same as mask's
if mask is not None:
output_mask_int_shape = K.int_shape(mask)
else:
output_mask_int_shape = K.compute_output_shape(
input_shape)[:-1]
output_mask_shape = self._get_shape_tuple(
(-1, input_length), output_mask, 1, output_mask_int_shape[1:])
output_mask = K.reshape(output_mask, output_mask_shape)
return output_mask
def compute_mask(self, inputs, mask=None):
"""Computes an output mask tensor for Embedding layer.
This is based on the inputs, mask, and the inner layer.
If batch size is specified:
Simply return the input `mask`. (An rnn-based implementation with
more than one rnn inputs is required but not supported in tf.keras yet.)
Otherwise we call `compute_mask` of the inner layer at each time step.
If the output mask at each time step is not `None`:
(E.g., inner layer is Masking or RNN)
Concatenate all of them and return the concatenation.
If the output mask at each time step is `None` and the input mask is not
`None`:(E.g., inner layer is Dense)
Reduce the input_mask to 2 dimensions and return it.
Otherwise (both the output mask and the input mask are `None`):
(E.g., `mask` is not used at all)
Return `None`.
Arguments:
inputs: Tensor with shape [batch size, timesteps, ...] indicating the
input to TimeDistributed. If static shape information is available for
"batch size", `mask` is returned unmodified.
mask: Either None (indicating no masking) or a Tensor indicating the
input mask for TimeDistributed. The shape can be static or dynamic.
Returns:
Either None (no masking), or a [batch size, timesteps, ...] Tensor with
an output mask for the TimeDistributed layer with the shape beyond the
second dimension being the value of the input mask shape(if the computed
output mask is none), an output mask with the shape beyond the first
dimension being the value of the mask shape(if mask is not None) or
output mask with the shape beyond the first dimension being the
value of the computed output shape.
"""
# cases need to call the layer.compute_mask when input_mask is None:
# Masking layer and Embedding layer with mask_zero
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, we currently do not handle mask explicitly
return mask
inner_mask = mask
if inner_mask is not None:
inner_mask_shape = self._get_shape_tuple((-1, ), mask, 2)
inner_mask = K.reshape(inner_mask, inner_mask_shape)
input_uid = generic_utils.object_list_uid(inputs)
inner_inputs = self._input_map.get(input_uid, inputs)
output_mask = self.layer.compute_mask(inner_inputs, inner_mask)
if output_mask is None:
if mask is None:
return None
# input_mask is not None, and output_mask is None:
# we should return a not-None mask
output_mask = mask
for _ in range(2, len(K.int_shape(mask))):
output_mask = K.any(output_mask, axis=-1)
else:
# output_mask is not None. We need to reshape it
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
output_mask_int_shape = K.int_shape(output_mask)
if output_mask_int_shape is None:
# if the output_mask does not have a static shape,
# its shape must be the same as mask's
if mask is not None:
output_mask_int_shape = K.int_shape(mask)
else:
output_mask_int_shape = K.compute_output_shape(
input_shape)[:-1]
output_mask_shape = self._get_shape_tuple(
(-1, input_length), output_mask, 1, output_mask_int_shape[1:])
output_mask = K.reshape(output_mask, output_mask_shape)
return output_mask
def call(self, inputs):
boolean_mask = K.any(
math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
return inputs * math_ops.cast(boolean_mask, inputs.dtype)
def compute_mask(self, inputs, mask=None): return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1)
def greater3_on_axis(x, axis):
x = K.sum(x, axis=axis)
x = K.greater(x, 3)
x = K.any(x)
return x
def compute_mask(self, inputs, mask=None): return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1)
def call(self, inputs):
boolean_mask = K.any(
math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
return inputs * math_ops.cast(boolean_mask, inputs.dtype)
def compute_mask(self, inputs, mask=None):
"""Computes an output mask tensor for Embedding layer.
This is based on the inputs, mask, and the inner layer.
If batch size is specified:
Simply return the input `mask`. (An rnn-based implementation with
more than one rnn inputs is required but not supported in tf.keras yet.)
Otherwise we call `compute_mask` of the inner layer at each time step.
If the output mask at each time step is not `None`:
(E.g., inner layer is Masking or RNN)
Concatenate all of them and return the concatenation.
If the output mask at each time step is `None` and the input mask is not
`None`:(E.g., inner layer is Dense)
Reduce the input_mask to 2 dimensions and return it.
Otherwise (both the output mask and the input mask are `None`):
(E.g., `mask` is not used at all)
Return `None`.
Args:
inputs: Tensor with shape [batch size, timesteps, ...] indicating the
input to TimeDistributed. If static shape information is available for
"batch size", `mask` is returned unmodified.
mask: Either None (indicating no masking) or a Tensor indicating the
input mask for TimeDistributed. The shape can be static or dynamic.
Returns:
Either None (no masking), or a [batch size, timesteps, ...] Tensor with
an output mask for the TimeDistributed layer with the shape beyond the
second dimension being the value of the input mask shape(if the computed
output mask is none), an output mask with the shape beyond the first
dimension being the value of the mask shape(if mask is not None) or
output mask with the shape beyond the first dimension being the
value of the computed output shape.
"""
# cases need to call the layer.compute_mask when input_mask is None:
# Masking layer and Embedding layer with mask_zero
input_shape = nest.map_structure(
lambda x: tensor_shape.TensorShape(backend.int_shape(x)), inputs)
input_shape = tf_utils.convert_shapes(input_shape, to_tuples=False)
batch_size = tf_utils.convert_shapes(input_shape)
batch_size = nest.flatten(batch_size)[0]
is_ragged_input = nest.map_structure(
lambda x: isinstance(x, ragged_tensor.RaggedTensor), inputs)
is_ragged_input = generic_utils.to_list(nest.flatten(is_ragged_input))
if batch_size and not self._always_use_reshape or any(is_ragged_input):
# batch size matters, we currently do not handle mask explicitly, or if
# the layer always uses reshape approach, or the input is a ragged tensor.
return mask
inner_mask = mask
if inner_mask is not None:
inner_mask_shape = self._get_shape_tuple((-1, ), mask, 2)
inner_mask = backend.reshape(inner_mask, inner_mask_shape)
inner_input_shape = nest.map_structure(
lambda tensor: self._get_shape_tuple((-1, ), tensor, 2), inputs)
inner_inputs = nest.map_structure_up_to(inputs, array_ops.reshape,
inputs, inner_input_shape)
output_mask = self.layer.compute_mask(inner_inputs, inner_mask)
if output_mask is None:
if mask is None:
return None
# input_mask is not None, and output_mask is None:
# we should return a not-None mask
output_mask = mask
for _ in range(2, len(backend.int_shape(mask))):
output_mask = backend.any(output_mask, axis=-1)
else:
# output_mask is not None. We need to reshape it
input_length = tf_utils.convert_shapes(input_shape)
input_length = nest.flatten(input_length)[1]
if not input_length:
input_length = nest.map_structure(
lambda x: backend.shape(x)[1], inputs)
input_length = nest.flatten(input_length)[0]
output_mask_int_shape = backend.int_shape(output_mask)
if output_mask_int_shape is None:
# if the output_mask does not have a static shape,
# its shape must be the same as mask's
if mask is not None:
output_mask_int_shape = backend.int_shape(mask)
else:
input_shape = generic_utils.to_list(
nest.flatten(input_shape))[0]
output_mask_int_shape = backend.compute_output_shape(
input_shape)[:-1]
output_mask_shape = self._get_shape_tuple(
(-1, input_length), output_mask, 1, output_mask_int_shape[1:])
output_mask = backend.reshape(output_mask, output_mask_shape)
return output_mask
def compute_mask(self, inputs, mask=None):
"""Computes an output mask tensor for Embedding layer.
This is based on the inputs, mask, and the inner layer.
If batch size is specified:
Simply return the input `mask`. (An rnn-based implementation with
more than one rnn inputs is required but not supported in tf.keras yet.)
Otherwise we call `compute_mask` of the inner layer at each time step.
If the output mask at each time step is not `None`:
(E.g., inner layer is Masking or RNN)
Concatenate all of them and return the concatenation.
If the output mask at each time step is `None` and the input mask is not
`None`:(E.g., inner layer is Dense)
Reduce the input_mask to 2 dimensions and return it.
Otherwise (both the output mask and the input mask are `None`):
(E.g., `mask` is not used at all)
Return `None`.
Arguments:
inputs: Tensor with shape [batch size, timesteps, ...] indicating the
input to TimeDistributed. If static shape information is available for
"batch size", `mask` is returned unmodified.
mask: Either None (indicating no masking) or a Tensor indicating the
input mask for TimeDistributed. The shape can be static or dynamic.
Returns:
Either None (no masking), or a [batch size, timesteps, ...] Tensor with
an output mask for the TimeDistributed layer with the shape beyond the
second dimension being the value of the input mask shape(if the computed
output mask is none), an output mask with the shape beyond the first
dimension being the value of the mask shape(if mask is not None) or
output mask with the shape beyond the first dimension being the
value of the computed output shape.
"""
# cases need to call the layer.compute_mask when input_mask is None:
# Masking layer and Embedding layer with mask_zero
input_shape = K.int_shape(inputs)
if input_shape[0]:
# batch size matters, we currently do not handle mask explicitly
return mask
inner_mask = mask
if inner_mask is not None:
inner_mask_shape = self._get_shape_tuple((-1,), mask, 2)
inner_mask = K.reshape(inner_mask, inner_mask_shape)
input_uid = generic_utils.object_list_uid(inputs)
inner_inputs = self._input_map.get(input_uid, inputs)
output_mask = self.layer.compute_mask(inner_inputs, inner_mask)
if output_mask is None:
if mask is None:
return None
# input_mask is not None, and output_mask is None:
# we should return a not-None mask
output_mask = mask
for _ in range(2, len(K.int_shape(mask))):
output_mask = K.any(output_mask, axis=-1)
else:
# output_mask is not None. We need to reshape it
input_length = input_shape[1]
if not input_length:
input_length = K.shape(inputs)[1]
output_mask_int_shape = K.int_shape(output_mask)
if output_mask_int_shape is None:
# if the output_mask does not have a static shape,
# its shape must be the same as mask's
if mask is not None:
output_mask_int_shape = K.int_shape(mask)
else:
output_mask_int_shape = K.compute_output_shape(input_shape)[:-1]
output_mask_shape = self._get_shape_tuple(
(-1, input_length), output_mask, 1, output_mask_int_shape[1:])
output_mask = K.reshape(output_mask, output_mask_shape)
return output_mask
def mask_loss(self, y_true, y_pred):
def _mask(y_true, y_pred, iou_threshold=0.5, mask_size=(28, 28)):
# split up the different predicted blobs
boxes = y_pred[:, :, :4]
masks = y_pred[:, :, 4:]
# split up the different blobs
annotations = y_true[:, :, :5]
width = K.cast(y_true[0, 0, 5], dtype='int32')
height = K.cast(y_true[0, 0, 6], dtype='int32')
masks_target = y_true[:, :, 7:]
# reshape the masks back to their original size
masks_target = K.reshape(masks_target,
(K.shape(masks_target)[0] *
K.shape(masks_target)[1], height, width))
masks = K.reshape(masks, (K.shape(masks)[0] * K.shape(masks)[1],
mask_size[0], mask_size[1], -1))
# batch size > 1 fix
boxes = K.reshape(boxes, (-1, K.shape(boxes)[2]))
annotations = K.reshape(annotations, (-1, K.shape(annotations)[2]))
# compute overlap of boxes with annotations
iou = overlap(boxes, annotations)
argmax_overlaps_inds = K.argmax(iou, axis=1)
max_iou = K.max(iou, axis=1)
# filter those with IoU > 0.5
indices = tf.where(K.greater_equal(max_iou, iou_threshold))
boxes = tf.gather_nd(boxes, indices)
masks = tf.gather_nd(masks, indices)
argmax_overlaps_inds = tf.gather_nd(argmax_overlaps_inds, indices)
argmax_overlaps_inds = K.cast(argmax_overlaps_inds, 'int32')
labels = K.gather(annotations[:, 4], argmax_overlaps_inds)
labels = K.cast(labels, 'int32')
# make normalized boxes
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
boxes = K.stack([
y1 / (K.cast(height, dtype=K.floatx()) - 1),
x1 / (K.cast(width, dtype=K.floatx()) - 1),
(y2 - 1) / (K.cast(height, dtype=K.floatx()) - 1),
(x2 - 1) / (K.cast(width, dtype=K.floatx()) - 1),
],
axis=1)
# crop and resize masks_target
# append a fake channel dimension
masks_target = K.expand_dims(masks_target, axis=3)
masks_target = tf.image.crop_and_resize(masks_target, boxes,
argmax_overlaps_inds,
mask_size)
# remove fake channel dimension
masks_target = masks_target[:, :, :, 0]
# gather the predicted masks using the annotation label
masks = tf.transpose(masks, (0, 3, 1, 2))
label_indices = K.stack([tf.range(K.shape(labels)[0]), labels],
axis=1)
masks = tf.gather_nd(masks, label_indices)
# compute mask loss
mask_loss = K.binary_crossentropy(masks_target, masks)
normalizer = K.shape(masks)[0] * K.shape(masks)[1] * K.shape(
masks)[2]
normalizer = K.maximum(K.cast(normalizer, K.floatx()), 1)
mask_loss = K.sum(mask_loss) / normalizer
return mask_loss
# if there are no masks annotations, return 0; else, compute the masks loss
return tf.cond(
K.any(K.equal(K.shape(y_true), 0)), lambda: K.cast_to_floatx(0.0),
lambda: _mask(y_true,
y_pred,
iou_threshold=self.iou_threshold,
mask_size=self.mask_size))
def is_winning(x):
return K.any(K.stack([greater3(x), diaggreater3(x), fulldiaggreater3(x)]))
def __call__(self,
*args,
inputs=None,
initial_state=None,
constants=None,
mask=None,
**kwargs):
inputs, initial_state, constants = _standardize_args(
inputs, initial_state, constants, self._num_constants)
# We allow different shapes of input, even None. It doesn't really matter
# because ultimately the input will be ignored except for the first step.
# Nevertheless, we expand the input to have a timesteps dimension. This
# is done simply for parent class calculations of output size, etc.
# Allow None as an input. We will create an array of zeros of appropriate shape.
if inputs is None:
if initial_state is not None:
# If LSTM then state might be a list.
_state = initial_state[0] if isinstance(
initial_state, list) else initial_state
batch_size = _state.shape[:-1]
inputs = K.zeros_like(_state[..., 0][..., tf.newaxis])
# inputs = 0 * _state[..., 0][..., tf.newaxis] # Assume dim=1 input
else:
# Neither inputs nor initial_state provided. This likely only happens
# when building/testing the layer.
inputs = tf.zeros(
(self.timesteps, 1, 1)) if self.time_major else tf.zeros(
(1, self.timesteps, 1))
# Allow 2D input, here reshape to 3D input
if len(K.int_shape(inputs)) < 3:
if self.time_major:
inputs = inputs[tf.newaxis, ...]
else:
inputs = inputs[..., tf.newaxis, :]
time_ax_ix, batch_ax_ix = (0, 1) if self.time_major else (-2, 0)
input_shape = K.int_shape(inputs)
input_timesteps = input_shape[time_ax_ix]
if mask is not None and K.any(~mask):
mask = nest.flatten(mask)[0]
# We assume mask has a time dimension and require it is same size as input
# (It doesn't make sense to use mask otherwise).
mask_shape = K.int_shape(mask)
# If the mask only has 1 item in the batch dim then tile it
if mask_shape[batch_ax_ix] == 1 and input_shape[batch_ax_ix] > 1:
if self.time_major:
bcast_or = tf.zeros((1, input_shape[batch_ax_ix], 1),
dtype=tf.bool)
else:
bcast_or = tf.zeros((input_shape[batch_ax_ix], 1, 1),
dtype=tf.bool)
mask = tf.math.logical_or(mask, bcast_or)
if mask_shape[time_ax_ix] == input_timesteps:
# Prepare slice parameters
# For head (kept)
h_sl_begin = [0 for _ in input_shape]
h_sl_sz = [-1 for _ in input_shape]
h_sl_sz[batch_ax_ix] = 1
# For tail (replaced)
t_sl_begin = [0 for _ in input_shape]
t_sl_sz = [-1 for _ in input_shape]
t_sl_sz[batch_ax_ix] = 1
# Collect input replacements in list
new_inputs = []
for batch_ix in range(input_shape[batch_ax_ix]):
samp_mask = mask[
...,
batch_ix, :] if self.time_major else mask[batch_ix]
if K.any(~samp_mask):
h_sl_begin[batch_ax_ix] = batch_ix
t_sl_begin[batch_ax_ix] = batch_ix
first_bad = tf.where(~samp_mask)[0, 0]
h_sl_sz[time_ax_ix] = first_bad # sz is 1-based
t_sl_begin[time_ax_ix] = first_bad
head = tf.slice(inputs, h_sl_begin, h_sl_sz)
tail = tf.slice(inputs, t_sl_begin, t_sl_sz)
if self.tile_input:
tile_samp = head[-1] if self.time_major else head[
..., -1, :]
else:
tile_samp = tf.zeros((1, input_shape[-1]))
new_row = tf.concat(
(head, tile_samp * K.ones_like(tail)),
axis=time_ax_ix)
new_inputs.append(new_row)
inputs = tf.concat(new_inputs, axis=batch_ax_ix)
# Fill/trim input time dimension to be self.timesteps
if input_timesteps > self.timesteps:
# Trim excess, if any
inputs = inputs[:self.timesteps,
...] if self.time_major else inputs[
..., :self.timesteps, :]
elif input_timesteps < self.timesteps:
# Take the last timestep as our starting point for the padding data
pad_sample = inputs[-1] if self.time_major else inputs[..., -1, :]
if not self.tile_input:
# zero out padding data if we aren't tiling
pad_sample = K.zeros_like(pad_sample)
# pad_sample = 0 * pad_sample
# Add the time axis back to our pad_sample
pad_sample = pad_sample[tf.newaxis,
...] if self.time_major else pad_sample[
..., tf.newaxis, :]
# How many more timestamps do we need?
pad_timestamps = self.timesteps - K.int_shape(inputs)[time_ax_ix]
# Tile pad_data using broadcast-add. Does this same line work for time_major and not?
pad_data = pad_sample + tf.zeros((pad_timestamps, 1))
inputs = tf.concat((inputs, pad_data), axis=time_ax_ix)
if not self._built_with_input:
self.build_with_input(inputs,
*args,
initial_state=initial_state,
constants=constants,
mask=mask,
**kwargs)
return super().__call__(inputs,
initial_state=initial_state,
constants=constants,
**kwargs)