def compute_mask(self, inputs, mask=None):
if len(inputs[1].shape) == 3:
output_mask = K.any(K.not_equal(inputs[1], self.mask_value),
axis=-1)
else:
output_mask = K.not_equal(inputs[1], self.mask_value)
return output_mask
def __init__(self, src, trg=None, pad=0):
self.src = src
self.src_mask = K.expand_dims(K.not_equal(src, pad), axis=-2)
if trg is not None:
self.trg = trg[:, :-1] # without last token of sentence
self.trg_y = trg[:, 1:] # without first token of sentence
self.trg_mask = self.make_std_mask(self.trg, pad)
self.ntokens = K.sum(
K.cast(K.not_equal(self.trg_y, pad), dtype='uint8'))
def __call__(self, y_true, y_pred, sample_weight=None):
""" Automatically calculates the weight of 0-1 activations using the number of
activations in y_true.
"""
num_ones = K.sum(y_true)
num_zeros = K.sum(K.cast_to_floatx(K.not_equal(y_true, 0.0)))
sw = K.cast_to_floatx(K.equal(y_true, 0.0)) * (num_ones / num_zeros)
sw = math_ops.add(sw, K.cast_to_floatx(K.not_equal(y_true, 0.0)) * 1)
return super().__call__(y_true, y_pred, sample_weight=sw)
def call(self, inputs):
if len(inputs[1].shape) == 3:
boolean_mask = K.any(K.not_equal(inputs[1], self.mask_value),
axis=-1,
keepdims=True)
else:
boolean_mask = K.expand_dims(
K.not_equal(inputs[1], self.mask_value))
return inputs[0] * K.cast(boolean_mask, K.dtype(inputs[0]))
def _get_triplet_mask(self, y_true: Tensor, pairwise_dist: Tensor) -> Tensor:
# mask label(a) != label(p)
mask1 = K.expand_dims(K.equal(K.expand_dims(y_true, 0), K.expand_dims(y_true, 1)), 2)
mask1 = K.cast(mask1, K.dtype(pairwise_dist))
# mask a == p
mask2 = K.expand_dims(K.not_equal(pairwise_dist, 0.0), 2)
mask2 = K.cast(mask2, K.dtype(pairwise_dist))
# mask label(n) == label(a)
mask3 = K.expand_dims(K.not_equal(K.expand_dims(y_true, 0), K.expand_dims(y_true, 1)), 1)
mask3 = K.cast(mask3, K.dtype(pairwise_dist))
return mask1 * mask2 * mask3
def exact_matched_accuracy(y_true, y_pred, mask_id):
true_ids = bk.argmax(y_true, axis=-1)
pred_ids = bk.argmax(y_pred, axis=-1)
maskBool = bk.not_equal(true_ids, mask_id)
maskInt64 = bk.cast(maskBool, 'int64')
diff = (true_ids - pred_ids) * maskInt64
matches = bk.cast(bk.not_equal(diff, bk.zeros_like(diff)), 'int64')
matches = bk.sum(matches, axis=-1)
matches = bk.cast(bk.equal(matches, bk.zeros_like(matches)), bk.floatx())
return bk.mean(matches)
def masked_loss(y_true, y_pred):
max_args = argmax(y_true)
mask = cast(not_equal(max_args, zeros_like(max_args)), dtype='float32')
loss = switch(mask,
categorical_crossentropy(y_true, y_pred, from_logits=True),
zeros_like(mask, dtype=floatx()))
return sum(loss) / (cast(sum(mask), dtype='float32') + epsilon())
def update_state(self, y_true, y_pred, sample_weight=None):
# if log_metrics:
# wandb_log_report(report)
# skip to count samples with label __unknown__
mask = K.cast(K.not_equal(y_true, 1), K.floatx())
if self.threshold is None:
threshold = tf.reduce_max(y_pred, axis=-1, keepdims=True)
# make sure [0, 0, 0] doesn't become [1, 1, 1]
# Use abs(x) > eps, instead of x != 0 to check for zero
y_pred = tf.logical_and(y_pred >= threshold,
tf.abs(y_pred) > 1e-12)
else:
y_pred = y_pred > self.threshold
y_true = K.one_hot(K.cast(K.flatten(y_true), tf.int32),
y_pred.shape[1])
y_true = tf.cast(y_true, self.dtype)
y_pred = tf.cast(y_pred, self.dtype)
# # skip counting samples where the PAD token is predicted
def _weighted_sum(val, sample_weight):
if sample_weight is not None:
val = tf.math.multiply(val, tf.expand_dims(sample_weight, 1))
return tf.reduce_sum(val * mask, axis=self.axis)[2:]
self.true_positives.assign_add(
_weighted_sum(y_pred * y_true, sample_weight))
self.false_positives.assign_add(
_weighted_sum(y_pred * (1 - y_true), sample_weight))
self.false_negatives.assign_add(
_weighted_sum((1 - y_pred) * y_true, sample_weight))
self.weights_intermediate.assign_add(
_weighted_sum(y_true, sample_weight))
def qscore(y_true, y_pred):
from tensorflow.keras import backend as K
error = K.cast(
K.not_equal(K.max(y_true, axis=-1),
K.cast(K.argmax(y_pred, axis=-1), K.floatx())), K.floatx())
error = K.sum(error) / K.sum(K.ones_like(error))
return -10.0 * 0.434294481 * K.log(error)
def decorator(self, x):
# Only call layer if there are input spikes. This is to prevent
# accumulation of bias.
self.impulse = tf.cond(k.any(k.not_equal(x, 0)), lambda: call(self, x),
lambda: k.zeros_like(self.mem))
return self.update_neurons()
def call(self, inputs, mask=None):
atoms, bonds, edges = inputs
# Import dimensions
num_samples = atoms.shape[0]
max_atoms = atoms.shape[1]
num_atom_features = atoms.shape[-1]
num_bond_features = bonds.shape[-1]
# Create a matrix that stores for each atom, the degree it is
# atom_degrees = K.sum(K.not_equal(edges, -1), axis=-1, keepdims=True)
# backend cast to floatx:
atom_degrees = K.sum(K.cast(K.not_equal(edges, -1), 'float64'),
axis=-1,
keepdims=True)
# For each atom, look up the features of it's neighbour
neighbour_atom_features = neighbour_lookup(atoms,
edges,
include_self=True)
# Sum along degree axis to get summed neighbour features
summed_atom_features = K.sum(neighbour_atom_features, axis=-2)
# Sum the edge features for each atom
summed_bond_features = K.sum(bonds, axis=-2)
# Concatenate the summed atom and bond features
# summed_features = K.concatenate([summed_atom_features, summed_bond_features], axis=-1)
# Tensorflow concat:
summed_features = tf.concat(
[summed_atom_features, summed_bond_features], axis=-1)
# For each degree we convolve with a different weight matrix
new_features_by_degree = []
for degree in range(self.max_degree):
# Create mask for this degree
atom_masks_this_degree = K.cast(K.equal(atom_degrees, degree),
K.floatx())
# Multiply with hidden merge layer
# (use time Distributed because we are dealing with 2D input/3D for batches)
# Add keras shape to let keras now the dimensions
summed_features._keras_shape = (None, max_atoms,
num_atom_features +
num_bond_features)
new_unmasked_features = self.inner_3D_layers[degree](
summed_features)
# Do explicit masking because TimeDistributed does not support masking
new_masked_features = new_unmasked_features * atom_masks_this_degree
new_features_by_degree.append(new_masked_features)
# Finally sum the features of all atoms
new_features = layers.add(new_features_by_degree)
return new_features
def f(y_true, y_pred):
mask_true = K.cast(K.not_equal(y_true, mask_value), K.floatx())
masked_squared_error = K.square(mask_true * (y_true - y_pred))
# in case mask_true is 0 everywhere, the error would be nan, therefore divide by at least 1
# this doesn't change anything as where sum(mask_true)==0, sum(masked_squared_error)==0 as well
masked_mse = K.sum(masked_squared_error, axis=-1) / K.maximum(K.sum(mask_true, axis=-1), 1)
return masked_mse
def precisionK(y_true, y_pred):
# works with non binary data as well as binary
y_true_class = K.argmax(y_true, axis=-1)
y_pred_class = K.argmax(y_pred, axis=-1)
TP = K.cast(K.equal(y_true_class, y_pred_class), dtype='int32')
nonzero_pred = K.cast(K.not_equal(y_pred_class, 0), dtype='int32')
return K.sum(TP * nonzero_pred) / K.maximum(K.sum(nonzero_pred), 1)
def loss(self, y_true, y_pred):
# We always delay import of Keras so that mhcflurry can be imported
# initially without tensorflow debug output, etc.
configure_tensorflow()
from tensorflow.keras import backend as K
y_true = K.flatten(y_true)
y_pred = K.flatten(y_pred)
# Handle (=) inequalities
diff1 = y_pred - y_true
diff1 *= K.cast(y_true >= 0.0, "float32")
diff1 *= K.cast(y_true <= 1.0, "float32")
# Handle (>) inequalities
diff2 = y_pred - (y_true - 2.0)
diff2 *= K.cast(y_true >= 2.0, "float32")
diff2 *= K.cast(y_true <= 3.0, "float32")
diff2 *= K.cast(diff2 < 0.0, "float32")
# Handle (<) inequalities
diff3 = y_pred - (y_true - 4.0)
diff3 *= K.cast(y_true >= 4.0, "float32")
diff3 *= K.cast(diff3 > 0.0, "float32")
denominator = K.maximum(
K.sum(K.cast(K.not_equal(y_true, 2.0), "float32"), 0),
1.0)
result = (
K.sum(K.square(diff1)) +
K.sum(K.square(diff2)) +
K.sum(K.square(diff3))) / denominator
return result
def inequaility_loss(y_true, y_pred):
## adapted from MHCflurry
from tensorflow.keras import backend as K
y_true = K.flatten(y_true)
y_pred = K.flatten(y_pred)
# Handle (=) inequalities
diff1 = y_pred - y_true
diff1 *= K.cast(y_true >= 0.0, "float32")
diff1 *= K.cast(y_true <= 1.0, "float32")
# Handle (>) inequalities
diff2 = y_pred - (y_true - 2.0)
diff2 *= K.cast(y_true >= 2.0, "float32")
diff2 *= K.cast(y_true <= 3.0, "float32")
diff2 *= K.cast(diff2 > 0.0, "float32")
# Handle (<) inequalities
diff3 = y_pred - (y_true - 4.0)
diff3 *= K.cast(y_true >= 4.0, "float32")
diff3 *= K.cast(diff3 > 0.0, "float32")
denominator = K.maximum(
K.sum(K.cast(K.not_equal(y_true, 2.0), "float32"), 0),
1.0)
result = (
K.sum(K.square(diff1)) +
K.sum(K.square(diff2)) +
K.sum(K.square(diff3))) / denominator
return result
def ignore_accuracy(self, y_true, y_pred):
pred = argmax(y_pred, axis=-1)
true = argmax(y_true, axis=-1)
ignore_mask = cast(not_equal(pred, 0), 'int32')
matches = cast(equal(true, pred), 'int32') * ignore_mask
accuracy = sum(matches) / maximum(sum(ignore_mask), 1)
return accuracy
def masked_accuracy(y_true, y_pred):
max_args = argmax(y_true)
mask = cast(not_equal(max_args, zeros_like(max_args)), dtype='float32')
points = switch(
mask,
cast(equal(argmax(y_true, -1), argmax(y_pred, -1)), dtype='float32'),
zeros_like(mask, dtype=floatx()))
return sum(points) / cast(sum(mask), dtype='float32')
def ignore_acc(y_true_class, y_pred_class, class_to_ignore=0):
y_pred_class = K.cast(K.argmax(y_pred_class, axis=-1), 'int32')
y_true_class = K.cast(y_true_class, 'int32')
ignore_mask = K.cast(K.not_equal(y_true_class, class_to_ignore), 'int32')
matches = K.cast(K.equal(y_true_class, y_pred_class),
'int32') * ignore_mask
accuracy = K.sum(matches) / K.maximum(K.sum(ignore_mask), 1)
return accuracy
def GetPadMask(q, k):
'''
shape: [B, Q, K]
'''
ones = K.expand_dims(K.ones_like(q, 'float32'), -1)
mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')
mask = K.batch_dot(ones, mask, axes=[2, 1])
return mask
def masked_mse(y_true, y_pred):
# masked function
mask_true = K.cast(K.not_equal(y_true, 0), K.floatx())
# masked squared error
masked_squared_error = K.square(mask_true * (y_true - y_pred))
masked_mse = K.sum(masked_squared_error, axis=-1) / K.maximum(
K.sum(mask_true, axis=-1), 1)
return masked_mse
def get_notnull_indices(tensor):
#y_pred_flat = tf.reshape(y_pred, [-1])
zero = K.constant(0, dtype=tf.float32)
where = K.not_equal(tensor, zero)
indices = tf.where(where)
return indices
def positions_func(inputs, pad=0):
"""
A layer filling i-th column of a 2D tensor with
1+ln(1+i) when it contains a meaningful symbol
and with 0 when it contains PAD
"""
position_inputs = K.cumsum(K.ones_like(inputs, dtype="float32"), axis=1)
position_inputs *= K.cast(K.not_equal(inputs, pad), "float32")
return K.log(1.0 + position_inputs)
def loss_cls(y_true, y_pred):
condition = K.not_equal(y_true, -1)
indices = tf.where(condition)
target = tf.gather_nd(y_true, indices)
output = tf.gather_nd(y_pred, indices)
loss = K.binary_crossentropy(target, output)
return K.mean(loss)
def compute_mask(self, inputs, mask=None):
if self.mode == self.MODE_EXPAND:
if self.mask_zero:
output_mask = K.not_equal(inputs, self.mask_zero)
else:
output_mask = None
else:
output_mask = mask
return output_mask
def get_pos_seq(self, x):
mask = K.cast(K.not_equal(x, 0), 'int32')
# Nok: Replace K.cumsum with operations those can be run in CUDA (to support Data-Parallel Multi-GPU training)
# pos = K.cumsum(K.ones_like(x, 'int32'), 1)
tensor_shape = shape_list(x)
pos = tf.add(tf.range(tensor_shape[1]), 1)
pos = tf.tile(pos, [tensor_shape[0]])
pos = tf.reshape(pos, tensor_shape)
return pos * mask
def custom_accuracy(y_true, y_pred):
y_true_class = K.argmax(y_true, axis=-1)
y_pred_class = K.argmax(y_pred, axis=-1)
ignore_mask = K.cast(K.not_equal(y_pred_class, to_ignore), 'int32')
matches = K.cast(K.equal(y_true_class, y_pred_class),
'int32') * ignore_mask
accuracy = K.sum(matches) / K.maximum(K.sum(ignore_mask), 1)
return accuracy
def f(y_true, y_pred):
error = y_true - y_pred
cond = K.abs(error) < clip_delta
mask_true = K.cast(K.not_equal(y_true, mask_value), K.floatx())
masked_squared_error = 0.5 * K.square(mask_true *
(y_true - y_pred))
linear_loss = mask_true * (clip_delta * K.abs(error) - 0.5 *
(clip_delta**2))
huber_loss = tf.where(cond, masked_squared_error, linear_loss)
return K.sum(huber_loss) / K.sum(mask_true)
def masked_mse(y, p, mask_val):
mask = K.cast(K.not_equal(y, mask_val), K.floatx())
if tf.__version__[0] == '2':
masked_loss = tf.losses.mse(y * mask, p * mask)
else:
mask = K.cast(mask, 'float32')
masked_loss = K.mean(tf.math.square(p * mask - y * mask), axis=-1)
# masked_loss = tf.compat.v1.losses.mean_squared_error(y*mask, p*mask)
return masked_loss
def mask_aware_mean(inputs):
# https://github.com/github/CodeSearchNet/blob/master/src/utils/tfutils.py#L107
# recreate the masks - all zero rows have been masked
mask = backend.not_equal(backend.sum(backend.abs(inputs), axis=2, keepdims=True), 0)
# number of that rows are not all zeros
num = backend.sum(backend.cast(mask, 'float32'), axis=1, keepdims=False)
# compute mask-aware mean of inputs
inputs_mean = backend.sum(inputs, axis=1, keepdims=False) / (num + 1E-8)
return inputs_mean
def call(self, inputs, mask=None):
atoms, bonds, edges = inputs
# Import dimensions
num_samples = atoms.shape[0]
max_atoms = atoms.shape[1]
num_atom_features = atoms.shape[-1]
num_bond_features = bonds.shape[-1]
# Create a matrix that stores for each atom, the degree it is, use it
# to create a general atom mask (unused atoms are 0 padded)
# We have to use the edge vector for this, because in theory, a convolution
# could lead to a zero vector for an atom that is present in the molecule
# atom_degrees = K.sum(K.not_equal(edges, -1), axis=-1, keepdims=True)
# backend cast to floatx:
atom_degrees = K.sum(K.cast(K.not_equal(edges, -1), 'float64'),
axis=-1,
keepdims=True)
general_atom_mask = K.cast(K.not_equal(atom_degrees, 0), K.floatx())
# Sum the edge features for each atom
summed_bond_features = K.sum(bonds, axis=-2)
# Concatenate the summed atom and bond features
# atoms_bonds_features = K.concatenate([atoms, summed_bond_features], axis=-1)
# Tensorflow concat:
atoms_bonds_features = tf.concat([atoms, summed_bond_features],
axis=-1)
# Compute fingerprint
atoms_bonds_features._keras_shape = (None, max_atoms,
num_atom_features +
num_bond_features)
fingerprint_out_unmasked = self.inner_3D_layer(atoms_bonds_features)
# Do explicit masking because TimeDistributed does not support masking
fingerprint_out_masked = fingerprint_out_unmasked * general_atom_mask
# Sum across all atoms
final_fp_out = K.sum(fingerprint_out_masked, axis=-2)
return final_fp_out
