def haraka_post_corrections(self, in_out):
"""
Change any obviously wrong haraka marks according to the character and its context.
:param in_out: input layer and prediction layers outputs.
:return: corrected predictions.
"""
inputs, pred_haraka, pred_shadda = in_out
if not self.rules_enabled:
return pred_haraka
char_index = K.argmax(inputs[:, -1], axis=-1)
# Force the correct haraka on some letters
forced_diac_chars = {CHAR2INDEX['إ']: 3}
for f_diac_char, f_diac in forced_diac_chars.items():
mask = K.reshape(K.cast(K.not_equal(char_index, f_diac_char), 'float32'), (-1, 1))
pred_haraka = mask * pred_haraka + (1 - mask) * K.one_hot(f_diac, K.int_shape(pred_haraka)[-1])
# Force the correct haraka before some letters
f_prev_diac_chars = {CHAR2INDEX['ى']: 1, CHAR2INDEX['ة']: 1}
prev_char_index = K.argmax(inputs[:, -2], axis=-1)
for fd_char, f_diac in f_prev_diac_chars.items():
mask = K.cast(K.not_equal(char_index[1:], fd_char), 'float32')
mask = K.reshape(K.concatenate([mask, K.ones((1,))], axis=0), (-1, 1))
pred_haraka = pred_haraka * mask + (1 - mask) * K.one_hot(f_diac, K.int_shape(pred_haraka)[-1])
# Allow only Fatha, Fathatan, or nothing before ا if it is in the end of the word
mask = K.reshape(K.concatenate([K.clip(
K.cast(K.not_equal(char_index[1:-1], CHAR2INDEX['ا']), 'float32') +
K.cast(K.not_equal(char_index[2:], CHAR2INDEX[' ']), 'float32'), 0, 1), K.ones((2,))], axis=0), (-1, 1))
pred_haraka = mask * pred_haraka + (1 - mask) * K.constant([1, 1, 0, 0, 0, 1, 0, 0], shape=(1, 8)) * pred_haraka
# Force Fatha before ا if it is not in the end of the word
mask = K.reshape(K.concatenate([K.clip(
K.cast(K.not_equal(char_index[1:-1], CHAR2INDEX['ا']), 'float32') +
K.cast(K.equal(char_index[2:], CHAR2INDEX[' ']), 'float32'), 0, 1), K.ones((2,))], axis=0), (-1, 1))
pred_haraka = mask * pred_haraka + (1 - mask) * K.one_hot(1, K.int_shape(pred_haraka)[-1])
# Force no sukun and tanween at the beginning of the word
mask = K.reshape(
K.concatenate([K.zeros((1,)), K.cast(K.not_equal(prev_char_index[1:], CHAR2INDEX[' ']), 'float32')],
axis=0), (-1, 1))
pred_haraka = mask * pred_haraka + (1 - mask) * K.constant([1, 1, 1, 1, 0, 0, 0, 0], shape=(1, 8)) * pred_haraka
# Allow tanween only at the end of the word
mask = K.reshape(K.concatenate([K.cast(K.not_equal(char_index[1:], CHAR2INDEX[' ']), 'float32'), K.zeros((1,))],
axis=0), (-1, 1))
pred_haraka = mask * K.constant([1, 1, 1, 1, 1, 0, 0, 0], shape=(1, 8)) * pred_haraka + (1 - mask) * pred_haraka
# Prohibit Fathatan on most letters
mask = K.reshape(K.concatenate([K.clip(
K.cast(K.not_equal(char_index[1:], CHAR2INDEX[' ']), 'float32') +
K.cast(K.not_equal(char_index[:-1], CHAR2INDEX['ء']), 'float32'), 0, 1), K.ones((1,))], axis=0), (-1, 1))
mask *= K.reshape(K.cast(K.not_equal(char_index, CHAR2INDEX['ة']), 'float32'), (-1, 1))
mask *= K.reshape(K.concatenate([K.clip(
K.cast(K.not_equal(char_index[1:-1], CHAR2INDEX['ا']), 'float32') +
K.cast(K.not_equal(char_index[2:], CHAR2INDEX[' ']), 'float32'), 0, 1), K.ones((2,))], axis=0), (-1, 1))
pred_haraka = mask * K.constant([1, 1, 1, 1, 1, 0, 1, 1], shape=(1, 8)) * pred_haraka + (1 - mask) * pred_haraka
# Drop haraka from the forbidden characters
forbidden_chars = [CHAR2INDEX[' '], CHAR2INDEX['0'], CHAR2INDEX['آ'], CHAR2INDEX['ى'], CHAR2INDEX['ا']]
mask = K.cast(K.not_equal(char_index, forbidden_chars[0]), 'float32')
for forbidden_char in forbidden_chars[1:]:
mask *= K.cast(K.not_equal(char_index, forbidden_char), 'float32')
mask = K.reshape(mask, (-1, 1))
pred_haraka = mask * pred_haraka + (1 - mask) * K.one_hot(0, K.int_shape(pred_haraka)[-1])
return pred_haraka
def build(self, input_shape):
k_dim = self.spatial_extent
in_ch = input_shape[-1]
# NOTE: assume channel-last inputs
# NOTE: make sure that all initializer seeds are different!
# U(1) and U(2): 1x1xKxK kernels; b(1) and b(2): 1x1xK channel-wise gate biases
self.u1 = self.add_weight(name='u1', shape=(1,1,in_ch,in_ch), trainable=True)
self.u2 = self.add_weight(name='u2', shape=(1,1,in_ch,in_ch), trainable=True)
self.b1 = self.add_weight(name='b1', shape=(1,1,in_ch), trainable=True)
self.b2 = self.add_weight(name='b2', shape=(1,1,in_ch), trainable=True)
orthogonal_init = keras.initializers.Orthogonal(seed=self.rand_seed)
randu_init = keras.initializers.RandomUniform(1.0, self.timesteps-1.0, seed=self.rand_seed+1)
self.u1.assign(orthogonal_init(self.u1.shape))
self.u2.assign(orthogonal_init(self.u2.shape))
self.b1.assign(K.log(randu_init(self.b1.shape))) # chronos init
self.b2.assign(-self.b1)
# one separate batchnorm layer for each timestep
self.bn = [keras.layers.BatchNormalization(momentum=0.001, epsilon=1e-03)
for _ in range(self.timesteps*4)]
# W: SxSxKxK shared inhibition/excitation kernel
self.w_inh = self.add_weight(name='w_inh', shape=(k_dim, k_dim, in_ch, in_ch), trainable=True)
self.w_exc = self.add_weight(name='w_exc', shape=(k_dim, k_dim, in_ch, in_ch), trainable=True)
self.w_inh.assign(orthogonal_init(self.w_inh.shape))
self.w_exc.assign(orthogonal_init(self.w_exc.shape))
if self.channel_sym: # channel symmetric init
self.w_inh.assign((self.w_inh + K.permute_dimensions(self.w_inh, (0,1,3,2))) * 0.5)
self.w_exc.assign((self.w_exc + K.permute_dimensions(self.w_exc, (0,1,3,2))) * 0.5)
# mu, alpha: channel-wise linear/quadratic control for inhibition
# kappa, omega, beta: channel-wise linear/quadratic control and additional gain for excitation
self.mu = self.add_weight(name='mu', shape=(1,1,in_ch), trainable=True)
self.alpha = self.add_weight(name='alpha', shape=(1,1,in_ch), trainable=True)
self.kappa = self.add_weight(name='kappa', shape=(1,1,in_ch), trainable=True)
self.omega = self.add_weight(name='omega', shape=(1,1,in_ch), trainable=True)
self.beta = self.add_weight(name='beta', shape=(1,1,in_ch), trainable=True)
self.mu.assign(K.ones(self.mu.shape) * 1.0)
self.alpha.assign(K.ones(self.alpha.shape) * 0.1)
self.kappa.assign(K.ones(self.kappa.shape) * 0.5)
self.omega.assign(K.ones(self.omega.shape) * 0.5)
self.beta.assign(K.ones(self.beta.shape) * 1.0)
# eta: timestep weights
glorot_init = keras.initializers.glorot_normal(seed=self.rand_seed+2)
if not self.batchnorm:
self.eta = self.add_weight(name='eta', shape=(self.timesteps,),
initializer=glorot_init, trainable=True)
super(hGRUCell, self).build(input_shape) # Be sure to call this at the end
def test_get_img_shape_on_2d_image():
n = 5
channels = 4
dim1 = 1
dim2 = 2
K.set_image_data_format('channels_first')
assert (n, channels, dim1, dim2) == utils.get_img_shape(
K.ones(shape=(n, channels, dim1, dim2)))
K.set_image_data_format('channels_last')
assert (n, channels, dim1, dim2) == utils.get_img_shape(
K.ones(shape=(n, dim1, dim2, channels)))
def get_updates(self, params, loss):
grads = self.get_gradients(loss, params)
shapes = [K.get_variable_shape(p) for p in params]
alphas = [
K.variable(K.ones(shape) * self.init_alpha) for shape in shapes
]
old_grads = [K.zeros(shape) for shape in shapes]
self.weights = alphas + old_grads
self.updates = []
for p, grad, old_grad, alpha in zip(params, grads, old_grads, alphas):
grad = K.sign(grad)
new_alpha = K.switch(
K.greater(grad * old_grad, 0),
K.minimum(alpha * self.scale_up, self.max_alpha),
K.switch(K.less(grad * old_grad, 0),
K.maximum(alpha * self.scale_down, self.min_alpha),
alpha))
grad = K.switch(K.less(grad * old_grad, 0), K.zeros_like(grad),
grad)
new_p = p - grad * new_alpha
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
self.updates.append(K.update(alpha, new_alpha))
self.updates.append(K.update(old_grad, grad))
return self.updates
def compute_p_c_z(self, z):
assert z.shape[1:] == (
self.latent_dim, ), 'z.shape[1:] {} != {}'.format(
z.shape[1:], (self.latent_dim, ))
Z = K.permute_dimensions(K.repeat(z, self.n_clusters), [0, 2, 1])
assert Z.shape[1:] == (self.latent_dim,
self.n_clusters), 'Z.shape[1:] {} != {}'.format(
Z.shape[1:],
(self.latent_dim, self.n_clusters))
u_tensor3 = self.compute_u_tensor3()
lambda_tensor3 = self.compute_lambda_tensor3()
assert self.theta_p.shape == (
self.n_clusters, ), 'self.theta_p.shape {} != {}'.format(
self.theta_p.shape, (self.n_clusters, ))
theta_tensor3 = K.expand_dims(
K.expand_dims(self.theta_p, axis=0), axis=0) * K.ones(
(self.batch_size, self.latent_dim, self.n_clusters))
assert theta_tensor3.shape == (
self.batch_size, self.latent_dim,
self.n_clusters), 'theta_tensor3.shape {} != {}'.format(
theta_tensor3.shape,
(self.batch_size, self.latent_dim, self.n_clusters))
p_c_z=K.exp(K.sum((K.log(theta_tensor3)-0.5*K.log(2*math.pi*lambda_tensor3)-\
K.square(Z-u_tensor3)/(2*lambda_tensor3)),axis=1))+1e-10
assert p_c_z.shape[1:] == (
self.n_clusters, ), 'p_c_z.shape[1:] {} != {}'.format(
p_c_z.shape[1:], (self.n_clusters, ))
return p_c_z / K.sum(p_c_z, axis=-1, keepdims=True)
def call(self, inputs, **kwargs):
# Generate random uniform tensor between [1-alpha, 1+alpha] for training and ones tensor for test (ReLU)
k = K.in_train_phase(
K.random_uniform(inputs.shape[1:], 1 - self.alpha, 1 + self.alpha),
K.ones(inputs.shape[1:]))
return keras.activations.relu(inputs * k)
def _custom_loss(y_true, y_pred):
"""Computes loss for single trigger word detection model.
The loss is the sum over samples and timesteps of the binary cross entropy
between target and prediction.
The only variation is that values of target lower than -0.001 are
interpreted as a don't care values. Where don't care value are present
the loss is forced to zero.
Args:
y_true (keras.backend.Tensor): target (shape = (#samples, #timesteps, 1))
y_pred (keras.backend.Tensor): predictions to match against the target,
same shape of y_true
Returns:
keras.backend.Tensor: Scalar cost.
"""
# COMPUTING BINARY CROSS-ENTROPY
one = K.ones(K.shape(y_true))
loss = -y_true * K.log(tf.clip_by_value(y_pred, 1e-10, 1.0)) - (
one - y_true) * K.log(tf.clip_by_value(one - y_pred, 1e-10, 1.0))
#SETTING TO ZERO WHERE TARGET IS "DON't CARE"
thres = tf.fill(K.shape(y_true), -0.001)
fil = tf.cast(K.greater(y_true, thres), tf.float32)
loss_filtered = loss * fil
#SUMMING OVER TIMESTEP AND SAMPLES
cost = K.sum(loss_filtered)
return cost
def __call__(self, w):
other_weights = K.cast(K.ones(K.shape(w))[:-1], K.floatx())
last_weight = K.cast(
K.equal(K.reshape(w[-1, :], (1, K.shape(w)[1])), 0.), K.floatx())
appended = K.concatenate([other_weights, last_weight], axis=0)
w *= appended
return w
def dice_coef_every_class(
y_true, y_pred
): # not used yet, because one metric must be one scalar from what I experienced
"""
dice coefficient for every class/label.
:param y_true: ground truth
:param y_pred: prediction results
:return: dice value
"""
alpha = 0.5
beta = 0.5
ones = K.ones(K.shape(y_true))
p0 = y_pred # proba that voxels are class i
p1 = ones - y_pred # proba that voxels are not class i
g0 = y_true
g1 = ones - y_true
num = K.sum(p0 * g0, (0, 1, 2, 3))
den = num + alpha * K.sum(p0 * g1, (0, 1, 2, 3)) + beta * K.sum(
p1 * g0, (0, 1, 2, 3))
T = K.sum(num /
den) # when summing over classes, T has dynamic range [0 Ncl]
Ncl = K.cast(K.shape(y_true)[-1], 'float32')
return Ncl - T
def sinc(band, t_right):
y_right = K.sin(
2 * math.pi * band * t_right) / (2 * math.pi * band * t_right)
# y_left = flip(y_right, 0) TODO remove if useless
y_left = K.reverse(y_right, 0)
y = K.concatenate([y_left, K.variable(K.ones(1)), y_right])
return y
def call(self, x):
x_orig = x
# x reshape
this_bs_int = K.shape(x)[0]
this_bs = tf.cast(this_bs_int, 'float32') # this batch size
prev_count = self.count
x = K.batch_flatten(x) # B x N
# update mean
new_mean, new_count = _mean_update(self.mean, self.count, x, self.cap)
# new C update. Should be B x N x N
x = K.expand_dims(x, -1)
C_delta = K.batch_dot(x, K.permute_dimensions(x, [0, 2, 1]))
# update cov
prev_cap = K.minimum(prev_count, self.cap)
C = self.cov * (prev_cap - 1) + K.sum(C_delta, 0)
new_cov = C / (prev_cap + this_bs - 1)
# updates
updates = [(self.count, new_count), (self.mean, new_mean), (self.cov, new_cov)]
self.add_update(updates, x_orig)
# prep for broadcasting :(
p = tf.concat((K.reshape(this_bs_int, (1,)), K.shape(self.cov)), 0)
z = K.ones(p)
return K.minimum(1., new_count/self.cap) * (z * K.expand_dims(new_cov, 0))
def test_DSSIM_channels_first():
prev_data = K.image_data_format()
K.set_image_data_format('channels_first')
for input_dim, kernel_size in zip([32, 33], [2, 3]):
input_shape = [3, input_dim, input_dim]
X = np.random.random_sample(4 * input_dim * input_dim * 3)
X = X.reshape([4] + input_shape)
y = np.random.random_sample(4 * input_dim * input_dim * 3)
y = y.reshape([4] + input_shape)
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', input_shape=input_shape,
activation='relu'))
model.add(Conv2D(3, (3, 3), padding='same', input_shape=input_shape,
activation='relu'))
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
model.compile(loss=DSSIMObjective(kernel_size=kernel_size), metrics=['mse'],
optimizer=adam)
model.fit(X, y, batch_size=2, epochs=1, shuffle='batch')
# Test same
x1 = K.constant(X, 'float32')
x2 = K.constant(X, 'float32')
dssim = DSSIMObjective(kernel_size=kernel_size)
assert_allclose(0.0, K.eval(dssim(x1, x2)), atol=1e-4)
# Test opposite
x1 = K.zeros([4] + input_shape)
x2 = K.ones([4] + input_shape)
dssim = DSSIMObjective(kernel_size=kernel_size)
assert_allclose(0.5, K.eval(dssim(x1, x2)), atol=1e-4)
K.set_image_data_format(prev_data)
def build(self, input_shape):
if self.data_format == 'channels_first':
self.seq_len = input_shape[-1]
else:
self.seq_len = input_shape[1]
self.ones = K.ones(self.seq_len, name='ones')
self.zeros = K.zeros(self.seq_len, name='zeros')
super().build(input_shape)
def build(self, input_shape):
if self.data_format == 'channels_first':
self.height, self.width = input_shape[2], input_shape[3]
else:
self.height, self.width = input_shape[1], input_shape[2]
self.ones = K.ones((self.height, self.width), name='ones')
self.zeros = K.zeros((self.height, self.width), name='zeros')
super().build(input_shape)
def custom_weighted_loss(dI):
if not (K.int_shape(dI)[0] == 'None'):
bf = K.ones((1, 1970))
w = bf - dI
def custom_loss(y_true, y_pred):
return w * K.binary_crossentropy(y_true, y_pred)
return custom_loss
def __create_model(self):
max_input_len = DATASET_CONFIG["MAX_INPUT_LENGTH"]
max_output_len = DATASET_CONFIG["MAX_DECODER_LENGTH"]
input_vocab_size = DATASET_CONFIG["INPUT_VOCABULARY_SIZE"]
output_vocab_size = DATASET_CONFIG["SPARQL_VOCABULARY_SIZE"]
# ENCODER
enc_input = Input(shape=(max_input_len), name="encoder_input")
enc_embedding = Embedding(input_vocab_size + 1,
self.embedding_dim,
mask_zero=True,
name="encoder_embedding")(enc_input)
enc_forward = LSTM(self.latent_dim,
return_sequences=True,
return_state=True,
dropout=0.8,
name="encoder_forward")
enc_backward = LSTM(self.latent_dim,
return_sequences=True,
return_state=True,
dropout=0.8,
go_backwards=True,
name="encoder_backward")
forward_outputs, forward_h, forward_c = enc_forward(enc_embedding)
backward_outputs, backward_h, backward_c = enc_backward(enc_embedding)
enc_outputs = Concatenate(name="encoder_out_concat")(
[forward_outputs, backward_outputs])
enc_state_h = Concatenate(name="encoder_state-h_concat")(
[forward_h, backward_h])
enc_state_c = Concatenate(name="encoder_state-c_concat")(
[forward_c, backward_c])
# TEMPLATE CLASSIFIER
# DECODER
constants = K.ones(
(K.shape(enc_input)[0], max_output_len, self.latent_dim * 2))
decoder = tf.keras.layers.RNN(CustomLSTMCell(self.latent_dim * 2,
enc_outputs.shape,
dropout=0.8),
return_sequences=True,
name="decoder")
dec_outputs = decoder(
constants, initial_state=[enc_state_h, enc_state_c, enc_outputs])
# Time-distributed classifier with tanh layer connecting to softmax layer
timedist_tanh = TimeDistributed(
Dense(self.latent_dim * 8, activation="tanh"))
timedist_softmax = TimeDistributed(
Dense(output_vocab_size + 1, activation="softmax"))
outputs = timedist_softmax(timedist_tanh(dec_outputs))
# nmt_model = Model([enc_input, dec_input], outputs)
nmt_model = Model(enc_input, outputs)
self.model = nmt_model
def spectral_norm(self, w, r=5):
w_shape = K.int_shape(w)
in_dim = np.prod(w_shape[:-1]).astype(int)
out_dim = w_shape[-1]
w = K.reshape(w, (in_dim, out_dim))
u = K.ones((1, in_dim))
for i in range(r):
v = K.l2_normalize(K.dot(u, w))
u = K.l2_normalize(K.dot(v, K.transpose(w)))
return K.sum(K.dot(K.dot(u, w), K.transpose(v)))
def custom_weighted_loss(dI, bias):
if not (K.int_shape(dI)[0] == 'None'):
bf = K.ones((1, 1970))
w = bf - dI
def custom_loss(y_true, y_pred):
loss = w * K.binary_crossentropy(y_true, y_pred)
return bias * (1 - y_true) * loss + (1 - bias) * y_true * loss
return custom_loss
def call(self,x):
x=x[0]
one=K.ones(shape=(self.c,self.c))
ret=kron_b1(x,one)
return ret
def call(self, inputs, **kwargs):
# Generate random uniform tensor between [1-alpha, 1+alpha] for training and ones tensor for test
k = K.in_train_phase(
K.random_uniform(inputs.shape[1:], 1 - self.alpha, 1 + self.alpha),
K.ones(inputs.shape[1:]))
pos = keras.activations.relu(inputs) * k
neg = -self.a * keras.activations.relu(-inputs)
out = pos + neg
return out
def _degree_matrix(self, vol_shape):
# get shape stats
ndims = len(vol_shape)
sz = [*vol_shape, ndims]
# prepare conv kernel
conv_fn = getattr(tf.nn, 'conv%dd' % ndims)
# prepare tf filter
z = K.ones([1] + sz)
filt_tf = tf.convert_to_tensor(self._adj_filt(ndims), dtype=tf.float32)
strides = [1] * (ndims + 2)
return conv_fn(z, filt_tf, strides, "SAME")
def DiceLoss(y_true, y_pred):
alpha, beta = 0.5, 0.5
ones = K.ones(K.shape(y_true))
p0 = y_pred
p1 = ones - y_pred
g0 = y_true
g1 = ones - y_true
num = K.sum(p0 * g0, (0, 1, 2))
den = num + alpha * K.sum(p0 * g1,
(0, 1, 2)) + beta * K.sum(p1 * g0, (0, 1, 2))
T = K.sum(num / den)
Ncl = K.cast(K.shape(y_true)[-1], 'float32')
return Ncl - T
def _composite_loss(y_true, y_pred):
"""Computes loss for multi trigger word detection model.
The target (both true and predicted) is twofold:
* the first feature goes to one after any trigger word being
pronounced.
* the remaining features are one per trigger word class going to one only
atfer a word of that specific class has been pronounced.
This loss computes sums over samples and timesteps the sum of two terms:
* The binary cross entropy of the first feature
* The multiclass cross entropy of the remaining features, but only for
where y_true is one, otherwise this term goes to 0 --> if no
trigger word has just finished, it does not matter which class will
be predicted...
Args:
y_true (keras.backend.Tensor): target (shape = (#samples, #timesteps, #positive_classes+1))
y_pred (keras.backend.Tensor): predictions to match against the target,
same shape of y_true
Returns:
keras.backend.Tensor: Scalar cost.
"""
Ytb = y_true[:,:,:1] # first feature is the bynary classification target for the task "any cmd vs no cmd". b stands for binary
Ytm = y_true[:,:,1:] # all other features are the one hot target for the task "which cmd". m stands for multiple
Ypb = y_pred[:,:,:1]
Ypm = y_pred[:,:,1:]
# COMPUTING BINARY CROSS-ENTROPY
one = K.ones(K.shape(Ytb))
Lb = -Ytb*K.log(tf.clip_by_value(Ypb,1e-10,1.0))-(one-Ytb)*K.log(tf.clip_by_value(one-Ypb,1e-10,1.0)) # binary loss
#SETTING BINARY CROSS-ENTROPY ZERO WHERE TARGET IS "DON't CARE"
thres = tf.fill(K.shape(Lb),-0.001)
fil = tf.cast(K.greater(Ytb, thres), tf.float32)
Lb_fil = Lb*fil
# COMPUTING MULTICLASS CROSS-ENTROPY
parts = []
for i in range(_n_classes):
parts.append(-Ytm[:,:,i:i+1]*K.log(tf.clip_by_value(Ypm[:,:,i:i+1],1e-10,1.0)))
Lm=tf.add_n(parts)
Lmm = Lm*Ytb
Cb = K.sum(Lb_fil)
Cm = K.sum(Lmm)
C = Cb+Cm
return C
def get_updates(self, params, loss):
grads = self.get_gradients(loss, params)
shapes = [K.shape(p) for p in params]
alphas = [
K.variable(K.ones(shape) * self.init_alpha) for shape in shapes
]
old_grads = [K.zeros(shape) for shape in shapes]
prev_weight_deltas = [K.zeros(shape) for shape in shapes]
self.weights = alphas + old_grads
self.updates = []
for param, grad, old_grad, prev_weight_delta, alpha in zip(
params, grads, old_grads, prev_weight_deltas, alphas):
# equation 4
new_alpha = K.switch(
K.greater(grad * old_grad, 0),
K.minimum(alpha * self.scale_up, self.max_alpha),
K.switch(K.less(grad * old_grad, 0),
K.maximum(alpha * self.scale_down, self.min_alpha),
alpha))
# equation 5
new_delta = K.switch(
K.greater(grad, 0), -new_alpha,
K.switch(K.less(grad, 0), new_alpha, K.zeros_like(new_alpha)))
# equation 7
weight_delta = K.switch(K.less(grad * old_grad, 0),
-prev_weight_delta, new_delta)
# equation 6
new_param = param + weight_delta
# reset gradient_{t-1} to 0 if gradient sign changed (so that we do
# not "double punish", see paragraph after equation 7)
grad = K.switch(K.less(grad * old_grad, 0), K.zeros_like(grad),
grad)
# Apply constraints
#if param in constraints:
# c = constraints[param]
# new_param = c(new_param)
self.updates.append(K.update(param, new_param))
self.updates.append(K.update(alpha, new_alpha))
self.updates.append(K.update(old_grad, grad))
self.updates.append(K.update(prev_weight_delta, weight_delta))
return self.updates
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if len(input_shape) != 5:
raise ValueError('Inputs should have rank 5, '
'received input shape: %s' % input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape.dims[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined, found None: %s' % input_shape)
input_dim = int(input_shape[channel_axis])
self.input_spec = InputSpec(ndim=5, axes={channel_axis: input_dim})
if self.data_format == 'channels_first':
depth = int(input_shape[2])
else:
depth = int(input_shape[1])
kernel_shape = (depth, self.filter_size, self.filter_size, input_dim,
1)
# self.kernel = self.add_weight(
# 'kernel',
# shape=kernel_shape,
# initializer=self.kernel_initializer,
# regularizer=self.kernel_regularizer,
# constraint=self.kernel_constraint,
# trainable=False,
# dtype=self.compute_dtype)
W = K.ones(kernel_shape, dtype=self.compute_dtype)
W = W / K.cast(K.prod(K.int_shape(W)), dtype=self.compute_dtype)
self.kernel = W
# self.set_weights([W])
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(depth, self.filter_size,
self.filter_size),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=False,
dtype=self.compute_dtype)
else:
self.bias = None
self.built = True
def loss_tv(mask, y_comp):
"""Total variation loss, used for smoothing the hole region, see. eq. 6"""
# Create dilated hole region using a 3x3 kernel of all 1s.
kernel = K.ones(shape=(3, 3, mask.shape[3], mask.shape[3]))
dilated_mask = K.conv2d(1-mask, kernel, data_format='channels_last', padding='same')
# Cast values to be [0., 1.], and compute dilated hole region of y_comp
dilated_mask = K.cast(K.greater(dilated_mask, 0), 'float32')
P = dilated_mask * y_comp
# Calculate total variation loss
a = l1(P[:,1:,:,:], P[:,:-1,:,:])
b = l1(P[:,:,1:,:], P[:,:,:-1,:])
return a+b
def loss(y_true, y_pred):
# Calculate the binary crossentropy
b_ce = K.binary_crossentropy(y_true, y_pred)
alpha = 0.5
b_ce = b_ce * K.ones((batch_size,320,512,4))[:,:,:,1].assign(K.ones((batch_size,320,512,4))[:,:,:,1] + alpha*K.cast(((y_true>[0.5,1,1,1])[:,:,:,0] == (y_pred>[1,0.5,1,1])[:,:,:,1]),'float32'))
b_ce = b_ce * K.ones((batch_size,320,512,4))[:,:,:,3].assign(K.ones((batch_size,320,512,4))[:,:,:,3] + alpha*K.cast(((y_true<[1,1,1,0.5])[:,:,:,3] == (y_pred>[1,1,1,0.5])[:,:,:,3]),'float32'))
# Apply the weights
weighted_b_ce = w * b_ce
jloss = sm.losses.JaccardLoss(class_weights=w)
dl = sm.losses.DiceLoss(class_weights=w)
l = dl(y_true,y_pred)
"""
tf.config.experimental_run_functions_eagerly(True)
@tf.function
def f(x):
if x > 1:
return 1
else:
return 0
ch1 = f(K.sum(y_true[:,:,:,2]))
ch2 = f(K.sum(y_true[:,:,:,3]))
if (ch1<1 and ch2>0) or (ch1>0 and ch2<1):
l = 1 - (1-l)*4/3
elif ch1 and ch2:
l = 1 - (1-l)*2
"""
# Return the mean error
return K.mean(weighted_b_ce) + l
def call(self, x):
# get new mean and count
this_bs_int = K.shape(x)[0]
new_mean, new_count = _mean_update(self.mean, self.count, x, self.cap)
# update op
updates = [(self.count, new_count), (self.mean, new_mean)]
self.add_update(updates, x)
# prep for broadcasting :(
p = tf.concat((K.reshape(this_bs_int, (1,)), K.shape(self.mean)), 0)
z = K.ones(p)
# the first few 1000 should not matter that much towards this cost
return K.minimum(1., new_count/self.cap) * (z * K.expand_dims(new_mean, 0))
def train(ep):
ones = K.ones(shape=(args.batch_size, 1))
# tf.function -> faster execution but cant debug.
@tf.function
def train_step(inp, tar):
# optimizer F
with tf.GradientTape() as tape:
outputs = net(inp)
loss = criterion(tar, outputs)
train_accuracy(tar, outputs)
train_loss(loss)
f_variables = net.downsampling_layers.trainable_variables
f_variables += net.drift.trainable_variables
f_variables += net.fc_layers.trainable_variables
gradients = tape.gradient(loss, f_variables)
optimizer_f.apply_gradients(zip(gradients, f_variables))
# optimizer G
# training with out-of-domain data
if args.training_out:
with tf.GradientTape() as tape:
g_variables = net.diffusion.trainable_variables
label = ones * real_label # real = 0.
loss_in = criterion2(label, net(inp, training_diffusion=True))
train_loss_in(loss_in)
gradients_in = tape.gradient(loss_in, g_variables)
optimizer_g.apply_gradients(zip(gradients_in, g_variables))
with tf.GradientTape() as tape:
label = label * 0 + fake_label # fake = 1.
inputs_out = 2 * K.random_normal((args.batch_size, args.imageSize, args.imageSize, 1)) + inp
loss_out = criterion2(label, net(inputs_out, training_diffusion=True))
train_loss_out(loss_out)
gradients_out = tape.gradient(loss_out, g_variables)
optimizer_g.apply_gradients(zip(gradients_out, g_variables))
print('\nEpoch: %d' % ep)
# training with in-domain data
for batch_idx, (inputs, targets) in enumerate(train_loader_in_domain):
inputs, targets = np.array(inputs), np.array(targets)
inputs = np.transpose(inputs, (0, 2, 3, 1))
train_step(inputs, targets)
print('Train epoch:{} \tLoss: {:.6f} | Loss_in: {:.8f}, Loss_out: {:.8f} | Acc: {:.4f}'
.format(ep, train_loss.result(), train_loss_in.result(), train_loss_out.result(),
train_accuracy.result()))
def build(self, input_shape):
"""Adapted from original _Conv() layer of Keras
param input_shape: list of dimensions for [img, mask]
"""
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[0][channel_axis] is None:
raise ValueError(
'The channel dimension of the inputs should be defined. Found `None`.'
)
self.input_dim = int(input_shape[0][channel_axis])
# Image kernel
kernel_shape = self.kernel_size + (self.input_dim, self.filters)
self.kernel = self.add_weight(shape=kernel_shape,
initializer=self.kernel_initializer,
name='img_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
# Mask kernel
self.kernel_mask = K.ones(shape=self.kernel_size +
(self.input_dim, self.filters))
# Calculate padding size to achieve zero-padding
self.pconv_padding = (
(int((self.kernel_size[0] - 1) / 2),
int((self.kernel_size[0] - 1) / 2)),
(int((self.kernel_size[0] - 1) / 2),
int((self.kernel_size[0] - 1) / 2)),
)
# Window size - used for normalization
self.window_size = self.kernel_size[0] * self.kernel_size[1]
if self.use_bias:
self.bias = self.add_weight(shape=(self.filters, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.built = True
