def test_sparse_concat(self):
x_d = np.array([0, 7, 2, 3], dtype=np.float32)
x_r = np.array([0, 2, 2, 3], dtype=np.int64)
x_c = np.array([4, 3, 2, 3], dtype=np.int64)
x_sparse_1 = sparse.csr_matrix((x_d, (x_r, x_c)), shape=(4, 5))
x_d = np.array([0, 7, 2, 3], dtype=np.float32)
x_r = np.array([0, 2, 2, 3], dtype=np.int64)
x_c = np.array([4, 3, 2, 3], dtype=np.int64)
x_sparse_2 = sparse.csr_matrix((x_d, (x_r, x_c)), shape=(4, 5))
x_dense_1 = x_sparse_1.toarray()
x_dense_2 = x_sparse_2.toarray()
backends = [KTF]
if KTH.th_sparse_module:
# Theano has some dependency issues for sparse
backends.append(KTH)
for K in backends:
k_s = K.concatenate([K.variable(x_sparse_1), K.variable(x_sparse_2)])
assert K.is_sparse(k_s)
k_s_d = K.eval(k_s)
k_d = K.eval(K.concatenate([K.variable(x_dense_1), K.variable(x_dense_2)]))
assert k_s_d.shape == k_d.shape
assert_allclose(k_s_d, k_d, atol=1e-05)
def test_metrics():
y_a = K.variable(np.random.random((6, 7)))
y_b = K.variable(np.random.random((6, 7)))
for metric in all_metrics:
output = metric(y_a, y_b)
print(metric.__name__)
assert K.eval(output).shape == (6,)
def build(self, input_shape):
assert len(input_shape) == 3
self.W = K.variable(self.init((input_shape[-1], self.attention_dim)))
self.b = K.variable(self.init((self.attention_dim, )))
self.u = K.variable(self.init((self.attention_dim, 1)))
self.trainable_weights = [self.W, self.b, self.u]
super(AttLayer, self).build(input_shape)
def test_merge_subtract():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
i3 = layers.Input(shape=(4, 5))
i4 = layers.Input(shape=(3, 5))
o = layers.subtract([i1, i2])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2], o)
subtract_layer = layers.Subtract()
o2 = subtract_layer([i1, i2])
assert subtract_layer.output_shape == (None, 4, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 4, 5)
assert_allclose(out, x1 - x2, atol=1e-4)
assert subtract_layer.compute_mask([i1, i2], [None, None]) is None
assert np.all(K.eval(subtract_layer.compute_mask(
[i1, i2], [K.variable(x1), K.variable(x2)])))
# Test invalid use case
with pytest.raises(ValueError):
subtract_layer.compute_mask([i1, i2], x1)
with pytest.raises(ValueError):
subtract_layer.compute_mask(i1, [None, None])
with pytest.raises(ValueError):
subtract_layer([i1, i2, i3])
with pytest.raises(ValueError):
subtract_layer([i1])
def __call__(self, loss):
from . import patches
output = self.layer.get_output(True)
assert K.ndim(output) == 4
batch_size = K.shape(output)[0] // 2
patch_size = self.patch_size
patch_stride = 1
generated = output[:batch_size, :, :, :]
content = output[batch_size:, :, :, :]
# extract patches from feature maps
generated_patches, generated_patches_norm = \
patches.make_patches(generated, patch_size, patch_stride)
content_patches, content_patches_norm = \
patches.make_patches(content, patch_size, patch_stride)
a_patches, a_patches_norm = \
patches.make_patches(K.variable(self.features_a), patch_size, patch_stride)
ap_patches, ap_patches_norm = \
patches.make_patches(K.variable(self.features_ap), patch_size, patch_stride)
# find best patches and calculate loss
patch_ids = patches.find_patch_matches(
content_patches, content_patches_norm,
a_patches / a_patches_norm)
best_analogy_patches = K.reshape(
ap_patches[patch_ids], K.shape(generated_patches))
loss += self.weight * K.sum(K.square(best_analogy_patches - generated_patches)) / patch_size ** 2
return loss
def test_convolution_2d_dim_ordering():
nb_filter = 4
nb_row = 3
nb_col = 2
stack_size = 3
np.random.seed(1337)
weights = [np.random.random((nb_filter, stack_size, nb_row, nb_col)),
np.random.random(nb_filter)]
input = np.random.random((1, stack_size, 10, 10))
layer = convolutional.Convolution2D(
nb_filter, nb_row, nb_col,
weights=weights,
input_shape=input.shape[1:],
dim_ordering='th')
layer.input = K.variable(input)
out_th = K.eval(layer.get_output(False))
input = np.transpose(input, (0, 2, 3, 1))
weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
layer = convolutional.Convolution2D(
nb_filter, nb_row, nb_col,
weights=weights,
input_shape=input.shape[1:],
dim_ordering='tf')
layer.input = K.variable(input)
out_tf = K.eval(layer.get_output(False))
assert_allclose(out_tf, np.transpose(out_th, (0, 2, 3, 1)), atol=1e-05)
def content_mean_square_error(self, y_true, y_pred):
y_pred = K.variable(value=y_pred)
y_true = K.variable(value=y_true)
_, filters, x_pos, y_pos = self.get_shape(y_pred)
y_pred = K.reshape(y_pred, (filters, x_pos * y_pos))
y_true = K.reshape(y_true, (filters, x_pos * y_pos))
return K.sum(K.square(y_pred - y_true))
def test_zero_padding_3d():
num_samples = 2
stack_size = 2
input_len_dim1 = 4
input_len_dim2 = 5
input_len_dim3 = 3
inputs = np.ones((num_samples,
input_len_dim1, input_len_dim2, input_len_dim3,
stack_size))
# basic test
for data_format in ['channels_first', 'channels_last']:
layer_test(convolutional.ZeroPadding3D,
kwargs={'padding': (2, 2, 2), 'data_format': data_format},
input_shape=inputs.shape)
layer_test(convolutional.ZeroPadding3D,
kwargs={'padding': ((1, 2), (3, 4), (0, 2)), 'data_format': data_format},
input_shape=inputs.shape)
# correctness test
layer = convolutional.ZeroPadding3D(padding=(2, 2, 2),
data_format=data_format)
layer.build(inputs.shape)
outputs = layer(K.variable(inputs))
np_output = K.eval(outputs)
if data_format == 'channels_last':
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, offset, :, :, :], 0.)
assert_allclose(np_output[:, :, offset, :, :], 0.)
assert_allclose(np_output[:, :, :, offset, :], 0.)
assert_allclose(np_output[:, 2:-2, 2:-2, 2:-2, :], 1.)
elif data_format == 'channels_first':
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, :, offset, :, :], 0.)
assert_allclose(np_output[:, :, :, offset, :], 0.)
assert_allclose(np_output[:, :, :, :, offset], 0.)
assert_allclose(np_output[:, :, 2:-2, 2:-2, 2:-2], 1.)
layer = convolutional.ZeroPadding3D(padding=((1, 2), (3, 4), (0, 2)),
data_format=data_format)
layer.build(inputs.shape)
outputs = layer(K.variable(inputs))
np_output = K.eval(outputs)
if data_format == 'channels_last':
for dim1_offset in [0, -1, -2]:
assert_allclose(np_output[:, dim1_offset, :, :, :], 0.)
for dim2_offset in [0, 1, 2, -1, -2, -3, -4]:
assert_allclose(np_output[:, :, dim2_offset, :, :], 0.)
for dim3_offset in [-1, -2]:
assert_allclose(np_output[:, :, :, dim3_offset, :], 0.)
assert_allclose(np_output[:, 1:-2, 3:-4, 0:-2, :], 1.)
elif data_format == 'channels_first':
for dim1_offset in [0, -1, -2]:
assert_allclose(np_output[:, :, dim1_offset, :, :], 0.)
for dim2_offset in [0, 1, 2, -1, -2, -3, -4]:
assert_allclose(np_output[:, :, :, dim2_offset, :], 0.)
for dim3_offset in [-1, -2]:
assert_allclose(np_output[:, :, :, :, dim3_offset], 0.)
assert_allclose(np_output[:, :, 1:-2, 3:-4, 0:-2], 1.)
def test_prelu():
from keras.layers.advanced_activations import PReLU
np.random.seed(1337)
inp = get_standard_values()
for train in [True, False]:
# test with custom weights
alphas = np.random.random(inp.shape)
layer = PReLU(weights=alphas, input_shape=inp.flatten().shape)
# calling build here causes an error, unclear if this is a bug
# layer.build()
layer.input = K.variable(inp)
outp = K.eval(layer.get_output(train))
assert_allclose(inp, outp)
layer.input = K.variable(-inp)
outp = K.eval(layer.get_output(train))
assert_allclose(-alphas*inp, outp)
# test with default weights
layer = PReLU(input_shape=inp.flatten().shape)
# layer.build()
layer.input = K.variable(inp)
outp = K.eval(layer.get_output(train))
assert_allclose(inp, outp)
layer.input = K.variable(-inp)
outp = K.eval(layer.get_output(train))
assert_allclose(0., alphas*outp)
layer.get_config()
def test_vgg_fc():
if K.image_data_format() == 'channels_first':
x1 = K.variable(np.random.random((1, 512, 14, 14)))
y1_shape = (1, 512, 8, 8)
else:
x1 = K.variable(np.random.random((1, 14, 14, 512)))
y1_shape = (1, 8, 8, 512)
def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
global add_tables
nb_filter = output_shape[0]
conv = Convolution2D(nb_filter, 3, 3, subsample=strides, border_mode="same")(x)
conv = BatchNormalization(axis=1)(conv)
conv = Activation("relu")(conv)
conv = Convolution2D(nb_filter, 3, 3, border_mode="same")(conv)
conv = BatchNormalization(axis=1)(conv)
if strides[0] >= 2:
x = AveragePooling2D(strides)(x)
if (output_shape[0] - input_shape[0]) > 0:
pad_shape = (1,
output_shape[0] - input_shape[0],
output_shape[1],
output_shape[2])
padding = K.ones(pad_shape)
padding = K.repeat_elements(padding, K.shape(x)[0], axis=0)
x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
output_shape=output_shape)(x)
_death_rate = K.variable(death_rate)
scale = K.ones_like(conv) - _death_rate
conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
output_shape=output_shape)(conv)
out = merge([conv, x], mode="sum")
out = Activation("relu")(out)
gate = K.variable(1, dtype="uint8")
add_tables += [{"death_rate": _death_rate, "gate": gate}]
return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
output_shape=output_shape)([out, x])
def __init__(self, lr=0.01, momentum=0., decay_block=1000, nesterov=False, **kwargs):
super(SGD_step_decay, self).__init__(**kwargs)
self.__dict__.update(locals())
self.iterations = K.variable(0.)
self.lr = K.variable(lr)
self.momentum = K.variable(momentum)
self.decay_block = K.variable(decay_block)
def _runner(layer_class):
"""
All the recurrent layers share the same interface,
so we can run through them with a single function.
"""
for ret_seq in [True, False]:
layer = layer_class(output_dim, return_sequences=ret_seq, weights=None, input_shape=(timesteps, input_dim))
layer.input = K.variable(np.ones((nb_samples, timesteps, input_dim)))
layer.get_config()
for train in [True, False]:
out = K.eval(layer.get_output(train))
# Make sure the output has the desired shape
if ret_seq:
assert out.shape == (nb_samples, timesteps, output_dim)
else:
assert out.shape == (nb_samples, output_dim)
mask = layer.get_output_mask(train)
# check statefulness
layer = layer_class(
output_dim, return_sequences=False, weights=None, batch_input_shape=(nb_samples, timesteps, input_dim)
)
layer.input = K.variable(np.ones((nb_samples, timesteps, input_dim)))
out = K.eval(layer.get_output(train))
assert out.shape == (nb_samples, output_dim)
def eigen_loss(y_true, y_pred):
y_true = tf.Print(y_true, [y_true], message='y_true', summarize=30)
y_pred = tf.Print(y_pred, [y_pred], message='y_pred', summarize=30)
y_true_clipped = K.clip(y_true, K.epsilon(), None)
y_pred_clipped = K.clip(y_pred, K.epsilon(), None)
first_log = K.log(y_pred_clipped + 1.)
second_log = K.log(y_true_clipped + 1.)
w_x = K.variable(np.array([[-1., 0., 1.],
[-1., 0., 1.],
[-1., 0., 1.]]).reshape(3, 3, 1, 1))
grad_x_pred = K.conv2d(first_log, w_x, padding='same')
grad_x_true = K.conv2d(second_log, w_x, padding='same')
w_y = K.variable(np.array([[-1., -1., -1.],
[0., 0., 0.],
[1., 1., 1.]]).reshape(3, 3, 1, 1))
grad_y_pred = K.conv2d(first_log, w_y, padding='same')
grad_y_true = K.conv2d(second_log, w_y, padding='same')
diff_x = grad_x_pred - grad_x_true
diff_y = grad_y_pred - grad_y_true
log_term = K.mean(K.square((first_log - second_log)), axis=-1)
sc_inv_term = K.square(K.mean((first_log - second_log),axis=-1))
grad_loss = K.mean(K.square(diff_x) + K.square(diff_y), axis=-1)
return log_term - (0.5 * sc_inv_term) + grad_loss
def loss(self, y_true, y_pred):
""" prepare a loss of the given metric/loss operating on non-bg data """
yt = y_true #.eval()
ytbg = np.where(yt == 0)
y_true_fix = K.variable(yt.flat(ytbg))
y_pred_fix = K.variable(y_pred.flat(ytbg))
return self.metric(y_true_fix, y_pred_fix)
def __init__(self, nb_labels,
weights=None,
input_type='prob',
dice_type='soft',
approx_hard_max=True,
vox_weights=None,
crop_indices=None,
area_reg=0.1): # regularization for bottom of Dice coeff
"""
input_type is 'prob', or 'max_label'
dice_type is hard or soft
approx_hard_max - see note below
Note: for hard dice, we grab the most likely label and then compute a
one-hot encoding for each voxel with respect to possible labels. To grab the most
likely labels, argmax() can be used, but only when Dice is used as a metric
For a Dice *loss*, argmax is not differentiable, and so we can't use it
Instead, we approximate the prob->one_hot translation when approx_hard_max is True.
"""
self.nb_labels = nb_labels
self.weights = None if weights is None else K.variable(weights)
self.vox_weights = None if vox_weights is None else K.variable(vox_weights)
self.input_type = input_type
self.dice_type = dice_type
self.approx_hard_max = approx_hard_max
self.area_reg = area_reg
self.crop_indices = crop_indices
if self.crop_indices is not None and vox_weights is not None:
self.vox_weights = utils.batch_gather(self.vox_weights, self.crop_indices)
def test_merge_add():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
i3 = layers.Input(shape=(4, 5))
o = layers.add([i1, i2, i3])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2, i3], o)
add_layer = layers.Add()
o2 = add_layer([i1, i2, i3])
assert add_layer.output_shape == (None, 4, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
x3 = np.random.random((2, 4, 5))
out = model.predict([x1, x2, x3])
assert out.shape == (2, 4, 5)
assert_allclose(out, x1 + x2 + x3, atol=1e-4)
assert add_layer.compute_mask([i1, i2, i3], [None, None, None]) is None
assert np.all(K.eval(add_layer.compute_mask(
[i1, i2, i3], [K.variable(x1), K.variable(x2), K.variable(x3)])))
# Test invalid use case
with pytest.raises(ValueError):
add_layer.compute_mask([i1, i2, i3], x1)
with pytest.raises(ValueError):
add_layer.compute_mask(i1, [None, None, None])
with pytest.raises(ValueError):
add_layer.compute_mask([i1, i2, i3], [None, None])
def call(self):
E = K.variable(np.random.random((1000,100)), name="entity_embeddings")
R = K.variable(np.random.random((10,10000)), name="relation_embeddings")
x = K.placeholder(shape=(1,3), name="spo")
y = K.placeholder(ndim=0, name="y")
batch_placeholder = K.cast(x, 'int32')[0]
# print(batch_placeholder.eval())
s, o, p = [batch_placeholder[i] for i in range(3)]
s2v = K.gather(E, s)
o2v = K.gather(E, o)
r2v = K.gather(R, p)
def ccorr(a, b):
return T.outer(a,b).flatten()
# return T.arctan(s2v) + T.arctan(o2v)
# return (s2v.dimshuffle('x', 'x', 0, 'x') + o2v.dimshuffle('x', 'x', 0, 'x')).flatten()
# return T.nnet.conv2d(a.dimshuffle('x', 'x', 0, 'x'), b.dimshuffle('x', 'x', 0, 'x'), None,
# None,
# filter_flip=True, border_mode='half')
# return self.ccorr1d_sc(a, b, border_mode='half')
eta = K.dot(r2v, ccorr(s2v, o2v))
# py = 1/(1+K.exp(-eta))
# l = -K.log(py)
# from theano import pp, function, printing
# grad = T.grad(eta, E)
# print(pp(grad))
# func = function([x], grad)
func = K.function([x, y], K.gradients(eta, [s2v, o2v, r2v, E, R]))
# for i in func.maker.fgraph.outputs:
# print(pp(i))
# print (T.grad(py, s2v))
print (func([[[1,2,3]], -1]))
def check_loss(_shape):
if _shape == '2d':
shape = (2, 3)
elif _shape == '3d':
shape = (2, 3, 1)
elif _shape == '4d':
shape = (8, 5, 6, 7)
elif _shape == '5d':
shape = (9, 8, 5, 6, 7)
y_a = 1.0*np.ones(shape)
y_b = 0.5+np.random.random(shape)
print(y_a)
print(y_b)
out1 = K.eval(_loss_tensor(K.variable(y_a), K.variable(y_b)))
print(out1)
out2 = _loss_np(y_a, y_b)
print(out2)
assert out1.shape == out2.shape
assert out1.shape == shape[:-1]
print(np.linalg.norm(out1))
print(np.linalg.norm(out2))
print(np.linalg.norm(out1-out2))
def test_sparse_categorical_crossentropy():
y_pred = K.variable(np.array([[0.3, 0.6, 0.1],
[0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([1, 2]))
expected_loss = - (np.log(0.6) + np.log(0.7)) / 2
loss = K.eval(losses.sparse_categorical_crossentropy(y_true, y_pred))
assert np.isclose(expected_loss, np.mean(loss))
def main(weights_path, base_path, base_file, style_path, style_file,
combo_path, img_width, img_height, iterations):
result_prefix = base_file[:-4] + '_' + style_file[:-4]
base_img_path = base_path + base_file
style_img_path = style_path + style_file
# get tensor representations of images
base_img = K.variable(preprocess_image(base_img_path,
img_width,
img_height))
style_img = K.variable(preprocess_image(style_img_path,
img_width,
img_height))
combo_img = K.placeholder((1, 3, img_width, img_height))
# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_img, style_img, combo_img],
axis=0)
print('Creating painting of {} in the style of {}'.format(base_file[:-4],
style_file[:-4]))
print('Loading model with VGG16 network weights...')
nn = model(weights_path, input_tensor, img_width, img_height)
loss, grads = calc_loss_grad(nn, combo_img, img_width, img_height)
evaluate = Evaluator(loss, grads, combo_img, img_width, img_height)
return optimizer(evaluate, img_width, img_height, combo_path,
result_prefix, iterations=iterations)
def test_zero_padding_1d():
num_samples = 2
input_dim = 2
num_steps = 5
shape = (num_samples, num_steps, input_dim)
input = np.ones(shape)
# basic test
layer_test(convolutional.ZeroPadding1D,
kwargs={'padding': 2},
input_shape=input.shape)
layer_test(convolutional.ZeroPadding1D,
kwargs={'padding': (1, 2)},
input_shape=input.shape)
# correctness test
layer = convolutional.ZeroPadding1D(padding=2)
layer.build(shape)
output = layer(K.variable(input))
np_output = K.eval(output)
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, offset, :], 0.)
assert_allclose(np_output[:, 2:-2, :], 1.)
layer = convolutional.ZeroPadding1D(padding=(1, 2))
layer.build(shape)
output = layer(K.variable(input))
np_output = K.eval(output)
for left_offset in [0]:
assert_allclose(np_output[:, left_offset, :], 0.)
for right_offset in [-1, -2]:
assert_allclose(np_output[:, right_offset, :], 0.)
assert_allclose(np_output[:, 1:-2, :], 1.)
layer.get_config()
def test_cropping_3d():
num_samples = 2
stack_size = 2
input_len_dim1 = 8
input_len_dim2 = 8
input_len_dim3 = 8
cropping = ((2, 2), (3, 3), (2, 3))
for data_format in ['channels_last', 'channels_first']:
if data_format == 'channels_first':
inputs = np.random.rand(num_samples, stack_size,
input_len_dim1, input_len_dim2, input_len_dim3)
else:
inputs = np.random.rand(num_samples,
input_len_dim1, input_len_dim2,
input_len_dim3, stack_size)
# basic test
layer_test(convolutional.Cropping3D,
kwargs={'cropping': cropping,
'data_format': data_format},
input_shape=inputs.shape)
# correctness test
layer = convolutional.Cropping3D(cropping=cropping,
data_format=data_format)
layer.build(inputs.shape)
output = layer(K.variable(inputs))
np_output = K.eval(output)
# compare with numpy
if data_format == 'channels_first':
expected_out = inputs[:,
:,
cropping[0][0]: -cropping[0][1],
cropping[1][0]: -cropping[1][1],
cropping[2][0]: -cropping[2][1]]
else:
expected_out = inputs[:,
cropping[0][0]: -cropping[0][1],
cropping[1][0]: -cropping[1][1],
cropping[2][0]: -cropping[2][1],
:]
assert_allclose(np_output, expected_out)
for data_format in ['channels_last', 'channels_first']:
if data_format == 'channels_first':
inputs = np.random.rand(num_samples, stack_size,
input_len_dim1, input_len_dim2, input_len_dim3)
else:
inputs = np.random.rand(num_samples,
input_len_dim1, input_len_dim2,
input_len_dim3, stack_size)
# another correctness test (no cropping)
cropping = ((0, 0), (0, 0), (0, 0))
layer = convolutional.Cropping3D(cropping=cropping,
data_format=data_format)
layer.build(inputs.shape)
output = layer(K.variable(inputs))
np_output = K.eval(output)
# compare with input
assert_allclose(np_output, inputs)
def test_zero_padding_2d():
num_samples = 2
stack_size = 2
input_num_row = 4
input_num_col = 5
for data_format in ['channels_first', 'channels_last']:
inputs = np.ones((num_samples, input_num_row, input_num_col, stack_size))
inputs = np.ones((num_samples, stack_size, input_num_row, input_num_col))
# basic test
layer_test(convolutional.ZeroPadding2D,
kwargs={'padding': (2, 2), 'data_format': data_format},
input_shape=inputs.shape)
layer_test(convolutional.ZeroPadding2D,
kwargs={'padding': ((1, 2), (3, 4)), 'data_format': data_format},
input_shape=inputs.shape)
# correctness test
layer = convolutional.ZeroPadding2D(padding=(2, 2),
data_format=data_format)
layer.build(inputs.shape)
output = layer(K.variable(inputs))
np_output = K.eval(output)
if data_format == 'channels_last':
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, offset, :, :], 0.)
assert_allclose(np_output[:, :, offset, :], 0.)
assert_allclose(np_output[:, 2:-2, 2:-2, :], 1.)
elif data_format == 'channels_first':
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, :, offset, :], 0.)
assert_allclose(np_output[:, :, :, offset], 0.)
assert_allclose(np_output[:, 2:-2, 2:-2, :], 1.)
layer = convolutional.ZeroPadding2D(padding=((1, 2), (3, 4)),
data_format=data_format)
layer.build(inputs.shape)
output = layer(K.variable(inputs))
np_output = K.eval(output)
if data_format == 'channels_last':
for top_offset in [0]:
assert_allclose(np_output[:, top_offset, :, :], 0.)
for bottom_offset in [-1, -2]:
assert_allclose(np_output[:, bottom_offset, :, :], 0.)
for left_offset in [0, 1, 2]:
assert_allclose(np_output[:, :, left_offset, :], 0.)
for right_offset in [-1, -2, -3, -4]:
assert_allclose(np_output[:, :, right_offset, :], 0.)
assert_allclose(np_output[:, 1:-2, 3:-4, :], 1.)
elif data_format == 'channels_first':
for top_offset in [0]:
assert_allclose(np_output[:, :, top_offset, :], 0.)
for bottom_offset in [-1, -2]:
assert_allclose(np_output[:, :, bottom_offset, :], 0.)
for left_offset in [0, 1, 2]:
assert_allclose(np_output[:, :, :, left_offset], 0.)
for right_offset in [-1, -2, -3, -4]:
assert_allclose(np_output[:, :, :, right_offset], 0.)
assert_allclose(np_output[:, :, 1:-2, 3:-4], 1.)
def __init__(self, threshold,
smooth_threshold,
sigma=1,
**kwargs):
self.sigma = sigma
self.threshold = K.variable(threshold)
self.smooth_threshold = K.variable(smooth_threshold)
super().__init__(**kwargs)
def get_mask_weight_blending(inputs, min=0, max=2):
input = concat(inputs)
return sequential([
Convolution2D(1, 3, 3),
Flatten(),
Dense(1),
LinearInBounds(K.variable(min), K.variable(2), clip=True),
], ns='mask_weight_blending')(input)
def __init__(self, lr=5, momentum=0., decay=0.,
nesterov=False, **kwargs):
super(SimpleSGD, self).__init__(**kwargs)
self.__dict__.update(locals())
self.iterations = K.variable(0.)
self.lr = K.variable(lr)
self.momentum = K.variable(momentum)
self.decay = K.variable(decay)
def __init__(self, E, R, d, rparam, input_shape=(3,), **kwargs):
bnd = math.sqrt(6) / math.sqrt(2*E)
from numpy.random import uniform
self.init = [K.variable(uniform(size=(E,d), low=-bnd, high=bnd), name="E"),
K.variable(uniform(size=(R,d*d), low=-bnd, high=bnd), name="R")]
self.rparam = rparam
kwargs["input_shape"] = input_shape
super(HolographicLayer, self).__init__(**kwargs)
def test_categorical_hinge():
y_pred = K.variable(np.array([[0.3, 0.2, 0.1],
[0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([[0, 1, 0],
[1, 0, 0]]))
expected_loss = ((0.3 - 0.2 + 1) + (0.7 - 0.1 + 1)) / 2.0
loss = K.eval(losses.categorical_hinge(y_true, y_pred))
assert np.isclose(expected_loss, np.mean(loss))
def __init__(self, lr=0.01, momentum=0., decay=0., nesterov=False,
*args, **kwargs):
super(SGD_exponential_decay, self).__init__(**kwargs)
self.__dict__.update(locals())
self.iterations = K.variable(0.)
self.lr = K.variable(lr)
self.momentum = K.variable(momentum)
self.decay = K.variable(decay)
def test_objective_shapes_2d(self, loss_fn):
y_a = K.variable(np.random.random((6, 7)))
y_b = K.variable(np.random.random((6, 7)))
objective_output = loss_fn(y_a, y_b)
assert K.eval(objective_output).shape == (6,)
def yolo_head(feats, anchors, num_classes):
"""Convert final layer features to bounding box parameters.
Parameters
----------
feats : tensor
Final convolutional layer features.
anchors : array-like
Anchor box widths and heights.
num_classes : int
Number of target classes.
Returns
-------
box_xy : tensor
x, y box predictions adjusted by spatial location in conv layer.
box_wh : tensor
w, h box predictions adjusted by anchors and conv spatial resolution.
box_conf : tensor
Probability estimate for whether each box contains any object.
box_class_pred : tensor
Probability distribution estimate for each box over class labels.
"""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.variable(anchors), [1, 1, 1, num_anchors, 2])
# Static implementation for fixed models.
# TODO: Remove or add option for static implementation.
# _, conv_height, conv_width, _ = K.int_shape(feats)
# conv_dims = K.variable([conv_width, conv_height])
# Dynamic implementation of conv dims for fully convolutional model.
conv_dims = K.shape(feats)[1:3] # assuming channels last
# In YOLO the height index is the inner most iteration.
conv_height_index = K.arange(0, stop=conv_dims[0])
conv_width_index = K.arange(0, stop=conv_dims[1])
conv_height_index = K.tile(conv_height_index, [conv_dims[1]])
# TODO: Repeat_elements and tf.split doesn't support dynamic splits.
# conv_width_index = K.repeat_elements(conv_width_index, conv_dims[1], axis=0)
conv_width_index = K.tile(K.expand_dims(conv_width_index, 0),
[conv_dims[0], 1])
conv_width_index = K.flatten(K.transpose(conv_width_index))
conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
conv_index = K.cast(conv_index, K.dtype(feats))
feats = K.reshape(
feats, [-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats))
# Static generation of conv_index:
# conv_index = np.array([_ for _ in np.ndindex(conv_width, conv_height)])
# conv_index = conv_index[:, [1, 0]] # swap columns for YOLO ordering.
# conv_index = K.variable(
# conv_index.reshape(1, conv_height, conv_width, 1, 2))
# feats = Reshape(
# (conv_dims[0], conv_dims[1], num_anchors, num_classes + 5))(feats)
box_confidence = K.sigmoid(feats[..., 4:5])
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_class_probs = K.softmax(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
# Note: YOLO iterates over height index before width index.
box_xy = (box_xy + conv_index) / conv_dims
box_wh = box_wh * anchors_tensor / conv_dims
return box_confidence, box_xy, box_wh, box_class_probs
def W_init(shape, name=None):
"""Initialize weights as in paper"""
values = np.random.normal(loc=0, scale=1e-2, size=shape)
return K.variable(values, name=name)
def test_zero_padding_2d():
nb_samples = 2
stack_size = 2
input_nb_row = 4
input_nb_col = 5
dim_ordering = K.image_dim_ordering()
assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
if dim_ordering == 'tf':
input = np.ones((nb_samples, input_nb_row, input_nb_col, stack_size))
elif dim_ordering == 'th':
input = np.ones((nb_samples, stack_size, input_nb_row, input_nb_col))
# basic test
layer_test(convolutional.ZeroPadding2D,
kwargs={'padding': (2, 2)},
input_shape=input.shape)
layer_test(convolutional.ZeroPadding2D,
kwargs={'padding': (1, 2, 3, 4)},
input_shape=input.shape)
layer_test(convolutional.ZeroPadding2D,
kwargs={
'padding': {
'top_pad': 1,
'bottom_pad': 2,
'left_pad': 3,
'right_pad': 4
}
},
input_shape=input.shape)
# correctness test
layer = convolutional.ZeroPadding2D(padding=(2, 2))
layer.build(input.shape)
output = layer(K.variable(input))
np_output = K.eval(output)
if dim_ordering == 'tf':
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, offset, :, :], 0.)
assert_allclose(np_output[:, :, offset, :], 0.)
assert_allclose(np_output[:, 2:-2, 2:-2, :], 1.)
elif dim_ordering == 'th':
for offset in [0, 1, -1, -2]:
assert_allclose(np_output[:, :, offset, :], 0.)
assert_allclose(np_output[:, :, :, offset], 0.)
assert_allclose(np_output[:, 2:-2, 2:-2, :], 1.)
layer = convolutional.ZeroPadding2D(padding=(1, 2, 3, 4))
layer.build(input.shape)
output = layer(K.variable(input))
np_output = K.eval(output)
if dim_ordering == 'tf':
for top_offset in [0]:
assert_allclose(np_output[:, top_offset, :, :], 0.)
for bottom_offset in [-1, -2]:
assert_allclose(np_output[:, bottom_offset, :, :], 0.)
for left_offset in [0, 1, 2]:
assert_allclose(np_output[:, :, left_offset, :], 0.)
for right_offset in [-1, -2, -3, -4]:
assert_allclose(np_output[:, :, right_offset, :], 0.)
assert_allclose(np_output[:, 1:-2, 3:-4, :], 1.)
elif dim_ordering == 'th':
for top_offset in [0]:
assert_allclose(np_output[:, :, top_offset, :], 0.)
for bottom_offset in [-1, -2]:
assert_allclose(np_output[:, :, bottom_offset, :], 0.)
for left_offset in [0, 1, 2]:
assert_allclose(np_output[:, :, :, left_offset], 0.)
for right_offset in [-1, -2, -3, -4]:
assert_allclose(np_output[:, :, :, right_offset], 0.)
assert_allclose(np_output[:, :, 1:-2, 3:-4], 1.)
layer.get_config()
mask_tensor[i, :, :] = mask
else:
mask_tensor[:, :, i] = mask
return mask_tensor
def pooling_func(x):
if pooltype == 1:
return AveragePooling2D((2, 2), strides=(2, 2))(x)
else:
return MaxPooling2D((2, 2), strides=(2, 2))(x)
# get tensor representations of our images
base_image = K.variable(preprocess_image(base_image_path, True, read_mode=read_mode))
style_reference_images = []
for style_path in style_image_paths:
style_reference_images.append(K.variable(preprocess_image(style_path)))
# this will contain our generated image
if K.image_dim_ordering() == 'th':
combination_image = K.placeholder((1, 3, img_width, img_height))
else:
combination_image = K.placeholder((1, img_width, img_height, 3))
image_tensors = [base_image]
for style_image_tensor in style_reference_images:
image_tensors.append(style_image_tensor)
image_tensors.append(combination_image)
if K.image_data_format() == 'channels_first':
x = x.reshape((3, img_nrows, img_ncols))
x = x.transpose((1, 2, 0))
else:
x = x.reshape((img_nrows, img_ncols, 3))
x[:, :, 0] += 103.939
x[:, :, 1] += 116.779
x[:, :, 2] += 123.68
x = x[:, :, ::-1] #BGR---> RGB
x = np.clip(x, 0, 255).astype('uint8')
return x
from keras import backend as K
target_image = K.variable(preprocess_image(target_image_path))
style_reference_image = K.variable(
preprocess_image(style_reference_image_path))
if K.image_data_format() == 'channels_first':
combination_image = K.placeholder((1, 3, img_nrows, img_ncols))
else:
combination_image = K.placeholder((1, img_nrows, img_ncols, 3))
#combination_image = K.placeholder((1,img_height,img_width,3))
input_tensor = K.concatenate(
[target_image, style_reference_image, combination_image], axis=0)
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
#定义内容loss
def my_init(shape, name=None):
value = np.random.random(shape)
return K.variable(value, name=name)
def tybalt_2layer_model(
learning_rate,
batch_size,
epochs,
kappa,
intermediate_dim,
latent_dim,
epsilon_std,
rnaseq,
base_dir,
analysis_name):
"""
Train 2-layer Tybalt model using input dataset
Arguments
----------
learning_rate: float
Step size used for gradient descent. In other words, it's how quickly the methods is learning
batch_size: int
Training is performed in batches. So this determines the number of samples to consider at a given time.
epochs: int
The number of times to train over the entire input dataset.
kappa: float
How fast to linearly ramp up KL loss
intermediate_dim: int
Size of the hidden layer
latent_dim: int
Size of the bottleneck layer
epsilon_std: float
Standard deviation of Normal distribution to sample latent space
rnaseq: pandas.dataframe
Gene expression data
base_dir: str
Parent directory where data/, scripts/, models/ are subdirectories
analysis_name: str
Name that will be used to create a subdirectory where results and models will be stored
Returns
--------
model_decoder_file, weights_decoder_file: .h5 file
Files used to generate decoding neural networks to use in downstream analysis
model_encoder_file, weights_encoder_file: .h5 file
Files used to generate encoding neural networks to use in downstream analysis
encoded_file: .txt file
File containing input data encoded into latent space using encoder neural network
"""
# The below is necessary in Python 3.2.3 onwards to
# have reproducible behavior for certain hash-based operations.
# See these references for further details:
# https://docs.python.org/3.4/using/cmdline.html#envvar-PYTHONHASHSEED
# https://github.com/keras-team/keras/issues/2280#issuecomment-306959926
randomState = 123
import os
os.environ['PYTHONHASHSEED'] = '0'
# The below is necessary for starting Numpy generated random numbers
# in a well-defined initial state.
np.random.seed(42)
# The below is necessary for starting core Python generated random numbers
# in a well-defined state.
rn.seed(12345)
# Force TensorFlow to use single thread.
# Multiple threads are a potential source of
# non-reproducible results.
# For further details, see: https://stackoverflow.com/questions/42022950/which-seeds-have-to-be-set-where-to-realize-100-reproducibility-of-training-res
session_conf = tf.ConfigProto(
intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
from keras import backend as K
# The below tf.set_random_seed() will make random number generation
# in the TensorFlow backend have a well-defined initial state.
# For further details, see: https://www.tensorflow.org/api_docs/python/tf/set_random_seed
tf.set_random_seed(1234)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)
# Load rnaseq data
rnaseq = rnaseq
# Initialize hyper parameters
original_dim = rnaseq.shape[1]
beta = K.variable(0)
stat_file = os.path.join(
base_dir,
"output",
"stats",
analysis_name,
"tybalt_2layer_{}latent_stats.tsv".format(latent_dim))
hist_plot_file = os.path.join(
base_dir,
"output",
"viz",
analysis_name,
"tybalt_2layer_{}latent_hist.png".format(latent_dim))
encoded_file = os.path.join(
base_dir,
"data",
"encoded",
analysis_name,
"train_input_2layer_{}latent_encoded.txt".format(latent_dim))
model_encoder_file = os.path.join(
base_dir,
"models",
analysis_name,
"tybalt_2layer_{}latent_encoder_model.h5".format(latent_dim))
weights_encoder_file = os.path.join(
base_dir,
"models",
analysis_name,
"tybalt_2layer_{}latent_encoder_weights.h5".format(latent_dim))
model_decoder_file = os.path.join(
base_dir,
"models",
analysis_name,
"tybalt_2layer_{}latent_decoder_model.h5".format(latent_dim))
weights_decoder_file = os.path.join(
base_dir,
"models",
analysis_name,
"tybalt_2layer_{}latent_decoder_weights.h5".format(latent_dim))
# Data initalizations
# Split 10% test set randomly
test_set_percent = 0.1
rnaseq_test_df = rnaseq.sample(
frac=test_set_percent, random_state=randomState)
rnaseq_train_df = rnaseq.drop(rnaseq_test_df.index)
# Create a placeholder for an encoded (original-dimensional)
rnaseq_input = Input(shape=(original_dim, ))
# Architecture of VAE
# ENCODER
# Input layer is compressed into a mean and log variance vector of size
# `latent_dim`. Each layer is initialized with glorot uniform weights and each
# step (dense connections, batch norm,and relu activation) are funneled
# separately
# Each vector of length `latent_dim` are connected to the rnaseq input tensor
# "z_mean_dense_linear" is the encoded representation of the input
# Take as input arrays of shape (*, original dim) and output arrays of shape (*, latent dim)
# Combine input from previous layer using linear summ
# Normalize the activations (combined weighted nodes of the previous layer)
# Transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1.
# Apply ReLU activation function to combine weighted nodes from previous layer
# relu = threshold cutoff (cutoff value will be learned)
# ReLU function filters noise
# X is encoded using Q(z|X) to yield mu(X), sigma(X) that describes latent space distribution
hidden_dense_linear = Dense(
intermediate_dim, kernel_initializer='glorot_uniform')(rnaseq_input)
hidden_dense_batchnorm = BatchNormalization()(hidden_dense_linear)
hidden_encoded = Activation('relu')(hidden_dense_batchnorm)
# Note:
# Normalize and relu filter at each layer adds non-linear component (relu is non-linear function)
# If architecture is layer-layer-normalization-relu then the computation is still linear
# Add additional layers in triplicate
z_mean_dense_linear = Dense(
latent_dim, kernel_initializer='glorot_uniform')(hidden_encoded)
z_mean_dense_batchnorm = BatchNormalization()(z_mean_dense_linear)
z_mean_encoded = Activation('relu')(z_mean_dense_batchnorm)
z_log_var_dense_linear = Dense(
latent_dim, kernel_initializer='glorot_uniform')(rnaseq_input)
z_log_var_dense_batchnorm = BatchNormalization()(z_log_var_dense_linear)
z_log_var_encoded = Activation('relu')(z_log_var_dense_batchnorm)
# Customized layer
# Returns the encoded and randomly sampled z vector
# Takes two keras layers as input to the custom sampling function layer with a
# latent_dim` output
#
# sampling():
# randomly sample similar points z from the latent normal distribution that is assumed to generate the data,
# via z = z_mean + exp(z_log_sigma) * epsilon, where epsilon is a random normal tensor
# z ~ Q(z|X)
# Note: there is a trick to reparameterize to standard normal distribution so that the space is differentiable and
# therefore gradient descent can be used
#
# Returns the encoded and randomly sampled z vector
# Takes two keras layers as input to the custom sampling function layer with a
# latent_dim` output
z = Lambda(sampling_maker(epsilon_std),
output_shape=(latent_dim, ))([z_mean_encoded, z_log_var_encoded])
# DECODER
# The decoding layer is much simpler with a single layer glorot uniform
# initialized and sigmoid activation
# Reconstruct P(X|z)
decoder_model = Sequential()
decoder_model.add(
Dense(intermediate_dim, activation='relu', input_dim=latent_dim))
decoder_model.add(Dense(original_dim, activation='sigmoid'))
rnaseq_reconstruct = decoder_model(z)
# CONNECTIONS
# fully-connected network
adam = optimizers.Adam(lr=learning_rate)
vae_layer = CustomVariationalLayer(original_dim, z_log_var_encoded, z_mean_encoded, beta)([
rnaseq_input, rnaseq_reconstruct])
vae = Model(rnaseq_input, vae_layer)
vae.compile(optimizer=adam, loss=None, loss_weights=[beta])
# Training
# fit Model
# hist: record of the training loss at each epoch
hist = vae.fit(
np.array(rnaseq_train_df),
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_data=(np.array(rnaseq_test_df), None),
callbacks=[WarmUpCallback(beta, kappa)])
# Use trained model to make predictions
encoder = Model(rnaseq_input, z_mean_encoded)
encoded_rnaseq_df = encoder.predict_on_batch(rnaseq)
encoded_rnaseq_df = pd.DataFrame(encoded_rnaseq_df, index=rnaseq.index)
encoded_rnaseq_df.columns.name = 'sample_id'
encoded_rnaseq_df.columns = encoded_rnaseq_df.columns + 1
# Visualize training performance
history_df = pd.DataFrame(hist.history)
ax = history_df.plot()
ax.set_xlabel('Epochs')
ax.set_ylabel('VAE Loss')
fig = ax.get_figure()
fig.savefig(hist_plot_file)
del ax, fig
# Output
# Save training performance
history_df = pd.DataFrame(hist.history)
history_df = history_df.assign(learning_rate=learning_rate)
history_df = history_df.assign(batch_size=batch_size)
history_df = history_df.assign(epochs=epochs)
history_df = history_df.assign(kappa=kappa)
history_df.to_csv(stat_file, sep='\t', index=False)
# Save latent space representation
encoded_rnaseq_df.to_csv(encoded_file, sep='\t')
# Save models
# (source) https://machinelearningmastery.com/save-load-keras-deep-learning-models/
# Save encoder model
encoder.save(model_encoder_file)
# serialize weights to HDF5
encoder.save_weights(weights_encoder_file)
# Save decoder model
# (source) https://github.com/greenelab/tybalt/blob/master/scripts/nbconverted/tybalt_vae.py
# can generate from any sampled z vector
decoder_input = Input(shape=(latent_dim, ))
_x_decoded_mean = decoder_model(decoder_input)
decoder = Model(decoder_input, _x_decoded_mean)
decoder.save(model_decoder_file)
# serialize weights to HDF5
decoder.save_weights(weights_decoder_file)
# Save weight matrix: how each gene contribute to each feature
# build a generator that can sample from the learned distribution
# can generate from any sampled z vector
decoder_input = Input(shape=(latent_dim, ))
x_decoded_mean = decoder_model(decoder_input)
decoder = Model(decoder_input, x_decoded_mean)
weights = []
for layer in decoder.layers:
weights.append(layer.get_weights())
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.conv_layers = {c: [] for c in ['i', 'f', 'c', 'o', 'a', 'ahat']}
for l in range(self.nb_layers):
for c in ['i', 'f', 'c', 'o']:
act = self.LSTM_activation if c == 'c' else self.LSTM_inner_activation
self.conv_layers[c].append(
Conv2D(self.R_stack_sizes[l],
self.R_filt_sizes[l],
padding='same',
activation=act,
data_format=self.data_format))
act = 'relu' if l == 0 else self.A_activation
self.conv_layers['ahat'].append(
Conv2D(self.stack_sizes[l],
self.Ahat_filt_sizes[l],
padding='same',
activation=act,
data_format=self.data_format))
if l < self.nb_layers - 1:
self.conv_layers['a'].append(
Conv2D(self.stack_sizes[l + 1],
self.A_filt_sizes[l],
padding='same',
activation=self.A_activation,
data_format=self.data_format))
self.upsample = UpSampling2D(data_format=self.data_format)
self.pool = MaxPooling2D(data_format=self.data_format)
self.trainable_weights = []
nb_row, nb_col = (
input_shape[-2],
input_shape[-1]) if self.data_format == 'channels_first' else (
input_shape[-3], input_shape[-2])
for c in sorted(self.conv_layers.keys()):
for l in range(len(self.conv_layers[c])):
ds_factor = 2**l
if c == 'ahat':
nb_channels = self.R_stack_sizes[l]
elif c == 'a':
nb_channels = 2 * self.stack_sizes[l]
else:
nb_channels = self.stack_sizes[l] * 2 + self.R_stack_sizes[
l]
if l < self.nb_layers - 1:
nb_channels += self.R_stack_sizes[l + 1]
in_shape = (input_shape[0], nb_channels, nb_row // ds_factor,
nb_col // ds_factor)
if self.data_format == 'channels_last':
in_shape = (in_shape[0], in_shape[2], in_shape[3],
in_shape[1])
with K.name_scope('layer_' + c + '_' + str(l)):
self.conv_layers[c][l].build(in_shape)
self.trainable_weights += self.conv_layers[c][
l].trainable_weights
self.states = [None] * self.nb_layers * 3
if self.extrap_start_time is not None:
self.t_extrap = K.variable(
self.extrap_start_time,
int if K.backend() != 'tensorflow' else 'int32')
self.states += [None] * 2 # [previous frame prediction, timestep]
from keras import backend as K
from keras.regularizers import ActivityRegularizer
import numpy as np
dummy_loss_val = K.variable(0.0)
softminus = lambda x: x - K.softplus(x)
# Dummy loss function which simply returns 0
# This is because we will be training the network using regularizers.
def dummy_loss(y_true, y_pred):
return dummy_loss_val
def psnr(y_true, y_pred):
assert y_true.shape == y_pred.shape, "Cannot calculate PSNR. Input shapes not same." \
" y_true shape = %s, y_pred shape = %s" % (str(y_true.shape),
str(y_pred.shape))
return -10. * np.log10(np.mean(np.square(y_pred - y_true)))
def PSNRLoss(y_true, y_pred):
"""
PSNR is Peek Signal to Noise Ratio, which is similar to mean squared error.
It can be calculated as
PSNR = 20 * log10(MAXp) - 10 * log10(MSE)
When providing an unscaled input, MAXp = 255. Therefore 20 * log10(255)== 48.1308036087.
class weighted_model:
alpha = 0.6
epoch_ctr = K.variable(0, name='epoch_ctr')
ramp_wt = K.variable(0, name='ramp_wt', dtype='float32')
ramp_period = 300
weight_max = 1.
class LossCallback(Callback):
def on_epoch_end(self, epoch, logs={}):
weighted_model.epoch_ctr = epoch
print(weighted_model.epoch_ctr)
weighted_model.ramp_wt = ramp_up_weight(weighted_model.ramp_period,
weighted_model.weight_max,
epoch)
def dice_coef(self, y_true, y_pred, smooth=1.):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
return (2. * intersection + smooth) / (K.sum(y_true_f) +
K.sum(y_pred_f) + smooth)
def dice_tb(
self,
input,
class_wt=1.,
):
def dice_loss(y_true, y_pred, smooth=1., axis=(1, 2, 3)):
supervised_flag = input[1, :, :, :, :]
unsupervised_gt = input[0, :, :, :, :]
alpha = 0.6
# unsupervised_gt = unsupervised_gt / (1 - alpha ** (self.epoch_ctr + 1))
y_true_final = tf.where(tf.equal(supervised_flag, 2),
unsupervised_gt, y_true)
supervised_flag = tf.where(tf.equal(supervised_flag, 2),
K.ones_like(supervised_flag),
supervised_flag)
y_true = y_true_final * supervised_flag
y_pred = y_pred * supervised_flag
intersection = K.sum(y_true * y_pred, axis=axis)
y_true_sum = K.sum(y_true, axis=axis)
y_pred_sum = K.sum(y_pred, axis=axis)
sign_pred = tf.where(tf.greater(y_pred, 0), K.ones_like(y_pred),
K.zeros_like(y_pred))
if tf.greater(intersection, 0) is not None:
c = K.sum(y_true * y_pred) / (K.sum(y_true * sign_pred) +
K.epsilon())
else:
c = 1
return -K.mean((2. * intersection + smooth) /
((c * y_pred_sum) + y_true_sum + smooth))
return dice_loss
def c_reg_dice_loss(self, input):
def dice_loss(y_true, y_pred, smooth=1., axis=(1, 2, 3)):
supervised_flag = input[1, :, :, :, :]
y_true = y_true * supervised_flag
y_pred = y_pred * supervised_flag
intersection = K.sum(y_true * y_pred, axis=axis)
y_true_sum = K.sum(y_true, axis=axis)
y_pred_sum = K.sum(y_pred, axis=axis)
sign_pred = tf.where(tf.greater(y_pred, 0), K.ones_like(y_pred),
K.zeros_like(y_pred))
if tf.greater(intersection, 0) is not None:
c = K.sum(y_true * y_pred) / (K.sum(y_true * sign_pred) +
K.epsilon())
else:
c = 1
return -K.mean((2. * intersection + smooth) /
((c * y_pred_sum) + y_true_sum + smooth))
return dice_loss
def c_dice_loss(self, y_true, y_pred, weight, smooth=1.):
y_true = y_true * weight
y_pred = y_pred * weight
intersection = K.sum(y_true * y_pred)
y_true_sum = K.sum(y_true)
y_pred_sum = K.sum(y_pred)
return -K.mean((2. * intersection + smooth) /
((y_pred_sum) + y_true_sum + smooth))
def unsup_dice_tb(self, input, class_wt=1.):
unsupervised_gt = input
# unsupervised_gt = unsupervised_gt / (1 - (0.6** (self.epoch_ctr + 1)))
def unsup_dice_loss(y_true, y_pred, smooth=1., axis=(1, 2, 3)):
y_true = unsupervised_gt
y_pred = y_pred
intersection = K.sum(y_true * y_pred, axis=axis)
y_true_sum = K.sum(y_true, axis=axis)
y_pred_sum = K.sum(y_pred, axis=axis)
sign_pred = tf.where(tf.greater(y_pred, 0), K.ones_like(y_pred),
K.zeros_like(y_pred))
if tf.greater(intersection, 0) is not None:
c = K.sum(y_true * y_pred) / (K.sum(y_true * sign_pred) +
K.epsilon())
else:
c = 1
avg_dice_coef = K.mean((2. * intersection + smooth) /
((c * y_pred_sum) + y_true_sum + smooth))
return 1 - avg_dice_coef
return unsup_dice_loss
def dice_loss(self, y_true, y_pred, weight, smooth=1., axis=(1, 2, 3)):
y_true = y_true * weight
y_pred = y_pred * weight
intersection = K.sum(y_true * y_pred, axis=axis)
y_true_sum = K.sum(y_true, axis=axis)
y_pred_sum = K.sum(y_pred, axis=axis)
avg_dice_coef = K.mean(
(2. * intersection + smooth) / (y_true_sum + y_pred_sum + smooth))
return -avg_dice_coef
def unsup_dice_loss(self,
y_true,
y_pred,
weight,
smooth=1.,
axis=(1, 2, 3)):
y_true = y_true
y_pred = y_pred
intersection = K.sum(y_true * y_pred, axis=axis)
y_true_sum = K.sum(y_true, axis=axis)
y_pred_sum = K.sum(y_pred, axis=axis)
avg_dice_coef = K.mean(
(2. * intersection + smooth) / (y_true_sum + y_pred_sum + smooth))
return 1 - avg_dice_coef
def unsup_c_dice_loss(self, y_true, y_pred, smooth=1., axis=(1, 2, 3)):
intersection = K.sum(y_true * y_pred, axis=axis)
y_true_sum = K.sum(y_true, axis=axis)
y_pred_sum = K.sum(y_pred, axis=axis)
sign_pred = tf.where(tf.greater(y_pred, 0), K.ones_like(y_pred),
K.zeros_like(y_pred))
if tf.greater(intersection, 0) is not None:
c = K.sum(
y_true * y_pred) / (K.sum(y_true * sign_pred) + K.epsilon())
else:
c = 1
avg_dice_coef = K.mean((2. * intersection + smooth) /
((c * y_pred_sum) + y_true_sum + smooth))
# return 1 / (1 + avg_dice_coef)
return 1 - avg_dice_coef
def semi_supervised_loss(self, input, unsup_loss_class_wt=1., alpha=0.6):
def loss_func(y_true, y_pred):
print(K.eval(self.epoch_ctr))
unsupervised_gt = input
unsupervised_gt = unsupervised_gt / (1 -
alpha**(self.epoch_ctr + 1))
# supervised_loss = self.c_dice_loss(y_true, y_pred, supervised_flag)
supervised_loss = -self.dice_coef(y_true, y_pred)
r = weighted_model.ramp_wt
unsupervised_loss = r * self.unsup_c_dice_loss(
unsupervised_gt, y_pred)
return supervised_loss + unsup_loss_class_wt * unsupervised_loss
return loss_func
def downLayer(self, inputLayer, filterSize, i, bn=False):
conv = Conv3D(filterSize, (3, 3, 3),
activation='relu',
padding='same',
name='conv' + str(i) + '_1')(inputLayer)
if bn:
conv = BatchNormalization()(conv)
conv = Conv3D(filterSize * 2, (3, 3, 3),
activation='relu',
padding='same',
name='conv' + str(i) + '_2')(conv)
if bn:
conv = BatchNormalization()(conv)
pool = MaxPooling3D(pool_size=(1, 2, 2))(conv)
return pool, conv
def upLayer(self,
inputLayer,
concatLayer,
filterSize,
i,
bn=False,
do=False):
up = Conv3DTranspose(filterSize, (2, 2, 2),
strides=(1, 2, 2),
activation='relu',
padding='same',
name='up' + str(i))(inputLayer)
# print( concatLayer.shape)
up = concatenate([up, concatLayer])
conv = Conv3D(int(filterSize / 2), (3, 3, 3),
activation='relu',
padding='same',
name='conv' + str(i) + '_1')(up)
if bn:
conv = BatchNormalization()(conv)
if do:
conv = Dropout(0.5, seed=3, name='Dropout_' + str(i))(conv)
conv = Conv3D(int(filterSize / 2), (3, 3, 3),
activation='relu',
padding='same',
name='conv' + str(i) + '_2')(conv)
if bn:
conv = BatchNormalization()(conv)
return conv
def build_model(self,
img_shape=(32, 168, 168),
learning_rate=5e-5,
gpu_id=None,
nb_gpus=None):
input_img = Input((*img_shape, 1), name='img_inp')
unsupervised_label = Input((*img_shape, 5), name='unsup_label_inp')
# supervised_flag = Input(shape=img_shape, name='flag_inp')
kernel_init = 'he_normal'
sfs = 16 # start filter size
bn = True
do = True
conv1, conv1_b_m = self.downLayer(input_img, sfs, 1, bn)
conv2, conv2_b_m = self.downLayer(conv1, sfs * 2, 2, bn)
conv3 = Conv3D(sfs * 4, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(3) + '_1')(conv2)
if bn:
conv3 = BatchNormalization()(conv3)
conv3 = Conv3D(sfs * 8, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(3) + '_2')(conv3)
if bn:
conv3 = BatchNormalization()(conv3)
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv3)
# conv3, conv3_b_m = downLayer(conv2, sfs*4, 3, bn)
conv4 = Conv3D(sfs * 16, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv4_1')(pool3)
if bn:
conv4 = BatchNormalization()(conv4)
if do:
conv4 = Dropout(0.5, seed=4, name='Dropout_' + str(4))(conv4)
conv4 = Conv3D(sfs * 16, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv4_2')(conv4)
if bn:
conv4 = BatchNormalization()(conv4)
# conv5 = upLayer(conv4, conv3_b_m, sfs*16, 5, bn, do)
up1 = Conv3DTranspose(sfs * 16, (2, 2, 2),
strides=(2, 2, 2),
activation='relu',
padding='same',
name='up' + str(5))(conv4)
up1 = concatenate([up1, conv3])
conv5 = Conv3D(int(sfs * 8), (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(5) + '_1')(up1)
if bn:
conv5 = BatchNormalization()(conv5)
if do:
conv5 = Dropout(0.5, seed=5, name='Dropout_' + str(5))(conv5)
conv5 = Conv3D(int(sfs * 8), (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(5) + '_2')(conv5)
if bn:
conv5 = BatchNormalization()(conv5)
conv6 = self.upLayer(conv5, conv2_b_m, sfs * 8, 6, bn, do)
conv7 = self.upLayer(conv6, conv1_b_m, sfs * 4, 7, bn, do)
conv_out = Conv3D(5, (1, 1, 1), name='conv_final_softmax')(conv7)
# conv_out = Lambda(lambda x: x / temp, name='scaling')(conv_out)
conv_out_sm = Activation('softmax')(conv_out)
pz_sm_out = Lambda(lambda x: x[:, :, :, :, 0], name='pz')(conv_out_sm)
cz_sm_out = Lambda(lambda x: x[:, :, :, :, 1], name='cz')(conv_out_sm)
us_sm_out = Lambda(lambda x: x[:, :, :, :, 2], name='us')(conv_out_sm)
afs_sm_out = Lambda(lambda x: x[:, :, :, :, 3],
name='afs')(conv_out_sm)
bg_sm_out = Lambda(lambda x: x[:, :, :, :, 4], name='bg')(conv_out_sm)
pz_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 0],
name='pzu')(unsupervised_label)
cz_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 1],
name='czu')(unsupervised_label)
us_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 2],
name='usu')(unsupervised_label)
afs_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 3],
name='afsu')(unsupervised_label)
bg_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 4],
name='bgu')(unsupervised_label)
optimizer = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999)
if (nb_gpus is None):
p_model = Model(
[input_img, unsupervised_label],
[pz_sm_out, cz_sm_out, us_sm_out, afs_sm_out, bg_sm_out])
p_model.compile(
optimizer=optimizer,
loss={
'pz':
self.semi_supervised_loss(pz_ensemble_pred,
unsup_loss_class_wt=1),
'cz':
self.semi_supervised_loss(cz_ensemble_pred, 1),
'us':
self.semi_supervised_loss(us_ensemble_pred, 2),
'afs':
self.semi_supervised_loss(afs_ensemble_pred, 2),
'bg':
self.semi_supervised_loss(bg_ensemble_pred, 1)
},
metrics={
'pz':
[self.dice_coef,
self.unsup_dice_tb(pz_ensemble_pred, 1)],
'cz':
[self.dice_coef,
self.unsup_dice_tb(cz_ensemble_pred, 1)],
'us':
[self.dice_coef,
self.unsup_dice_tb(us_ensemble_pred, 2)],
'afs':
[self.dice_coef,
self.unsup_dice_tb(afs_ensemble_pred, 2)],
'bg':
[self.dice_coef,
self.unsup_dice_tb(bg_ensemble_pred, 1)]
})
else:
with tf.device(gpu_id):
model = Model(
[input_img, unsupervised_label],
[pz_sm_out, cz_sm_out, us_sm_out, afs_sm_out, bg_sm_out])
p_model = multi_gpu_model(model, gpus=nb_gpus)
p_model.compile(
optimizer=optimizer,
loss={
'pz':
self.semi_supervised_loss(pz_ensemble_pred,
unsup_loss_class_wt=1),
'cz':
self.semi_supervised_loss(cz_ensemble_pred, 1),
'us':
self.semi_supervised_loss(us_ensemble_pred, 2),
'afs':
self.semi_supervised_loss(afs_ensemble_pred, 2),
'bg':
self.semi_supervised_loss(bg_ensemble_pred, 1)
},
metrics={
'pz': [
self.dice_coef,
self.unsup_dice_tb(pz_ensemble_pred, 1)
],
'cz': [
self.dice_coef,
self.unsup_dice_tb(cz_ensemble_pred, 1)
],
'us': [
self.dice_coef,
self.unsup_dice_tb(us_ensemble_pred, 2)
],
'afs': [
self.dice_coef,
self.unsup_dice_tb(afs_ensemble_pred, 2)
],
'bg': [
self.dice_coef,
self.unsup_dice_tb(bg_ensemble_pred, 1)
]
},
loss_weights={
'pz': 1,
'cz': 1,
'us': 2,
'afs': 2,
'bg': 1
})
return p_model
def get_base_image(img_nrows, img_ncols):
return K.variable(load_image("../images/mat.jpg", img_nrows, img_ncols))
def __init__(self, weights):
self.config = {'weights': list(weights)}
self.length = len(weights)
self.weights = K.variable(weights, name='acc_weights')
self.__name__ = 'waccOA'
def get_style_image(img_nrows, img_ncols):
return K.variable(load_image("../images/munch.jpg", img_nrows, img_ncols))
def build(self, input_shape):
initial_weight_value = np.random.random_sample(
(int(input_shape[1]), int(input_shape[2])))
self.W = K.variable(initial_weight_value)
self.trainable_weights = [self.W]
def call(self, x, mask=None):
# TODO: validate input shape
assert (len(x) == 3)
L_flat = x[0]
mu = x[1]
a = x[2]
if self.mode == 'full':
# Create L and L^T matrix, which we use to construct the positive-definite matrix P.
L = None
LT = None
if K.backend() == 'theano':
import theano.tensor as T
import theano
def fn(x, L_acc, LT_acc):
x_ = K.zeros((self.nb_actions, self.nb_actions))
x_ = T.set_subtensor(x_[np.tril_indices(self.nb_actions)], x)
diag = K.exp(T.diag(x_)) + K.epsilon()
x_ = T.set_subtensor(x_[np.diag_indices(self.nb_actions)], diag)
return x_, x_.T
outputs_info = [
K.zeros((self.nb_actions, self.nb_actions)),
K.zeros((self.nb_actions, self.nb_actions)),
]
results, _ = theano.scan(fn=fn, sequences=L_flat, outputs_info=outputs_info)
L, LT = results
elif K.backend() == 'tensorflow':
import tensorflow as tf
# Number of elements in a triangular matrix.
nb_elems = (self.nb_actions * self.nb_actions + self.nb_actions) // 2
# Create mask for the diagonal elements in L_flat. This is used to exponentiate
# only the diagonal elements, which is done before gathering.
diag_indeces = [0]
for row in range(1, self.nb_actions):
diag_indeces.append(diag_indeces[-1] + (row + 1))
diag_mask = np.zeros(1 + nb_elems) # +1 for the leading zero
diag_mask[np.array(diag_indeces) + 1] = 1
diag_mask = K.variable(diag_mask)
# Add leading zero element to each element in the L_flat. We use this zero
# element when gathering L_flat into a lower triangular matrix L.
nb_rows = tf.shape(L_flat)[0]
zeros = tf.expand_dims(tf.tile(K.zeros((1,)), [nb_rows]), 1)
try:
# Old TF behavior.
L_flat = tf.concat(1, [zeros, L_flat])
except TypeError:
# New TF behavior
L_flat = tf.concat([zeros, L_flat], 1)
# Create mask that can be used to gather elements from L_flat and put them
# into a lower triangular matrix.
tril_mask = np.zeros((self.nb_actions, self.nb_actions), dtype='int32')
tril_mask[np.tril_indices(self.nb_actions)] = range(1, nb_elems + 1)
# Finally, process each element of the batch.
init = [
K.zeros((self.nb_actions, self.nb_actions)),
K.zeros((self.nb_actions, self.nb_actions)),
]
def fn(a, x):
# Exponentiate everything. This is much easier than only exponentiating
# the diagonal elements, and, usually, the action space is relatively low.
x_ = K.exp(x) + K.epsilon()
# Only keep the diagonal elements.
x_ *= diag_mask
# Add the original, non-diagonal elements.
x_ += x * (1. - diag_mask)
# Finally, gather everything into a lower triangular matrix.
L_ = tf.gather(x_, tril_mask)
return [L_, tf.transpose(L_)]
tmp = tf.scan(fn, L_flat, initializer=init)
if isinstance(tmp, (list, tuple)):
# TensorFlow 0.10 now returns a tuple of tensors.
L, LT = tmp
else:
# Old TensorFlow < 0.10 returns a shared tensor.
L = tmp[:, 0, :, :]
LT = tmp[:, 1, :, :]
else:
raise RuntimeError('Unknown Keras backend "{}".'.format(K.backend()))
assert L is not None
assert LT is not None
P = K.batch_dot(L, LT)
elif self.mode == 'diag':
if K.backend() == 'theano':
import theano.tensor as T
import theano
def fn(x, P_acc):
x_ = K.zeros((self.nb_actions, self.nb_actions))
x_ = T.set_subtensor(x_[np.diag_indices(self.nb_actions)], x)
return x_
outputs_info = [
K.zeros((self.nb_actions, self.nb_actions)),
]
P, _ = theano.scan(fn=fn, sequences=L_flat, outputs_info=outputs_info)
elif K.backend() == 'tensorflow':
import tensorflow as tf
# Create mask that can be used to gather elements from L_flat and put them
# into a diagonal matrix.
diag_mask = np.zeros((self.nb_actions, self.nb_actions), dtype='int32')
diag_mask[np.diag_indices(self.nb_actions)] = range(1, self.nb_actions + 1)
# Add leading zero element to each element in the L_flat. We use this zero
# element when gathering L_flat into a lower triangular matrix L.
nb_rows = tf.shape(L_flat)[0]
zeros = tf.expand_dims(tf.tile(K.zeros((1,)), [nb_rows]), 1)
try:
# Old TF behavior.
L_flat = tf.concat(1, [zeros, L_flat])
except TypeError:
# New TF behavior
L_flat = tf.concat([zeros, L_flat], 1)
# Finally, process each element of the batch.
def fn(a, x):
x_ = tf.gather(x, diag_mask)
return x_
P = tf.scan(fn, L_flat, initializer=K.zeros((self.nb_actions, self.nb_actions)))
else:
raise RuntimeError('Unknown Keras backend "{}".'.format(K.backend()))
assert P is not None
assert K.ndim(P) == 3
# Combine a, mu and P into a scalar (over the batches). What we compute here is
# -.5 * (a - mu)^T * P * (a - mu), where * denotes the dot-product. Unfortunately
# TensorFlow handles vector * P slightly suboptimal, hence we convert the vectors to
# 1xd/dx1 matrices and finally flatten the resulting 1x1 matrix into a scalar. All
# operations happen over the batch size, which is dimension 0.
prod = K.batch_dot(K.expand_dims(a - mu, 1), P)
prod = K.batch_dot(prod, K.expand_dims(a - mu, -1))
A = -.5 * K.batch_flatten(prod)
assert K.ndim(A) == 2
return A
import copy
from keras import backend as K
from keras.losses import binary_crossentropy, categorical_crossentropy, mse
from keras.models import Model, load_model
from keras.layers import Input, Dense, Lambda, Concatenate, Reshape
from keras.layers.core import Dense, Activation, Flatten, RepeatVector
from keras.layers.wrappers import TimeDistributed
from keras.layers.recurrent import GRU
from keras.layers.convolutional import Convolution1D
import tensorflow as tf
import zinc_grammar as G
masks_K = K.variable(G.masks)
ind_of_ind_K = K.variable(G.ind_of_ind)
MAX_LEN = 277
MAX_LEN_FUNCTIONAL = 200
DIM = G.D
class MoleculeVAE():
autoencoder = None
def create(self,
charset,
max_length=MAX_LEN,
max_length_functional=MAX_LEN_FUNCTIONAL,
latent_rep_size=2,
weights_file=None):
charset_length = len(charset)
if K.image_data_format() == 'channels_first':
x = x.reshape((3, img_nrows, img_ncols))
x = x.transpose((1, 2, 0))
else:
x = x.reshape((img_nrows, img_ncols, 3))
# Remove zero-center by mean pixel
x[:, :, 0] += 103.939
x[:, :, 1] += 116.779
x[:, :, 2] += 123.68
# 'BGR'->'RGB'
x = x[:, :, ::-1]
x = np.clip(x, 0, 255).astype('uint8')
return x
# get tensor representations of our images
base_image = K.variable(preprocess_image(base_image_path))
style_reference_image = K.variable(preprocess_image(style_reference_image_path))
# this will contain our generated image
if K.image_data_format() == 'channels_first':
combination_image = K.placeholder((1, 3, img_nrows, img_ncols))
else:
combination_image = K.placeholder((1, img_nrows, img_ncols, 3))
# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_image, style_reference_image, combination_image], axis=0)
# build the VGG19 network with our 3 images as input
# the model will be loaded with pre-trained ImageNet weights
model = vgg19.VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False)
print('Model loaded.')
training_generator.min_randomized_snr_db = min_randomized_snr_db
training_generator.max_randomized_snr_db = max_randomized_snr_db
else:
training_generator.randomize_SNR = True
training_generator.SNRdB = 0
input_train, output_train = training_generator.get_examples(
num_training_examples)
input_val, output_val = training_generator.get_examples(
num_validation_examples)
global H_normalization_factor
H_normalization_factor = np.sqrt(Nr * Nt)
global K_H_normalization_factor
K_H_normalization_factor = K.variable(H_normalization_factor, dtype=np.float32)
if False:
# real / compl as new dimension
input_shape = (Nr, numSamplesPerExample, 2)
output_dim = (Nr, Nt, 2)
else:
# real / compl as twice number of rows
input_shape = (numSamplesPerExample, 2 * (Nr))
output_dim = (2 * Nr, Nt)
numInputs = np.prod(input_shape)
numOutputs = np.prod(output_dim)
print(numInputs, " ", numOutputs)
shape = im.shape[1:]
#AQ UNDA AIRCHIO ROMELI CONV LAYERIDAN GINDA AGEBA
vgg_model = custom_vgg(shape, 13)
symbolic_conv_outputs = [
layer.get_output_at(1) for layer in vgg_model.layers
if layer.name.endswith('conv1')
]
#now are are doing actual model and it also has a good shortcut
multi_model_output = Model(vgg_model.input, symbolic_conv_outputs)
layer_outputs = [K.variable(k) for k in multi_model_output.predict(im)]
loss = 0
for symbolic, numerical in zip(symbolic_conv_outputs, layer_outputs):
loss += style_loss(symbolic[0], numerical[0])
grad = K.gradients(loss, multi_model_output.input)
loss_grad = K.function(inputs=[multi_model_output.input],
outputs=[loss] + grad)
img = minimize(wrapper, 10, batch_shape)
img = process(img)
plt.imshow(img)
symbolic_conv_outputs = [
layer.get_output_at(1) for layer in vgg.layers \
if layer.name.endswith('conv1')
]
# pick the earlier layers for
# a more "localized" representation
# this is opposed to the content model
# where the later layers represent a more "global" structure
# symbolic_conv_outputs = symbolic_conv_outputs[:2]
# make a big model that outputs multiple layers' outputs
multi_output_model = Model(vgg.input, symbolic_conv_outputs)
# calculate the targets that are output at each layer
style_layers_outputs = [K.variable(y) for y in multi_output_model.predict(x)]
# calculate the total style loss
loss = 0
for symbolic, actual in zip(symbolic_conv_outputs, style_layers_outputs):
# gram_matrix() expects a (H, W, C) as input
loss += style_loss(symbolic[0], actual[0])
grads = K.gradients(loss, multi_output_model.input)
# just like theano.function
get_loss_and_grads = K.function(
inputs=[multi_output_model.input],
outputs=[loss] + grads
)
K.set_learning_phase(0)
model = inception_v3.InceptionV3(weights='imagenet', include_top=False)
#勾配率
layer_contributions = {
'mixed2': 0.4,
'mixed3': 3.,
'mixed4': 2.,
'mixed5': 1.5,
}
layer_dict = dict([(layer.name, layer) for layer in model.layers])
loss = K.variable(0.)
for layer_name in layer_contributions:
coeff = layer_contributions[layer_name]
activation = layer_dict[layer_name].output
scaling = K.prod(K.cast(K.shape(activation), 'float32'))
loss += coeff * K.sum(K.square(activation[:, 2:-2, 2:-2, :])) / scaling
#生成された画像（dream）を保持するテンソル
dream = model.input
#ドリームの損失関数の勾配を計算
grads = K.gradients(loss, dream)[0]
def __init__(self, weights, unsuper_weight=0, unsuper_channel=-1):
self.config = {'weights': list(weights), 'unsuper_weight':unsuper_weight, 'unsuper_channel':unsuper_channel}
self.unsuper_weight = K.variable(unsuper_weight)
self.supervised_loss = wcceOA(weights)
self.unsupervised_loss = entrONA(unsuper_channel)
self.__name__ = 'wlossOA'
def _b_init(self, shape, name=None):
"""Initialize bias as in paper"""
values = numpy.random.normal(loc=0.5, scale=1e-2, size=shape)
return K.variable(values, name=name)
def __init__(self):
self.gamma = K.variable(2.)
def __init__(self, unsuper_channel=-1):
self.config = {'unsuper_channel': unsuper_channel}
self.__name__ = 'entrONA'
self.unsuper_channel = K.variable(unsuper_channel, dtype='int')
def unifinv_init(shape, dtype=None):
return K.variable(stats.uniform.ppf(numpy.random.rand(*shape), loc=-.5, scale=(.5 - -.5)),
dtype=dtype)
def call(self, x, mask=None):
if hasattr(x, '_keras_shape'):
input_shape = x._keras_shape
elif hasattr(K, 'int_shape'):
input_shape = K.int_shape(x)
# --------------------------------- #
# 获取输入进来的特征层的宽和高
# 比如38x38
# --------------------------------- #
layer_width = input_shape[self.waxis]
layer_height = input_shape[self.haxis]
# --------------------------------- #
# 获取输入进来的图片的宽和高
# 比如300x300
# --------------------------------- #
img_width = self.img_size[1]
img_height = self.img_size[0]
box_widths = []
box_heights = []
# --------------------------------- #
# self.aspect_ratios一般有两个值
# [1, 1, 2, 1/2]
# [1, 1, 2, 1/2, 3, 1/3]
# --------------------------------- #
for ar in self.aspect_ratios:
# 首先添加一个较小的正方形
if ar == 1 and len(box_widths) == 0:
box_widths.append(self.min_size)
box_heights.append(self.min_size)
# 然后添加一个较大的正方形
elif ar == 1 and len(box_widths) > 0:
box_widths.append(np.sqrt(self.min_size * self.max_size))
box_heights.append(np.sqrt(self.min_size * self.max_size))
# 然后添加长方形
elif ar != 1:
box_widths.append(self.min_size * np.sqrt(ar))
box_heights.append(self.min_size / np.sqrt(ar))
# --------------------------------- #
# 获得所有先验框的宽高1/2
# --------------------------------- #
box_widths = 0.5 * np.array(box_widths)
box_heights = 0.5 * np.array(box_heights)
# --------------------------------- #
# 每一个特征层对应的步长
# --------------------------------- #
step_x = img_width / layer_width
step_y = img_height / layer_height
# --------------------------------- #
# 生成网格中心
# --------------------------------- #
linx = np.linspace(0.5 * step_x, img_width - 0.5 * step_x,
layer_width)
liny = np.linspace(0.5 * step_y, img_height - 0.5 * step_y,
layer_height)
centers_x, centers_y = np.meshgrid(linx, liny)
centers_x = centers_x.reshape(-1, 1)
centers_y = centers_y.reshape(-1, 1)
# 每一个先验框需要两个(centers_x, centers_y)，前一个用来计算左上角，后一个计算右下角
num_priors_ = len(self.aspect_ratios)
prior_boxes = np.concatenate((centers_x, centers_y), axis=1)
prior_boxes = np.tile(prior_boxes, (1, 2 * num_priors_))
# 获得先验框的左上角和右下角
prior_boxes[:, ::4] -= box_widths
prior_boxes[:, 1::4] -= box_heights
prior_boxes[:, 2::4] += box_widths
prior_boxes[:, 3::4] += box_heights
# --------------------------------- #
# 将先验框变成小数的形式
# 归一化
# --------------------------------- #
prior_boxes[:, ::2] /= img_width
prior_boxes[:, 1::2] /= img_height
prior_boxes = prior_boxes.reshape(-1, 4)
prior_boxes = np.minimum(np.maximum(prior_boxes, 0.0), 1.0)
num_boxes = len(prior_boxes)
if len(self.variances) == 1:
variances = np.ones((num_boxes, 4)) * self.variances[0]
elif len(self.variances) == 4:
variances = np.tile(self.variances, (num_boxes, 1))
else:
raise Exception('Must provide one or four variances.')
prior_boxes = np.concatenate((prior_boxes, variances), axis=1)
prior_boxes_tensor = K.expand_dims(K.variable(prior_boxes), 0)
pattern = [tf.shape(x)[0], 1, 1]
prior_boxes_tensor = tf.tile(prior_boxes_tensor, pattern)
return prior_boxes_tensor
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
shape = (input_shape[self.axis], )
init_gamma = self.scale * np.ones(shape)
self.gamma = K.variable(init_gamma, name='{}_gamma'.format(self.name))
self.trainable_weights = [self.gamma]
def test_sparse_metrics():
for metric in all_sparse_metrics:
y_a = K.variable(np.random.randint(0, 7, (6,)), dtype=K.floatx())
y_b = K.variable(np.random.random((6, 7)), dtype=K.floatx())
assert K.eval(metric(y_a, y_b)).shape == (6,)
