def test_function(self):
val = np.random.random((4, 2))
input_val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
yth = KTH.placeholder(ndim=2)
ytf = KTF.placeholder(ndim=2)
exp_th = KTH.square(xth) + yth
exp_tf = KTF.square(xtf) + ytf
update_th = xth * 2
update_tf = xtf * 2
fth = KTH.function([yth], [exp_th], updates=[(xth, update_th)])
ftf = KTF.function([ytf], [exp_tf], updates=[(xtf, update_tf)])
function_outputs_th = fth([input_val])[0]
function_outputs_tf = ftf([input_val])[0]
assert function_outputs_th.shape == function_outputs_tf.shape
assert_allclose(function_outputs_th, function_outputs_tf, atol=1e-05)
new_val_th = KTH.get_value(xth)
new_val_tf = KTF.get_value(xtf)
assert new_val_th.shape == new_val_tf.shape
assert_allclose(new_val_th, new_val_tf, atol=1e-05)
def test_function(self):
val = np.random.random((4, 2))
input_val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
yth = KTH.placeholder(ndim=2)
ytf = KTF.placeholder(ndim=2)
exp_th = KTH.square(xth) + yth
exp_tf = KTF.square(xtf) + ytf
update_th = xth * 2
update_tf = xtf * 2
fth = KTH.function([yth], [exp_th], updates=[(xth, update_th)])
ftf = KTF.function([ytf], [exp_tf], updates=[(xtf, update_tf)])
function_outputs_th = fth([input_val])[0]
function_outputs_tf = ftf([input_val])[0]
assert function_outputs_th.shape == function_outputs_tf.shape
assert_allclose(function_outputs_th, function_outputs_tf, atol=1e-05)
new_val_th = KTH.get_value(xth)
new_val_tf = KTF.get_value(xtf)
assert new_val_th.shape == new_val_tf.shape
assert_allclose(new_val_th, new_val_tf, atol=1e-05)
def _get_nth_layer_output(self, model, n_layer, X, train=1):
'''
Returns output of nth layer in a given model.
:param model: keras model to get an intermediate value out of
:param n_layer: the layer number to get the value of
:param X: input data for which layer value should be computed and returned.
:param train: (1/0): 1 to use the same setting as training (for example, with Dropout, etc.), 0 to use the same setting as testing phase for the model.
:return the value of n_layer in the given model, input, and setting
'''
get_nth_layer_output = K.function(
[model.layers[0].input, K.learning_phase()],
[model.layers[n_layer].output])
return get_nth_layer_output([X, train])[0]
def get_tensorflow_decoder(output_tensor, beam_size=1024):
""" The TensorFlow implementation of the CTC decoder. """
def get_length(tensor):
lengths = tf.reduce_sum(tf.ones_like(tensor), 1)
return tf.cast(lengths, tf.int32)
sequence_length = get_length(tf.reduce_max(output_tensor, 2))
top_k_decoded, _ = K.ctc_decode(output_tensor,
sequence_length,
greedy=False,
beam_width=beam_size)
decoder = K.function([output_tensor], [top_k_decoded[0]])
return decoder
def grad_cam(model,inputdata):
inputdata = np.reshape(inputdata, [1, 300, 1])
#try using book
model_output = model.output[:,0]
last_conv_layer= model.get_layer('conv1d_1')
grads = K.gradients(model_output,last_conv_layer.output)[0]
pooled_grads = K.mean(grads,axis=(0,1))
iterate = K.function([model.input],
[pooled_grads,last_conv_layer.output])
pooled_grads_value,conv_layer_output_value = iterate([inputdata])
for i in range(len(pooled_grads_value)):
conv_layer_output_value[:,:,i] *= pooled_grads_value[i]
grad_cam = np.average(conv_layer_output_value, 0)
cam_data = []
for i in range(len(grad_cam)):
cam_data.append(np.average(grad_cam[i,:]))
cam_data = np.reshape(cam_data,[1,len(cam_data)])
from scipy.signal import savgol_filter
#test_cam2 = savgol_filter(cam_data, 3,0)
test_cam2 = np.resize(cam_data,[1,300])
"""
fig = plt.figure(figsize=(20, 10))
ax0 = plt.subplot2grid((1, 1), (0, 0), colspan=1)
plt.yticks(fontsize=15)
ax0.plot(inputdata.flatten(),c='blue')
ax0_2 = ax0.twinx()
ax0_2.imshow(test_cam2,cmap='gist_heat',aspect='auto',alpha=0.4)
#plt.show()
"""
return test_cam2
def get_activations(model,
data,
layer,
batch_size=256,
flatten=True,
cropsize=0,
verbose=0):
# get_layer_output = K.function([model.layers[0].input, K.learning_phase()],
# [model.layers[layername].output if type(layername) is type(int) else model.get_layer(layername).output])
#
if type(layer) is str:
layerindex = None
layername = layer
else:
layerindex = layer
layername = None
# print (layername, layerindex)
get_layer_output = K.function(
[model.layers[0].input, K.learning_phase()],
[model.get_layer(name=layername, index=layerindex).output])
activations = []
batch_size = int(batch_size)
for i in tqdm(range(int(np.ceil(len(data) / batch_size))),
desc='Feature extraction') if verbose == 1 else range(
int(np.ceil(len(data) / batch_size))):
batch = np.array(data[i * batch_size:(i + 1) * batch_size],
dtype=np.float32)
if cropsize != 0:
diff = (batch.shape[1] - cropsize) // 2
batch = batch[:, diff:-diff, :]
act = get_layer_output([batch, 0])[0]
activations.extend(act)
activations = np.array(activations, dtype=np.float32)
if flatten: activations = activations.reshape([len(activations), -1])
return activations
def predict_validation(self):
print("Predicting validation data...")
tmp = np.load('/net/duna/scratch1/aasensio/deepLearning/opticalFlow/database/normalization.npz')
min_i, max_i, min_v, max_v = tmp['arr_0'], tmp['arr_1'], tmp['arr_2'], tmp['arr_3']
f = io.readsav(self.observations)
out = f[self.name_of_variable]
self.median_i = np.median(out[:,100:-100,100:-100])
input_validation = np.zeros((1,self.nx,self.ny,2), dtype='float32')
input_validation[0,:,:,0] = out[0:1,100:100+self.nx,100:100+self.ny] / self.median_i
input_validation[0,:,:,1] = out[1:2,100:100+self.nx,100:100+self.ny] / self.median_i
# ff = io.readsav(self.observations)
# im = ff['cont']
# x = np.arange(self.nx)
# y = np.arange(self.ny)
start = time.time()
out = self.model.predict_generator(self.validation_generator(), self.n_frames, max_q_size=1)
end = time.time()
print("Prediction took {0} seconds...".format(end-start))
fun = ktf.function([self.model.layers[0].input],[self.model.layers[1].output])
output = np.squeeze(fun([input_validation])[0][0,200:300,200:300,:]).reshape((100,100,8,8))
f, ax = pl.subplots(nrows=2, ncols=2, figsize=(12,12))
ax[0,0].imshow(output[:,:,0,0] / np.median(output[:,:,0,0]))
ax[0,1].imshow(output[:,:,4,0] / np.median(output[:,:,4,0]))
ax[1,0].imshow(output[:,:,3,4] / np.median(output[:,:,3,4]))
ax[1,1].imshow(output[:,:,2,2] / np.median(output[:,:,2,2]))
pl.show()
#
stop()
def grad_cam(self, input_model, image, category_index, layer_name):
nb_classes = 5 # 18
target_layer = lambda x: self.target_category_loss(x, category_index, nb_classes)
data_ = []
for i in input_model.layers:
if i.name == 'add_marge':
print('add_marge',i)
data_.append(i)
# レイヤー指定
# モデルの推論を実行すると、予測クラス以外の値は0になる
x = input_model.layers[9].output
print('input',input_model.layers[9].name)
x = Lambda(target_layer, output_shape=self.target_category_loss_output_shape)(x)
model = keras.models.Model(input_model.layers[0].input, x)
data = []
for i in model.layers:
if i.name == 'prediction_branch':
print('prediction',i)
data.append(i)
conv_output = model.layers[3].output #3
print('layer4', conv_output)
# print(conv_output = [l for l in input_model.layers if l.name == layer_name][0].output)
# 予測クラス以外の値は0になっている ・予測の損失
# sumをとり予測クラスの値のみを抽出
loss = KTF.sum(model.layers[9].output)
print('nameeeeee final', model.layers[9].name)
# 予測クラスの値から最後のconv層までの勾配を算出する関数を定義
#
grads = self.normalize(KTF.gradients(loss, conv_output)[0])
gradient_function = KTF.function([model.layers[0].input], [conv_output, grads])
# 定義した勾配計算用の関数で算出
output, grads_val = gradient_function([image])
output, grads_val = output[0, :], grads_val[0, :, :, :]
# 最後のconv層のチャンネル毎に勾配の平均を算出し
# かくチャンネルの重要度とする
# GAP
weights = np.mean(grads_val, axis=(0, 1))
cam = np.ones(output.shape[0: 2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * (output[:, :, i]) # 255*(output[:,:,i])
cam = cv2.resize(cam, (300, 300))
# 負の値を0に変換。処理はReluと同意
cam = np.maximum(cam, 0)
# 値を0-1に正規化
heatmap = cam / np.max(cam)
# Return to BGR [0..255] from the preprocessed image
image = image[0, :]
image -= np.min(image)
image = np.minimum(image, 255)
cam1 = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
# ヒートマップと入力画像の重ね合わせ
cam = np.float32(cam1) + np.float32(image)
cam = 255 * cam / np.max(cam)
# ヒートマップのみ
cam1 = cv2.resize(cam1, (300, 300))
cv2.imwrite('/Users/matsunagamasaaki/Desktop/gimage.png', image)
cv2.imwrite('/Users/matsunagamasaaki/Desktop/gcam.png', cam1)
return np.uint8(cam), heatmap, cam1
# Improvement 4
# Geometric weighted scaling of style loss
loss += (style_weights[j] / (2**(nb_layers - (i + 1)))) * sl
loss += total_variation_weight * total_variation_loss(combination_image)
# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, combination_image)
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
f_outputs = K.function([combination_image], outputs)
def eval_loss_and_grads(x):
if K.image_dim_ordering() == 'th':
x = x.reshape((1, 3, img_width, img_height))
else:
x = x.reshape((1, img_width, img_height, 3))
outs = f_outputs([x])
loss_value = outs[0]
if len(outs[1:]) == 1:
grad_values = outs[1].flatten().astype('float64')
else:
grad_values = np.array(outs[1:]).flatten().astype('float64')
return loss_value, grad_values
def grad_cam(self, input_model, image, category_index, layer_name,
boxcoords):
nb_classes = 5 #6#18
# bounding box boords
xmin = boxcoords[0]
ymin = boxcoords[1]
xmax = boxcoords[2]
ymax = boxcoords[3]
target_layer = lambda x: self.target_category_loss(
x, category_index, nb_classes)
# レイヤー指定
x = input_model.layers[-3].output
x = Lambda(target_layer,
output_shape=self.target_category_loss_output_shape)(x)
model = keras.models.Model(input_model.layers[0].input, x)
conv_output = model.layers[30].output #model.layers[5].output
#print(conv_output = [l for l in input_model.layers if l.name == layer_name][0].output)
loss = KTF.sum(model.layers[-3].output)
grads = self.normalize(KTF.gradients(loss, conv_output)[0])
gradient_function = KTF.function([model.layers[0].input],
[conv_output, grads])
output, grads_val = gradient_function([image])
output, grads_val = output[0, :], grads_val[0, :, :, :]
#多分GAP
weights = np.mean(grads_val, axis=(0, 1))
cam = np.ones(output.shape[0:2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * (255 * output[:, :, i])
cam = cv2.resize(cam, (300, 300))
cam = np.maximum(cam, 0)
heatmap = cam / np.max(cam)
#Return to BGR [0..255] from the preprocessed image
image = image[0, :]
image -= np.min(image)
image = np.minimum(image, 255)
cam1 = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
# boundingbox以外のヒートマップ領域を消す
cam1 = cv2.resize(cam1, (720, 405))
cam1[:, 0:xmin, :] = 0 # left
cam1[0:ymin, :, :] = 0 # top
cam1[ymax:-1, :, :] = 0 # bottom
cam1[:, xmax:-1, :] = 0
print('PPPPPPPPP', np.shape(cam1))
cam1 = cv2.resize(cam1, (300, 300))
heatmap = cv2.resize(heatmap, (720, 405))
heatmap[:, 0:xmin] = 0 # left
heatmap[0:ymin, :] = 0 # top
heatmap[ymax:-1, :] = 0 # bottom
heatmap[:, xmax:-1] = 0
heatmap = cv2.resize(heatmap, (300, 300))
# 視認性を上げるため入力画像を重ねる
cam = np.float32(cam1) + np.float32(image)
cam = 255 * cam / np.max(cam)
return np.uint8(cam), heatmap
for layer_name in style_layers:
layer_features = outputs_dict[layer_name]
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[2, :, :, :]
sl = style_loss(style_reference_features, combination_features)
loss += (style_weight / len(style_layers)) * sl
# In[18]:
loss += total_variation_weight * total_variation_loss(combination_image)
# In[19]:
grads = KTF.gradients(loss, combination_image)[0]
fetch_loss_and_grads = KTF.function([combination_image], [loss, grads])
# In[20]:
class Evaluator(object):
def __init__(self):
self.loss_value = None
self.grads_values = None
def loss(self, x):
assert self.loss_value is None
x = x.reshape((1, img_height, img_width, 3))
outs = fetch_loss_and_grads([x])
loss_value = outs[0]
grad_values = outs[1].flatten().astype('float64')
def _compile_learning(self):
# ミニバッチの状態で入力できるようにするplaceholder
# s = K.placeholder(shape=tuple([None] + [self.history_len] + self.state_shape)) # history?
s = K.placeholder(shape=tuple([None] + self.state_shape))
a = K.placeholder(ndim=1, dtype='int32')
r = K.placeholder(ndim=2, dtype='float32')
# s2 = K.placeholder(shape=tuple([None] + [self.history_len] + self.state_shape))
s2 = K.placeholder(shape=tuple([None] + self.state_shape))
t = K.placeholder(ndim=1, dtype='float32')
updates = []
costs = 0
# costs_arr = np.zeros(len(self.networks))
costs_list = []
qs = []
q2s = []
# 構築したネットワーク分だけ処理
for i in range(len(self.networks)):
local_s = s
local_s2 = s2
# remove_features → 未実装
# 推論値 s: Stをinputとして
qs.append(self.networks[i](local_s))
# 教師値 s: St+1をinputとして
q2s.append(self.target_networks[i](local_s2))
if self.use_hra:
# cost = lossの計算
# cost = self._compute_cost(qs[-1], a, r[:, i], t, q2s[-1])
cost = self._compute_cost(qs[-1], a, r[:, i], t, q2s[-1])
optimizer = RMSprop(lr=self.learning_rate,
rho=.95,
epsilon=1e-7)
# 学習設定
updates += optimizer.get_updates(
params=self.networks[i].trainable_weights, loss=cost)
# self.networks[i].compile(loss=cost, optimizer=optimizer)
# costの合計
costs += cost
# 各costが格納されたリスト
costs_list.append(cost)
# costs_arr[i] = cost
# target_netのweightを更新
target_updates = []
for network, target_network in zip(self.networks,
self.target_networks):
for target_weight, network_weight in zip(
target_network.trainable_weights,
network.trainable_weights):
target_updates.append(K.update(target_weight,
network_weight)) # from, to
# kerasの関数のインスタンスを作成 updates: 更新する命令のリスト．
# self._train_on_batch = K.function(inputs=[s, a, r, s2, t], outputs=[costs], updates=updates)
self._train_on_batch = K.function(inputs=[s, a, r, s2, t],
outputs=costs_list,
updates=updates)
self.predict_network = K.function(inputs=[s], outputs=qs)
self.predict_target_network = K.function(inputs=[s], outputs=qs)
self.update_weights = K.function(inputs=[],
outputs=[],
updates=target_updates)
