def test_function(self): val = np.random.random((4, 2)) input_val = np.random.random((4, 2)) xth = KTH.variable(val) xtf = KTF.variable(val) yth = KTH.placeholder(ndim=2) ytf = KTF.placeholder(ndim=2) exp_th = KTH.square(xth) + yth exp_tf = KTF.square(xtf) + ytf update_th = xth * 2 update_tf = xtf * 2 fth = KTH.function([yth], [exp_th], updates=[(xth, update_th)]) ftf = KTF.function([ytf], [exp_tf], updates=[(xtf, update_tf)]) function_outputs_th = fth([input_val])[0] function_outputs_tf = ftf([input_val])[0] assert function_outputs_th.shape == function_outputs_tf.shape assert_allclose(function_outputs_th, function_outputs_tf, atol=1e-05) new_val_th = KTH.get_value(xth) new_val_tf = KTF.get_value(xtf) assert new_val_th.shape == new_val_tf.shape assert_allclose(new_val_th, new_val_tf, atol=1e-05)
def _get_nth_layer_output(self, model, n_layer, X, train=1): ''' Returns output of nth layer in a given model. :param model: keras model to get an intermediate value out of :param n_layer: the layer number to get the value of :param X: input data for which layer value should be computed and returned. :param train: (1/0): 1 to use the same setting as training (for example, with Dropout, etc.), 0 to use the same setting as testing phase for the model. :return the value of n_layer in the given model, input, and setting ''' get_nth_layer_output = K.function( [model.layers[0].input, K.learning_phase()], [model.layers[n_layer].output]) return get_nth_layer_output([X, train])[0]
def get_tensorflow_decoder(output_tensor, beam_size=1024): """ The TensorFlow implementation of the CTC decoder. """ def get_length(tensor): lengths = tf.reduce_sum(tf.ones_like(tensor), 1) return tf.cast(lengths, tf.int32) sequence_length = get_length(tf.reduce_max(output_tensor, 2)) top_k_decoded, _ = K.ctc_decode(output_tensor, sequence_length, greedy=False, beam_width=beam_size) decoder = K.function([output_tensor], [top_k_decoded[0]]) return decoder
def grad_cam(model,inputdata): inputdata = np.reshape(inputdata, [1, 300, 1]) #try using book model_output = model.output[:,0] last_conv_layer= model.get_layer('conv1d_1') grads = K.gradients(model_output,last_conv_layer.output)[0] pooled_grads = K.mean(grads,axis=(0,1)) iterate = K.function([model.input], [pooled_grads,last_conv_layer.output]) pooled_grads_value,conv_layer_output_value = iterate([inputdata]) for i in range(len(pooled_grads_value)): conv_layer_output_value[:,:,i] *= pooled_grads_value[i] grad_cam = np.average(conv_layer_output_value, 0) cam_data = [] for i in range(len(grad_cam)): cam_data.append(np.average(grad_cam[i,:])) cam_data = np.reshape(cam_data,[1,len(cam_data)]) from scipy.signal import savgol_filter #test_cam2 = savgol_filter(cam_data, 3,0) test_cam2 = np.resize(cam_data,[1,300]) """ fig = plt.figure(figsize=(20, 10)) ax0 = plt.subplot2grid((1, 1), (0, 0), colspan=1) plt.yticks(fontsize=15) ax0.plot(inputdata.flatten(),c='blue') ax0_2 = ax0.twinx() ax0_2.imshow(test_cam2,cmap='gist_heat',aspect='auto',alpha=0.4) #plt.show() """ return test_cam2
def get_activations(model, data, layer, batch_size=256, flatten=True, cropsize=0, verbose=0): # get_layer_output = K.function([model.layers[0].input, K.learning_phase()], # [model.layers[layername].output if type(layername) is type(int) else model.get_layer(layername).output]) # if type(layer) is str: layerindex = None layername = layer else: layerindex = layer layername = None # print (layername, layerindex) get_layer_output = K.function( [model.layers[0].input, K.learning_phase()], [model.get_layer(name=layername, index=layerindex).output]) activations = [] batch_size = int(batch_size) for i in tqdm(range(int(np.ceil(len(data) / batch_size))), desc='Feature extraction') if verbose == 1 else range( int(np.ceil(len(data) / batch_size))): batch = np.array(data[i * batch_size:(i + 1) * batch_size], dtype=np.float32) if cropsize != 0: diff = (batch.shape[1] - cropsize) // 2 batch = batch[:, diff:-diff, :] act = get_layer_output([batch, 0])[0] activations.extend(act) activations = np.array(activations, dtype=np.float32) if flatten: activations = activations.reshape([len(activations), -1]) return activations
def predict_validation(self): print("Predicting validation data...") tmp = np.load('/net/duna/scratch1/aasensio/deepLearning/opticalFlow/database/normalization.npz') min_i, max_i, min_v, max_v = tmp['arr_0'], tmp['arr_1'], tmp['arr_2'], tmp['arr_3'] f = io.readsav(self.observations) out = f[self.name_of_variable] self.median_i = np.median(out[:,100:-100,100:-100]) input_validation = np.zeros((1,self.nx,self.ny,2), dtype='float32') input_validation[0,:,:,0] = out[0:1,100:100+self.nx,100:100+self.ny] / self.median_i input_validation[0,:,:,1] = out[1:2,100:100+self.nx,100:100+self.ny] / self.median_i # ff = io.readsav(self.observations) # im = ff['cont'] # x = np.arange(self.nx) # y = np.arange(self.ny) start = time.time() out = self.model.predict_generator(self.validation_generator(), self.n_frames, max_q_size=1) end = time.time() print("Prediction took {0} seconds...".format(end-start)) fun = ktf.function([self.model.layers[0].input],[self.model.layers[1].output]) output = np.squeeze(fun([input_validation])[0][0,200:300,200:300,:]).reshape((100,100,8,8)) f, ax = pl.subplots(nrows=2, ncols=2, figsize=(12,12)) ax[0,0].imshow(output[:,:,0,0] / np.median(output[:,:,0,0])) ax[0,1].imshow(output[:,:,4,0] / np.median(output[:,:,4,0])) ax[1,0].imshow(output[:,:,3,4] / np.median(output[:,:,3,4])) ax[1,1].imshow(output[:,:,2,2] / np.median(output[:,:,2,2])) pl.show() # stop()
def grad_cam(self, input_model, image, category_index, layer_name): nb_classes = 5 # 18 target_layer = lambda x: self.target_category_loss(x, category_index, nb_classes) data_ = [] for i in input_model.layers: if i.name == 'add_marge': print('add_marge',i) data_.append(i) # レイヤー指定 # モデルの推論を実行すると、予測クラス以外の値は0になる x = input_model.layers[9].output print('input',input_model.layers[9].name) x = Lambda(target_layer, output_shape=self.target_category_loss_output_shape)(x) model = keras.models.Model(input_model.layers[0].input, x) data = [] for i in model.layers: if i.name == 'prediction_branch': print('prediction',i) data.append(i) conv_output = model.layers[3].output #3 print('layer4', conv_output) # print(conv_output = [l for l in input_model.layers if l.name == layer_name][0].output) # 予測クラス以外の値は0になっている ・予測の損失 # sumをとり予測クラスの値のみを抽出 loss = KTF.sum(model.layers[9].output) print('nameeeeee final', model.layers[9].name) # 予測クラスの値から最後のconv層までの勾配を算出する関数を定義 # grads = self.normalize(KTF.gradients(loss, conv_output)[0]) gradient_function = KTF.function([model.layers[0].input], [conv_output, grads]) # 定義した勾配計算用の関数で算出 output, grads_val = gradient_function([image]) output, grads_val = output[0, :], grads_val[0, :, :, :] # 最後のconv層のチャンネル毎に勾配の平均を算出し # かくチャンネルの重要度とする # GAP weights = np.mean(grads_val, axis=(0, 1)) cam = np.ones(output.shape[0: 2], dtype=np.float32) for i, w in enumerate(weights): cam += w * (output[:, :, i]) # 255*(output[:,:,i]) cam = cv2.resize(cam, (300, 300)) # 負の値を0に変換。処理はReluと同意 cam = np.maximum(cam, 0) # 値を0-1に正規化 heatmap = cam / np.max(cam) # Return to BGR [0..255] from the preprocessed image image = image[0, :] image -= np.min(image) image = np.minimum(image, 255) cam1 = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET) # ヒートマップと入力画像の重ね合わせ cam = np.float32(cam1) + np.float32(image) cam = 255 * cam / np.max(cam) # ヒートマップのみ cam1 = cv2.resize(cam1, (300, 300)) cv2.imwrite('/Users/matsunagamasaaki/Desktop/gimage.png', image) cv2.imwrite('/Users/matsunagamasaaki/Desktop/gcam.png', cam1) return np.uint8(cam), heatmap, cam1
# Improvement 4 # Geometric weighted scaling of style loss loss += (style_weights[j] / (2**(nb_layers - (i + 1)))) * sl loss += total_variation_weight * total_variation_loss(combination_image) # get the gradients of the generated image wrt the loss grads = K.gradients(loss, combination_image) outputs = [loss] if type(grads) in {list, tuple}: outputs += grads else: outputs.append(grads) f_outputs = K.function([combination_image], outputs) def eval_loss_and_grads(x): if K.image_dim_ordering() == 'th': x = x.reshape((1, 3, img_width, img_height)) else: x = x.reshape((1, img_width, img_height, 3)) outs = f_outputs([x]) loss_value = outs[0] if len(outs[1:]) == 1: grad_values = outs[1].flatten().astype('float64') else: grad_values = np.array(outs[1:]).flatten().astype('float64') return loss_value, grad_values
def grad_cam(self, input_model, image, category_index, layer_name, boxcoords): nb_classes = 5 #6#18 # bounding box boords xmin = boxcoords[0] ymin = boxcoords[1] xmax = boxcoords[2] ymax = boxcoords[3] target_layer = lambda x: self.target_category_loss( x, category_index, nb_classes) # レイヤー指定 x = input_model.layers[-3].output x = Lambda(target_layer, output_shape=self.target_category_loss_output_shape)(x) model = keras.models.Model(input_model.layers[0].input, x) conv_output = model.layers[30].output #model.layers[5].output #print(conv_output = [l for l in input_model.layers if l.name == layer_name][0].output) loss = KTF.sum(model.layers[-3].output) grads = self.normalize(KTF.gradients(loss, conv_output)[0]) gradient_function = KTF.function([model.layers[0].input], [conv_output, grads]) output, grads_val = gradient_function([image]) output, grads_val = output[0, :], grads_val[0, :, :, :] #多分GAP weights = np.mean(grads_val, axis=(0, 1)) cam = np.ones(output.shape[0:2], dtype=np.float32) for i, w in enumerate(weights): cam += w * (255 * output[:, :, i]) cam = cv2.resize(cam, (300, 300)) cam = np.maximum(cam, 0) heatmap = cam / np.max(cam) #Return to BGR [0..255] from the preprocessed image image = image[0, :] image -= np.min(image) image = np.minimum(image, 255) cam1 = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET) # boundingbox以外のヒートマップ領域を消す cam1 = cv2.resize(cam1, (720, 405)) cam1[:, 0:xmin, :] = 0 # left cam1[0:ymin, :, :] = 0 # top cam1[ymax:-1, :, :] = 0 # bottom cam1[:, xmax:-1, :] = 0 print('PPPPPPPPP', np.shape(cam1)) cam1 = cv2.resize(cam1, (300, 300)) heatmap = cv2.resize(heatmap, (720, 405)) heatmap[:, 0:xmin] = 0 # left heatmap[0:ymin, :] = 0 # top heatmap[ymax:-1, :] = 0 # bottom heatmap[:, xmax:-1] = 0 heatmap = cv2.resize(heatmap, (300, 300)) # 視認性を上げるため入力画像を重ねる cam = np.float32(cam1) + np.float32(image) cam = 255 * cam / np.max(cam) return np.uint8(cam), heatmap
for layer_name in style_layers: layer_features = outputs_dict[layer_name] style_reference_features = layer_features[1, :, :, :] combination_features = layer_features[2, :, :, :] sl = style_loss(style_reference_features, combination_features) loss += (style_weight / len(style_layers)) * sl # In[18]: loss += total_variation_weight * total_variation_loss(combination_image) # In[19]: grads = KTF.gradients(loss, combination_image)[0] fetch_loss_and_grads = KTF.function([combination_image], [loss, grads]) # In[20]: class Evaluator(object): def __init__(self): self.loss_value = None self.grads_values = None def loss(self, x): assert self.loss_value is None x = x.reshape((1, img_height, img_width, 3)) outs = fetch_loss_and_grads([x]) loss_value = outs[0] grad_values = outs[1].flatten().astype('float64')
def _compile_learning(self): # ミニバッチの状態で入力できるようにするplaceholder # s = K.placeholder(shape=tuple([None] + [self.history_len] + self.state_shape)) # history? s = K.placeholder(shape=tuple([None] + self.state_shape)) a = K.placeholder(ndim=1, dtype='int32') r = K.placeholder(ndim=2, dtype='float32') # s2 = K.placeholder(shape=tuple([None] + [self.history_len] + self.state_shape)) s2 = K.placeholder(shape=tuple([None] + self.state_shape)) t = K.placeholder(ndim=1, dtype='float32') updates = [] costs = 0 # costs_arr = np.zeros(len(self.networks)) costs_list = [] qs = [] q2s = [] # 構築したネットワーク分だけ処理 for i in range(len(self.networks)): local_s = s local_s2 = s2 # remove_features → 未実装 # 推論値 s: Stをinputとして qs.append(self.networks[i](local_s)) # 教師値 s: St+1をinputとして q2s.append(self.target_networks[i](local_s2)) if self.use_hra: # cost = lossの計算 # cost = self._compute_cost(qs[-1], a, r[:, i], t, q2s[-1]) cost = self._compute_cost(qs[-1], a, r[:, i], t, q2s[-1]) optimizer = RMSprop(lr=self.learning_rate, rho=.95, epsilon=1e-7) # 学習設定 updates += optimizer.get_updates( params=self.networks[i].trainable_weights, loss=cost) # self.networks[i].compile(loss=cost, optimizer=optimizer) # costの合計 costs += cost # 各costが格納されたリスト costs_list.append(cost) # costs_arr[i] = cost # target_netのweightを更新 target_updates = [] for network, target_network in zip(self.networks, self.target_networks): for target_weight, network_weight in zip( target_network.trainable_weights, network.trainable_weights): target_updates.append(K.update(target_weight, network_weight)) # from, to # kerasの関数のインスタンスを作成 updates: 更新する命令のリスト. # self._train_on_batch = K.function(inputs=[s, a, r, s2, t], outputs=[costs], updates=updates) self._train_on_batch = K.function(inputs=[s, a, r, s2, t], outputs=costs_list, updates=updates) self.predict_network = K.function(inputs=[s], outputs=qs) self.predict_target_network = K.function(inputs=[s], outputs=qs) self.update_weights = K.function(inputs=[], outputs=[], updates=target_updates)