def conv_output(model, layer_name, img):
"""Get the output of conv layer.
Args:
model: keras model.
layer_name: name of layer in the model.
img: processed input image.
Returns:
intermediate_output: feature map.
"""
# this is the placeholder for the input images
input_img = model.input
try:
# this is the placeholder for the conv output
out_conv = model.get_layer(layer_name).output
except:
raise Exception('Not layer named {}!'.format(layer_name))
# get the intermediate layer model
intermediate_layer_model = Model(inputs=input_img, outputs=out_conv)
# get the output of intermediate layer model
intermediate_output = intermediate_layer_model.predict(img)
return intermediate_output[0]
def output_heatmap(model, last_conv_layer, img):
"""Get the heatmap for image.
Args:
model: keras model.
last_conv_layer: name of last conv layer in the model.
img: processed input image.
Returns:
heatmap: heatmap.
"""
# predict the image class
preds = model.predict(img)
# find the class index
index = np.argmax(preds[0])
# This is the entry in the prediction vector
target_output = model.output[:, index]
# get the last conv layer
last_conv_layer = model.get_layer(last_conv_layer)
# compute the gradient of the output feature map with this target class
grads = K.gradients(target_output, last_conv_layer.output)[0]
# mean the gradient over a specific feature map channel
pooled_grads = K.mean(grads, axis=(0, 1, 2))
# this function returns the output of last_conv_layer and grads
# given the input picture
iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([img])
# We multiply each channel in the feature map array
# by "how important this channel is" with regard to the target class
for i in range(conv_layer_output_value.shape[-1]):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
# The channel-wise mean of the resulting feature map
# is our heatmap of class activation
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
return heatmap
def grad_cam(model, x, category_index, layer_name):
"""
Args:
model: model
x: image input
category_index: category index 类
layer_name: last convolution layer name
"""
# get category loss
class_output = model.output[:, category_index]
# layer output
convolution_output = model.get_layer(layer_name).output
# get gradients
grads = K.gradients(class_output, convolution_output)[0]
# get convolution output and gradients for input
gradient_function = K.function([model.input], [convolution_output, grads])
output, grads_val = gradient_function([x])
output, grads_val = output[0], grads_val[0]
# avg
weights = np.mean(grads_val, axis=(0, 1))
cam = np.dot(output, weights)
# create heat map
cam = cv2.resize(cam, (x.shape[1], x.shape[2]), cv2.INTER_LINEAR)
cam = np.maximum(cam, 0)
heatmap = cam / np.max(cam)
# Return to BGR [0..255] from the preprocessed image
image_rgb = x[0, :]
image_rgb -= np.min(image_rgb)
image_rgb = np.minimum(image_rgb, 255)
cam = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
cam = np.float32(cam) + np.float32(image_rgb)
cam = 255 * cam / np.max(cam)
return np.uint8(cam), heatmap
def build_singer_model(model, train_list, feat_mean, feat_std, mel_path, num_singers, forBuildingModel):
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
if args.pretrained :
layer_output = model.get_layer('model_1').get_output_at(0)
get_last_layer_outputs = K.function([model.get_layer('model_1').get_input_at(0), K.learning_phase()], [layer_output])
else :
get_last_layer_outputs = K.function([model.layers[0].input, K.learning_phase()], [layer_dict['global_average_pooling1d_1'].output])
artist_to_track_model = dict.fromkeys(range(0,500), None)
for i in range(len(train_list)):
artist_id, feat_path, start_frame = train_list[i]
feat = np.load(os.path.join(mel_path, feat_path))
if artist_to_track_model[artist_id] == None:
artist_to_track_model[artist_id] = {}
tmp_feat = feat[:, start_frame:start_frame + config.input_frame_len]
tmp_feat = tmp_feat.T
tmp_feat -= feat_mean
tmp_feat /= feat_std
tmp_feat = np.expand_dims(tmp_feat, 0)
pred = get_last_layer_outputs([tmp_feat, 0])[0]
pred = pred[0]
try:
artist_to_track_model[artist_id][feat_path].append(pred)
except:
artist_to_track_model[artist_id][feat_path] = []
artist_to_track_model[artist_id][feat_path].append(pred)
if forBuildingModel:
embs = []
artist_track_answer = []
for k,track_dict in artist_to_track_model.items():
artist_all_feat= []
count_tracks = 0
for tid,v in track_dict.items():
artist_all_feat.extend(v)
for _ in range(len(v)):
artist_track_answer.append(k)
count_tracks +=1
artist_all_feat = np.array(artist_all_feat)
mean = np.mean(artist_all_feat, axis=0)
embs.append(mean)
track_answer = []
embs = np.array(embs)
return embs, artist_track_answer
else :
embs = []
track_answer = []
for k,track_dict in artist_to_track_model.items():
for tid,v in track_dict.items():
v = np.array(v)
mean = np.mean(v, axis=0)
embs.append(mean)
track_answer.append(k)
embs = np.array(embs)
track_answer = np.array(track_answer)
return embs, track_answer
data_chunk_size = 1024
test_batches, x_test, y_test = next(
BatchGenerator(test_images_path,
test_labels_path).generate(data_chunk_size))
for model_name in os.listdir(trained_models_path):
model = model.get_model('LSTM')
model.load_weights(os.path.join(trained_models_path, model_name))
print(model_name)
print(test_batches[0])
print(test_batches[1])
get_encoder_output = K.function([model.input],
[model.get_layer('flatten_layer').output])
get_decoder_output = K.function([model.get_layer('flatten_layer').output],
[model.output])
encoder_output = get_encoder_output([x_test])[0]
joined_scores = np.array([(encoder_output[0] + encoder_output[1]) / 2],
dtype=encoder_output.dtype)
decoder_output = get_decoder_output([joined_scores])[0]
for predicted_classes, y_true in zip(decoder_output, y_test):
words_pred = [
vocabulary.vocabulary[np.argmax(c)].encode('utf-8')
for c in predicted_classes
]
print(words_pred)