def predict(weight_path, video):
global lipnet
global adam
global spell
global decoder
if lipnet is None:
lipnet = LipNet(img_c=3,
img_w=100,
img_h=50,
frames_n=75,
absolute_max_string_len=32,
output_size=28)
adam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
lipnet.model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=adam)
lipnet.model.load_weights(weight_path)
spell = Spell(path=PREDICT_DICTIONARY)
decoder = Decoder(greedy=PREDICT_GREEDY,
beam_width=PREDICT_BEAM_WIDTH,
postprocessors=[labels_to_text, spell.sentence])
X_data = np.array([video.data]).astype(np.float32) / 255
input_length = np.array([len(video.data)])
y_pred = lipnet.predict(X_data)
result = decoder.decode(y_pred, input_length)[0]
show_video_subtitle(video.face, result)
print result
def show_results(_video, _align, video, align):
show_video_subtitle(frames=_video.face, subtitle=_align.sentence)
print "Video: "
print _video.length
print np.array_equiv(_video.mouth, video.mouth),
print np.array_equiv(_video.data, video.data),
print np.array_equiv(_video.face, video.face)
print "Align: "
print labels_to_text(_align.padded_label.astype(np.int))
print _align.padded_label
print _align.label_length
print np.array_equiv(_align.sentence, align.sentence),
print np.array_equiv(_align.label, align.label),
print np.array_equiv(_align.padded_label, align.padded_label)
input_length = np.array([len(video.data)])
layer_name = 'dense1'
layer_idx = [
idx for idx, layer in enumerate(lipnet.model.layers)
if layer.name == layer_name
][0]
y_pred = lipnet.predict(X_data)
result = decoder.decode(y_pred, input_length)[0]
heatmap = visualize_saliency(lipnet.model, layer_idx, range(0, 28),
video.data)
return (heatmap, result)
if __name__ == '__main__':
if len(sys.argv) == 3:
video, result = predict(sys.argv[1], sys.argv[2])
elif len(sys.argv) == 4:
video, result = predict(sys.argv[1], sys.argv[2], sys.argv[3])
elif len(sys.argv) == 5:
video, result = predict(sys.argv[1], sys.argv[2], sys.argv[3],
sys.argv[4])
else:
video, result = None, ""
if video is not None:
show_video_subtitle(video.face, result)
y_pred = lipnet.predict(X_data)
result = decoder.decode(y_pred, input_length)[0]
return (video, result)
if __name__ == '__main__':
video = []
result = []
video, result = process_video(sys.argv[1], sys.argv[2])
list_v_r = zip(video, result)
for item in list_v_r:
if item[0] is not None:
show_video_subtitle(item[0].face, item[1])
stripe = "-" * len(item[1])
print ""
print " --{}- ".format(stripe)
print "[ DECODED ] |> {} |".format(item[1])
print " --{}- ".format(stripe)
'''
for v,r in video,result:
if v is not None:
#show_video_subtitle(v.face, r)
stripe = "-" * len(r)
print ""