def main():
show_images = False
diagnose_plots = False
model_dir = './models'
global backend
# override backend if provided as an input arg
if len(sys.argv) > 1:
if 'tf' in sys.argv[1].lower():
backend = 'tf'
else:
backend = 'th'
print "[Info] Using backend={}".format(backend)
if backend == 'th':
model_weight_filename = os.path.join(model_dir, 'sports1M_weights_th.h5')
model_json_filename = os.path.join(model_dir, 'sports1M_weights_th.json')
else:
model_weight_filename = os.path.join(model_dir, 'sports1M_weights_tf.h5')
model_json_filename = os.path.join(model_dir, 'sports1M_weights_tf.json')
print("[Info] Reading model architecture...")
model = model_from_json(open(model_json_filename, 'r').read())
#model = c3d_model.get_model(backend=backend)
# visualize model
model_img_filename = os.path.join(model_dir, 'c3d_model.png')
if not os.path.exists(model_img_filename):
from keras.utils import plot_model
plot_model(model, to_file=model_img_filename)
print("[Info] Loading model weights...")
model.load_weights(model_weight_filename)
print("[Info] Loading model weights -- DONE!")
model.compile(loss='mean_squared_error', optimizer='sgd')
print("[Info] Loading labels...")
with open('sports1m/labels.txt', 'r') as f:
labels = [line.strip() for line in f.readlines()]
print('Total labels: {}'.format(len(labels)))
print("[Info] Loading a sample video...")
#cap = cv2.VideoCapture('dM06AMFLsrc.mp4')
cap = cv2.VideoCapture('test.mp4')
vid = []
while True:
ret, img = cap.read()
if not ret:
break
vid.append(cv2.resize(img, (171, 128)))
vid = np.array(vid, dtype=np.float32)
#plt.imshow(vid[2000]/256)
#plt.show()
print(vid.shape)
# sample 16-frame clip
#start_frame = 100
#start_frame = 2000
start_frame = 0
X = vid[start_frame:(start_frame + 16), :, :, :]
print("shape is ", X.shape)
#diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images)
# subtract mean
mean_cube = np.load('models/train01_16_128_171_mean.npy')
mean_cube = np.transpose(mean_cube, (1, 2, 3, 0))
print(mean_cube.shape)
#diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images)
X -= mean_cube
#diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images)
# center crop
X = X[:, 8:120, 30:142, :] # (l, h, w, c)
#diagnose(X, verbose=True, label='Center-cropped X', plots=show_images)
if backend == 'th':
X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112)
else:
pass # input_shape = (16,112,112,3)
# get activations for intermediate layers if needed
inspect_layers = [
# 'fc6',
'fc7',
]
####################################################################################
############################My Code#################################################
sys.path.insert(0,'/home/supunK/GIT/c3d-keras/models' )
from give_arr import arr
out = arr()
####################################################################################
####################################################################################
for layer in inspect_layers:
int_model = c3d_model.get_int_model(model=model, layer=layer, backend=backend)
int_output = int_model.predict_on_batch(np.array([X]))
int_output = int_output[0, ...]
print "[Debug] at layer={}: output.shape={}".format(layer, int_output.shape)
diagnose(int_output,
verbose=True,
label='{} activation'.format(layer),
plots=diagnose_plots,
backend=backend)
#print(type(int_output))
#out = int_output.reshape(1,1,4096)
print(out.shape)
pca = PCA(n_components=500)
pca.fit(out)
def main():
show_images = False
diagnose_plots = False
model_dir = './models'
global backend
# override backend if provided as an input arg
if len(sys.argv) > 1:
if 'tf' in sys.argv[1].lower():
backend = 'tf'
else:
backend = 'th'
print "[Info] Using backend={}".format(backend)
if backend == 'th':
model_weight_filename = os.path.join(model_dir,
'sports1M_weights_th.h5')
model_json_filename = os.path.join(model_dir,
'sports1M_weights_th.json')
else:
model_weight_filename = os.path.join(model_dir,
'sports1M_weights_tf.h5')
model_json_filename = os.path.join(model_dir,
'sports1M_weights_tf.json')
print("[Info] Reading model architecture...")
model = model_from_json(open(model_json_filename, 'r').read())
#model = c3d_model.get_model(backend=backend)
# visualize model
model_img_filename = os.path.join(model_dir, 'c3d_model.png')
if not os.path.exists(model_img_filename):
from keras.utils.visualize_util import plot
plot(model, to_file=model_img_filename)
print("[Info] Loading model weights...")
model.load_weights(model_weight_filename)
print("[Info] Loading model weights -- DONE!")
model.compile(loss='mean_squared_error', optimizer='sgd')
print("[Info] Loading labels...")
with open('sports1m/labels.txt', 'r') as f:
labels = [line.strip() for line in f.readlines()]
print('Total labels: {}'.format(len(labels)))
print("[Info] Loading a sample video...")
cap = cv2.VideoCapture('dM06AMFLsrc.mp4')
vid = []
while True:
ret, img = cap.read()
if not ret:
break
vid.append(cv2.resize(img, (171, 128)))
vid = np.array(vid, dtype=np.float32)
#plt.imshow(vid[2000]/256)
#plt.show()
# sample 16-frame clip
#start_frame = 100
start_frame = 2000
X = vid[start_frame:(start_frame + 16), :, :, :]
#diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images)
# subtract mean
mean_cube = np.load('models/train01_16_128_171_mean.npy')
mean_cube = np.transpose(mean_cube, (1, 2, 3, 0))
#diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images)
X -= mean_cube
#diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images)
# center crop
X = X[:, 8:120, 30:142, :] # (l, h, w, c)
#diagnose(X, verbose=True, label='Center-cropped X', plots=show_images)
if backend == 'th':
X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112)
else:
pass # input_shape = (16,112,112,3)
# get activations for intermediate layers if needed
inspect_layers = [
# 'fc6',
# 'fc7',
]
for layer in inspect_layers:
int_model = c3d_model.get_int_model(model=model,
layer=layer,
backend=backend)
int_output = int_model.predict_on_batch(np.array([X]))
int_output = int_output[0, ...]
print "[Debug] at layer={}: output.shape={}".format(
layer, int_output.shape)
diagnose(int_output,
verbose=True,
label='{} activation'.format(layer),
plots=diagnose_plots,
backend=backend)
# inference
output = model.predict_on_batch(np.array([X]))
# show results
print(
'You will see a plot of class probabilities. Please close it, and '
'you\'ll see top 5 classification results.')
plt.plot(output[0])
plt.title('Probability')
plt.show()
print('Position of maximum probability: {}'.format(output[0].argmax()))
print('Maximum probability: {:.5f}'.format(max(output[0])))
print('Corresponding label: {}'.format(labels[output[0].argmax()]))
# sort top five predictions from softmax output
top_inds = output[0].argsort(
)[::-1][:5] # reverse sort and take five largest items
print('\nTop 5 probabilities and labels:')
for i in top_inds:
print('{1}: {0:.5f}'.format(output[0][i], labels[i]))
X = X[:, 8:120, 30:142, :] # (l, h, w, c)
#diagnose(X, verbose=True, label='Center-cropped X', plots=show_images)
if backend == 'th':
X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112)
else:
pass # input_shape = (16,112,112,3)
# get activations for intermediate layers if needed
inspect_layers = [
# 'fc6',
'fc7',
]
for layer in inspect_layers:
int_model = c3d_model.get_int_model(model=model,
layer=layer,
backend=backend)
int_output = int_model.predict_on_batch(np.array([X]))
int_output = int_output[0, ...]
#print "[Debug] at layer={}: output.shape={}".format(layer, int_output.shape)
diagnose(int_output,
verbose=True,
label='{} activation'.format(layer),
plots=diagnose_plots,
backend=backend)
new_out = int_output.reshape(1, -1)
reduced = np.dot(new_out, PCAs)
#vid_feat = [vid_feat,[reduced]]
#print(vid_feat)
break
vid.append(cv2.resize(img, (171, 128)))
vid = np.array(vid, dtype=np.float32)
#plt.imshow(vid[2000]/256)
#plt.show()
# sample 16-frame clip
#start_frame = 100
#start_frame = 2000
start = 0
end = vid.shape[0]
print('total number of frames are %d'%end)
mean_cube = np.load('models/train01_16_128_171_mean.npy')
mean_cube = np.transpose(mean_cube, (1, 2, 3, 0))
stack = []
inspect_layers = ['fc7']
int_model = c3d_model.get_int_model(model=model, layer='fc7', backend=backend)
for start_frame in range(start,end,8):
print('frame no: %d/%d'%(start_frame,end))
X = vid[start_frame:(start_frame + 16), :, :, :]
if(X.shape[0]<16):
continue
# subtract mean
#print(mean_cube.shape)
#diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images)
#X -= mean_cube
#diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images)
# center crop
X = X[:, 8:120, 30:142, :] # (l, h, w, c)
#diagnose(X, verbose=True, label='Center-cropped X', plots=show_images)
def main():
show_images = False
diagnose_plots = False
model_dir = './models'
global backend
# override backend if provided as an input arg
if len(sys.argv) > 1:
if 'tf' in sys.argv[1].lower():
backend = 'tf'
else:
backend = 'th'
print("[Info] Using backend={}".format(backend))
if backend == 'th':
model_weight_filename = os.path.join(model_dir,
'sports1M_weights_th.h5')
model_json_filename = os.path.join(model_dir,
'sports1M_weights_th.json')
else:
model_weight_filename = os.path.join(model_dir, h5model)
model_json_filename = os.path.join(model_dir, jsonmodel)
print("[Info] Reading model architecture...")
#model = model_from_json(open(model_json_filename, 'r').read())
#model = c3d_model.get_model(backend=backend)
model = one_network.deep_model()
# visualize model
model_img_filename = os.path.join(model_dir, 'one_network_model.png')
if not os.path.exists(model_img_filename):
from keras.utils import plot_model
plot_model(model, to_file=model_img_filename)
print("[Info] Loading model weights...")
model.load_weights(model_weight_filename)
print("[Info] Loading model weights -- DONE!")
model.compile(loss='mean_squared_error', optimizer='sgd')
print("[Info] Loading labels...")
with open(
'/media/kmhosny/01CFE6D64EF8ED00/datasets/ucfTrainTestlist/classInd.txt',
'r') as f:
labels = [line.strip() for line in f.readlines()]
print('Total labels: {}'.format(len(labels)))
print("[Info] Loading a sample video...")
cap = cv2.VideoCapture('test_video.mp4')
vid = []
while True:
ret, img = cap.read()
if not ret:
break
vid.append(cv2.resize(img, (112, 112)))
vid = np.array(vid, dtype=np.float32)
#plt.imshow(vid[2000]/256)
#plt.show()
# sample 16-frame clip
#start_frame = 100
start_frame = 5000
print(vid.shape)
X = vid[start_frame:(start_frame + 16), :, :, :]
image_index = random.randint(0, len(vid) - 1)
im = vid[image_index]
print(X.shape)
#diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images)
# subtract mean
#mean_cube = np.load('models/train01_16_128_171_mean.npy')
#mean_cube = np.transpose(mean_cube, (1, 2, 3, 0))
#diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images)
#X -= mean_cube
#diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images)
# center crop
#X = X[:, 8:120, 30:142, :] # (l, h, w, c)
print('after crop ' + str(X.shape))
#diagnose(X, verbose=True, label='Center-cropped X', plots=show_images)
if backend == 'th':
X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112)
else:
pass # input_shape = (16,112,112,3)
# get activations for intermediate layers if needed
inspect_layers = [
# 'fc6',
# 'fc7',
]
for layer in inspect_layers:
int_model = c3d_model.get_int_model(
model=model, layer=layer, backend=backend)
int_output = int_model.predict_on_batch(np.array([X]))
int_output = int_output[0, ...]
print("[Debug] at layer={}: output.shape={}".format(
layer, int_output.shape))
diagnose(
int_output,
verbose=True,
label='{} activation'.format(layer),
plots=diagnose_plots,
backend=backend)
# inference
print(model.summary())
# intermediate_layer_model = Model(
# inputs=model.input, outputs=model.get_layer(name='pool5').output)
# intermediate_output = intermediate_layer_model.predict(np.array([X]))
print("before prediction", np.array([X]).shape)
output = model.predict_on_batch([np.array([X]), np.array([im])])
# print(intermediate_output.shape)
# show results
print('Saving class probabilitities in probabilities.png')
plt.plot(output[0])
plt.title('Probability')
plt.savefig("probabilities.png")
print('Position of maximum probability: {}'.format(output[0].argmax()))
print('Maximum probability: {:.5f}'.format(max(output[0])))
print('Corresponding label: {}'.format(labels[output[0].argmax()]))
# sort top five predictions from softmax output
top_inds = output[0].argsort(
)[::-1][:5] # reverse sort and take five largest items
print('\nTop 5 probabilities and labels:')
for i in top_inds:
print('{1}: {0:.5f}'.format(output[0][i], labels[i]))
def main():
show_images = False
diagnose_plots = False
model_dir = './models'
global backend
# override backend if provided as an input arg
if len(sys.argv) > 1:
if 'tf' in sys.argv[1].lower():
backend = 'tf'
else:
backend = 'th'
print "[Info] Using backend={}".format(backend)
if backend == 'th':
model_weight_filename = os.path.join(model_dir,
'sports1M_weights_th.h5')
model_json_filename = os.path.join(model_dir,
'sports1M_weights_th.json')
else:
model_weight_filename = os.path.join(model_dir,
'sports1M_weights_tf.h5')
model_json_filename = os.path.join(model_dir,
'sports1M_weights_tf.json')
print("[Info] Reading model architecture...")
model = model_from_json(open(model_json_filename, 'r').read())
#model = c3d_model.get_model(backend=backend)
# visualize model
model_img_filename = os.path.join(model_dir, 'c3d_model.png')
if not os.path.exists(model_img_filename):
from keras.utils import plot_model
plot_model(model, to_file=model_img_filename)
print("[Info] Loading model weights...")
model.load_weights(model_weight_filename)
print("[Info] Loading model weights -- DONE!")
model.compile(loss='mean_squared_error', optimizer='sgd')
print("[Info] Loading labels...")
with open('sports1m/labels.txt', 'r') as f:
labels = [line.strip() for line in f.readlines()]
print('Total labels: {}'.format(len(labels)))
print("[Info] Loading a sample video...")
fold_names = [f for f in os.listdir(data_dir)]
n = 0
for video_id, video_dir in enumerate(fold_names):
n += 1
# print os.path.join(feature_dir,video_dir[2:])
# print os.path.isfile(os.path.join(feature_dir,video_dir[2:]))
feature_name = video_dir[2:] + '.npy'
if os.path.isfile(os.path.join(feature_dir, feature_name)):
print n, '/', len(fold_names), ' ', video_dir, ' existed'
else:
start = time.time()
video_path = os.path.join(data_dir, video_dir)
sort_img = key_frame(video_path, mFrame, top_k) # key frame
# sort_img = np.sort(os.listdir(video_path))
# print len(sort_img)
# if len(sort_img)<min_len:
# min_len = len(sort_img)
# min_id = video_idfea
# min_dir = video_dir
# print min_len, min_id, min_dir
# shutil.rmtree(os.path.join(data_dir,min_dir))
video_img = []
for imgs in sort_img:
img = cv2.imread(os.path.join(video_path, imgs))
# print img.shape
video_img.append(cv2.resize(img, (171, 128)))
video_img = np.array(video_img, dtype=np.float32)
# print video_img
video_img -= np.array([89.62182617, 97.80722809, 101.37097168])
# print video_img
# print 'video_id','video_img', video_img.shape
X = np.zeros((1, 16, 112, 112, 3))
# for i in range((len(sort_img)/8)-1):
# img16 = np.expand_dims(video_img[i*8:(i+2)*8,8:120,30:142,:], axis=0)
# print 'img16', img16.shape
# X = np.concatenate((X, img16),axis=0)
# print 'X', X.shape
img16 = np.expand_dims(video_img[0:16, 8:120, 30:142, :], axis=0)
X = img16
print 'X', X.shape
int_model1 = c3d_model.get_int_model(model=model,
layer='fc6',
backend=backend)
# int_model2 = c3d_model.get_int_model(model=model, layer='fc7', backend=backend)
# int_model3 = c3d_model.get_int_model(model=model, layer='pool5', backend=backend)
# int_model4 = c3d_model.get_int_model(model=model, layer='conv5b', backend=backend)
# int_model5 = c3d_model.get_int_model(model=model, layer='conv5a', backend=backend)
# int_model6 = c3d_model.get_int_model(model=model, layer='pool4', backend=backend)
# int_model7 = c3d_model.get_int_model(model=model, layer='conv4b', backend=backend)
# int_model8 = c3d_model.get_int_model(model=model, layer='conv4a', backend=backend)
# int_model9 = c3d_model.get_int_model(model=model, layer='conv3b', backend=backend)
# int_model10 = c3d_model.get_int_model(model=model, layer='conv3a', backend=backend)
int_output1 = int_model1.predict_on_batch(X)
# int_output2 = int_model2.predict_on_batch(X)
# int_output3 = int_model3.predict_on_batch(X)
# int_output4 = int_model4.predict_on_batch(X)
# int_output5 = int_model5.predict_on_batch(X)
# int_output6 = int_model6.predict_on_batch(X)
# int_output7 = int_model7.predict_on_batch(X)
# int_output8 = int_model8.predict_on_batch(X)
# int_output9 = int_model9.predict_on_batch(X)
# int_output10 = int_model10.predict_on_batch(X)
# int_output = int_output[0, ...]
end = time.time()
video_time = end - start
print n, '/', len(
fold_names
), ' ', video_dir, ': ', int_output1.shape, ' | ', ' time: ', video_time, 's'
np.save(os.path.join(feature_dir, video_dir[2:]), int_output1)
