def main(): show_images = False diagnose_plots = False model_dir = './models' global backend # override backend if provided as an input arg if len(sys.argv) > 1: if 'tf' in sys.argv[1].lower(): backend = 'tf' else: backend = 'th' print "[Info] Using backend={}".format(backend) if backend == 'th': model_weight_filename = os.path.join(model_dir, 'sports1M_weights_th.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_th.json') else: model_weight_filename = os.path.join(model_dir, 'sports1M_weights_tf.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_tf.json') print("[Info] Reading model architecture...") model = model_from_json(open(model_json_filename, 'r').read()) #model = c3d_model.get_model(backend=backend) # visualize model model_img_filename = os.path.join(model_dir, 'c3d_model.png') if not os.path.exists(model_img_filename): from keras.utils import plot_model plot_model(model, to_file=model_img_filename) print("[Info] Loading model weights...") model.load_weights(model_weight_filename) print("[Info] Loading model weights -- DONE!") model.compile(loss='mean_squared_error', optimizer='sgd') print("[Info] Loading labels...") with open('sports1m/labels.txt', 'r') as f: labels = [line.strip() for line in f.readlines()] print('Total labels: {}'.format(len(labels))) print("[Info] Loading a sample video...") #cap = cv2.VideoCapture('dM06AMFLsrc.mp4') cap = cv2.VideoCapture('test.mp4') vid = [] while True: ret, img = cap.read() if not ret: break vid.append(cv2.resize(img, (171, 128))) vid = np.array(vid, dtype=np.float32) #plt.imshow(vid[2000]/256) #plt.show() print(vid.shape) # sample 16-frame clip #start_frame = 100 #start_frame = 2000 start_frame = 0 X = vid[start_frame:(start_frame + 16), :, :, :] print("shape is ", X.shape) #diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images) # subtract mean mean_cube = np.load('models/train01_16_128_171_mean.npy') mean_cube = np.transpose(mean_cube, (1, 2, 3, 0)) print(mean_cube.shape) #diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images) X -= mean_cube #diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images) # center crop X = X[:, 8:120, 30:142, :] # (l, h, w, c) #diagnose(X, verbose=True, label='Center-cropped X', plots=show_images) if backend == 'th': X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112) else: pass # input_shape = (16,112,112,3) # get activations for intermediate layers if needed inspect_layers = [ # 'fc6', 'fc7', ] #################################################################################### ############################My Code################################################# sys.path.insert(0,'/home/supunK/GIT/c3d-keras/models' ) from give_arr import arr out = arr() #################################################################################### #################################################################################### for layer in inspect_layers: int_model = c3d_model.get_int_model(model=model, layer=layer, backend=backend) int_output = int_model.predict_on_batch(np.array([X])) int_output = int_output[0, ...] print "[Debug] at layer={}: output.shape={}".format(layer, int_output.shape) diagnose(int_output, verbose=True, label='{} activation'.format(layer), plots=diagnose_plots, backend=backend) #print(type(int_output)) #out = int_output.reshape(1,1,4096) print(out.shape) pca = PCA(n_components=500) pca.fit(out)
def main(): show_images = False diagnose_plots = False model_dir = './models' global backend # override backend if provided as an input arg if len(sys.argv) > 1: if 'tf' in sys.argv[1].lower(): backend = 'tf' else: backend = 'th' print "[Info] Using backend={}".format(backend) if backend == 'th': model_weight_filename = os.path.join(model_dir, 'sports1M_weights_th.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_th.json') else: model_weight_filename = os.path.join(model_dir, 'sports1M_weights_tf.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_tf.json') print("[Info] Reading model architecture...") model = model_from_json(open(model_json_filename, 'r').read()) #model = c3d_model.get_model(backend=backend) # visualize model model_img_filename = os.path.join(model_dir, 'c3d_model.png') if not os.path.exists(model_img_filename): from keras.utils.visualize_util import plot plot(model, to_file=model_img_filename) print("[Info] Loading model weights...") model.load_weights(model_weight_filename) print("[Info] Loading model weights -- DONE!") model.compile(loss='mean_squared_error', optimizer='sgd') print("[Info] Loading labels...") with open('sports1m/labels.txt', 'r') as f: labels = [line.strip() for line in f.readlines()] print('Total labels: {}'.format(len(labels))) print("[Info] Loading a sample video...") cap = cv2.VideoCapture('dM06AMFLsrc.mp4') vid = [] while True: ret, img = cap.read() if not ret: break vid.append(cv2.resize(img, (171, 128))) vid = np.array(vid, dtype=np.float32) #plt.imshow(vid[2000]/256) #plt.show() # sample 16-frame clip #start_frame = 100 start_frame = 2000 X = vid[start_frame:(start_frame + 16), :, :, :] #diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images) # subtract mean mean_cube = np.load('models/train01_16_128_171_mean.npy') mean_cube = np.transpose(mean_cube, (1, 2, 3, 0)) #diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images) X -= mean_cube #diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images) # center crop X = X[:, 8:120, 30:142, :] # (l, h, w, c) #diagnose(X, verbose=True, label='Center-cropped X', plots=show_images) if backend == 'th': X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112) else: pass # input_shape = (16,112,112,3) # get activations for intermediate layers if needed inspect_layers = [ # 'fc6', # 'fc7', ] for layer in inspect_layers: int_model = c3d_model.get_int_model(model=model, layer=layer, backend=backend) int_output = int_model.predict_on_batch(np.array([X])) int_output = int_output[0, ...] print "[Debug] at layer={}: output.shape={}".format( layer, int_output.shape) diagnose(int_output, verbose=True, label='{} activation'.format(layer), plots=diagnose_plots, backend=backend) # inference output = model.predict_on_batch(np.array([X])) # show results print( 'You will see a plot of class probabilities. Please close it, and ' 'you\'ll see top 5 classification results.') plt.plot(output[0]) plt.title('Probability') plt.show() print('Position of maximum probability: {}'.format(output[0].argmax())) print('Maximum probability: {:.5f}'.format(max(output[0]))) print('Corresponding label: {}'.format(labels[output[0].argmax()])) # sort top five predictions from softmax output top_inds = output[0].argsort( )[::-1][:5] # reverse sort and take five largest items print('\nTop 5 probabilities and labels:') for i in top_inds: print('{1}: {0:.5f}'.format(output[0][i], labels[i]))
X = X[:, 8:120, 30:142, :] # (l, h, w, c) #diagnose(X, verbose=True, label='Center-cropped X', plots=show_images) if backend == 'th': X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112) else: pass # input_shape = (16,112,112,3) # get activations for intermediate layers if needed inspect_layers = [ # 'fc6', 'fc7', ] for layer in inspect_layers: int_model = c3d_model.get_int_model(model=model, layer=layer, backend=backend) int_output = int_model.predict_on_batch(np.array([X])) int_output = int_output[0, ...] #print "[Debug] at layer={}: output.shape={}".format(layer, int_output.shape) diagnose(int_output, verbose=True, label='{} activation'.format(layer), plots=diagnose_plots, backend=backend) new_out = int_output.reshape(1, -1) reduced = np.dot(new_out, PCAs) #vid_feat = [vid_feat,[reduced]] #print(vid_feat)
break vid.append(cv2.resize(img, (171, 128))) vid = np.array(vid, dtype=np.float32) #plt.imshow(vid[2000]/256) #plt.show() # sample 16-frame clip #start_frame = 100 #start_frame = 2000 start = 0 end = vid.shape[0] print('total number of frames are %d'%end) mean_cube = np.load('models/train01_16_128_171_mean.npy') mean_cube = np.transpose(mean_cube, (1, 2, 3, 0)) stack = [] inspect_layers = ['fc7'] int_model = c3d_model.get_int_model(model=model, layer='fc7', backend=backend) for start_frame in range(start,end,8): print('frame no: %d/%d'%(start_frame,end)) X = vid[start_frame:(start_frame + 16), :, :, :] if(X.shape[0]<16): continue # subtract mean #print(mean_cube.shape) #diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images) #X -= mean_cube #diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images) # center crop X = X[:, 8:120, 30:142, :] # (l, h, w, c) #diagnose(X, verbose=True, label='Center-cropped X', plots=show_images)
def main(): show_images = False diagnose_plots = False model_dir = './models' global backend # override backend if provided as an input arg if len(sys.argv) > 1: if 'tf' in sys.argv[1].lower(): backend = 'tf' else: backend = 'th' print("[Info] Using backend={}".format(backend)) if backend == 'th': model_weight_filename = os.path.join(model_dir, 'sports1M_weights_th.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_th.json') else: model_weight_filename = os.path.join(model_dir, h5model) model_json_filename = os.path.join(model_dir, jsonmodel) print("[Info] Reading model architecture...") #model = model_from_json(open(model_json_filename, 'r').read()) #model = c3d_model.get_model(backend=backend) model = one_network.deep_model() # visualize model model_img_filename = os.path.join(model_dir, 'one_network_model.png') if not os.path.exists(model_img_filename): from keras.utils import plot_model plot_model(model, to_file=model_img_filename) print("[Info] Loading model weights...") model.load_weights(model_weight_filename) print("[Info] Loading model weights -- DONE!") model.compile(loss='mean_squared_error', optimizer='sgd') print("[Info] Loading labels...") with open( '/media/kmhosny/01CFE6D64EF8ED00/datasets/ucfTrainTestlist/classInd.txt', 'r') as f: labels = [line.strip() for line in f.readlines()] print('Total labels: {}'.format(len(labels))) print("[Info] Loading a sample video...") cap = cv2.VideoCapture('test_video.mp4') vid = [] while True: ret, img = cap.read() if not ret: break vid.append(cv2.resize(img, (112, 112))) vid = np.array(vid, dtype=np.float32) #plt.imshow(vid[2000]/256) #plt.show() # sample 16-frame clip #start_frame = 100 start_frame = 5000 print(vid.shape) X = vid[start_frame:(start_frame + 16), :, :, :] image_index = random.randint(0, len(vid) - 1) im = vid[image_index] print(X.shape) #diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images) # subtract mean #mean_cube = np.load('models/train01_16_128_171_mean.npy') #mean_cube = np.transpose(mean_cube, (1, 2, 3, 0)) #diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images) #X -= mean_cube #diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images) # center crop #X = X[:, 8:120, 30:142, :] # (l, h, w, c) print('after crop ' + str(X.shape)) #diagnose(X, verbose=True, label='Center-cropped X', plots=show_images) if backend == 'th': X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112) else: pass # input_shape = (16,112,112,3) # get activations for intermediate layers if needed inspect_layers = [ # 'fc6', # 'fc7', ] for layer in inspect_layers: int_model = c3d_model.get_int_model( model=model, layer=layer, backend=backend) int_output = int_model.predict_on_batch(np.array([X])) int_output = int_output[0, ...] print("[Debug] at layer={}: output.shape={}".format( layer, int_output.shape)) diagnose( int_output, verbose=True, label='{} activation'.format(layer), plots=diagnose_plots, backend=backend) # inference print(model.summary()) # intermediate_layer_model = Model( # inputs=model.input, outputs=model.get_layer(name='pool5').output) # intermediate_output = intermediate_layer_model.predict(np.array([X])) print("before prediction", np.array([X]).shape) output = model.predict_on_batch([np.array([X]), np.array([im])]) # print(intermediate_output.shape) # show results print('Saving class probabilitities in probabilities.png') plt.plot(output[0]) plt.title('Probability') plt.savefig("probabilities.png") print('Position of maximum probability: {}'.format(output[0].argmax())) print('Maximum probability: {:.5f}'.format(max(output[0]))) print('Corresponding label: {}'.format(labels[output[0].argmax()])) # sort top five predictions from softmax output top_inds = output[0].argsort( )[::-1][:5] # reverse sort and take five largest items print('\nTop 5 probabilities and labels:') for i in top_inds: print('{1}: {0:.5f}'.format(output[0][i], labels[i]))
def main(): show_images = False diagnose_plots = False model_dir = './models' global backend # override backend if provided as an input arg if len(sys.argv) > 1: if 'tf' in sys.argv[1].lower(): backend = 'tf' else: backend = 'th' print "[Info] Using backend={}".format(backend) if backend == 'th': model_weight_filename = os.path.join(model_dir, 'sports1M_weights_th.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_th.json') else: model_weight_filename = os.path.join(model_dir, 'sports1M_weights_tf.h5') model_json_filename = os.path.join(model_dir, 'sports1M_weights_tf.json') print("[Info] Reading model architecture...") model = model_from_json(open(model_json_filename, 'r').read()) #model = c3d_model.get_model(backend=backend) # visualize model model_img_filename = os.path.join(model_dir, 'c3d_model.png') if not os.path.exists(model_img_filename): from keras.utils import plot_model plot_model(model, to_file=model_img_filename) print("[Info] Loading model weights...") model.load_weights(model_weight_filename) print("[Info] Loading model weights -- DONE!") model.compile(loss='mean_squared_error', optimizer='sgd') print("[Info] Loading labels...") with open('sports1m/labels.txt', 'r') as f: labels = [line.strip() for line in f.readlines()] print('Total labels: {}'.format(len(labels))) print("[Info] Loading a sample video...") fold_names = [f for f in os.listdir(data_dir)] n = 0 for video_id, video_dir in enumerate(fold_names): n += 1 # print os.path.join(feature_dir,video_dir[2:]) # print os.path.isfile(os.path.join(feature_dir,video_dir[2:])) feature_name = video_dir[2:] + '.npy' if os.path.isfile(os.path.join(feature_dir, feature_name)): print n, '/', len(fold_names), ' ', video_dir, ' existed' else: start = time.time() video_path = os.path.join(data_dir, video_dir) sort_img = key_frame(video_path, mFrame, top_k) # key frame # sort_img = np.sort(os.listdir(video_path)) # print len(sort_img) # if len(sort_img)<min_len: # min_len = len(sort_img) # min_id = video_idfea # min_dir = video_dir # print min_len, min_id, min_dir # shutil.rmtree(os.path.join(data_dir,min_dir)) video_img = [] for imgs in sort_img: img = cv2.imread(os.path.join(video_path, imgs)) # print img.shape video_img.append(cv2.resize(img, (171, 128))) video_img = np.array(video_img, dtype=np.float32) # print video_img video_img -= np.array([89.62182617, 97.80722809, 101.37097168]) # print video_img # print 'video_id','video_img', video_img.shape X = np.zeros((1, 16, 112, 112, 3)) # for i in range((len(sort_img)/8)-1): # img16 = np.expand_dims(video_img[i*8:(i+2)*8,8:120,30:142,:], axis=0) # print 'img16', img16.shape # X = np.concatenate((X, img16),axis=0) # print 'X', X.shape img16 = np.expand_dims(video_img[0:16, 8:120, 30:142, :], axis=0) X = img16 print 'X', X.shape int_model1 = c3d_model.get_int_model(model=model, layer='fc6', backend=backend) # int_model2 = c3d_model.get_int_model(model=model, layer='fc7', backend=backend) # int_model3 = c3d_model.get_int_model(model=model, layer='pool5', backend=backend) # int_model4 = c3d_model.get_int_model(model=model, layer='conv5b', backend=backend) # int_model5 = c3d_model.get_int_model(model=model, layer='conv5a', backend=backend) # int_model6 = c3d_model.get_int_model(model=model, layer='pool4', backend=backend) # int_model7 = c3d_model.get_int_model(model=model, layer='conv4b', backend=backend) # int_model8 = c3d_model.get_int_model(model=model, layer='conv4a', backend=backend) # int_model9 = c3d_model.get_int_model(model=model, layer='conv3b', backend=backend) # int_model10 = c3d_model.get_int_model(model=model, layer='conv3a', backend=backend) int_output1 = int_model1.predict_on_batch(X) # int_output2 = int_model2.predict_on_batch(X) # int_output3 = int_model3.predict_on_batch(X) # int_output4 = int_model4.predict_on_batch(X) # int_output5 = int_model5.predict_on_batch(X) # int_output6 = int_model6.predict_on_batch(X) # int_output7 = int_model7.predict_on_batch(X) # int_output8 = int_model8.predict_on_batch(X) # int_output9 = int_model9.predict_on_batch(X) # int_output10 = int_model10.predict_on_batch(X) # int_output = int_output[0, ...] end = time.time() video_time = end - start print n, '/', len( fold_names ), ' ', video_dir, ': ', int_output1.shape, ' | ', ' time: ', video_time, 's' np.save(os.path.join(feature_dir, video_dir[2:]), int_output1)