def extract_features(directory):
# load the model
model = Xception()
# re-structure the model
model.layers.pop()
model = Model(inputs=model.inputs, outputs=model.layers[-1].output)
# summarize
print(model.summary())
# extract features from each photo
features = dict()
for name in listdir(directory):
# load an image from file
filename = directory + '/' + name
image = load_img(filename, target_size=(299, 299))
# convert the image pixels to a numpy array
image = img_to_array(image)
# reshape data for the model
image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
# prepare the image for the VGG model
image = preprocess_input(image)
# get features
feature = model.predict(image, verbose=0)
# get image id
image_id = name.split('.')[0]
# store feature
features[image_id] = feature
print('>%s' % name)
return features
def test_pipeline(self):
""" Pipeline should provide correct function composition """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
xcpt_model = Xception(weights="imagenet")
stages = [('spimage', gfac.buildSpImageConverter(SparkMode.RGB_FLOAT32)),
('xception', GraphFunction.fromKeras(xcpt_model))]
piped_model = GraphFunction.fromList(stages)
for fpath in img_fpaths:
target_size = tuple(xcpt_model.input.shape.as_list()[1:-1])
img = load_img(fpath, target_size=target_size)
img_arr = np.expand_dims(img_to_array(img), axis=0)
img_input = xcpt.preprocess_input(img_arr)
preds_ref = xcpt_model.predict(img_input)
spimg_input_dict = imageArrayToStruct(img_input).asDict()
spimg_input_dict['data'] = bytes(spimg_input_dict['data'])
with IsolatedSession() as issn:
# Need blank import scope name so that spimg fields match the input names
feeds, fetches = issn.importGraphFunction(piped_model, prefix="")
feed_dict = dict((tnsr, spimg_input_dict[tfx.op_name(issn.graph, tnsr)]) for tnsr in feeds)
preds_tgt = issn.run(fetches[0], feed_dict=feed_dict)
# Uncomment the line below to see the graph
# tfx.write_visualization_html(issn.graph,
# NamedTemporaryFile(prefix="gdef", suffix=".html").name)
self.assertTrue(np.all(preds_tgt == preds_ref))
def test_pipeline(self):
""" Pipeline should provide correct function composition """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
xcpt_model = Xception(weights="imagenet")
stages = [('spimage',
gfac.buildSpImageConverter(SparkMode.RGB_FLOAT32)),
('xception', GraphFunction.fromKeras(xcpt_model))]
piped_model = GraphFunction.fromList(stages)
for fpath in img_fpaths:
target_size = tuple(xcpt_model.input.shape.as_list()[1:-1])
img = load_img(fpath, target_size=target_size)
img_arr = np.expand_dims(img_to_array(img), axis=0)
img_input = xcpt.preprocess_input(img_arr)
preds_ref = xcpt_model.predict(img_input)
spimg_input_dict = imageArrayToStruct(img_input).asDict()
spimg_input_dict['data'] = bytes(spimg_input_dict['data'])
with IsolatedSession() as issn:
# Need blank import scope name so that spimg fields match the input names
feeds, fetches = issn.importGraphFunction(piped_model,
prefix="")
feed_dict = dict(
(tnsr, spimg_input_dict[tfx.op_name(tnsr, issn.graph)])
for tnsr in feeds)
preds_tgt = issn.run(fetches[0], feed_dict=feed_dict)
# Uncomment the line below to see the graph
# tfx.write_visualization_html(issn.graph,
# NamedTemporaryFile(prefix="gdef", suffix=".html").name)
self.assertTrue(np.all(preds_tgt == preds_ref))
class XceptNet:
def __init__(self):
self.input_shape = (299, 299, 3)
self.weight = 'imagenet'
self.pooling = 'max'
self.model = Xception(weights=self.weight,
input_shape=(self.input_shape[0],
self.input_shape[1],
self.input_shape[2]),
pooling=self.pooling,
include_top=False)
self.model.predict(np.zeros((1, 299, 299, 3)))
def extract_feat(self, img_path):
img = image.load_img(img_path,
target_size=(self.input_shape[0],
self.input_shape[1]))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input(img)
feat = self.model.predict(img)
norm_feat = feat[0] / LA.norm(feat[0])
norm_feat = [i.item() for i in norm_feat]
return norm_feat
def xception_featurize(file):
# load model
model = Xception(include_top=True, weights='imagenet')
img_path = file
img = load_img(img_path, target_size=(299, 299))
x = img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
features = model.predict(x)
# print(features.shape)
features=np.ndarray.flatten(features)
# feature shape = (25088,)
labels=list()
for i in range(len(features)):
labels.append('xception_feature_%s'%(str(i+1)))
return features, labels
def extract_features(filename):
# load the model
model = Xception()
# re-structure the model
model.layers.pop()
model = Model(inputs=model.inputs, outputs=model.layers[-1].output)
# load the photo
image = load_img(filename, target_size=(224, 224))
# convert the image pixels to a numpy array
image = img_to_array(image)
# reshape data for the model
image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
# prepare the image for the VGG model
image = preprocess_input(image)
# get features
feature = model.predict(image, verbose=0)
return feature
def extract_features_with_data_augmentation(datagen, directory, sample_count,
num_of_classes, batch_size):
conv_base = Xception(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
print("Conv Base Dims:", conv_base.output.shape)
features = np.zeros(
shape=(sample_count, conv_base.output.shape[1],
conv_base.output.shape[2],
conv_base.output.shape[3])) # output shape of conv base
labels = np.zeros(shape=(sample_count,
num_of_classes)) # number of classes
generator = datagen.flow_from_directory(directory,
target_size=(224, 224),
batch_size=batch_size,
class_mode='categorical')
i = 0
for inputs_batch, labels_batch in generator:
features_batch = conv_base.predict(inputs_batch, verbose=1)
print("Feature batch size:", len(features_batch))
if len(features_batch) == batch_size:
features[i * batch_size:(i + 1) * batch_size] = features_batch
labels[i * batch_size:(i + 1) * batch_size] = labels_batch
else:
print("Processing last batch:", len(features_batch))
tmp_batch_size = len(features_batch)
features[i * batch_size:(i * batch_size) +
tmp_batch_size] = features_batch
labels[i * batch_size:(i * batch_size) +
tmp_batch_size] = labels_batch
i += 1
if i * batch_size >= sample_count:
break
return features, labels
from keras.applications.inception_v3 import preprocess_input
from keras.preprocessing.image import img_to_array
from keras.preprocessing.image import load_img
import numpy as np
import argparse
import cv2
##################################################################
## initialize the input image shape (224x224 pixels) along with the pre-processing function
# inputShape = (224, 224)
# preprocess = imagenet_utils.preprocess_input
## inception, xception 用下面的
inputShape = (299, 299)
preprocess = preprocess_input
##################################################################
## loading model
model = Xception(weights="imagenet")
##################################################################
## loading image; 尺寸要和上面的一样
image = load_img("/Users/coder352/github/jImage/Dream_Afar/Acanalonia conica planthopper.jpg", target_size=inputShape)
image = img_to_array(image) # a NumPy array of shape (inputShape[0], inputShape[1], 3)
image = np.expand_dims(image, axis=0) # we need to expand the dimension by making the shape (1, inputShape[0], inputShape[1], 3)
image = preprocess(image) # pre-process the image using the appropriate function based on the model that has been loaded (i.e., mean subtraction, scaling, etc.)
preds = model.predict(image)
P = imagenet_utils.decode_predictions(preds)
# loop over the predictions and display the rank-5 predictions +
# probabilities to our terminal
for (i, (imagenetID, label, prob)) in enumerate(P[0]):
print("{}. {}: {:.2f}%".format(i + 1, label, prob * 100))
from args import parser
args = parser.parse_args()
path_save_Xception = args.Xception_features_path
path_audio_img = args.audio_images_folder_path
path_save_rgb = os.path.join(path_audio_img, 'RGB_sound.npy')
RGB_sound = np.load(path_save_rgb)
print('shape RGB sound', RGB_sound.shape)
model = Xception(weights='imagenet', include_top=False)
preds = model.predict(RGB_sound)
print('pred_reduc taille', preds.shape)
modelbis = Sequential()
modelbis.add(ZeroPadding2D((1, 1), input_shape=(10, 10, 2048)))
modelbis.add(MaxPooling2D((2, 2), strides=(2, 2)))
pred_reduc_pool = modelbis.predict(preds)
print('pool pred_reduc taille', pred_reduc_pool.shape)
pred_reduc_pool = pred_reduc_pool.reshape((-1, 2048 * 6 * 6))
print('concatenate in size', pred_reduc_pool.shape)
from keras.layers import Conv2D
from keras.layers import SeparableConv2D
from keras.layers import MaxPooling2D
from keras.layers import GlobalAveragePooling2D
from keras.layers import GlobalMaxPooling2D
from keras.engine.topology import get_source_inputs
from keras.utils.data_utils import get_file
from keras import backend as K
from keras.applications.imagenet_utils import decode_predictions
from keras.applications.imagenet_utils import _obtain_input_shape
from keras.applications import Xception
from keras.applications.xception import preprocess_input
TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels.h5'
TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'
if __name__ == '__main__':
model = Xception(include_top=True, weights='imagenet')
img_path = 'data\\dogscats\\train\\cats\\cat.10013.jpg'
img = image.load_img(img_path, target_size=(299, 299))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
print('Input image shape:', x.shape)
preds = model.predict(x)
print(np.argmax(preds))
print('Predicted:', decode_predictions(preds, 1)) # ('n02123394', 'Persian_cat', 0.91428012)
def inception_get_features(run_evaluation_model=False):
def run_model():
print("here")
with tf.device('/gpu:0'):
model = Sequential()
model.add(Dense(1024, input_shape=(2048, ), activation='relu'))
model.add(Dense(512, input_shape=(1024, ), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, input_shape=(64, )))
model.add(Dense(num_classes, activation='softmax'))
adamax = Adamax()
model.compile(loss='categorical_crossentropy',
optimizer=adamax,
metrics=['accuracy'])
model.fit(x_train_feature_map,
y_train,
validation_data=(x_train_feature_map, y_train),
nb_epoch=70,
batch_size=64)
score = model.evaluate(x_test_feature_map,
y_public_test,
batch_size=64)
print("Result")
print(score)
raw_data = pd.read_csv(raw_data_csv_file_name)
emotion = raw_data[['emotion']]
pixels = raw_data[['pixels']]
# Get data in matrix form
(x_train_matrix, x_public_test_matrix, x_private_test_matrix, y_train,
y_public_test,
y_private_test) = get_data_in_matrix_format(emotion, pixels)
# Only used train and public test for now
x_train_matrix = x_train_matrix.astype('float32')
x_public_test_matrix = x_public_test_matrix.astype('float32')
# Put values between 1 and 0
x_train_matrix = x_train_matrix / 255.0
x_public_test_matrix = x_public_test_matrix / 255.0
# 7 Classes
num_classes = y_train.shape[1]
if os.path.exists("./pre_saved_features/inceptiontrainfeatures.npy"):
x_train_feature_map = np.load(
"./pre_saved_features/inceptiontrainfeatures.npy")
x_test_feature_map = np.load(
"./pre_saved_features/inceptiontestfeatures.npy")
else:
xception_pre_trained = Xception(include_top=False,
input_shape=(96, 96, 3),
pooling='avg',
weights='imagenet')
x_train_feature_map = np.empty([N_TEST, 2048])
f_l = np.empty([int(N_TEST), 48 * 2, 48 * 2, 3])
for index, item in enumerate(f_l): # Refill the list
im = Image.fromarray(x_train_matrix[index])
resized_image = im.resize((48 * 2, 48 * 2), Image.NEAREST)
arr = np.reshape(
np.fromiter(iter(resized_image.getdata()), np.uint8), (96, 96))
for d in range(3):
item[:, :, d] = arr
picture_train_features = xception_pre_trained.predict(f_l)
del f_l
# BUILD NEW TRAIN FEATURE INPUT
for idx_pic, picture in enumerate(picture_train_features):
x_train_feature_map[idx_pic] = picture
print("converting test features")
f_t = np.empty([3588, 96, 96, 3])
for index, item in enumerate(f_t): # Refill the list
im = Image.fromarray(x_public_test_matrix[index])
resized_image = im.resize((48 * 2, 48 * 2), Image.NEAREST)
arr = np.reshape(
np.fromiter(iter(resized_image.getdata()), np.uint8), (96, 96))
for d in range(3):
item[:, :, d] = arr
picture_test_features = xception_pre_trained.predict(f_t)
del f_t
# BUILD NEW TEST
x_test_feature_map = np.empty([3588, 2048])
for idx_pic, picture in enumerate(picture_test_features):
x_test_feature_map[idx_pic] = picture
np.save("./pre_saved_features/inceptiontestfeatures",
x_test_feature_map)
np.save("./pre_saved_features/inceptiontrainfeatures",
x_train_feature_map)
if run_evaluation_model:
run_model()
return x_train_feature_map, x_test_feature_map
def extract_features(input_dir,
output_dir,
model_type='inceptionv3',
batch_size=32):
"""
Extracts features from a CNN trained on ImageNet classification from all
videos in a directory.
Args:
input_dir (str): Input directory of videos to extract from.
output_dir (str): Directory where features should be stored.
model_type (str): Model type to use.
batch_size (int): Batch size to use when processing.
"""
input_dir = os.path.expanduser(input_dir)
output_dir = os.path.expanduser(output_dir)
if not os.path.isdir(input_dir):
sys.stderr.write("Input directory '%s' does not exist!\n" % input_dir)
sys.exit(1)
# Load desired ImageNet model
# Note: import Keras only when needed so we don't waste time revving up
# Theano/TensorFlow needlessly in case of an error
model = None
input_shape = (224, 224)
if model_type.lower() == 'inceptionv3':
from keras.applications import InceptionV3
model = InceptionV3(include_top=True, weights='imagenet')
elif model_type.lower() == 'xception':
from keras.applications import Xception
model = Xception(include_top=True, weights='imagenet')
elif model_type.lower() == 'resnet50':
from keras.applications import ResNet50
model = ResNet50(include_top=True, weights='imagenet')
elif model_type.lower() == 'vgg16':
from keras.applications import VGG16
model = VGG16(include_top=True, weights='imagenet')
elif model_type.lower() == 'vgg19':
from keras.applications import VGG19
model = VGG19(include_top=True, weights='imagenet')
else:
sys.stderr.write("'%s' is not a valid ImageNet model.\n" % model_type)
sys.exit(1)
if model_type.lower() == 'inceptionv3' or model_type.lower() == 'xception':
shape = (299, 299)
# Get outputs of model from layer just before softmax predictions
from keras.models import Model
model = Model(model.inputs, output=model.layers[-2].output)
# Create output directories
visual_dir = os.path.join(output_dir, 'visual') # RGB features
#motion_dir = os.path.join(output_dir, 'motion') # Spatiotemporal features
#opflow_dir = os.path.join(output_dir, 'opflow') # Optical flow features
for directory in [visual_dir]: #, motion_dir, opflow_dir]:
if not os.path.exists(directory):
os.makedirs(directory)
# Find all videos that need to have features extracted
def is_video(x):
return x.endswith('.mp4') or x.endswith('.avi') or x.endswith('.mov')
vis_existing = [x.split('.')[0] for x in os.listdir(visual_dir)]
#mot_existing = [os.path.splitext(x)[0] for x in os.listdir(motion_dir)]
#flo_existing = [os.path.splitext(x)[0] for x in os.listdir(opflow_dir)]
video_filenames = [
x for x in sorted(os.listdir(input_dir))
if is_video(x) and os.path.splitext(x)[0] not in vis_existing
]
# Go through each video and extract features
from keras.applications.imagenet_utils import preprocess_input
for video_filename in tqdm(video_filenames):
# Open video clip for reading
try:
clip = VideoFileClip(os.path.join(input_dir, video_filename))
except Exception as e:
sys.stderr.write("Unable to read '%s'. Skipping...\n" %
video_filename)
sys.stderr.write("Exception: {}\n".format(e))
continue
# Sample frames at 1fps
fps = int(np.round(clip.fps))
frames = [
scipy.misc.imresize(crop_center(x.astype(np.float32)), shape)
for idx, x in enumerate(clip.iter_frames())
if idx % fps == fps // 2
]
n_frames = len(frames)
frames_arr = np.empty((n_frames, ) + shape + (3, ), dtype=np.float32)
for idx, frame in enumerate(frames):
frames_arr[idx, :, :, :] = frame
frames_arr = preprocess_input(frames_arr)
features = model.predict(frames_arr, batch_size=batch_size)
name, _ = os.path.splitext(video_filename)
feat_filepath = os.path.join(visual_dir, name + '.npy')
with open(feat_filepath, 'wb') as f:
np.save(f, features)
def get_features_CNN(image_list, outpath, method='VGG16', da=False):
"""
Extracts image features using CNN
Arguments:
- image_list: list, image set
- model: str, VGG16, VGG16DAB, Xception or XceptionDAB
Returns:
- features: list, contains tuples with image path + histogram of features
"""
features = []
if method in ['VGG16', 'VGG16DAB', 'VGG16H']:
print('Loading network...')
model = VGG16(weights='imagenet', include_top=False, pooling='avg')
model.summary()
for im in tqdm(image_list):
image = Image.open(im)
if method == 'VGG16DAB':
image = imagetoDAB(image)
if method == 'VGG16H':
image = imagetoDAB(image, h=True)
image = numpy.asarray(image)
image = numpy.expand_dims(image, axis=0)
image = imagenet_utils.preprocess_input(image)
curr_feat = model.predict(image)
curr_feat = curr_feat.flatten()
features.append((im, curr_feat))
if method in ['Xception', 'XceptionDAB', 'XceptionH']:
print('Loading network...')
if da:
outdir = os.path.join(outpath, model)
if not os.path.exists(outdir):
if method == 'Xception':
domain_adaption(image_list, outdir, 224, pdl1=True)
if method == 'XceptionDAB':
domain_adaption(image_list,
outdir,
224,
pdl1=True,
dab=True)
if method == 'XceptionH':
domain_adaption(image_list, outdir, 224, pdl1=True, h=True)
weights_dir = os.path.join(outdir, 'weights')
model = load_model(outdir, 5)
model.summary()
else:
model = Xception(weights='imagenet',
include_top=False,
pooling='avg',
input_shape=(224, 224, 3))
for im in tqdm(image_list):
image = Image.open(im)
if method == 'XceptionDAB':
image = imagetoDAB(image)
if method == 'XceptionH':
image = imagetoDAB(image, h=True)
image = numpy.asarray(image)
if image.shape == (224, 224, 3):
image = numpy.expand_dims(image, axis=0)
image = preprocess_input(image)
curr_feat = model.predict(image)
curr_feat = curr_feat.flatten()
features.append((im, curr_feat))
return features
from keras.layers import SeparableConv2D
from keras.layers import MaxPooling2D
from keras.layers import GlobalAveragePooling2D
from keras.layers import GlobalMaxPooling2D
from keras.engine.topology import get_source_inputs
from keras.utils.data_utils import get_file
from keras import backend as K
from keras.applications.imagenet_utils import decode_predictions
from keras.applications.imagenet_utils import _obtain_input_shape
from keras.applications import Xception
from keras.applications.xception import preprocess_input
TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels.h5'
TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'
if __name__ == '__main__':
model = Xception(include_top=True, weights='imagenet')
img_path = 'data\\dogscats\\train\\cats\\cat.10013.jpg'
img = image.load_img(img_path, target_size=(299, 299))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
print('Input image shape:', x.shape)
preds = model.predict(x)
print(np.argmax(preds))
print('Predicted:',
decode_predictions(preds,
1)) # ('n02123394', 'Persian_cat', 0.91428012)
#convert the image to array
img_array = img_to_array(img)
#convert the image into a 4 dimensional Tensor
#convert from (height, width, channels), (batchsize, height, width, channels)
img_array = np.expand_dims(img_array, axis=0)
#preprocess the input image array
img_array = preprocess_input(img_array)
#Load the model from internet / computer
#approximately 91 MB
pretrained_model = Xception(weights="imagenet")
#predict using predict() method
prediction = pretrained_model.predict(img_array)
#decode the prediction
actual_prediction = imagenet_utils.decode_predictions(prediction)
print("predicted object is:")
print(actual_prediction[0][0][1])
print("with accuracy")
print(actual_prediction[0][0][2]*100)
#display image and the prediction text over it
disp_img = cv2.imread(img_path)
#display prediction text over the image
cv2.putText(disp_img, actual_prediction[0][0][1], (20,20), cv2.FONT_HERSHEY_TRIPLEX , 0.8, (0,0,0))
#show the image
cv2.imshow("Prediction",disp_img)
def extract_features(input_dir, output_dir, model_type='inceptionv3', batch_size=32):
"""
Extracts features from a CNN trained on ImageNet classification from all
videos in a directory.
Args:
input_dir (str): Input directory of videos to extract from.
output_dir (str): Directory where features should be stored.
model_type (str): Model type to use.
batch_size (int): Batch size to use when processing.
"""
input_dir = os.path.expanduser(input_dir)
output_dir = os.path.expanduser(output_dir)
if not os.path.isdir(input_dir):
sys.stderr.write("Input directory '%s' does not exist!\n" % input_dir)
sys.exit(1)
# Load desired ImageNet model
# Note: import Keras only when needed so we don't waste time revving up
# Theano/TensorFlow needlessly in case of an error
model = None
input_shape = (224, 224)
if model_type.lower() == 'inceptionv3':
from keras.applications import InceptionV3
model = InceptionV3(include_top=True, weights='imagenet')
elif model_type.lower() == 'xception':
from keras.applications import Xception
model = Xception(include_top=True, weights='imagenet')
elif model_type.lower() == 'resnet50':
from keras.applications import ResNet50
model = ResNet50(include_top=True, weights='imagenet')
elif model_type.lower() == 'vgg16':
from keras.applications import VGG16
model = VGG16(include_top=True, weights='imagenet')
elif model_type.lower() == 'vgg19':
from keras.applications import VGG19
model = VGG19(include_top=True, weights='imagenet')
else:
sys.stderr.write("'%s' is not a valid ImageNet model.\n" % model_type)
sys.exit(1)
if model_type.lower() == 'inceptionv3' or model_type.lower() == 'xception':
shape = (299, 299)
# Get outputs of model from layer just before softmax predictions
from keras.models import Model
model = Model(model.inputs, output=model.layers[-2].output)
# Create output directories
visual_dir = os.path.join(output_dir, 'visual') # RGB features
#motion_dir = os.path.join(output_dir, 'motion') # Spatiotemporal features
#opflow_dir = os.path.join(output_dir, 'opflow') # Optical flow features
for directory in [visual_dir]:#, motion_dir, opflow_dir]:
if not os.path.exists(directory):
os.makedirs(directory)
# Find all videos that need to have features extracted
def is_video(x):
return x.endswith('.mp4') or x.endswith('.avi') or x.endswith('.mov')
vis_existing = [x.split('.')[0] for x in os.listdir(visual_dir)]
#mot_existing = [os.path.splitext(x)[0] for x in os.listdir(motion_dir)]
#flo_existing = [os.path.splitext(x)[0] for x in os.listdir(opflow_dir)]
video_filenames = [x for x in sorted(os.listdir(input_dir))
if is_video(x) and os.path.splitext(x)[0] not in vis_existing]
# Go through each video and extract features
from keras.applications.imagenet_utils import preprocess_input
for video_filename in tqdm(video_filenames):
# Open video clip for reading
try:
clip = VideoFileClip( os.path.join(input_dir, video_filename) )
except Exception as e:
sys.stderr.write("Unable to read '%s'. Skipping...\n" % video_filename)
sys.stderr.write("Exception: {}\n".format(e))
continue
# Sample frames at 1fps
fps = int( np.round(clip.fps) )
frames = [scipy.misc.imresize(crop_center(x.astype(np.float32)), shape)
for idx, x in enumerate(clip.iter_frames()) if idx % fps == fps//2]
n_frames = len(frames)
frames_arr = np.empty((n_frames,)+shape+(3,), dtype=np.float32)
for idx, frame in enumerate(frames):
frames_arr[idx,:,:,:] = frame
frames_arr = preprocess_input(frames_arr)
features = model.predict(frames_arr, batch_size=batch_size)
name, _ = os.path.splitext(video_filename)
feat_filepath = os.path.join(visual_dir, name+'.npy')
with open(feat_filepath, 'wb') as f:
np.save(f, features)
def get_features_CNN(image_list, method='VGG16'):
"""
Extracts image features using CNN
Arguments:
- image_list: list, image set
- method: str, VGG16 or Xception
Returns:
- features: list, contains tuples with image path + histogram of features
"""
features = []
list_features_batch = []
if method in ['VGG16', 'VGG16DAB']:
print('Loading network...')
model = VGG16(weights='imagenet', include_top=False, pooling='avg')
model.summary()
x = 0
n = 0
for im in tqdm(image_list):
image = imread(im)
if method == 'VGG16DAB':
image = imagetoDAB(image)
image = numpy.asarray(image)
image = numpy.expand_dims(image, axis=0)
image = imagenet_utils.preprocess_input(image)
curr_feat = model.predict(image)
curr_feat = curr_feat.flatten()
features.append((im, curr_feat))
n += 1
# When list has 100000 images + features, it is saved with pickle and a new list starts
if n == 100000:
features_batch = os.path.join(
outpath, 'features_{}_batch{}.p'.format(method, x))
with open(features_batch, "wb") as f:
pickle.dump(features, f)
list_features_batch.append(features_batch)
features = []
n = 0
x += 1
features_batch = os.path.join(
outpath, 'features_{}_batch{}.p'.format(method, x))
with open(features_batch, "wb") as f:
pickle.dump(features, f)
list_features_batch.append(features_batch)
if method in ['Xception', 'XceptionDAB']:
print('Loading network...')
model = Xception(weights='imagenet', include_top=False, pooling='avg')
model.summary()
x = 0
n = 0
for im in tqdm(image_list):
image = imread(im)
if method == 'XceptionDAB':
image = imagetoDAB(image)
image = numpy.asarray(image)
image = numpy.expand_dims(image, axis=0)
image = preprocess_input(image)
curr_feat = model.predict(image)
curr_feat = curr_feat.flatten()
features.append((im, curr_feat))
n += 1
# When list has 100000 images + features, it is saved with pickle and a new list starts
if n == 100000:
features_batch = os.path.join(
outpath, 'features_{}_batch{}.p'.format(method, x))
with open(features_batch, "wb") as f:
pickle.dump(features, f)
list_features_batch.append(features_batch)
features = []
n = 0
x += 1
features_batch = os.path.join(
outpath, 'features_{}_batch{}.p'.format(method, x))
with open(features_batch, "wb") as f:
pickle.dump(features, f)
list_features_batch.append(features_batch)
return list_features_batch
verbose=1,
callbacks=[checkpoint, lr_scheduler, csv_logger])
###################################### end version basic and balance ###################################################
####################################### run on test set ################################################################
## only run for testing by adding parameter 'test' when running script
elif modus == 'test':
y_test_matrix = to_categorical(y_test, len(labeltonumber))
if worker == 'single':
print(model.metrics_names)
#model.load_weights(save_modeldirectory + '/Xception_genus_pad_version1.1/Xception.109.0.964.hdf5')
model.load_weights(save_modeldirectory + '/{}'.format(weightfile))
accuracy = model.evaluate(x=X_test, y=y_test_matrix)
## get predicted labels for test set
y_prob = model.predict(X_test)
y_pred = y_prob.argmax(axis=-1)
elif worker == 'parallel':
print(parallel_model.metrics_names)
parallel_model.load_weights(save_modeldirectory +
'/{}'.format(weightfile))
accuracy = parallel_model.evaluate(x=X_test, y=y_test_matrix)
## get predicted labels for test set
y_prob = parallel_model.predict(X_test)
y_pred = y_prob.argmax(axis=-1)
print('loss: {}, accuracy: {}'.format(accuracy[0], accuracy[1]))
## get precision, recall, f1-score and support for each class predicted on test set
classreport = classification_report(y_test, y_pred, output_dict=True)
## print which label belongs to which species/genus
for idx, label in enumerate(labeltonumber):
def extract_split_features(input_vid,
output_dir,
begin_fid,
end_fid,
model_type='inceptionv3',
batch_size=32):
name, _ = os.path.splitext(input_vid)
name = name.split('/')[-1]
output_dir = os.path.join(output_dir, name) # RGB features
# motion_dir = os.path.join(output_dir, 'motion') # Spatiotemporal features
# opflow_dir = os.path.join(output_dir, 'opflow') # Optical flow features
for directory in [output_dir]: # , motion_dir, opflow_dir]:
if not os.path.exists(directory):
os.makedirs(directory)
if model_type.lower() == 'inceptionv3':
from keras.applications import InceptionV3
model = InceptionV3(include_top=True, weights='imagenet')
elif model_type.lower() == 'xception':
from keras.applications import Xception
model = Xception(include_top=True, weights='imagenet')
elif model_type.lower() == 'resnet50':
from keras.applications import ResNet50
model = ResNet50(include_top=True, weights='imagenet')
elif model_type.lower() == 'vgg16':
from keras.applications import VGG16
model = VGG16(include_top=True, weights='imagenet')
elif model_type.lower() == 'vgg19':
from keras.applications import VGG19
model = VGG19(include_top=True, weights='imagenet')
else:
sys.stderr.write("'%s' is not a valid ImageNet model.\n" % model_type)
sys.exit(1)
if model_type.lower() == 'inceptionv3' or model_type.lower() == 'xception':
shape = (299, 299)
# Get outputs of model from layer just before softmax predictions
from keras.models import Model
model = Model(model.inputs, output=model.layers[-2].output)
clip = VideoFileClip(input_vid)
frames = [
scipy.misc.imresize(crop_center(x.astype(np.float32)), shape)
for idx, x in enumerate(clip.iter_frames())
]
from keras.applications.imagenet_utils import preprocess_input
n_frames = end_fid - begin_fid + 1
frames_arr = np.empty((n_frames, ) + shape + (3, ), dtype=np.float32)
fid = 0
for idx, frame in enumerate(frames):
if begin_fid <= idx <= end_fid:
frames_arr[fid, :, :, :] = frame
fid += 1
# print(frames_arr)
frames_arr = preprocess_input(frames_arr)
features = model.predict(frames_arr, batch_size=batch_size)
feat_filepath = os.path.join(
str(output_dir) + '/' + str(name) + '_' + str(begin_fid) + '_' +
str(end_fid) + '.npy')
print("Save to: " + feat_filepath)
with open(feat_filepath, 'wb') as f:
np.save(f, features)
return features
