def extract_feat(self, img):
"""
最后一层卷积特征
Parameters:
img:图片
Return:
norm_feat:归一化后的特征数据
"""
target_size = (self.input_shape[0], self.input_shape[1])
img = cv2.resize(img, dsize=target_size)
img = np.expand_dims(img, axis=0)
img = img.astype('float64')
# img = img.reshape((self.input_shape[0], self.input_shape[1],3))
if self.recognition_model_type == "VGG16":
# img = img.astype('float64')
img = preprocess_input_vgg(img)
elif self.recognition_model_type == "ResNet50":
# img = img.astype('float64')
img = preprocess_input_resnet(img)
elif self.recognition_model_type == "DenseNet121":
# img = img.astype('float64')
img = preprocess_input_densenet(img)
else:
raise ValueError("The pretrained model is wrong")
# print(feat.shape)
feat = self.model.predict(img)
norm_feat = feat[0] / LA.norm(feat[0])
return norm_feat
def vgg_extract_feat(self, img_path):
img = image.load_img(img_path, target_size=(self.input_shape[0], self.input_shape[1]))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input_vgg(img)
feat = self.model_vgg.predict(img)
# print(feat.shape)
norm_feat = feat[0]/LA.norm(feat[0])
return norm_feat
def vgg_extract_feat(self, img_path):
img = image.load_img(img_path,
target_size=(self.input_shape[0],
self.input_shape[1])) #加载图片
img = image.img_to_array(img) #转换为数组
img = np.expand_dims(img, axis=0) #转换为np数组
img = preprocess_input_vgg(img) #进行VGG处理
feat = self.model_vgg.predict(img) #提取特征
norm_feat = feat[0] / LA.norm(feat[0]) #求归一化特征向量
return norm_feat
def extract_feature(self, img_path):
img = image.load_img(img_path,
target_size=(self.input_shape[0],
self.input_shape[1]))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input_vgg(img)
feat = self.model_vgg.predict(img)
norm_feat = feat[0] / LA.norm(feat[0])
norm_feat = [i.item() for i in norm_feat]
return norm_feat
def vgg_extract_feat(img_path, model, graph, sess):
with sess.as_default():
with graph.as_default():
img = image.load_img(img_path, target_size=(input_shape[0], input_shape[1]))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input_vgg(img)
feat = model.predict(img)
norm_feat = feat[0] / LA.norm(feat[0])
norm_feat = [i.item() for i in norm_feat]
return norm_feat
def binary_classifier_vgg(img, model_file):
model_vgg_loaded = keras.models.load_model(model_file)
image = img.convert('RGB').resize(input_shape, Image.ANTIALIAS)
image.load()
image_array = np.array(image)
expanded_image_array = np.expand_dims(image_array, axis=0)
preprocessed_expanded_image_array = preprocess_input_vgg(
expanded_image_array)
pred = model_vgg_loaded.predict(preprocessed_expanded_image_array)
output_label = [1 if x > 0.5 else 0 for x in pred]
return output_label[0]
def extract_with_session(img_path, model, sess, graph):
with sess.as_default():
with graph.as_default():
img = image.load_img(img_path, target_size=(224, 224))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input_vgg(img)
feat = model.predict(img)
norm_feat = feat[0] / LA.norm(feat[0])
norm_feat = [i.item() for i in norm_feat]
img_name = os.path.split(img_path)[1]
return norm_feat, img_name
def get_data_batches(path: str=_DATASET_PATH,
image_dim: Tuple=_IMAGE_DIM,
batch_size: int=32,
color: bool=False) -> Generator:
# Load data
data = pd.read_csv(path, usecols=['emotion', 'pixels'])
data_size = len(data.pixels)
# Load label data
labels = np.asarray(data.emotion, dtype=np.int32)
# Create a features array of (batch_size, *image_dim, channels)
for ii in range(0, data_size, batch_size):
# We want all data not just full batches
if data_size - ii < batch_size:
batch_size = data_size - ii
if color:
features = np.empty(
shape=(batch_size, *image_dim, 3),
dtype=np.float32)
else:
features = np.empty(
shape=(batch_size, *image_dim),
dtype=np.float32)
for i, row in enumerate(data.pixels[ii:ii + batch_size]):
image_array = np.fromstring(
str(row),
dtype=np.uint8,
sep=' ').reshape((48, 48))
# Convert to color
if color:
image = cv.cvtColor(image_array, cv.COLOR_GRAY2RGB)
else:
image = image_array
# Resize if needed
if image_dim != (48, 48):
image = cv.resize(image, image_dim)
features[i] = image
# VGG Preprocess
if color:
features = preprocess_input_vgg(features)
else:
features = preprocess_input(features)
yield features, labels[ii:ii + batch_size]
def vgg_extract_feat(self, img_path):
norm_feat = []
with self.graph.as_default():
set_session(self.session)
img = image.load_img(img_path,
target_size=(self.input_shape[0],
self.input_shape[1]))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input_vgg(img)
feat = self.model_vgg.predict(img)
norm_feat = feat[0] / LA.norm(feat[0])
norm_feat = [i.item() for i in norm_feat]
return norm_feat
def predict(self, img):
""" # Arguments
img: a numpy array
# Returns
A dict containing predictions
"""
img = Image.fromarray(img)
img = img.resize((224, 224))
x = keras_image.img_to_array(img)[:, :, :3]
x = np.expand_dims(x, axis=0)
x = preprocess_input_vgg(x)
features = self.model.predict(x)
predictions = decode_predictions_vgg(features)[0]
clean_predictions = [{'score': str(k), 'class': j} for (i, j, k) in predictions]
return json.dumps(clean_predictions)
def prepare_image(self, image):
"""
Setter method defined to prepare image for classification and store it.
Parameters:
-----------
- image: image object (width image, height image, channels image)
Return:
-------
- preprocessed image
"""
# crop the image with detection windows
img_crop = image[self.detection_rows, self.detection_columns]
# resize to expected size for model
img_resized = cv2.resize(img_crop, (224, 224))
img_expanded = np.expand_dims(img_resized, axis=0)
preprocessed_image = preprocess_input_vgg(img_expanded)
return preprocessed_image
def get_data(path: str=_DATASET_PATH,
image_dim: Tuple=_IMAGE_DIM,
color: bool=False) -> Tuple[np.ndarray, np.ndarray]:
# Load data
data = pd.read_csv(path, usecols=['emotion', 'pixels'])
data_size = len(data.pixels)
if color:
features = np.zeros(
shape=(data_size, *image_dim, 3),
dtype=np.float32)
else:
features = np.zeros(
shape=(data_size, *image_dim),
dtype=np.float32)
labels = np.asarray(data.emotion, dtype=np.int32)
for idx, row in enumerate(data.pixels):
image_array = np.fromstring(
str(row),
dtype=np.uint8,
sep=' ').reshape((48, 48))
# Convert to color
if color:
image = cv.cvtColor(image_array, cv.COLOR_GRAY2RGB)
else:
image = image_array
# Resize if needed
if image_dim != (48, 48):
image = cv.resize(image, image_dim)
features[idx] = image
# VGG Preprocess
if color:
features = preprocess_input_vgg(features)
else:
features = preprocess_input(features)
return features, labels
def predict(self, img):
""" # Arguments
img: a numpy array
# Returns
A dict containing predictions
"""
img = Image.fromarray(img)
img = img.resize((224, 224))
x = keras_image.img_to_array(img)[:, :, :3]
x = np.expand_dims(x, axis=0)
x = preprocess_input_vgg(x)
features = self.model.predict(x)
predictions = decode_predictions_vgg(features)[0]
clean_predictions = [{
'score': str(k),
'class': j
} for (i, j, k) in predictions]
return json.dumps(clean_predictions)
def vgg_extract_feat_batch(self, img_path):
imgs = []
norm_feats = []
names = []
img_list = get_imlist(img_path)
for i,pa in enumerate(img_list):
img = image.load_img(pa, target_size=(self.input_shape[0], self.input_shape[1]))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input_vgg(img)
imgs.append(img)
img_name = os.path.split(pa)[1]
names.append(img_name.encode())
imgs = np.concatenate([x for x in imgs])
feats = self.model_vgg.predict(imgs,batch_size=32)
# print(feats)
count = 0
for feat in feats:
count += 1
norm_feat = feat / LA.norm(feat)
norm_feat = [i.item() for i in norm_feat]
norm_feats.append(norm_feat)
return norm_feats, names
def main():
# init variables
x, y, w, h = 400, 150, 200, 200
x2, y2, w2, h2 = 10, 370, 100, 100
wfinal, hfinal = 100, 100
key = 0
model_path = '../models'
# Load saved models
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
cnn_model = load_model(os.path.join(model_path, 'model_cnn.h5'))
cnn_preproc_model = load_model(os.path.join(model_path, 'model_cnn_preproc.h5'))
vgg16_model = load_model(os.path.join(model_path, 'model_vgg16.h5'))
ml_model = pickle.load(open(os.path.join(model_path, 'model_classic.sav'), 'rb'))
pca_model = pickle.load(open(os.path.join(model_path, 'model_classic_pca.sav'), 'rb'))
# Get Webcam stream
cam = cv2.VideoCapture(0)
cv2.namedWindow('WebCam', cv2.WINDOW_NORMAL)
while key != ord('q'):
ret, frame = cam.read()
frame = cv2.flip(frame, 1)
window = copy.deepcopy(frame)
# Define a ROI, show bounds as a rectangle
roi = frame[y:y + h, x:x + w]
cv2.rectangle(img=window, pt1=(x, y), pt2=(x + w, y + h), color=(0, 255, 0), thickness=2)
# Image to predict
to_predict = cv2.resize(roi, (wfinal, hfinal))
to_predict = cv2.cvtColor(to_predict, cv2.COLOR_BGR2GRAY)
key = cv2.waitKey(5) & 0xff
# predict using ml
myimg = to_predict
myimg = myimg.reshape(1, wfinal * hfinal)
myimg_pca = pca_model.transform(myimg)
result = ml_model.predict(myimg_pca)
pred = result[0]
cv2.putText(window, "ml model : %d" % pred, (400, 140), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
# predict using cnn
myimg = to_predict.reshape(1, hfinal, wfinal, 1)
myclass = cnn_model.predict(myimg)
pred = np.argmax(myclass)
cv2.putText(window, "cnn model : %d" % pred, (400, 380), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 0, 0), 2)
# predict using cnn with pre processing
myimg = cv2.GaussianBlur(to_predict, ksize=(1, 1), sigmaX=1000).astype('uint8')
myimg = cv2.adaptiveThreshold(myimg, maxValue=255, adaptiveMethod=cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
thresholdType=cv2.THRESH_BINARY, blockSize=5, C=3)
myimg = myimg.reshape(1, hfinal, wfinal, 1)
myclass = cnn_preproc_model.predict(myimg)
pred = np.argmax(myclass)
cv2.putText(window, "cnn preproc : %d" % pred, (400, 410), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (128, 255, 0), 2)
window[y2:y2 + h2, x2:x2 + w2] = cv2.cvtColor(myimg.reshape(hfinal, wfinal, 1), cv2.COLOR_GRAY2BGR)
# predict using vgg16
myimg = cv2.cvtColor(to_predict, cv2.COLOR_GRAY2BGR)
myimg = myimg.reshape(1, hfinal, wfinal, 3)
myimg = preprocess_input_vgg(myimg)
myclass = vgg16_model.predict(myimg)
pred = np.argmax(myclass)
cv2.putText(window, "vgg16 model : %d" % pred, (400, 440), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 0, 128), 2)
# Display image
cv2.imshow('WebCam', window)
# Release webcam
cam.release()
