def crop_or_pad(image, size=(299, 299)):
image = image.astype(np.float32)
h, w = image.shape[:2]
window = (0, 0, h, w)
scale = 1
padding = [(0, 0), (0, 0), (0, 0)]
image_max = max(h, w)
scale = float(min(size)) / image_max
image = cv2.resize(image, (int(w * scale), int(h * scale)))
h, w = image.shape[:2]
top_pad = (size[1] - h) // 2
bottom_pad = size[1] - h - top_pad
left_pad = (size[0] - w) // 2
right_pad = size[0] - w - left_pad
padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
image = np.pad(image, padding, mode='constant', constant_values=0)
# Fix image normalization
if np.nanmax(image) > 1:
image = np.divide(image, 255.)
return image
def run_dlib_shape(image):
# in this function, load the image, detect the landmarks of the face, and then return the image and the landmarks
# load the input image, resize it, and convert it to grayscale
resized_image = image.astype('uint8')
gray = cv2.cvtColor(resized_image, cv2.COLOR_BGR2GRAY)
gray = gray.astype('uint8')
# detect faces in the grayscale image
rects = detector(gray, 1)
num_faces = len(rects)
if num_faces == 0:
return None, resized_image
face_areas = np.zeros((1, num_faces))
face_shapes = np.zeros((136, num_faces), dtype=np.int64)
# loop over the face detections
for (i, rect) in enumerate(rects):
# determine the facial landmarks for the face region, then convert the facial landmark (x, y)-coordinates to a NumPy array
temp_shape = predictor(gray, rect)
temp_shape = shape_to_np(temp_shape)
# convert dlib's rectangle to a OpenCV-style bounding box
(x, y, w, h) = rect_to_bb(rect)
face_shapes[:, i] = np.reshape(temp_shape, [136])
face_areas[0, i] = w * h
# find largest face and keep it
dlibout = np.reshape(np.transpose(face_shapes[:, np.argmax(face_areas)]),
[68, 2])
return dlibout, resized_image
def image_preprocess(imagePath):
image = cv2.imread(imagePath)
image = cv2.resize(image, (128, 128))
image = image.astype("float") / 255.0
image = img_to_array(image)
image = np.expand_dims(image, axis=0)
return image
def display_instances(image, masks, ax=None, show_mask=True):
auto_show = False
N = masks.shape[0]
if not ax:
_, ax = plt.subplots(1, figsize=(20, 8))
auto_show = True
colors = random_colors(N)
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
ax.set_title("Results")
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
mask = masks[i]
if show_mask:
masked_image = apply_mask(masked_image, mask, color)
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor='none', edgecolor=color)
ax.add_patch(p)
masked_image = cv2.addWeighted(image, 0.9, masked_image.astype(np.float32),
0.1, 0)
ax.imshow(masked_image)
if auto_show:
plt.show()
def get_picture_matrix(task_all_train, config):
sentence_number = 0
train_picture = []
y_train = []
for index, row in task_all_train.iterrows(): #获取Humour，Sarcasm，offensive标签
image_name = row[0]
Humour_label = int(row[1])
Sarcasm_label = int(row[2])
offensive_label = int(row[3])
#不属于
# other_label = 1
if Humour_label != 0:
Humour_label = 1
if Sarcasm_label != 0:
Sarcasm_label = 1
if offensive_label != 0:
offensive_label = 1
img_path = source_path + image_name
image = keras.preprocessing.image.load_img(img_path,
target_size=(224, 224))
image = keras.preprocessing.image.img_to_array(image)
image = np.expand_dims(image, axis=0)
image = image.astype('float32')
pic_vector = image / 255. # 去均值中心化，preprocess_input函数详细功能见注释
train_picture.append(pic_vector)
y_train.append([Humour_label, Sarcasm_label, offensive_label])
sentence_number += 1
train_picture = np.vstack(train_picture)
y_train = np.array(y_train)
print(Counter(y_train[:, 0]))
print(Counter(y_train[:, 1]))
print(Counter(y_train[:, 2]))
return train_picture, y_train
def plot_images(self, save2file=False, samples=16, noise=None, step=0):
filename = 'posters.png'
if noise is None:
noise = np.random.uniform(-1.0, 1.0, size=[samples, 100])
else:
filename = "posters_%d.png" % step
images = self.generator.predict(noise)
plt.figure(figsize=(10, 10))
print("images[0] shape")
print(images.shape[0])
for i in range(images.shape[0]):
plt.subplot(4, 4, i + 1)
image = images[i, :, :, :]
if (self.channels == 1):
image = np.reshape(image, [self.img_rows, self.img_cols])
plt.imshow(image, cmap='gray')
else:
image = 255 * np.reshape(
image, [self.img_rows, self.img_cols, self.channels])
plt.imshow(image.astype('uint8'))
plt.axis('off')
plt.tight_layout()
if save2file:
plt.savefig(filename)
plt.close('all')
else:
plt.show()
def generate_data_val(directory, batch_size):
"""Replaces Keras' native ImageDataGenerator."""
i = 0
file_list = os.listdir(directory)
random.shuffle(file_list)
while True:
image_batch = []
labels = []
labels2 = []
for b in range(batch_size):
if i == len(file_list):
i = 0
#random.shuffle(file_list)
sample = file_list[i]
#print(sample)
i += 1
image = cv2.resize(cv2.imread(directory + sample), (112, 112))
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image_batch.append(image.astype('float32') / 255.)
face_class = sample.split(';')[0]
#print(face_class)
labels.append(face_class)
labels2 = keras.utils.to_categorical(labels, num_classes=10002)
#print(face_class)
yield [np.array(image_batch), np.array(labels2)], np.array(labels2)
def run_dlib_shape(image):
resized_image = image.astype('uint8')
gray = cv2.cvtColor(resized_image, cv2.COLOR_BGR2GRAY)
gray = gray.astype('uint8')
# detect faces in the grayscale image
rects = detector(gray, 1)
num_faces = len(rects)
if num_faces == 0:
return None, resized_image
face_areas = np.zeros((1, num_faces))
face_shapes = np.zeros((136, num_faces), dtype=np.int64)
# loop over the face detections
for (i, rect) in enumerate(rects):
temp_shape = predictor(gray, rect)
temp_shape = shape_to_np(temp_shape)
(x, y, w, h) = rect_to_bb(rect)
face_shapes[:, i] = np.reshape(temp_shape, [136])
face_areas[0, i] = w * h
dlibout = np.reshape(np.transpose(face_shapes[:, np.argmax(face_areas)]),
[68, 2])
return dlibout, resized_image
def load_image(self, image_index):
coco_image = self.coco.loadImgs(self.image_ids[image_index])[0]
path = os.path.join(self.data_dir, 'images', self.set_name, coco_image['file_name'])
image = cv2.imread(path, cv2.IMREAD_COLOR)
image, image_scale = keras_retinanet.preprocessing.image.resize_image(image, min_side=self.image_min_side, max_side=self.image_max_side)
# set ground truth boxes
annotations_ids = self.coco.getAnnIds(imgIds=coco_image['id'], iscrowd=False)
# some images appear to miss annotations (like image with id 257034)
if len(annotations_ids) == 0:
return None
# parse annotations
annotations = self.coco.loadAnns(annotations_ids)
boxes = np.zeros((0, 5), dtype=keras.backend.floatx())
for idx, a in enumerate(annotations):
box = np.zeros((1, 5), dtype=keras.backend.floatx())
box[0, :4] = a['bbox']
box[0, 4] = a['category_id'] - 1 # start from 0
boxes = np.append(boxes, box, axis=0)
# transform from [x, y, w, h] to [x1, y1, x2, y2]
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
# scale the ground truth boxes to the selected image scale
boxes[:, :4] *= image_scale
# convert to batches (currently only batch_size = 1 is allowed)
image_batch = np.expand_dims(image.astype(keras.backend.floatx()), axis=0)
boxes_batch = np.expand_dims(boxes, axis=0)
# randomly transform images and boxes simultaneously
image_batch, boxes_batch = keras_retinanet.preprocessing.image.random_transform_batch(image_batch, boxes_batch, self.image_data_generator)
# generate the label and regression targets
labels, regression_targets = anchor_targets(image, boxes_batch[0], self.num_classes)
regression_targets = np.append(regression_targets, labels, axis=1)
# convert target to batch (currently only batch_size = 1 is allowed)
regression_batch = np.expand_dims(regression_targets, axis=0)
labels_batch = np.expand_dims(labels, axis=0)
# convert the image to zero-mean
image_batch = keras_retinanet.preprocessing.image.preprocess_input(image_batch)
image_batch = self.image_data_generator.standardize(image_batch)
return {
'image' : image,
'image_scale' : image_scale,
'coco_index' : coco_image['id'],
'boxes_batch' : boxes_batch,
'image_batch' : image_batch,
'regression_batch' : regression_batch,
'labels_batch' : labels_batch,
}
def feature_extraction(image):
# preprocessing
im = image.astype(np.float32) / 255.
# TODO: real feature extraction
im = np.reshape(
im, (1, np.prod(im.shape)
)) # we need a vector [1 x num_features] (depends on classifier)
features = im
return features
def testing(imagepath, model, xmeanxvar): #generate yhat vector
allval = np.fromstring(xmeanxvar, sep=' ')
xmean = allval[0].astype('float32')
xvar = allval[1].astype('float32')
image = np.array([np.array(Image.open(imagepath))])
image = image.astype('float32')
image = (image - xmean) / xvar
classes = model.predict_proba(image, batch_size=1)
return classes
def predict_type_of_vehicle(image):
class_name = [ 'Bus','Car','Limousine','minivan','motorcycle','Truck']
# class_name = ['Ambulance','Barge','Bicycle','Boat', 'Bus','Car','Cart','Caterpillar','Helicopter','Limousine','Motorcycle','Segway','Snowmobile','Tank','Taxi','Truck','Van']
image = cv2.resize(image, dsize=(224, 224))
image = image.astype('float')*1./255
image = np.expand_dims(image, axis=0)
predict = modelMobile.predict(image)
vehicle_type = class_name[np.argmax(predict)]
return vehicle_type
def adjust_light(image):
seed = random.random()
if seed > 0.5:
gamma = random.random() * 3 + 0.5
invGamma = 1.0 / gamma
table = np.array([((i / 255.0)**invGamma) * 255
for i in np.arange(0, 256)]).astype(np.uint8)
image = cv2.LUT(image.astype(np.uint8), table).astype(np.uint8)
return image
def predict(self, predict_image):
image = resize_with_pad(predict_image)
image = image.astype('float32')
image /= 255
result = self.model.predict(np.array([image]))[0]
whois = 0 if (result[0] > result[1]) else 1
return whois
def subtract_mean(image):
r_mean = 125.7
b_mean = 93.1
g_mean = 121.4
(B, G, R) = cv2.split(image.astype("float32"))
R -= r_mean
G -= g_mean
B -= b_mean
return cv2.merge([B, G, R])
def plot_image_big(image):
# Ensure the pixel-values are between 0 and 255.
image = np.clip(image, 0.0, 255.0)
# Convert pixels to bytes.
image = image.astype(np.uint8)
# Convert to a PIL-image and display it.
plt.figure()
plt.imshow(PIL.Image.fromarray(image))
def save_image(image, filename):
# Ensure the pixel-values are between 0 and 255.
image = np.clip(image, 0.0, 255.0)
# Convert to bytes.
image = image.astype(np.uint8)
# Write the image-file in jpeg-format.
with open(filename, 'wb') as file:
PIL.Image.fromarray(image).save(file, 'jpeg')
def ad_blur(image):
seed = random.random()
if seed > 0.5:
image = image / 127.5 - 1
image[:, :, 0] = median(image[:, :, 0], disk(3))
image[:, :, 1] = median(image[:, :, 1], disk(3))
image[:, :, 2] = median(image[:, :, 2], disk(3))
image = (image + 1) * 127.5
image = image.astype(np.uint8)
return image
def predict(image):
# load the image, pre-process it, and store it in the data list
image = cv2.imread(image)
image = cv2.resize(image, (64, 64))
image = image.astype("float") / 255.0
image = img_to_array(image)
image = np.expand_dims(image, axis=0)
result = model.predict(image)[0]
idx = np.argmax(result)
if (idx == 0):
return 1
else:
return -1
def get_picture_numpy(image_names):
picture_list = []
for index, row in image_names.iterrows():
image_name = row["Image_name"]
img_path = source_path + image_name
image = keras.preprocessing.image.load_img(img_path,
target_size=(224, 224))
image = keras.preprocessing.image.img_to_array(image)
image = np.expand_dims(image, axis=0)
image = image.astype('float32')
pic_vector = image / 255
picture_list.append(pic_vector)
picture_list = np.vstack(picture_list)
return picture_list
def run_dlib_shape(self, image):
"""
Face and landmark detection takes place here.
Take an image, resize and converts grayscale, before looking for faces
and landmark features.
Takes the largest face detected and vectorise it.
argument:
image -- image path
return:
dlibout -- landmark data
resized_image
vectorised_landmarks
"""
resized_image = image.astype('uint8')
gray = cv2.cvtColor(resized_image, cv2.COLOR_BGR2GRAY)
gray = gray.astype('uint8')
# detect faces in the grayscale image
rects = self.detector(gray, 1)
num_faces = len(rects)
#print(num_faces)
if num_faces == 0:
return None, resized_image, None
face_areas = np.zeros((1, num_faces))
face_shapes = np.zeros((136, num_faces), dtype=np.int64)
# loop over the face detections
for (i, rect) in enumerate(rects):
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
temp_shape = self.predictor(gray, rect)
temp_shape = self.shape_to_np(temp_shape)
# convert dlib's rectangle to a OpenCV-style bounding box
# [i.e., (x, y, w, h)],
# (x, y, w, h) = face_utils.rect_to_bb(rect)
(x, y, w, h) = self.rect_to_bb(rect)
face_shapes[:, i] = np.reshape(temp_shape, [136])
face_areas[0, i] = w * h
# find largest face and keep
dlibout = np.reshape(
np.transpose(face_shapes[:, np.argmax(face_areas)]), [68, 2])
vectorised_landmarks = self.process_landmarks(dlibout)
return dlibout, resized_image, vectorised_landmarks
def decode_image_pillow(img_path):
### Going to create image vector via PIL
## does not work
start = timer()
image = np.asarray(Image.open(img_path).convert('RGB'))
image = image.astype('f')
end = timer()
print("decode time=", end - start)
#('decode time=', 0.023459911346435547)
# and without conversion as in
# https://github.com/fizyr/keras-retinanet/blob/master/keras_retinanet/utils/image.py
# it is not able to detect one person object
#Label', 'person', ' at ', array([ 0, 258, 308, 473]), ' Score ', 0.8666027)
#('Label', 'person', ' at ', array([239, 98, 427, 378]), ' Score ', 0.745802)
#('Label', 'tie', ' at ', array([312, 199, 340, 314]), ' Score ', 0.7037207)
return image
def generate_data(directory, batch_size):
"""Replaces Keras' native ImageDataGenerator."""
i = 0
file_list = os.listdir(directory)
while True:
image_batch = []
for b in range(batch_size):
if i == len(file_list):
i = 0
random.shuffle(file_list)
sample = file_list[i]
i += 1
image = cv2.resize(cv2.imread(sample[0]), INPUT_SHAPE)
image_batch.append((image.astype(float) - 128) / 128)
yield np.array(image_batch)
def test(model, data):
x_test, y_test = data
y_pred, x_recon = model.predict([x_test, y_test], batch_size=100)
print('-'*50)
print('Test acc:', numpy.sum(numpy.argmax(y_pred, 1) == numpy.argmax(y_test, 1))/y_test.shape[0])
import matplotlib.pyplot as plt
from PIL import Image
img = combine_images(numpy.concatenate([x_test[:50],x_recon[:50]]))
image = img * 255
Image.fromarray(image.astype(numpy.uint8)).save("real_and_recon.png")
print()
print('Reconstructed images are saved to ./real_and_recon.png')
print('-'*50)
plt.imshow(plt.imread("real_and_recon.png", ))
plt.show()
def plotImgs(x, y):
ax = []
columns = 3
rows = 3
w = 100
h = 100
fig = plt.figure(figsize=(9, 13))
for j in range(columns * rows):
i = np.random.randint(0, len(x))
image = x[i] * 255
title = "foo"
# create subplot and append to ax
ax.append(fig.add_subplot(rows, columns, j + 1))
ax[-1].set_title(title)
#print(image.shape)
plt.imshow(image.astype('uint8'))
plt.show()
def plotImgs(x, title):
columns = 4
rows = 4
w = 100
h = 100
fig = plt.figure(figsize=(9, 13))
for j in range( columns*rows ):
image = x[j]
if j>7:
image*=255.0
subplot=fig.add_subplot(rows, columns, j+1)
subplot.set_title(labels[j//4])
if j<4 or j>7:
plt.imshow(image.astype('uint8'))
else:
plt.imshow(image)
plt.title(title)
plt.show()
def augment_cnn(input_path='./dataset/Train',
output_path='./dataset/Augmented',
count=10):
''' Augments images and saves them into a new directory '''
for Images in os.listdir(input_path):
gen = ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
zca_epsilon=1e-06,
rotation_range=30,
width_shift_range=30,
height_shift_range=30,
brightness_range=[0.6, 0.8],
shear_range=4.0,
zoom_range=[0.8, 1.0],
channel_shift_range=10,
fill_mode='reflect',
cval=0.0,
horizontal_flip=True,
vertical_flip=True,
rescale=1. / 255,
preprocessing_function=None,
data_format='channels_last',
validation_split=0.2,
#interpolation_order=1,
dtype='float32')
img = load_img('{}/{}'.format(input_path, Images))
name = Images[:-4]
size = img.size
image = img_to_array(img)
image = image.reshape(1, size[1], size[0], 3)
image = image.astype('float32')
gen.fit(image)
images_flow = gen.flow(image, batch_size=1)
for i, new_images in enumerate(images_flow):
new_image = array_to_img(new_images[0], scale=True)
output = '{}/Aug_{}-{}.jpg'.format(output_path, name, i + 1)
print(output)
new_image.save(output)
if i >= count - 1: break
def get_features(image): # Gets features of face
"""
This function loads the image, detects the landmarks of the face
:return:
dlibout: an array containing 68 landmark points
resized_image: an array containing processed images
"""
# load the input image, resize it, and convert it to grayscale
resized_image = image.astype('uint8')
gray = cv2.cvtColor(resized_image, cv2.COLOR_BGR2GRAY)
gray = gray.astype('uint8')
# detect faces in the grayscale image
rects = detector(gray, 1)
num_faces = len(rects)
if num_faces == 0:
return None, resized_image
face_areas = np.zeros((1, num_faces))
face_shapes = np.zeros((136, num_faces), dtype=np.int64)
# loop over the face detections
for (i, rect) in enumerate(rects):
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
temp_shape = predictor(gray, rect)
temp_shape = shape_to_np(temp_shape)
# convert dlib's rectangle to a OpenCV-style bounding box
# [i.e., (x, y, w, h)],
# (x, y, w, h) = face_utils.rect_to_bb(rect)
(x, y, w, h) = rect_to_bb(rect)
face_shapes[:, i] = np.reshape(temp_shape, [136])
face_areas[0, i] = w * h
# find largest face and keep
dlibout = np.reshape(np.transpose(face_shapes[:, np.argmax(face_areas)]),
[68, 2])
return dlibout, resized_image
def image_colorfulness(image):
# split the image into its respective RGB components
(B, G, R) = cv2.split(image.astype("float"))
# compute rg = R - G
rg = np.absolute(R - G)
# compute yb = 0.5 * (R + G) - B
yb = np.absolute(0.5 * (R + G) - B)
# compute the mean and standard deviation of both `rg` and `yb`
(rbMean, rbStd) = (np.mean(rg), np.std(rg))
(ybMean, ybStd) = (np.mean(yb), np.std(yb))
# combine the mean and standard deviations
stdRoot = np.sqrt((rbStd**2) + (ybStd**2))
meanRoot = np.sqrt((rbMean**2) + (ybMean**2))
# derive the "colorfulness" metric and return it
return stdRoot + (0.3 * meanRoot)
def show_image(image, grayscale=True, ax=None, title=''):
if ax is None:
plt.figure()
plt.axis('off')
if len(image.shape) == 2 or grayscale == True:
if len(image.shape) == 3:
image = np.sum(np.abs(image), axis=2)
vmax = np.percentile(image, 99)
vmin = np.min(image)
plt.imshow(image, cmap=plt.cm.gray, vmin=vmin, vmax=vmax)
plt.title(title)
print('if')
else:
image = image + 127.5
image = image.astype('uint8')
plt.imshow(image)
plt.title(title)
print('else')
cv2.imwrite('mask.png', image)
