def draw_debug_rects(self, image):
if utils.is_gray(image):
face_color = 255
left_eye_color = 255
right_eye_color = 255
nose_color = 255
mouth_color = 255
else:
face_color = (255, 255, 255)
left_eye_color = (0, 0, 255)
right_eye_color = (0, 255, 255)
mouth_color = (255, 0, 0)
nose_color = (0, 255, 0)
for face in self._faces:
rects.outline_rect(image, face.face_rect, face_color)
rects.outline_rect(image, face.left_eye_rect, left_eye_color)
rects.outline_rect(image, face.right_eye_rect, right_eye_color)
rects.outline_rect(image, face.nose_rect, nose_color)
rects.outline_rect(image, face.mouth_rect, mouth_color)
def norm(image, bg_avg):
''' Normalize each by subtracting the background pixel mean differenc '''
log.info('Normalizing image')
gray = True if is_gray(image) else False
image = img_as_float(image)
filter = get_filter(image)
masked = image.copy()
if gray:
masked[~filter] = 0
else:
masked[~filter] = [0, 0, 0]
diff = bg_avg - np.mean(masked, axis=(0, 1))
log.info('Background diff: ' + str(diff))
normed = image + diff
if gray:
normed[normed > 1.0] = 1.0
normed[normed < -1.0] = -1.0
else:
# TODO: find out how to scale the color normalized to [0,255]
pass
return normed
def update(self, image):
self._faces = []
if utils.is_gray(image):
image = cv2.equalizeHist(image)
else:
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
cv2.equalizeHist(image, image)
min_size = utils.width_height_devided_by(image, 8)
face_rects = self._face_classifier.detectMultiScale(image,
self.scale_factor,
self.min_neighbors,
self.flags, min_size)
if face_rects is not None:
for face_rect in face_rects:
face = Face()
face.face_rect = face_rect
x, y, w, h = face_rect
search_rect = (x + w / 7, y, w * 2 / 7, h / 2)
face.left_eye_rect = self._detect_one_object(self._eye_classifier, image, search_rect, 64)
search_rect = (x + w * 4 / 7, y, w * 2 / 7, h / 2)
face.right_eye_rect = self._detect_one_object(self._eye_classifier, image, search_rect, 64)
search_rect = (x + w / 4, y + h /4, w / 2, h / 2)
face.nose_rect = self._detect_one_object(self._nose_classifier, image, search_rect, 32)
search_rect = (x + w / 6, y + h * 2 / 3, w * 2 / 3, h / 3)
face.mouth_rect = self._detect_one_object(self._mouth_classifier, image, search_rect, 16)
self._faces.append(face)
def segment_image(image):
log.info('Segmenting image')
filter = get_filter(image)
if is_gray(image):
image[filter] = 0
else:
image[filter] = [0, 0, 0]
return image
def draw_debug_text(self, image):
"""Draw text indicating the number of detected faces."""
if utils.is_gray(image):
text_colour = 255
else:
text_colour = (255, 255, 255)
found = "Faces: {}".format(len(self.faces))
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(image, found, (50, 50), font, 1, (0, 0, 0), 2, cv2.LINE_AA)
def get_filter(image):
# TODO: test this function
if not is_gray(image):
image = rgb2gray(image)
thresh = threshold_otsu(image)
# clear speckled black pixels in the image
filter = binary_closing(image > thresh, selem=square(10))
# clear speckled white pixels in the image
filter = binary_opening(filter, selem=square(15))
return filter
def show(self, frame):
frame_size = frame.shape[1::-1]
if utils.is_gray(frame):
conversion_type = cv2.COLOR_GRAY2RGB
else:
conversion_type = cv2.COLOR_BGR2RGB
rgb_frame = cv2.cvtColor(frame, conversion_type)
pyg_frame = pygame.image.frombuffer(rgb_frame.tostring(), frame_size,
'RGB')
display_surface = pygame.display.set_mode(frame_size)
display_surface.blit(pyg_frame, (0, 0))
pygame.display.flip()
def show(self, frame):
frame_size = frame.shape[1::-1]
if utils.is_gray(frame):
conversion_type = cv2.COLOR_GRAY2RGB
else:
conversion_type = cv2.COLOR_BGR2RGB
rgb_frame = cv2.cvtColor(frame, conversion_type)
pyg_frame = pygame.image.frombuffer(
rgb_frame.tostring(), frame_size, 'RGB')
display_surface = pygame.display.set_mode(frame_size)
display_surface.blit(pyg_frame, (0, 0))
pygame.display.flip()
def update(self, image):
"""updates tracked face features"""
self._faces = []
if (utils.is_gray(image)):
image = cv2.equalizeHist(image)
else:
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image = cv2.equalizeHist(image)
min_size = utils.widthHeightDividedBy(image, 8)
face_rects = self._face_classifier.detectMultiScale(
image, self.scale_factor, self.min_neighbors, self.flags, min_size)
if (face_rects is not None):
for face_rect in face_rects:
face = Face()
face.face_rect = face_rect
x, y, w, h = face_rect
# look for an eye in the upper-left part of the face
search_rect = (x+w//7, y, w*2//7, h//2)
face.left_eye_rect = self._detect_one_object(
self._eye_classifier, image, search_rect, 64
)
# look for an eye in the upper-right part of the face
search_rect = (x + (w*4)//7, y, w*2//7, h//2)
face.right_eye_rect = self._detect_one_object(
self._eye_classifier, image, search_rect, 64
)
# look for an node in the middle part of the face
search_rect = (x+w//4, y+h//4, w*2, h//2)
face.nose_rect = self._detect_one_object(
self._nose_classifier, image, search_rect, 32
)
# look for an mouth in the lower-middle part of the face
search_rect = (x+w//6, y + (h*2)//3, w*2//3, h//3)
face.mouth_rect = self._detect_one_object(
self._mouth_classifier, image, search_rect, 16
)
self._faces.append(face)
def update(self, image):
"""Update the tracked facial features."""
self._faces = []
if utils.is_gray(image):
image = cv2.equalizeHist(image)
else:
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
cv2.equalizeHist(image, image)
min_size = utils.width_height_divided_by(image, 8)
face_rects = self._face_classifier.detectMultiScale(
image,
scaleFactor=self.scale_factor,
minNeighbors=self.min_neighbours,
minSize=min_size)
if face_rects is not None:
for face_rect in face_rects:
face = Face()
face.face_rect = face_rect
x, y, w, h = face_rect
# Seek an eye in the upper-left part of the face.
search_rect = (x + w / 7, y, w * 2 / 7, h / 2)
face.left_eye_rect = self._detect_one_object(
self._eye_classifier, image, search_rect, 64)
# Seek an eye in the upper-right part of the face.
search_rect = (x + w * 4 / 7, y, w * 2 / 7, h / 2)
face.right_eye_rect = self._detect_one_object(
self._eye_classifier, image, search_rect, 64)
# Seek a nose in the middle part of the face.
search_rect = (x + w / 4, y + h / 4, w / 2, h / 2)
face.nose_rect = self._detect_one_object(
self._nose_classifier, image, search_rect, 32)
# Seek a mouth in the lower-middle part of the face.
search_rect = (x + w / 6, y + h * 2 / 3, w * 2 / 3, h / 3)
face.mouth_rect = self._detect_one_object(
self._mouth_classifier, image, search_rect, 16)
self._faces.append(face)
def update(self, image):
"""Update the tracked facial features"""
self._faces = []
if utils.is_gray(image):
image = cv2.equalizeHist(image)
else:
image = cv2.cvtColor(image, cv2.cv.CV_BGR2GRAY)
cv2.equalizeHist(image, image)
min_size = utils.width_height_divided_by(image, 8)
face_rects = self._face_classifier.detectMultiScale(
image, self.scale_factor, self.min_neighbors, self.flags, min_size)
if face_rects is not None:
for face_rect in face_rects:
face = Face()
face.face_rect = face_rect
x, y, w, h = face_rect
# seek a left eye
search_rect = (x + w * 4 / 7, y, w * 2 / 7, h / 2)
face.left_eye_rect = self._detect_one_object(
self._eye_classifier, image, search_rect, 64)
# seek a right eye
search_rect = (x + w / 7, y, w * 2 / 7, h / 2)
face.right_eye_rect = self._detect_one_object(
self._eye_classifier, image, search_rect, 64)
# seek a nose
search_rect = (x + w / 4, y + h / 4, w / 2, h / 2)
face.nose_rect = self._detect_one_object(
self._nose_classifier, image, search_rect, 32)
# seek a mouth
search_rect = (x + w / 6, y + h * 2 / 3, w * 2 / 3, h / 3)
face.mouth_rect = self._detect_one_object(
self._mouth_classifier, image, search_rect, 16)
self._faces.append(face)
def draw_debug_rects(self, image):
"""Draw rectangles around the tracked facial features."""
if utils.is_gray(image):
face_colour = 255
left_eye_colour = 255
right_eye_colour = 255
nose_colour = 255
mouth_colour = 255
else:
face_colour = (255, 255, 255) # white
left_eye_colour = (0, 0, 255) # red
right_eye_colour = (0, 255, 255) # yellow
nose_colour = (0, 255, 0) # green
mouth_colour = (255, 0, 0) # blue
for face in self.faces:
rects.outline_rect(image, face.face_rect, face_colour)
rects.outline_rect(image, face.left_eye_rect, left_eye_colour)
rects.outline_rect(image, face.right_eye_rect, right_eye_colour)
rects.outline_rect(image, face.nose_rect, nose_colour)
rects.outline_rect(image, face.mouth_rect, mouth_colour)
def plot_bbx(image, bboxes):
''' Plot red bounding boxes and return plot object '''
fig, ax = plt.subplots(figsize=(6, 6))
if is_gray(image):
ax.imshow(image, cmap='gray')
else:
ax.imshow(image)
for i, (minr, minc, maxr, maxc) in enumerate(bboxes):
rect = mpatches.Rectangle((minc, minr),
maxc - minc,
maxr - minr,
fill=False,
edgecolor='red',
linewidth=2)
ax.add_patch(rect)
ax.text(minc - 20, minr - 20, str(i))
return plt
def dataset(indir, outdir, anno_file, validation_split, segment):
'''
Generate a dataset for CNN training
params
* indir: directory of preprocessed images
* outdir: directory to output the images to
* anno_file: the path to the file of annotations
'''
sp.run(['mkdir', '-p', outdir])
fh = logging.FileHandler(osp.join(outdir, 'log'))
fh.setLevel(logging.WARNING)
log.addHandler(fh)
data = osp.join(outdir, 'data')
sp.run(['mkdir', '-p', data])
sp.run(['mkdir', '-p', osp.join(data, 'train')])
# sp.run(['mkdir', '-p', osp.join(data, 'valid')]) # handled by split_val
for i in range(1, 6):
sp.run(['mkdir', '-p', osp.join(data, 'train', str(i))])
bbox_dir = osp.join(outdir, 'bboxes')
bbox_err_dir = osp.join(outdir, 'err_bboxes')
sp.run(['mkdir', '-p', bbox_dir])
sp.run(['mkdir', '-p', bbox_err_dir])
sp.run([
'echo', 'filename,rating\n', '>',
osp.join(outdir, 'annotations.csv')
])
bbox_err_count = 0
try:
annotations = pd.read_csv(anno_file)
# convert the ratings string to list
annotations['ratings'] = annotations['ratings'].apply(eval)
except FileNotFoundError as fnfe:
log.error(fnfe)
exit()
annotation_file = open(osp.join(outdir, 'annotations.csv'), 'a')
annotation_file.write("filename,rating\n")
annotations_summary = {'1': 0, '2': 0, '3': 0, '4': 0, '5': 0}
for i, row in annotations.iterrows():
log.info("Processing " + row['filename'])
image_path = osp.join(indir, row['filename'])
if not osp.isfile(image_path):
log.error("Could not locate " + str(image_path))
continue
try:
image = imread(image_path)
if is_gray(image):
image = gray2rgb(image)
except Exception as e:
log.error("Failed to load " + image_path)
log.error(e)
tb.print_exc()
continue
bboxes = get_sorted_bboxes(image)
plot_bbx(image, bboxes)
if len(bboxes) != row['num_objects']:
log.error("Count of objects in image did not match: " +
row['filename'])
out_fname = osp.join(bbox_err_dir, row['filename'])
bbox_err_count += 1
plt.savefig(out_fname)
plt.close('all')
continue
else:
out_fname = osp.join(bbox_dir, row['filename'])
plt.savefig(out_fname)
plt.close('all')
# squash 2d list to 1d
ratings = [entry for line in row['ratings'] for entry in line]
if segment:
image = segment_image(image)
log.info("Getting thumbnails")
for j, bbox in enumerate(bboxes):
anno = ratings[j]
minr, minc, maxr, maxc = bbox
thumbnail = image[minr:maxr, minc:maxc]
out_fname = osp.join(outdir, 'data', 'train', str(anno),
str(j) + "_" + row['filename'])
imsave(out_fname, thumbnail)
annotation_file.write("{},{}\n".format(
str(j) + "_" + row['filename'], anno))
annotations_summary[anno] += 1
annotation_file.close()
log.info("Generating validation dataset")
split_val(data, validation_split)
log.info("SUMMARY:")
log.info("Number of bbox errors: " + str(bbox_err_count))
for i in range(1, 6):
log.info("Number of annotations with rating {}: {}".format(
i, annotations_summary[str(i)]))
num_samples = sum(list(annotations_summary.values()))
log.info("Total number of images: {}".format(num_samples))
return
