def is_skin_hair_block(self, block, block_type):
'''
Return true if the block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
is a skin/hair-block - it is if a majority (as per the threshold attribute) of the pixels in the block are
skin/hair colored. 'block_type' says whether we are testing for a skin block ('s') or a hair block ('h).
'''
# Your code
pyimage_object=PyImage(self.filename)
k=pyimage_object.size()
pixel_x=block[0]
pixel_y=block[1]
lst_skin=[]
lst_hair=[]
temp_x=0
while(pixel_x<k[0]):
temp_y=0
while(pixel_y<k[1] and temp_y<self.block_size):
rgb_a=pyimage_object.get_rgba(pixel_x,pixel_y)
pixel_color=Color(rgb_a)
if(self.is_skin(pixel_color)):
lst_skin.append((pixel_x,pixel_y))
elif(self.is_hair(pixel_color)):
lst_hair.append((pixel_x,pixel_y))
pixel_y+=1
temp_y+=1
pixel_x+=1
temp_x+=1
if(len(lst_skin)>self.majority*self.block_size and block_type=='s'):
return True
elif(len(lst_hair)>self.majority*self.block_size and block_type=='h'):
return True
else:
return False
def add_neighbour_blocks(self, block, graph):
'''
Given a block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
and a graph (could be a hair or a skin graph), add edges from the current block to its neighbours
on the image that are already nodes of the graph
Check blocks to the left, top-left and top of the current block and if any of these blocks is in the
graph (means the neighbour is also of the same type - skin or hair) add an edge from the current block
to the neighbour.
'''
# Your code
pyimage_object=PyImage(self.filename)
k=pyimage_object.size()
if(self.is_skin_hair_block(block, 's')):
if((block[0]-self.block_size)<k[0] and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0]-self.block_size,block[1]), 's')):
graph.add_edge(block, (block[0]-self.block_size,block[1]))
if((block[1]-self.block_size)<k[1] and (block[1]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 's')):
graph.add_edge(block, (block[0],block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]-self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 's')):
graph.add_edge(block, (block[0]-self.block_size,block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]+self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]+self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 's')):
graph.add_edge(block, (block[0]+self.block_size,block[1]-self.block_size))
if(self.is_skin_hair_block(block, 'h')):
if((block[0]-self.block_size)<k[0] and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0]-self.block_size,block[1]), 'h')):
graph.add_edge(block, (block[0]-self.block_size,block[1]))
if((block[1]-self.block_size)<k[1] and (block[1]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 'h')):
graph.add_edge(block, (block[0],block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]-self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 'h')):
graph.add_edge(block, (block[0]-self.block_size,block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]+self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]+self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 'h')):
graph.add_edge(block, (block[0]+self.block_size,block[1]-self.block_size))
def loadImage(self, image):
'''This function should open the image contained in the specified file
and reset the class' variables'''
# Reset variables
self.msFilter = None
self.msSegment = None
# Load image
self.original = PyImage()
self.original.loadImage(image)
def loadImage(self, image):
'''This function should initialize the class by loading an image from
the function call. It then places that image at the pyramid's lowest
level, and specifies no pixels were lost in this process.'''
# Reset pyramid and loss info
if self.pyramid:
self.pyramid = []
self.info_loss = []
# Create image, append things
img = PyImage()
img.loadImage(image)
self.pyramid.append(img)
self.info_loss.append((False, False))
def __init__(self,
filename,
block_size=11,
min_component_size=11,
majority=0.56):
'''
Constructor - keeps input image filename, image read from the file as a PyImage object, block size (in pixels),
threshold to decide how many skin color pixels are required to declare a block as a skin-block
and min number of blocks required for a component. The majority argument says what fraction of
the block pixels must be skin/hair colored for the block to be a skin/hair block - the default value is
0.5 (half).
'''
self.image = PyImage(filename)
self.block_sz = block_size
self.min_blocks = min_component_size
self.fraction_pixel = majority
def loadImage(self, image):
'''This function should open the image contained in the specified file
and reset the class' variables'''
# Reset variables
self.msFilter = None
self.msSegment = None
# Load image
self.original = PyImage()
self.original.loadImage(image)
def __init__(self, filename, block_size = 5, min_component_size = 10, majority = 0.5):
'''
Constructor - keeps input image filename, image read from the file as a PyImage object, block size (in pixels),
threshold to decide how many skin color pixels are required to declare a block as a skin-block
and min number of blocks required for a component. The majority argument says what fraction of
the block pixels must be skin/hair colored for the block to be a skin/hair block - the default value is
0.5 (half).
'''
# Your code
self.filename=filename
self.block_size=block_size
self.majority=majority
self.min_component_size=min_component_size
self.pyimage_obj = PyImage(filename)
def make_block_graph(self):
'''
Return the skin and hair graphs - nodes are the skin/hair blocks respectively
Initialize skin and hair graphs. For every block if it is a skin(hair) block
add edges to its neighbour skin(hair) blocks in the corresponding graph
For this to work the blocks have to be traversed in the top->bottom, left->right order
'''
# Your code
graph_object_skin=Graph(None)
graph_object_hair=Graph(None)
pyimage_object=PyImage(self.filename)
k=pyimage_object.size()
i=0
while(i<k[0]):
# 3
j=0
while(j<k[1]):
# 4
if(self.is_skin_hair_block((i,j),'s')):
graph_object_skin.add_node((i,j))
self.add_neighbour_blocks((i,j),graph_object_skin)
j+=self.block_size
i+=self.block_size
i=0
while(i<k[0]):
# 5
j=0
while(j<k[1]):
# 6
if(self.is_skin_hair_block((i,j),'h')):
graph_object_hair.add_node((i,j))
self.add_neighbour_blocks((i,j),graph_object_hair)
j+=self.block_size
i+=self.block_size
return graph_object_skin, graph_object_hair
def loadFile(self, filepath, nFrames, pyramid=0):
'''This function should load frames of a video with a regular file
expression to read filenames. The number of frames is known beforehand
to ease computing.'''
# Update variables based on input
self.pyramid = pyramid
self.nFrames = nFrames
self.filepath = filepath
# Separate name from extension
path = filepath.split('.')
extension = '.' + path[-1]
path = "".join(path[:-1]) + '-'
# For each frame that will be processed, we will load a single file
for i in np.arange(1, nFrames + 1):
print "DEBUG: Loading frame", i
# Load each frame
frame = PyImage()
frame.loadFile(path + str(i) + extension)
frame.img = frame.img.convert("RGB")
frame.updatePixels()
# Compute its grayscale counterpart
grayscaleFrame = frame.copy()
grayscaleFrame.img = grayscaleFrame.img.convert("L")
grayscaleFrame.updatePixels()
# Append frames to lists
self.frames.append(frame)
self.grayscaleFrames.append(grayscaleFrame)
# If pyramids will be used, start a pyramid for every grayscale
# frame to be used later on
if pyramid:
print "DEBUG: Computing pyramids for frame", i
framePyramid = GaussPyramid(pyramid)
framePyramid.loadImage(grayscaleFrame.img)
framePyramid.reduceMax()
self.pyramids.append(framePyramid)
def loadFile(self, filepath, nFrames, pyramid=0):
'''This function should load frames of a video with a regular file
expression to read filenames. The number of frames is known beforehand
to ease computing.'''
# Update variables based on input
self.pyramid = pyramid
self.nFrames = nFrames
self.filepath = filepath
# Separate name from extension
path = filepath.split('.')
extension = '.' + path[-1]
path = "".join(path[:-1]) + '-'
# For each frame that will be processed, we will load a single file
for i in np.arange(1, nFrames+1):
print "DEBUG: Loading frame", i
# Load each frame
frame = PyImage()
frame.loadFile(path+str(i)+extension)
frame.img = frame.img.convert("RGB")
frame.updatePixels()
# Compute its grayscale counterpart
grayscaleFrame = frame.copy()
grayscaleFrame.img = grayscaleFrame.img.convert("L")
grayscaleFrame.updatePixels()
# Append frames to lists
self.frames.append(frame)
self.grayscaleFrames.append(grayscaleFrame)
# If pyramids will be used, start a pyramid for every grayscale
# frame to be used later on
if pyramid:
print "DEBUG: Computing pyramids for frame", i
framePyramid = GaussPyramid(pyramid)
framePyramid.loadImage(grayscaleFrame.img)
framePyramid.reduceMax()
self.pyramids.append(framePyramid)
class FaceDetector(object):
'''
classdocs
'''
def __init__(self,
filename,
block_size=5,
min_component_size=15,
majority=0.5):
'''
Constructor - keeps input image filename, image read from the file as a PyImage object, block size (in pixels),
threshold to decide how many skin color pixels are required to declare a block as a skin-block
and min number of blocks required for a component. The majority argument says what fraction of
the block pixels must be skin/hair colored for the block to be a skin/hair block - the default value is
0.5 (half).
'''
# Your code
self.filename = filename
self.block_size = block_size
self.min_component_size = min_component_size
self.majority = majority
self.skin = Graph()
self.hair = Graph()
self.pyim = PyImage(filename)
def skin_green_limits(self, red):
'''
Return the limits of normalized green given the normalized red component as a tuple (min, max)
'''
return ((-0.776 * red * red + 0.5601 * red + 0.1766),
(-1.376 * red * red + 1.0743 * red + 0.1452))
def is_skin(self, pixel_color):
'''
Given the pixel color (as a Color object) return True if it represents the skin color
Color is skin if hue in degrees is (> 240 or less than or equal to 20) and
green is in the green limits and it is not white
'''
# Your code
color = Color(pixel_color)
hue = color.hue_degrees()
W = (color.R - 0.33) * (color.R - 0.33) + (color.G - 0.33) * (color.G -
0.33)
green_limits = self.skin_green_limits(color.R)
cond1 = (hue > 240 or hue <= 20)
cond2 = green_limits[0] < color.G and color.G < green_limits[1]
cond3 = color.rgb_abs()[0] < 230 and color.rgb_abs(
)[1] < 230 and color.rgb_abs()[2] < 230
cond4 = 0.6 > color.R and color.R > 0.2
if cond1 and cond2 and cond3 and cond4 and W > 0.0004:
return True
return False
def is_hair(self, pixel_color):
'''
Return True if the pixel color represents hair - it is if intensity < 80 and ((B-G)<15 or (B-R)<15 or
hue is between 20 and 40)
'''
# Your code
color = Color(pixel_color)
hue = color.hue_degrees()
R = pixel_color[0]
G = pixel_color[1]
B = pixel_color[2]
if ((20 <= hue and hue <= 40) or (B - G) < 15 or
(B - R) < 15) and color.intensity < 80:
return True
return False
def is_skin_hair_block(self, block, block_type):
'''
Return true if the block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
is a skin/hair-block - it is if a majority (as per the threshold attribute) of the pixels in the block are
skin/hair colored. 'block_type' says whether we are testing for a skin block ('s') or a hair block ('h).
'''
# Your code
count_pixels = 0
for pixely in range(block[1], block[1] + self.block_size):
for pixelx in range(block[0], block[0] + self.block_size):
if pixelx < self.pyim.size()[0] and pixely < self.pyim.size(
)[1]:
color = self.pyim.get_rgba(pixelx, pixely)
if block_type == 's' and self.is_skin(color):
count_pixels += 1
elif block_type == 'h' and self.is_hair(color):
count_pixels += 1
if ((count_pixels * 1.0) /
(self.block_size * self.block_size * 1.0)) >= self.majority:
return True
return False
def add_neighbour_blocks(self, block, graph):
'''
Given a block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
and a graph (could be a hair or a skin graph), add edges from the current block to its neighbours
on the image that are already nodes of the graph
Check blocks to the left, top-left and top of the current block and if any of these blocks is in the
graph (means the neighbour is also of the same type - skin or hair) add an edge from the current block
to the neighbour.
'''
# Your code
valid_rangeX = range(self.pyim.size()[0])
valid_rangeY = range(self.pyim.size()[1])
length = self.block_size
if self.is_skin_hair_block(block, 's'):
if (block[0] - length) in valid_rangeX and self.is_skin_hair_block(
(block[0] - length, block[1]), 's'):
self.skin.add_edge((block[0] - length, block[1]), block)
if (block[0] - length) in valid_rangeX and (
block[1] -
length) in valid_rangeY and self.is_skin_hair_block(
(block[0] - length, block[1] - length), 's'):
self.skin.add_edge((block[0] - length, block[1] - length),
block)
if (block[1] - length) in valid_rangeY and self.is_skin_hair_block(
(block[0], block[1] - length), 's'):
self.skin.add_edge((block[0], block[1] - length), block)
if (block[0] + length) in valid_rangeX and (
block[1] -
length) in valid_rangeY and self.is_skin_hair_block(
(block[0] + length, block[1] - length), 's'):
self.skin.add_edge((block[0] + length, block[1] - length),
block)
elif self.is_skin_hair_block(block, 'h'):
#self.hair.add_node(block)
if (block[0] - length) in valid_rangeX and self.is_skin_hair_block(
(block[0] - length, block[1]), 'h'):
self.hair.add_edge((block[0] - length, block[1]), block)
if (block[0] - length) in valid_rangeX and (
block[1] -
length) in valid_rangeY and self.is_skin_hair_block(
(block[0] - length, block[1] - length), 'h'):
self.hair.add_edge((block[0] - length, block[1] - length),
block)
if (block[1] - length) in valid_rangeY and self.is_skin_hair_block(
(block[0], block[1] - length), 'h'):
self.hair.add_edge((block[0], block[1] - length), block)
if (block[0] + length) in valid_rangeX and (
block[1] -
length) in valid_rangeY and self.is_skin_hair_block(
(block[0] + length, block[1] - length), 'h'):
self.hair.add_edge((block[0] + length, block[1] - length),
block)
def make_block_graph(self):
'''
Return the skin and hair graphs - nodes are the skin/hair blocks respectively
Initialize skin and hair graphs. For every block if it is a skin(hair) block
add edges to its neighbour skin(hair) blocks in the corresponding graph
For this to work the blocks have to be traversed in the top->bottom, left->right order
'''
# Your code
for col in range(0, self.pyim.size()[1], self.block_size):
for row in range(0, self.pyim.size()[0], self.block_size):
if self.is_skin_hair_block((row, col), 's'):
self.skin.add_node((row, col))
self.add_neighbour_blocks((row, col), self.skin.adjacency)
elif self.is_skin_hair_block((row, col), 'h'):
self.hair.add_node((row, col))
self.add_neighbour_blocks((row, col), self.hair.adjacency)
def find_bounding_box(self, component):
'''
Return the bounding box - a box is a pair of tuples - ((minx, miny), (maxx, maxy)) for the component
Argument 'component' - is just the list of blocks in that component where each block is represented by the
coordinates of its top-left pixel.
'''
# Your code
minx = miny = maxx = maxy = component[0][0]
for tup in component:
if tup[0] > maxx:
maxx = tup[0]
if tup[0] < minx:
minx = tup[0]
if tup[1] > maxy:
maxy = tup[1]
if tup[1] < miny:
miny = tup[1]
return ((minx, miny), (maxx, maxy))
def skin_hair_match(self, skin_box, hair_box):
'''
Return True if the skin-box and hair-box given are matching according to one of the pre-defined patterns
'''
# Your code
hair_minx = hair_box[0][0]
hair_miny = hair_box[0][1]
hair_maxx = hair_box[1][0]
hair_maxy = hair_box[1][1]
skin_minx = skin_box[0][0]
skin_miny = skin_box[0][1]
skin_maxx = skin_box[1][0]
skin_maxy = skin_box[1][1]
'''if skin_miny == hair_maxy :
return True
if skin_miny > hair_miny and skin_minx > hair_minx and skin_maxy < hair_maxy and skin_maxx > hair_maxx:
return True
if hair_miny < skin_miny and skin_miny < hair_maxy:
return True
if hair_maxy == skin_maxy :
return True
if skin_miny <= hair_miny and hair_maxy <= skin_maxy:
return True'''
return True
def detect_faces(self):
'''
Main method - to detect faces in the image that this class was initialized with
Return list of face boxes - a box is a pair of tuples - ((minx, miny), (maxx, maxy))
Algo: (i) Make block graph (ii) get the connected components of the graph (iii) filter the connected components
(iv) find bounding box for each component (v) Look for matches between face and hair bounding boxes
Return the list of face boxes that have matching hair boxes
'''
# Your code
index = 0
self.make_block_graph()
skin_components = self.skin.get_connected_components()
hair_components = self.hair.get_connected_components()
skin_components = skin_components[::-1]
hair_components = hair_components[::-1]
for comp in skin_components:
if len(comp) >= self.min_component_size:
break
index += 1
skin_components = skin_components[index:]
index = 0
for comp in hair_components:
if len(comp) >= self.min_component_size:
break
index += 1
hair_components = hair_components[index:]
skin_boxes = []
hair_boxes = []
for component in skin_components:
skin_boxes.append(self.find_bounding_box(component))
for component in hair_components:
hair_boxes.append(self.find_bounding_box(component))
faces = []
for skin_box in skin_boxes:
for hair_box in hair_boxes:
if self.skin_hair_match(skin_box, hair_box):
faces.append(skin_box)
return faces, hair_boxes
def mark_box(self, box, color):
'''
Mark the box (same as in the above methods) with a given color (given as a raw triple)
This is just a one-pixel wide line showing the box.
'''
# Your code
minx = box[0][0]
miny = box[0][1]
maxx = box[1][0]
maxy = box[1][1]
if minx > self.pyim.size()[0]:
minx = self.pyim.size()[0] - 1
if maxx > self.pyim.size()[0]:
maxx = self.pyim.size()[0] - 1
if miny > self.pyim.size()[1]:
miny = self.pyim.size()[1] - 1
if maxy > self.pyim.size()[1]:
maxy = self.pyim.size()[1] - 1
for pixel in range(minx, maxx + 1):
self.pyim.set(pixel, miny, color)
for pixel in range(minx, maxx + 1):
self.pyim.set(pixel, maxy, color)
for pixel in range(miny, maxy + 1):
self.pyim.set(minx, pixel, color)
for pixel in range(miny, maxy + 1):
self.pyim.set(maxx, pixel, color)
def mark_faces(self, marked_file):
'''
Detect faces and mark each face detected -- mark the bounding box of each face in red
and save the marked image in a new file
'''
# Your code
faces, hair = self.detect_faces()
for box in faces:
self.mark_box(box, (255, 0, 0))
'''for box in hair:
self.mark_box(box, (0, 255, 0))'''
self.pyim.save(marked_file)
class MeanShift(object):
'''This class should host all images related to the process of performing a
meanshift segmentation on an image, housing functions that will perform
such operation. It consists of a base image and two derived images, a
filtered one and a segmented one.'''
def __init__(self):
self.original = None
self.msFilter = None
self.msSegment = None
# ------------------------------------------------------------------------
# Input and Output functions
# ------------------------------------------------------------------------
def loadFile(self, filepath):
'''This function should open the image contained in the specified file
and reset the class' variables'''
# Reset variables
self.msFilter = None
self.msSegment = None
# Load image
self.original = PyImage()
self.original.loadFile(filepath)
def loadImage(self, image):
'''This function should open the image contained in the specified file
and reset the class' variables'''
# Reset variables
self.msFilter = None
self.msSegment = None
# Load image
self.original = PyImage()
self.original.loadImage(image)
def saveFile(self, filepath):
'''This function should save to disk the class' current state, by
saving not only the original file, but also the meanshifted ones, if
they exist.'''
# Separate name from extension
path = filepath.split('.')
extension = '.' + path[-1]
path = "".join(path[:-1]) + '-'
# Save original
self.original.saveFile(path + "original" + extension)
# Save meanshift filter
if self.msFilter is not None:
self.msFilter.saveFile(path + "filter" + extension)
# Save meanshift segment
if self.msSegment is not None:
self.msSegment.saveFile(path + "segment" + extension)
# ------------------------------------------------------------------------
# Meanshift functions
# ------------------------------------------------------------------------
def kernelIteration(self, tid, hs, hr, k, kernels, newKernels, eps,
kdTree, checks, ttot, img, valid):
'''This function is responsible for computing the new kernel position
for kernels [id*size:(id+1)*size]'''
for j in np.arange(tid*(img.height)/ttot, (tid+1)*(img.height)/ttot):
for i in np.arange(img.width):
# If not valid, do not iterate again
if not valid[j][i]:
newKernels[j][i] = kernels[j][i]
checks[j][i] = 0
continue
# Initialize variables
mainSum = np.zeros((5), dtype="float64")
weightSum = 0.0
base = kernels[j][i]
# Query kdtree for the k nearest neighbors
dist, index = kdTree.query(base,
k+1,
distance_upper_bound=hr,
n_jobs=-1)
# Iterate each pair distance, index found on the query
for d, ind in itertools.izip(dist[1:], index[1:]):
# Get 2D indexes from 1D one
if ind == kdTree.n:
continue
y = ind/img.width
x = ind - y*img.width
data = kernels[y][x]
# Compute spatial component
weightS = data[:2] - base[:2]
weightS = weightS/(1.0 * hs)
weightS = -sum(weightS ** 2)
weightS = np.exp(weightS)
# Compute color component
weightR = data[2:] - base[2:]
weightR = weightR/(1.0 * hr)
weightR = -sum(weightR ** 2)
weightR = np.exp(weightR)
# Resulting weight is product
weight = weightS * weightR
# Add things to variables
weightSum += weight
mainSum += data * weight
# Once done, result is average, if zero, kernel doesnt move
if weightSum == 0:
print i, j
newKernels[j][i] = kernels[j][i]
checks[j][i] = 0
valid[j][i] = False
continue
result = mainSum / weightSum
newKernels[j][i] = result
norm = np.linalg.norm(result - base)
checks[j][i] = norm
if norm < eps:
valid[j][i] = False
def meanshiftFilter(self, hs, hr, k, eps, lim, ops):
'''This function should apply a meanshift filter throughout the image.
It takes in several arguments, including softening coefficients hs and
hr, respectively for regular space and color space; k, the number of
nearest neighbors to look at when filtering a local kernel; and eps,
the threshold used to stop the iterating.'''
# Make copies of original image
img = self.original.copy()
kerns = self.original.copy()
# Initialize kernel matrix
kernels = np.empty((img.height, img.width, 5), dtype="float64")
# Fill first dimension with kernel's Y coordinate value
for j in np.arange(img.height):
kernels[j, :, 0] = j
# Fill second dimension with kernel's X coordinate value
for i in np.arange(img.width):
kernels[:, i, 1] = i
# Fill remaining dimensions with LUV values for each pixel
# kerns.convertRGBtoLUV()
kerns.pixels = kerns.pixels.astype("float64")
kernels[:, :, 2:] = kerns.pixels
# Initialize variables
checks = np.zeros((img.height, img.width))
boolchecks = checks.astype("bool")
size = img.height * img.width
kdTree = None
newKernels = None
firstKernel = kernels.copy()
# Initialize list of valid points for iteration
valid = np.ones((img.height, img.width), dtype="bool")
# Iteration count
count = 0
# Wait until all vectors' magnitudes go below threshold
while False in boolchecks:
# Interrupt if exceeps iteration limit
count += 1
if count > ops:
break
# Iniitalize vectors for this iteration
newKernels = np.empty((img.height, img.width, 5), dtype="float64")
# Initialize kdtree with linearized matrix and optimized for space
kdTree = sp.spatial.cKDTree(kernels.reshape((size, 5)),
compact_nodes=True,
balanced_tree=True)
# Initialize threads, iterate every kernel
threads = []
for i in range(4):
t = threading.Thread(name=str(i), target=self.kernelIteration,
args=(i, hs, hr, k, kernels, newKernels,
eps, kdTree, checks, 4, img, valid))
t.setDaemon(True) # Doesn't block main program
threads.append(t)
t.start()
# Wait for threads to finish
for t in threads:
t.join()
# Update kernels
kernels = newKernels
print "DEBUG: Checks min/avg/max:", checks.min(), \
np.mean(checks), checks.max()
# Maps check values to epsilon
boolchecks = checks <= eps
# Check how many kernels did not move
stopped = np.count_nonzero(boolchecks)
print "DEBUG: Fixed kernels:", stopped
# Interrupt if limit is achieved
if stopped >= (lim*size):
break
# Once done, copy the original pixel matrix
kernelsCopy = kernels.astype("uint64")
colors = img.pixels.copy()
# Iterate every pixel, changing its color from the one given by its
# kernel after iterating. Rounds down no matter the decimal part
for j in np.arange(img.height):
for i in np.arange(img.width):
y, x = kernelsCopy[j][i][:2]
img.pixels[j][i] = colors[y][x]
# Store result in class variable
img.updateImage()
self.msFilter = img.copy()
def meanshiftSegment(self, hr, filepath=None):
'''This function should perform a segmentation in an image based on
each pixel's color. It groups together neighbor pixels that have
similar colors. By default, it uses the filtered image in the class
instance, but it can load a file specified as a argument.'''
# Check where to load image from, default is class instance
if self.msFilter is None:
# Check if filepath is specified
if filepath is None:
# Neither sources are valid, abort
print "\nERROR: Please either " + \
"specify a file or run meanshiftFilter.\n"
return
else:
# Load image from filepath
img = PyImage()
img.loadFile(filepath)
else:
# Load image from class
img = self.msFilter.copy()
# Start group matrix with zeros
groups = np.zeros((img.height, img.width), dtype="int32")
groups -= 1
pixels = [] # List of pixels per group
colors = [] # Average color per group
lastGroup = 0
# Iterate pixels, assigning group to each pixel
for j in np.arange(img.height):
for i in np.arange(img.width):
# If pixel has no group, set a new group for it
if groups[j][i] == -1:
groups[j][i] = lastGroup
lastGroup += 1
pixels.append([(j, i)])
colors.append(img.pixels[j][i].astype("float64"))
# Get pixel neighbors
neighbors = []
if j:
neighbors.append((j-1, i, img.pixels[j-1][i]))
if i:
neighbors.append((j-1, i-1, img.pixels[j-1][i-1]))
if i < img.width - 1:
neighbors.append((j-1, i+1, img.pixels[j-1][i+1]))
if j < img.height - 1:
neighbors.append((j+1, i, img.pixels[j+1][i]))
if i:
neighbors.append((j+1, i-1, img.pixels[j+1][i-1]))
if i < img.width - 1:
neighbors.append((j+1, i+1, img.pixels[j+1][i+1]))
if i:
neighbors.append((j, i-1, img.pixels[j][i-1]))
if i < img.width - 1:
neighbors.append((j, i+1, img.pixels[j][i+1]))
# For each neighbor, check if color is similar
group = groups[j][i]
for neighbor in neighbors:
# Compute color difference
cDiff = colors[group] - neighbor[2]
cDiff = sum(cDiff ** 2)
cDiff **= 0.5
if cDiff < hr:
# Color is similar in all 3 channels, put neighbor as
# same group as current pixel
groups[neighbor[0]][neighbor[1]] = group
oldGroupLen = len(pixels[group])
pixels[group].append((neighbor[0], neighbor[1]))
color = colors[group]
color *= oldGroupLen
color += neighbor[2]
color /= oldGroupLen + 1
colors[group] = color
print lastGroup
# Iterate groups
for g in range(lastGroup):
# Iterate pixels, updating their colors
color = colors[g].astype("uint8")
for pixel in pixels[g]:
img.pixels[pixel[0]][pixel[1]] = color
# Store result
self.msSegment = img
def meanshiftSegment(self, hr, filepath=None):
'''This function should perform a segmentation in an image based on
each pixel's color. It groups together neighbor pixels that have
similar colors. By default, it uses the filtered image in the class
instance, but it can load a file specified as a argument.'''
# Check where to load image from, default is class instance
if self.msFilter is None:
# Check if filepath is specified
if filepath is None:
# Neither sources are valid, abort
print "\nERROR: Please either " + \
"specify a file or run meanshiftFilter.\n"
return
else:
# Load image from filepath
img = PyImage()
img.loadFile(filepath)
else:
# Load image from class
img = self.msFilter.copy()
# Start group matrix with zeros
groups = np.zeros((img.height, img.width), dtype="int32")
groups -= 1
pixels = [] # List of pixels per group
colors = [] # Average color per group
lastGroup = 0
# Iterate pixels, assigning group to each pixel
for j in np.arange(img.height):
for i in np.arange(img.width):
# If pixel has no group, set a new group for it
if groups[j][i] == -1:
groups[j][i] = lastGroup
lastGroup += 1
pixels.append([(j, i)])
colors.append(img.pixels[j][i].astype("float64"))
# Get pixel neighbors
neighbors = []
if j:
neighbors.append((j-1, i, img.pixels[j-1][i]))
if i:
neighbors.append((j-1, i-1, img.pixels[j-1][i-1]))
if i < img.width - 1:
neighbors.append((j-1, i+1, img.pixels[j-1][i+1]))
if j < img.height - 1:
neighbors.append((j+1, i, img.pixels[j+1][i]))
if i:
neighbors.append((j+1, i-1, img.pixels[j+1][i-1]))
if i < img.width - 1:
neighbors.append((j+1, i+1, img.pixels[j+1][i+1]))
if i:
neighbors.append((j, i-1, img.pixels[j][i-1]))
if i < img.width - 1:
neighbors.append((j, i+1, img.pixels[j][i+1]))
# For each neighbor, check if color is similar
group = groups[j][i]
for neighbor in neighbors:
# Compute color difference
cDiff = colors[group] - neighbor[2]
cDiff = sum(cDiff ** 2)
cDiff **= 0.5
if cDiff < hr:
# Color is similar in all 3 channels, put neighbor as
# same group as current pixel
groups[neighbor[0]][neighbor[1]] = group
oldGroupLen = len(pixels[group])
pixels[group].append((neighbor[0], neighbor[1]))
color = colors[group]
color *= oldGroupLen
color += neighbor[2]
color /= oldGroupLen + 1
colors[group] = color
print lastGroup
# Iterate groups
for g in range(lastGroup):
# Iterate pixels, updating their colors
color = colors[g].astype("uint8")
for pixel in pixels[g]:
img.pixels[pixel[0]][pixel[1]] = color
# Store result
self.msSegment = img
class FaceDetector(object):
'''
classdocs
'''
def __init__(self, filename, block_size = 5, min_component_size = 10, majority = 0.5):
'''
Constructor - keeps input image filename, image read from the file as a PyImage object, block size (in pixels),
threshold to decide how many skin color pixels are required to declare a block as a skin-block
and min number of blocks required for a component. The majority argument says what fraction of
the block pixels must be skin/hair colored for the block to be a skin/hair block - the default value is
0.5 (half).
'''
# Your code
self.filename=filename
self.block_size=block_size
self.majority=majority
self.min_component_size=min_component_size
self.pyimage_obj = PyImage(filename)
def skin_green_limits(self, red):
'''
Return the limits of normalized green given the normalized red component as a tuple (min, max)
'''
return ((-0.776*red*red + 0.5601*red + 0.18), (-1.376*red*red + 1.0743*red + 0.2))
def is_skin(self, pixel_color):
'''
Given the pixel color (as a Color object) return True if it represents the skin color
Color is skin if hue in degrees is (> 240 or less than or equal to 20) and
green is in the green limits and it is not white
'''
# Your code
pix_color=pixel_color.hue_degrees()
green=pixel_color.g
limit=self.skin_green_limits(pixel_color.r)
if(pix_color!=255):
if(pix_color>240 or pix_color<=20):
if (green>=limit[0] and green<=limit[1]):
return True
return False
def is_hair(self, pixel_color):
'''
Return True if the pixel color represents hair - it is if intensity < 80 and ((B-G)<15 or (B-R)<15 or
hue is between 20 and 40)
'''
# Your code
rgb=pixel_color.rgb_abs()
if(pixel_color.intensity<80):
if((rgb[2]-rgb[1])<15 or (rgb[2]-rgb[0])<15 or pixel_color.hue() in range(20,40)):
return True
return False
def is_skin_hair_block(self, block, block_type):
'''
Return true if the block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
is a skin/hair-block - it is if a majority (as per the threshold attribute) of the pixels in the block are
skin/hair colored. 'block_type' says whether we are testing for a skin block ('s') or a hair block ('h).
'''
# Your code
pyimage_object=PyImage(self.filename)
k=pyimage_object.size()
pixel_x=block[0]
pixel_y=block[1]
lst_skin=[]
lst_hair=[]
temp_x=0
while(pixel_x<k[0]):
temp_y=0
while(pixel_y<k[1] and temp_y<self.block_size):
rgb_a=pyimage_object.get_rgba(pixel_x,pixel_y)
pixel_color=Color(rgb_a)
if(self.is_skin(pixel_color)):
lst_skin.append((pixel_x,pixel_y))
elif(self.is_hair(pixel_color)):
lst_hair.append((pixel_x,pixel_y))
pixel_y+=1
temp_y+=1
pixel_x+=1
temp_x+=1
if(len(lst_skin)>self.majority*self.block_size and block_type=='s'):
return True
elif(len(lst_hair)>self.majority*self.block_size and block_type=='h'):
return True
else:
return False
def add_neighbour_blocks(self, block, graph):
'''
Given a block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
and a graph (could be a hair or a skin graph), add edges from the current block to its neighbours
on the image that are already nodes of the graph
Check blocks to the left, top-left and top of the current block and if any of these blocks is in the
graph (means the neighbour is also of the same type - skin or hair) add an edge from the current block
to the neighbour.
'''
# Your code
pyimage_object=PyImage(self.filename)
k=pyimage_object.size()
if(self.is_skin_hair_block(block, 's')):
if((block[0]-self.block_size)<k[0] and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0]-self.block_size,block[1]), 's')):
graph.add_edge(block, (block[0]-self.block_size,block[1]))
if((block[1]-self.block_size)<k[1] and (block[1]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 's')):
graph.add_edge(block, (block[0],block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]-self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 's')):
graph.add_edge(block, (block[0]-self.block_size,block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]+self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]+self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 's')):
graph.add_edge(block, (block[0]+self.block_size,block[1]-self.block_size))
if(self.is_skin_hair_block(block, 'h')):
if((block[0]-self.block_size)<k[0] and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0]-self.block_size,block[1]), 'h')):
graph.add_edge(block, (block[0]-self.block_size,block[1]))
if((block[1]-self.block_size)<k[1] and (block[1]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 'h')):
graph.add_edge(block, (block[0],block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]-self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]-self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 'h')):
graph.add_edge(block, (block[0]-self.block_size,block[1]-self.block_size))
if((block[1]-self.block_size)<k[1] and (block[0]+self.block_size)<k[0] and (block[1]-self.block_size)>=0 and (block[0]+self.block_size)>=0 and self.is_skin_hair_block((block[0],block[1]-self.block_size), 'h')):
graph.add_edge(block, (block[0]+self.block_size,block[1]-self.block_size))
def make_block_graph(self):
'''
Return the skin and hair graphs - nodes are the skin/hair blocks respectively
Initialize skin and hair graphs. For every block if it is a skin(hair) block
add edges to its neighbour skin(hair) blocks in the corresponding graph
For this to work the blocks have to be traversed in the top->bottom, left->right order
'''
# Your code
graph_object_skin=Graph(None)
graph_object_hair=Graph(None)
pyimage_object=PyImage(self.filename)
k=pyimage_object.size()
i=0
while(i<k[0]):
# 3
j=0
while(j<k[1]):
# 4
if(self.is_skin_hair_block((i,j),'s')):
graph_object_skin.add_node((i,j))
self.add_neighbour_blocks((i,j),graph_object_skin)
j+=self.block_size
i+=self.block_size
i=0
while(i<k[0]):
# 5
j=0
while(j<k[1]):
# 6
if(self.is_skin_hair_block((i,j),'h')):
graph_object_hair.add_node((i,j))
self.add_neighbour_blocks((i,j),graph_object_hair)
j+=self.block_size
i+=self.block_size
return graph_object_skin, graph_object_hair
def find_bounding_box(self, component):
'''
Return the bounding box - a box is a pair of tuples - ((minx, miny), (maxx, maxy)) for the component
Argument 'component' - is just the list of blocks in that component where each block is represented by the
coordinates of its top-left pixel.
'''
# Your code
temp_y=[]
temp_x=[]
size_component=len(component)
i=0
while(i<size_component):
# 7
temp_y+=[component[i][1]]
i+=1
i=0
while(i<size_component):
# 7
temp_x+=[component[i][0]]
i+=1
return ((min(temp_x),min(temp_y)),(max(temp_x),max(temp_y)))
def skin_hair_match(self, skin_box, hair_box):
'''
Return True if the skin-box and hair-box given are matching according to one of the pre-defined patterns
'''
# Your code
skin_coord=self.find_bounding_box(skin_box)
hair_coord=self.find_bounding_box(hair_box)
min_x_skin=skin_coord[0][0]
min_y_skin=skin_coord[0][1]
max_x_skin=skin_coord[1][0]
max_y_skin=skin_coord[1][1]
min_x_hair=hair_coord[0][0]
min_y_hair=hair_coord[0][1]
max_x_hair=hair_coord[1][0]
max_y_hair=hair_coord[1][1]
if min_x_skin>min_x_hair and max_x_skin<max_x_hair and min_y_skin>min_y_hair and max_y_skin<max_y_hair :
return True #2
elif min_x_skin>min_x_hair and max_x_skin<max_x_hair and min_y_skin>min_y_hair and max_y_skin>max_y_hair :
return True #3
elif min_x_skin>min_x_hair and max_x_skin<max_x_hair and min_y_skin>min_y_hair and max_y_skin==max_y_hair :
return True #4
elif min_x_skin<min_x_hair and max_x_skin<max_x_hair and min_y_skin>min_y_hair and max_y_skin>max_y_hair :
return True #5
elif min_x_skin>min_x_hair and max_x_skin>max_x_hair and min_y_skin>min_y_hair and max_y_skin>max_y_hair :
return True #6
elif min_x_skin==min_x_hair and max_x_skin<max_x_hair and min_y_skin>min_y_hair and max_y_skin>max_y_hair :
return True #7
elif min_x_skin>min_x_hair and max_x_skin==max_x_hair and min_y_skin>min_y_hair and max_y_skin>max_y_hair :
return True #8
elif min_x_skin>min_x_hair and max_x_skin<max_x_hair and min_y_skin<min_y_hair and max_y_skin>max_y_hair :
return True #9
elif min_x_skin>min_x_hair and max_x_skin>max_x_hair and min_y_skin<min_y_hair and max_y_skin>max_y_hair :
return True #10
elif min_x_skin<min_x_hair and max_x_skin<max_x_hair and min_y_skin<min_y_hair and max_y_skin>max_y_hair :
return True #11
elif min_x_skin==min_x_hair and max_x_skin<max_x_hair and min_y_skin>min_y_hair and max_y_skin==max_y_hair :
return True #12
elif min_x_skin>min_x_hair and max_x_skin--max_x_hair and min_y_skin>min_y_hair and max_y_skin==max_y_hair :
return True #13
elif min_x_skin<min_x_hair and max_x_skin>max_x_hair and min_y_skin==max_y_hair:
return True #14
elif min_x_skin<min_x_hair and max_x_skin==max_x_hair and min_y_skin==max_y_hair:
return True #15
elif min_x_skin==min_x_hair and max_x_skin>max_x_hair and min_y_skin==max_y_hair:
return True #16
elif min_x_skin==min_x_hair and max_x_skin==max_x_hair and min_y_skin==max_y_hair:
return True #1
else:
return False
def detect_faces(self):
'''
Main method - to detect faces in the image that this class was initialized with
Return list of face boxes - a box is a pair of tuples - ((minx, miny), (maxx, maxy))
Algo: (i) Make block graph (ii) get the connected components of the graph (iii) filter the connected components
(iv) find bounding box for each component (v) Look for matches between face and hair bounding boxes
Return the list of face boxes that have matching hair boxes
'''
# Your code
#1
skin_object, hair_object=self.make_block_graph()
skin_list=skin_object.get_connected_components()
hair_list=hair_object.get_connected_components()
final_list=[]
skin_list_final=[]
hair_list_final=[]
for block in skin_list:
if(len(block)>self.min_component_size):
bounding_tuple=self.find_bounding_box(block)
skin_list_final.append(bounding_tuple)
for block in hair_list:
if(len(block)>self.min_component_size):
bounding_tuple=self.find_bounding_box(block)
hair_list_final.append(bounding_tuple)
for s in skin_list_final:
for h in hair_list_final:
if(self.skin_hair_match(s,h)==True):
if s not in final_list:
final_list.append(s)
#print final_list
return final_list
def mark_box(self, box, color):
'''
Mark the box (same as in the above methods) with a given color (given as a raw triple)
This is just a one-pixel wide line showing the box.
'''
# Your code
min_x=box[0][0]
min_y=box[0][1]
max_x=box[1][0]
max_y=box[1][1]
i=min_x
while(i<max_x):
# 8
self.pyimage_obj.set(i,min_y,(254,0,0))
self.pyimage_obj.set(i,max_y,(254,0,0))
i+=1
i=min_y
while(i<max_y):
# 9
self.pyimage_obj.set(min_x,i,(254,0,0))
self.pyimage_obj.set(max_x,i,color)
i+=1
def mark_faces(self, marked_file):
'''
Detect faces and mark each face detected -- mark the bounding box of each face in red
and save the marked image in a new file
'''
# Your code
face_list=self.detect_faces()
for i in face_list:
self.mark_box(i, (254,0,0))
self.pyimage_obj.save(marked_file)
class FaceDetector(object):
'''
classdocs
'''
def __init__(self, filename, block_size = 5,
min_component_size = 10, majority = 0.5):
'''
Constructor - keeps input image filename, image read from the file as a PyImage object, block size (in pixels),
threshold to decide how many skin color pixels are required to declare a block as a skin-block
and min number of blocks required for a component. The majority argument says what fraction of
the block pixels must be skin/hair colored for the block to be a skin/hair block - the default value is
0.5 (half).
'''
self.input_image = PyImage(filename)
self.block_size = block_size
self.threshold = block_size * block_size * majority
self.component_length = min_component_size
self.majority = majority
def skin_green_limits(self, red):
'''
Return the limits of normalized green given the normalized red component as a tuple (min, max)
'''
return ((-0.776*red.norm_r*red.norm_r + 0.5601*red.norm_r + 0.18),
(-1.376*red.norm_r*red.norm_r + 1.0743*red.norm_r + 0.2))
def is_skin(self, pixel_color):
'''
Given the pixel color (as a Color object) return True if it represents the skin color
Color is skin if hue in degrees is (> 240 or less than or equal to 20) and
green is in the green limits and it is not white
'''
if((pixel_color.red > 95) and (pixel_color.green > 40)
and (pixel_color.blue > 20)
and ((max(pixel_color.red, pixel_color.green, pixel_color.blue)-
min(pixel_color.red, pixel_color.green, pixel_color.blue)) > 15 )
and (pixel_color.red-pixel_color.green > 15)
and (pixel_color.red > pixel_color.green)\
and (pixel_color.red > pixel_color.blue)):
return True
def is_hair(self, pixel_color):
'''
Return True if the pixel color represents hair - it is if intensity < 80 and ((B-G)<15 or (B-R)<15 or
hue is between 20 and 40)
'''
if (pixel_color.intensity < 80
and ((pixel_color.norm_b - pixel_color.norm_g < 15)
or (pixel_color.norm_b - pixel_color.norm_r < 15)
or pixel_color.hue() in range (20, 41))):
return True
return False
def is_skin_hair_block(self, block, block_type):
'''
Return true if the block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
is a skin/hair-block - it is if a majority (as per the threshold attribute) of the pixels in the block are
skin/hair colored. 'block_type' says whether we are testing for a skin block ('s') or a hair block ('h).
'''
block_check = 0
pixelx = block[0]
pixely = block[1]
for pixelx in range (pixelx, pixelx + self.block_size ):
for pixely in range (pixely, pixely + self.block_size):
if self.input_image.get_rgba(pixelx, pixely) == None:
continue
color_block = Color(self.input_image.get_rgba(pixelx, pixely))
if (self.is_skin(color_block) and block_type == 's'):
block_check += 1
elif (self.is_hair(color_block) and block_type == 'h'):
block_check += 1
return block_check >= self.threshold
def add_neighbour_blocks(self, block, graph):
'''
Given a block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
and a graph (could be a hair or a skin graph), add edges from the current block to its neighbours
on the image that are already nodes of the graph
Check blocks to the left, top-left and top of the current block and if any of these blocks is in the
graph (means the neighbour is also of the same type - skin or hair) add an edge from the current block
to the neighbour.
'''
for direction in ((-self.block_size, 0),
(-self.block_size, -self.block_size),
(0, -self.block_size),
(self.block_size, -self.block_size)):
if (graph.is_node((block[0] + direction[0],
block[1] + direction[1]))):
graph.add_directed_edge(block, (block[0] + direction[0],
block[1] + direction[1]))
def make_block_graph(self):
'''
Return the skin and hair graphs - nodes are the skin/hair blocks respectively
Initialize skin and hair graphs. For every block if it is a skin(hair) block
add edges to its neighbour skin(hair) blocks in the corresponding graph
For this to work the blocks have to be traversed in the top->bottom, left->right order
'''
comparison = lambda x, y : (1 if (x[0] > y[0])
else (0 if x[0]==y[0] else -1))
skin_graph = Graph(comparison)
hair_graph = Graph(comparison)
pixelx = 0
pixely = 0
while pixely <= (self.input_image.size()[1] - self.block_size):
while pixelx <= (self.input_image.size()[0] - self.block_size):
if self.is_skin_hair_block((pixelx, pixely), 's'):
skin_graph.add_node((pixelx, pixely))
self.add_neighbour_blocks((pixelx, pixely), skin_graph)
elif self.is_skin_hair_block((pixelx, pixely), 'h'):
hair_graph.add_node((pixelx, pixely))
self.add_neighbour_blocks((pixelx, pixely), hair_graph)
pixelx += self.block_size
pixelx = 0
pixely += self.block_size
return skin_graph, hair_graph
def find_bounding_box(self, component):
'''
Return the bounding box - a box is a pair of tuples - ((minx, miny), (maxx, maxy)) for the component
Argument 'component' - is just the list of blocks in that component where each block is represented by the
coordinates of its top-left pixel.
'''
pixelx_list = []
pixely_list = []
for nodes in component:
pixelx_list.append(nodes[0])
pixely_list.append(nodes[1])
return (((min(pixelx_list), min(pixely_list)),
(max(pixelx_list), max(pixely_list))))
def inside(self, corners, box):
'''
Check if a given coordinate lies inside the given box
'''
#print corners
for corner in corners:
if ((corner[0] >= box[0][0] and corner[1] >= box[0][1])
and (corner[0] <= box[1][0] and corner[1] <= box[1][1])):
return True
return False
def skin_hair_match(self, skin_box, hair_box):
'''
Return True if the skin-box and hair-box given are matching according to one of the pre-defined patterns
'''
corner = [(skin_box[0][0], skin_box[0][1]),
(skin_box[0][0], skin_box[1][1]),
(skin_box[1][0], skin_box[0][1]),
(skin_box[1][0], skin_box[1][1])]
if (self.inside(corner, hair_box)):
return True
elif skin_box[0][1] == hair_box[1][1]:
return True
else:
return False
def detect_faces(self):
'''
Main method - to detect faces in the image that this class was initialized with
Return list of face boxes - a box is a pair of tuples - ((minx, miny), (maxx, maxy))
Algo: (i) Make block graph (ii) get the connected components of the graph (iii) filter the connected components
(iv) find bounding box for each component (v) Look for matches between face and hair bounding boxes
Return the list of face boxes that have matching hair boxes
'''
skin_graph, hair_graph = self.make_block_graph()
skin_components = skin_graph.get_connected_components()
hair_components = hair_graph.get_connected_components()
filtered_skin = []
filtered_hair = []
for component in skin_components:
if len(component) >= self.component_length:
filtered_skin.append(component)
for component in hair_components:
if len(component) >= self.component_length:
filtered_hair.append(component)
bound_face = []
bound_hair = []
for block in filtered_skin:
bound_face.append(self.find_bounding_box(block))
for block in filtered_hair:
bound_hair.append(self.find_bounding_box(block))
del filtered_skin, filtered_hair
face_box = []
for component in bound_face:
for match in bound_hair:
if (self.skin_hair_match(component, match)
== True):
face_box.append(component)
return face_box
def mark_box(self, box, color):
'''
Mark the box (same as in the above methods) with a given color (given as a raw triple)
This is just a one-pixel wide line showing the box.
'''
for pixelx in range(box[0][0], box[1][0]):
self.input_image.set(pixelx, box[0][1], color)
for pixelx in range(box[0][0], box[1][0]):
self.input_image.set(pixelx, box[1][1], color)
for pixely in range(box[0][1], box[1][1]):
self.input_image.set(box[0][0], pixely, color)
for pixely in range(box[0][1], box[1][1]):
self.input_image.set(box[1][0], pixely, color)
def mark_faces(self, marked_file):
'''
Detect faces and mark each face detected -- mark the bounding box of each face in red
and save the marked image in a new file
'''
face_box = self.detect_faces()
for box in face_box:
self.mark_box(box, (255, 0, 0))
self.input_image.save(marked_file)
def meanshiftSegment(self, hr, filepath=None):
'''This function should perform a segmentation in an image based on
each pixel's color. It groups together neighbor pixels that have
similar colors. By default, it uses the filtered image in the class
instance, but it can load a file specified as a argument.'''
# Check where to load image from, default is class instance
if self.msFilter is None:
# Check if filepath is specified
if filepath is None:
# Neither sources are valid, abort
print "\nERROR: Please either " + \
"specify a file or run meanshiftFilter.\n"
return
else:
# Load image from filepath
img = PyImage()
img.loadFile(filepath)
else:
# Load image from class
img = self.msFilter.copy()
# Start group matrix with zeros
groups = np.zeros((img.height, img.width), dtype="int32")
groups -= 1
pixels = [] # List of pixels per group
colors = [] # Average color per group
lastGroup = 0
# Iterate pixels, assigning group to each pixel
for j in np.arange(img.height):
for i in np.arange(img.width):
# If pixel has no group, set a new group for it
if groups[j][i] == -1:
groups[j][i] = lastGroup
lastGroup += 1
pixels.append([(j, i)])
colors.append(img.pixels[j][i].astype("float64"))
# Get pixel neighbors
neighbors = []
if j:
neighbors.append((j - 1, i, img.pixels[j - 1][i]))
if i:
neighbors.append(
(j - 1, i - 1, img.pixels[j - 1][i - 1]))
if i < img.width - 1:
neighbors.append(
(j - 1, i + 1, img.pixels[j - 1][i + 1]))
if j < img.height - 1:
neighbors.append((j + 1, i, img.pixels[j + 1][i]))
if i:
neighbors.append(
(j + 1, i - 1, img.pixels[j + 1][i - 1]))
if i < img.width - 1:
neighbors.append(
(j + 1, i + 1, img.pixels[j + 1][i + 1]))
if i:
neighbors.append((j, i - 1, img.pixels[j][i - 1]))
if i < img.width - 1:
neighbors.append((j, i + 1, img.pixels[j][i + 1]))
# For each neighbor, check if color is similar
group = groups[j][i]
for neighbor in neighbors:
# Compute color difference
cDiff = colors[group] - neighbor[2]
cDiff = sum(cDiff**2)
cDiff **= 0.5
if cDiff < hr:
# Color is similar in all 3 channels, put neighbor as
# same group as current pixel
groups[neighbor[0]][neighbor[1]] = group
oldGroupLen = len(pixels[group])
pixels[group].append((neighbor[0], neighbor[1]))
color = colors[group]
color *= oldGroupLen
color += neighbor[2]
color /= oldGroupLen + 1
colors[group] = color
print lastGroup
# Iterate groups
for g in range(lastGroup):
# Iterate pixels, updating their colors
color = colors[g].astype("uint8")
for pixel in pixels[g]:
img.pixels[pixel[0]][pixel[1]] = color
# Store result
self.msSegment = img
class MeanShift(object):
'''This class should host all images related to the process of performing a
meanshift segmentation on an image, housing functions that will perform
such operation. It consists of a base image and two derived images, a
filtered one and a segmented one.'''
def __init__(self):
self.original = None
self.msFilter = None
self.msSegment = None
# ------------------------------------------------------------------------
# Input and Output functions
# ------------------------------------------------------------------------
def loadFile(self, filepath):
'''This function should open the image contained in the specified file
and reset the class' variables'''
# Reset variables
self.msFilter = None
self.msSegment = None
# Load image
self.original = PyImage()
self.original.loadFile(filepath)
def loadImage(self, image):
'''This function should open the image contained in the specified file
and reset the class' variables'''
# Reset variables
self.msFilter = None
self.msSegment = None
# Load image
self.original = PyImage()
self.original.loadImage(image)
def saveFile(self, filepath):
'''This function should save to disk the class' current state, by
saving not only the original file, but also the meanshifted ones, if
they exist.'''
# Separate name from extension
path = filepath.split('.')
extension = '.' + path[-1]
path = "".join(path[:-1]) + '-'
# Save original
self.original.saveFile(path + "original" + extension)
# Save meanshift filter
if self.msFilter is not None:
self.msFilter.saveFile(path + "filter" + extension)
# Save meanshift segment
if self.msSegment is not None:
self.msSegment.saveFile(path + "segment" + extension)
# ------------------------------------------------------------------------
# Meanshift functions
# ------------------------------------------------------------------------
def kernelIteration(self, tid, hs, hr, k, kernels, newKernels, eps, kdTree,
checks, ttot, img, valid):
'''This function is responsible for computing the new kernel position
for kernels [id*size:(id+1)*size]'''
for j in np.arange(tid * (img.height) / ttot,
(tid + 1) * (img.height) / ttot):
for i in np.arange(img.width):
# If not valid, do not iterate again
if not valid[j][i]:
newKernels[j][i] = kernels[j][i]
checks[j][i] = 0
continue
# Initialize variables
mainSum = np.zeros((5), dtype="float64")
weightSum = 0.0
base = kernels[j][i]
# Query kdtree for the k nearest neighbors
dist, index = kdTree.query(base,
k + 1,
distance_upper_bound=hr,
n_jobs=-1)
# Iterate each pair distance, index found on the query
for d, ind in itertools.izip(dist[1:], index[1:]):
# Get 2D indexes from 1D one
if ind == kdTree.n:
continue
y = ind / img.width
x = ind - y * img.width
data = kernels[y][x]
# Compute spatial component
weightS = data[:2] - base[:2]
weightS = weightS / (1.0 * hs)
weightS = -sum(weightS**2)
weightS = np.exp(weightS)
# Compute color component
weightR = data[2:] - base[2:]
weightR = weightR / (1.0 * hr)
weightR = -sum(weightR**2)
weightR = np.exp(weightR)
# Resulting weight is product
weight = weightS * weightR
# Add things to variables
weightSum += weight
mainSum += data * weight
# Once done, result is average, if zero, kernel doesnt move
if weightSum == 0:
print i, j
newKernels[j][i] = kernels[j][i]
checks[j][i] = 0
valid[j][i] = False
continue
result = mainSum / weightSum
newKernels[j][i] = result
norm = np.linalg.norm(result - base)
checks[j][i] = norm
if norm < eps:
valid[j][i] = False
def meanshiftFilter(self, hs, hr, k, eps, lim, ops):
'''This function should apply a meanshift filter throughout the image.
It takes in several arguments, including softening coefficients hs and
hr, respectively for regular space and color space; k, the number of
nearest neighbors to look at when filtering a local kernel; and eps,
the threshold used to stop the iterating.'''
# Make copies of original image
img = self.original.copy()
kerns = self.original.copy()
# Initialize kernel matrix
kernels = np.empty((img.height, img.width, 5), dtype="float64")
# Fill first dimension with kernel's Y coordinate value
for j in np.arange(img.height):
kernels[j, :, 0] = j
# Fill second dimension with kernel's X coordinate value
for i in np.arange(img.width):
kernels[:, i, 1] = i
# Fill remaining dimensions with LUV values for each pixel
# kerns.convertRGBtoLUV()
kerns.pixels = kerns.pixels.astype("float64")
kernels[:, :, 2:] = kerns.pixels
# Initialize variables
checks = np.zeros((img.height, img.width))
boolchecks = checks.astype("bool")
size = img.height * img.width
kdTree = None
newKernels = None
firstKernel = kernels.copy()
# Initialize list of valid points for iteration
valid = np.ones((img.height, img.width), dtype="bool")
# Iteration count
count = 0
# Wait until all vectors' magnitudes go below threshold
while False in boolchecks:
# Interrupt if exceeps iteration limit
count += 1
if count > ops:
break
# Iniitalize vectors for this iteration
newKernels = np.empty((img.height, img.width, 5), dtype="float64")
# Initialize kdtree with linearized matrix and optimized for space
kdTree = sp.spatial.cKDTree(kernels.reshape((size, 5)),
compact_nodes=True,
balanced_tree=True)
# Initialize threads, iterate every kernel
threads = []
for i in range(4):
t = threading.Thread(name=str(i),
target=self.kernelIteration,
args=(i, hs, hr, k, kernels, newKernels,
eps, kdTree, checks, 4, img, valid))
t.setDaemon(True) # Doesn't block main program
threads.append(t)
t.start()
# Wait for threads to finish
for t in threads:
t.join()
# Update kernels
kernels = newKernels
print "DEBUG: Checks min/avg/max:", checks.min(), \
np.mean(checks), checks.max()
# Maps check values to epsilon
boolchecks = checks <= eps
# Check how many kernels did not move
stopped = np.count_nonzero(boolchecks)
print "DEBUG: Fixed kernels:", stopped
# Interrupt if limit is achieved
if stopped >= (lim * size):
break
# Once done, copy the original pixel matrix
kernelsCopy = kernels.astype("uint64")
colors = img.pixels.copy()
# Iterate every pixel, changing its color from the one given by its
# kernel after iterating. Rounds down no matter the decimal part
for j in np.arange(img.height):
for i in np.arange(img.width):
y, x = kernelsCopy[j][i][:2]
img.pixels[j][i] = colors[y][x]
# Store result in class variable
img.updateImage()
self.msFilter = img.copy()
def meanshiftSegment(self, hr, filepath=None):
'''This function should perform a segmentation in an image based on
each pixel's color. It groups together neighbor pixels that have
similar colors. By default, it uses the filtered image in the class
instance, but it can load a file specified as a argument.'''
# Check where to load image from, default is class instance
if self.msFilter is None:
# Check if filepath is specified
if filepath is None:
# Neither sources are valid, abort
print "\nERROR: Please either " + \
"specify a file or run meanshiftFilter.\n"
return
else:
# Load image from filepath
img = PyImage()
img.loadFile(filepath)
else:
# Load image from class
img = self.msFilter.copy()
# Start group matrix with zeros
groups = np.zeros((img.height, img.width), dtype="int32")
groups -= 1
pixels = [] # List of pixels per group
colors = [] # Average color per group
lastGroup = 0
# Iterate pixels, assigning group to each pixel
for j in np.arange(img.height):
for i in np.arange(img.width):
# If pixel has no group, set a new group for it
if groups[j][i] == -1:
groups[j][i] = lastGroup
lastGroup += 1
pixels.append([(j, i)])
colors.append(img.pixels[j][i].astype("float64"))
# Get pixel neighbors
neighbors = []
if j:
neighbors.append((j - 1, i, img.pixels[j - 1][i]))
if i:
neighbors.append(
(j - 1, i - 1, img.pixels[j - 1][i - 1]))
if i < img.width - 1:
neighbors.append(
(j - 1, i + 1, img.pixels[j - 1][i + 1]))
if j < img.height - 1:
neighbors.append((j + 1, i, img.pixels[j + 1][i]))
if i:
neighbors.append(
(j + 1, i - 1, img.pixels[j + 1][i - 1]))
if i < img.width - 1:
neighbors.append(
(j + 1, i + 1, img.pixels[j + 1][i + 1]))
if i:
neighbors.append((j, i - 1, img.pixels[j][i - 1]))
if i < img.width - 1:
neighbors.append((j, i + 1, img.pixels[j][i + 1]))
# For each neighbor, check if color is similar
group = groups[j][i]
for neighbor in neighbors:
# Compute color difference
cDiff = colors[group] - neighbor[2]
cDiff = sum(cDiff**2)
cDiff **= 0.5
if cDiff < hr:
# Color is similar in all 3 channels, put neighbor as
# same group as current pixel
groups[neighbor[0]][neighbor[1]] = group
oldGroupLen = len(pixels[group])
pixels[group].append((neighbor[0], neighbor[1]))
color = colors[group]
color *= oldGroupLen
color += neighbor[2]
color /= oldGroupLen + 1
colors[group] = color
print lastGroup
# Iterate groups
for g in range(lastGroup):
# Iterate pixels, updating their colors
color = colors[g].astype("uint8")
for pixel in pixels[g]:
img.pixels[pixel[0]][pixel[1]] = color
# Store result
self.msSegment = img
class FaceDetector(object):
'''
classdocs
'''
def __init__(self,
filename,
block_size=11,
min_component_size=11,
majority=0.56):
'''
Constructor - keeps input image filename, image read from the file as a PyImage object, block size (in pixels),
threshold to decide how many skin color pixels are required to declare a block as a skin-block
and min number of blocks required for a component. The majority argument says what fraction of
the block pixels must be skin/hair colored for the block to be a skin/hair block - the default value is
0.5 (half).
'''
self.image = PyImage(filename)
self.block_sz = block_size
self.min_blocks = min_component_size
self.fraction_pixel = majority
def skin_green_limits(self, red):
'''
Return the limits of normalized green given the normalized red component as a tuple (min, max)
'''
return ((-0.776 * red * red + 0.5601 * red + 0.18),
(-1.376 * red * red + 1.0743 * red + 0.2))
def is_skin(self, pixel_color):
'''
Given the pixel color (as a Color object) return True if it represents the skin color
Color is skin if hue in degrees is (> 240 or less than or equal to 20) and
green is in the green limits and it is not white
'''
hue = pixel_color.hue_degrees()
abs_rgb = pixel_color.rgb_abs()
green = pixel_color.g
green_limits = self.skin_green_limits(pixel_color.r)
if ((hue > 240 or hue <= 20)
and (green > green_limits[0] and green < green_limits[1])
and abs_rgb != (255, 255, 255)):
return True
def is_hair(self, pixel_color):
'''
Return True if the pixel color represents hair - it is if intensity < 80 and ((B-G)<15 or (B-R)<15 or
hue is between 20 and 40)
'''
hue = pixel_color.hue_degrees()
intensity = pixel_color.intensity
abs_rgb = pixel_color.rgb_abs()
if (intensity < 80 and (abs_rgb[2] - abs_rgb[1] < 15)
or (abs_rgb[2] - abs_rgb[0] < 15) or (hue > 20 and hue < 40)):
return True
def is_skin_hair_block(self, block, block_type):
'''
Return true if the block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
is a skin/hair-block - it is if a majority (as per the threshold attribute) of the pixels in the block are
skin/hair colored. 'block_type' says whether we are testing for a skin block ('s') or a hair block ('h).
'''
total_pixels = 0
if block_type == 's':
num_pixels = 0
for width in range(block[0], block[0] + self.block_sz):
for height in range(block[1], block[1] + self.block_sz):
if 0 <= width < self.image.size(
)[0] and 0 <= height < self.image.size()[1]:
total_pixels += 1
pixel_cl = Color(self.image.get_rgba(width, height))
if self.is_skin(pixel_cl):
num_pixels += 1
else:
num_pixels = 0
for width in range(block[0], block[0] + self.block_sz):
for height in range(block[1], block[1] + self.block_sz):
if 0 <= width < self.image.size(
)[0] and 0 <= height < self.image.size()[1]:
total_pixels += 1
pixel_cl = Color(self.image.get_rgba(width, height))
if self.is_hair(pixel_cl):
num_pixels += 1
if (float(num_pixels) / total_pixels >= self.fraction_pixel):
return True
def add_neighbour_blocks(self, block, graph):
'''
Given a block (given by the argument 'block' which is the coordinate-tuple for the top-left corner)
and a graph (could be a hair or a skin graph), add edges from the current block to its neighbours
on the image that are already nodes of the graph
Check blocks to the left, top-left and top of the current block and if any of these blocks is in the
graph (means the neighbour is also of the same type - skin or hair) add an edge from the current block
to the neighbour.
'''
left_x = block[0] - self.block_sz
left_y = block[1]
topleft_x = block[0] - self.block_sz
topleft_y = block[1] - self.block_sz
top_x = block[0]
top_y = block[1] - self.block_sz
topright_x = block[0] + self.block_sz
topright_y = block[1] - self.block_sz
for (block_x, block_y) in ((left_x, left_y), (topleft_x, topleft_y),
(top_x, top_y), (topright_x, topright_y)):
if 0 <= block_x < self.image.size(
)[0] and 0 <= block_y < self.image.size()[1]:
block2 = (block_x, block_y)
graph.add_edge(block, block2)
def make_block_graph(self):
'''
Return the skin and hair graphs - nodes are the skin/hair blocks respectively
Initialize skin and hair graphs. For every block if it is a skin(hair) block
add edges to its neighbour skin(hair) blocks in the corresponding graph
For this to work the blocks have to be traversed in the top->bottom, left->right order
'''
skin_graph = Graph()
hair_graph = Graph()
for x_cordinate in range(0, self.image.size()[0] - 1, self.block_sz):
for y_cordinate in range(0,
self.image.size()[1] - 1, self.block_sz):
if self.is_skin_hair_block((x_cordinate, y_cordinate), 's'):
skin_graph.add_node((x_cordinate, y_cordinate))
self.add_neighbour_blocks((x_cordinate, y_cordinate),
skin_graph)
if self.is_skin_hair_block((x_cordinate, y_cordinate), 'h'):
hair_graph.add_node((x_cordinate, y_cordinate))
self.add_neighbour_blocks((x_cordinate, y_cordinate),
hair_graph)
return skin_graph, hair_graph
def find_bounding_box(self, component):
'''
Return the bounding box - a box is a pair of tuples - ((minx, miny), (maxx, maxy)) for the component
Argument 'component' - is just the list of blocks in that component where each block is represented by the
coordinates of its top-left pixel.
'''
minx = component[0][0]
miny = component[0][1]
maxx = component[0][0]
maxy = component[0][1]
for i in range(0, len(component), 1):
if component[i][0] < minx:
minx = component[i][0]
if component[i][1] < miny:
miny = component[i][1]
if component[i][0] > maxx:
maxx = component[i][0]
if component[i][1] > maxy:
maxy = component[i][1]
return ((minx, miny), (maxx, maxy))
def skin_hair_match(self, skin_box, hair_box):
'''
Return True if the skin-box and hair-box given are matching according to one of the pre-defined patterns
'''
if skin_box[0][0] in range(hair_box[0][0], hair_box[1][0]):
if skin_box[1][0] in range(hair_box[0][0], hair_box[1][0]):
if skin_box[0][1] in range(hair_box[0][1], hair_box[1][1]):
return True #pattern 1,2,3,4,7,8,12,13
if skin_box[0][1] in range(hair_box[0][1], hair_box[1][1]):
if skin_box[0][0] in range(hair_box[0][0], hair_box[1][0]):
return True #pattern 6
elif skin_box[1][0] in range(hair_box[0][0], hair_box[1][0]):
return True #pattern 5
if hair_box[0][1] in range(skin_box[0][1], skin_box[1][1]):
if hair_box[1][1] in range(skin_box[0][1], skin_box[1][1]):
if hair_box[1][0] in range(skin_box[0][0], skin_box[1][0]):
return True
elif hair_box[0][0] in range(skin_box[0][0], skin_box[1][0]):
return True # 9, 10, 11
def detect_faces(self):
'''
Main method - to detect faces in the image that this class was initialized with
Return list of face boxes - a box is a pair of tuples - ((minx, miny), (maxx, maxy))
Algo: (i) Make block graph (ii) get the connected components of the graph (iii) filter the connected components
(iv) find bounding box for each component (v) Look for matches between face and hair bounding boxes
Return the list of face boxes that have matching hair boxes
'''
skin_graph, hair_graph = self.make_block_graph()
skin_comp1 = skin_graph.get_connected_components()
hair_comp1 = hair_graph.get_connected_components()
skin_comp = []
hair_comp = []
list1 = len(skin_comp1)
for i in range(0, list1):
if len(skin_comp1[i]) > self.min_blocks:
skin_comp += [skin_comp1[i]]
list1 -= 1
list1 = len(hair_comp1)
for i in range(0, list1):
if len(skin_comp1[i]) > self.min_blocks:
hair_comp += [hair_comp1[i]]
list1 -= 1
face_list = []
for skin in skin_comp:
for hair in hair_comp:
skin_box = self.find_bounding_box(skin)
hair_box = self.find_bounding_box(hair)
if self.skin_hair_match(skin_box, hair_box):
face_list.append(skin_box)
return face_list
def mark_box(self, box, color):
'''
Mark the box (same as in the above methods) with a given color (given as a raw triple)
This is just a one-pixel wide line showing the box.
'''
for i in range(box[0][0], box[1][0]):
self.image.set(i, box[0][1], color)
self.image.set(i, box[1][1], color)
for i in range(box[0][1], box[1][1]):
self.image.set(box[0][0], i, color)
self.image.set(box[1][0], i, color)
def mark_faces(self, marked_file):
'''
Detect faces and mark each face detected -- mark the bounding box of each face in red
and save the marked image in a new file
'''
face_list = self.detect_faces()
for i in face_list:
self.mark_box(i, (255, 0, 0))
self.image.save(marked_file)
