def tryPointMaskerDilatedOnSamples(model):
folder = 'downloads/samples/'
for fname in os.listdir(folder):
if fname.endswith(('png', 'jpg')):
im = cv2.imread(folder + fname)
im = cv2.resize(im, (224, 224))
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
width = len(im[0])
height = len(im)
masks = getNormalizedMasksFromImage(model, im)
coords = utils.getCoordsFromPointMasks(masks, width, height,
'mean')
max_coords = utils.getCoordsFromPointMasks(masks, width, height,
'max')
#for i in range(len(masks)):
# utils.visualizeMask
# utils.visualizeCoords(, [[y_avg, x_avg]])
"""masks /= np.max(masks, axis=[1,2])
summed = np.sum(masks, axis=0)
summed = cv2.resize(summed, (height, width))
summed = np.minimum(1, summed)
"""
masks = np.moveaxis(masks, 0, -1)
#utils.visualizeCoordMasks(im, masks)
utils.visualizeCoords(im, coords)
def tryPointMaskerCascaded(model, batch_generator, sample_names=None):
if sample_names == None:
sample_names = batch_generator.all_names
for sample_name in sample_names:
inputs, _ = batch_generator.getTrainingPairFromName(sample_name)
#X, Y = batch_generator.getBatchFromNames([sample_name])
print inputs
im = inputs[0]
width = len(im[0])
height = len(im)
print 'sample name: ' + sample_name
base_masks, residual_masks = getNormalizedCascadedMasksFromImage(
model, im)
base_coords = utils.getCoordsFromPointMasks(base_masks, width, height,
'mean')
residual_coords = utils.getCoordsFromPointMasks(
residual_masks, 28, 28, 'mean')
#max_coords = utils.getCoordsFromPointMasks(base_masks, width, height, 'max')
coords = np.add(base_coords, residual_coords) - 28 / 2.0 - 4.0
#for i in range(len(residual_masks)):
# plt.imshow(residual_masks[i])
# plt.show()
utils.visualizeCoords(im, coords)
utils.visualizeCoords(im, np.concatenate([base_coords, coords],
axis=0),
np.arange(0, len(coords)))
def tryPointMaskerCascadedOnSamples(model,
folder,
output_folder,
compare_coords=None):
if not os.path.exists(output_folder):
os.mkdir(output_folder)
for fname in os.listdir(folder):
if fname.endswith(('png', 'jpg')):
im = cv2.imread(folder + '/' + fname)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
print folder + '/' + fname
im = cv2.resize(im, (224, 224))
width = len(im[0])
height = len(im)
t1 = time.time()
base_masks, residual_masks = getNormalizedCascadedMasksFromImage(
model, im)
print time.time() - t1
base_coords = utils.getCoordsFromPointMasks(
base_masks, width, height, 'mean')
residual_coords = utils.getCoordsFromPointMasks(
residual_masks, 28, 28, 'mean')
#max_coords = utils.getCoordsFromPointMasks(base_masks, width, height, 'max')
coords = np.add(base_coords, residual_coords) - 28 / 2.0
# scale back
full_im = cv2.imread(folder + '/' + fname)
full_im = cv2.cvtColor(full_im, cv2.COLOR_BGR2RGB)
scale_width = len(full_im[0]) / float(len(im[0]))
scale_height = len(full_im) / float(len(im))
print scale_width
coords[:, 0] *= scale_height
coords[:, 1] *= scale_width
#for i in range(len(residual_masks)):
# plt.imshow(residual_masks[i])
# plt.show()
#utils.visualizeCoords(im, base_coords)
output_path = output_folder + '/' + fname
key = fname[:-4]
if key in compare_coords:
utils.visualizeCoords(full_im,
np.concatenate(
[compare_coords[key], coords],
axis=0),
np.arange(0, len(compare_coords[key])),
output_name=output_path)
else:
utils.visualizeCoords(full_im, coords, output_name=output_path)
def tryPointMaskerVanilla(model, batch_generator, sample_names=None):
if sample_names == None:
sample_names = batch_generator.all_names
for sample_name in sample_names:
inputs, _ = batch_generator.getPair(sample_name)
X, Y = batch_generator.getBatchFromNames([sample_name])
im = inputs[0]
print_str = 'sample name: ' + sample_name
if batch_generator.isInValSet(sample_name):
print_str += ', from val set'
else:
print_str += ', from train set'
outputs = model.predict_on_batch(X)
preds = np.squeeze(outputs[1])
preds = utils.imSoftmax(preds)
preds = np.moveaxis(preds, -1, 0)
preds = [normalizeMask(pred) for pred in preds]
coords = utils.getCoordsFromPointMasks(preds, len(im[0]), len(im))
utils.visualizeCoords(im, coords)
print print_str
"""
summed = 80 * np.sum(preds, axis=0)
summed = cv2.resize(summed, (len(im), len(im[0])), interpolation=cv2.INTER_LINEAR)
helenUtils.visualizeMask(im, summed)
"""
"""for i in range(3):
pred = preds[i]
label = labels[i]
l = len(pred)
c = np.zeros((l, l, 3))
pred = np.maximum(pred - np.mean(pred), 0)
c[:,:,0] = pred / np.max(pred)
c[:,:,2] = label
plt.imshow(c)
plt.show()
#plt.show()
#plt.imshow(label)
#plt.show()
"""
"""
def videoTest(model):
cap = cv2.VideoCapture(0)
num_frames = 0
start = time.clock()
im_len = 640
#face_cascade = cv2.CascadeClassifier('downloads/haarcascades/haarcascade_frontalface_default.xml')
#predictor = dlib.shape_predictor('downloads/dlib/shape_predictor_68_face_landmarks.dat')
#eye_cascade = cv2.CascadeClassifier('downloads/haarcascades/haarcascade_eye.xml')
while (True):
# Capture frame-by-frame
ret, frame = cap.read()
frame = cv2.resize(frame, (720, 480), interpolation=cv2.INTER_CUBIC)
"""
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray,
scaleFactor=1.05,
minNeighbors=5,
minSize=(100, 100),
)
for (x,y,w,h) in faces:
cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)
roi_gray = gray[y:y+h, x:x+w]
roi_color = frame[y:y+h, x:x+w]
preds = getCoordsFromImage(model, roi_color)
for coord in preds:
final_coord = (x + int(round(coord[1])), y + int(round(coord[0])))
cv2.circle(frame, final_coord, 1, (0,255,0), thickness=1, lineType=8, shift=0)
"""
"""
dlib_rect = dlib.rectangle(140, 0, 540, len(frame))
detected_landmarks = predictor(frame, dlib_rect).parts()
landmarks = np.array([[p.x, p.y] for p in detected_landmarks])
min_x = np.min(landmarks[:,0])
min_y = np.min(landmarks[:,1])
max_x = np.max(landmarks[:,0])
max_y = np.max(landmarks[:,1])
cv2.rectangle(frame,(min_x,min_y),(max_x,max_y),(255,0,0),2)
y, x = min_y, min_x
h = max_y - min_y
w = max_x - min_x
print 'width and height: ' + str(w) + str(h)
if w <= 0 or h <= 0 or x < 0 or x > len(frame[0]) or y < 0 or y > len(frame):
continue
for coord in landmarks:
cv2.circle(frame, (int(coord[0]), int(coord[1])), 1, (0, 0, 255), thickness=1, lineType=8, shift=0)
"""
w = h = int(0.6 * len(frame))
y = int(0.5 * (len(frame) - h))
x = int(0.5 * (len(frame[0]) - w))
cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2)
roi_color = frame[y:y + h, x:x + w]
roi_color = cv2.cvtColor(roi_color, cv2.COLOR_BGR2RGB)
#preds = getCoordsFromImage(model, roi_color)
base_masks, residual_masks = getNormalizedCascadedMasksFromImage(
model, roi_color)
base_coords = utils.getCoordsFromPointMasks(base_masks, w, h, 'mean')
residual_coords = utils.getCoordsFromPointMasks(
residual_masks, 28, 28, 'mean')
#max_coords = utils.getCoordsFromPointMasks(base_masks, width, height, 'max')
preds = np.add(base_coords, residual_coords) - 28 / 2.0
for coord in preds:
#final_coord = (x + int(round(w / 224.0 * coord[1])), y + int(h / 224.0 * round(coord[0])))
final_coord = (x + int(coord[1]), y + int(coord[0]))
cv2.circle(frame,
final_coord,
1, (0, 255, 0),
thickness=1,
lineType=8,
shift=0)
#eyes = eye_cascade.detectMultiScale(roi_gray)
#for (ex,ey,ew,eh) in eyes:
# cv2.rectangle(roi_color,(ex,ey),(ex+ew,ey+eh),(0,255,0),2)
# Display the resulting frame
cv2.imshow('frame', frame)
num_frames += 1
if num_frames % 50 == 0:
print 'Fps : ' + str(num_frames / (time.clock() - start))
num_frames = 0
start = time.clock()
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
def getCoordsFromImage(model, im, mode='mean'):
width = len(im[0])
height = len(im)
preds = getNormalizedMasksFromImage(model, im)
coords = utils.getCoordsFromPointMasks(preds, width, height, mode)
return coords
def testNormalizedDistanceError(model, batch_generator, ibug_version=True):
"""
Parameters
----------
batch_generator:
Should be class PointsBatchGenerator.
"""
X, Y = batch_generator.getAllData()
ims, labels = X[0], Y[0]
overall_avg = 0.0
all_avgs = []
"""
for i in range(len(ims)):
im = ims[i]
factors = np.expand_dims([len(im), len(im[0])], axis=0)
label = labels[i] * factors
utils.visualizeCoords(ims[i], label)
"""
#for i in range(len(ims)):
# utils.visualizeCoords(ims[i], 224 * labels[i])
print 'Processing test set of images, counted ' + str(len(ims)) + ': '
t1 = time.time()
for i in range(len(ims)):
im = ims[i]
#labels[i] = helenUtils.normalizeCoords(labels[i], len(im[0]), len(im))
eye_dist = helenUtils.getEyeDistance(labels[i],
ibug_version=ibug_version)
lip_labels = helenUtils.getLipCoords(labels[i],
1.0,
ibug_version=ibug_version)
# vanilla version
#lip_preds = np.array(getCoordsFromImage(model, im))
# cascaded version
base_masks, residual_masks = getNormalizedCascadedMasksFromImage(
model, im)
base_coords = utils.getCoordsFromPointMasks(base_masks, 224, 224,
'mean')
residual_coords = utils.getCoordsFromPointMasks(
residual_masks, 28, 28, 'mean')
#max_coords = utils.getCoordsFromPointMasks(base_masks, width, height, 'max')
lip_preds = np.add(base_coords, residual_coords) - 28 / 2.0
lip_preds[:, 0] /= float(len(im))
lip_preds[:, 1] /= float(len(im[0]))
cur_avg = 0.0
for j in range(0, len(lip_preds)):
diff = lip_preds[j] - lip_labels[j]
dist = np.sqrt(diff.dot(diff))
normalized = dist / eye_dist
cur_avg += normalized
cur_avg /= len(lip_preds)
# expand to image dimensions to visualize
factors = np.expand_dims([len(im), len(im[0])], axis=0)
lip_preds *= factors
lip_labels *= factors
leye_coord = helenUtils.getLeyeCenter(labels[i],
ibug_version=ibug_version)
reye_coord = helenUtils.getReyeCenter(labels[i],
ibug_version=ibug_version)
leye_coord *= np.squeeze(factors)
reye_coord *= np.squeeze(factors)
all_coords = np.concatenate([lip_preds, lip_labels], axis=0)
all_coords = np.concatenate([all_coords, [leye_coord], [reye_coord]],
axis=0)
pred_indices = np.arange(0, len(all_coords) / 2)
#utils.visualizeCoords(im, all_coords, pred_indices)
#print 'eye to eye distance: ' + str(eye_dist)
#print 'avg error across all points: ' + str(cur_avg)
overall_avg += cur_avg
all_avgs.append(cur_avg)
utils.informProgress(i, len(ims))
processing_time = time.time() - t1
all_avgs = sorted(all_avgs)
overall_avg /= len(ims)
print '\nMedian normalized landmark error: ' + str(
all_avgs[len(all_avgs) / 2])
print 'Average normalized landmark error: ' + str(overall_avg)
print 'Total processing time: ' + str(processing_time)
print 'Per-image processing time: ' + str(
processing_time / float(len(ims)))