def get_train_calib_data(k = 12):
'''
for calibration net
return X - features
y - labels
cnt - count of examples
'''
sp.random.seed(42)
X_data,y_data = [],[]
suff = str(k)
c = 0
X_name = 'train_data_icalib_'+ suff + '.npz'
y_name = 'labels_icalib_'+ suff + '.npz'
label = -1
dbpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW'
dbpath = join(dbpath,'aflw.sqlite')
rfpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW\\img'
conn = sqlite3.connect(dbpath)
c = 0
for file_id,x,y,ra,rb,theta in conn.execute('SELECT file_id,x,y,ra,rb,theta FROM Faces NATURAL JOIN FaceEllipse'):
fpath = join(rfpath,file_id)
frame = fr.get_frame(fpath)
x1,y1,x2,y2 = util.ellipse2bbox(a = ra, b = rb, angle = theta, cx = x, cy = y)
x = x1
y = y1
h = abs(y2-y1)
w = abs(x2-x1)
no_neg = sp.all(sp.array([x,y,h,w]) > 0) ## ignore a bad data in sql table
if frame != None and no_neg:
y,x,w,h = [int(e) for e in (y,x,w,h)]
face = fr.get_patch(frame,y,x,(w,h))
#fr.write_frame('F:\\1\\' + str(c) + 'orig',face)
c += 1
for ((new_y,new_x,new_w,new_h),label) in [(util.calib(y,x,w,h,k),k) for k in sp.random.randint(0,45,5)]:
face = fr.get_patch(frame,new_y,new_x,(new_w,new_h))
no_neg_calib = sp.all(sp.array([new_x,new_y,new_h,new_w]) > 0)
face_r,good_example = Datasets.sample_resize(face,k,k)
if good_example and no_neg_calib:
#fr.write_frame('F:\\1\\' + str(c) + 'calib_'+str(label) ,face)
print('face:',fpath,label)
vec = fr.frame_to_vect(face_r)
X_data.append(vec)
y_data.append(label)
y_data = sp.array(y_data)
sp.savez(y_name,y_data)
X_data = sp.array(X_data)
sp.savez(X_name,X_data)
return X_data,y_data
def get_aflw_face_data(k = 12):
dbpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW'
dbpath = join(dbpath,'aflw.sqlite')
rfpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW\\img'
conn = sqlite3.connect(dbpath)
X = []
for file_id,x,y,h,w in conn.execute('SELECT file_id,x,y,h,w FROM Faces NATURAL JOIN FaceRect'):
fpath = join(rfpath,file_id)
frame = fr.get_frame(fpath)
no_neg = sp.all(sp.array([x,y,h,w]) > 0) ## ignore a bad data in sql table
if frame != None and no_neg:
face = fr.get_patch(frame,y,x,(h,w))
face_r,good_example = Datasets.sample_resize(face,k)
if good_example:
print('face:',fpath)
vec = fr.frame_to_vect(face_r)
X.append(vec)
face_flip = fr.flip_frame(face)
face_flip_r = fr.resize_frame(face_flip,(k,k))
vec = fr.frame_to_vect(face_flip_r)
X.append(vec)
#fr.write_frame('F:\\1\\'+'flip'+str(file_id),face_flip)
#fr.write_frame('F:\\1\\'+str(file_id),face)
X = sp.array(X)
y = sp.ones(len(X))
return X,y
def get_aflw_face_data(k=12):
dbpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW'
dbpath = join(dbpath, 'aflw.sqlite')
rfpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW\\img'
conn = sqlite3.connect(dbpath)
X = []
for file_id, x, y, h, w in conn.execute(
'SELECT file_id,x,y,h,w FROM Faces NATURAL JOIN FaceRect'):
fpath = join(rfpath, file_id)
frame = fr.get_frame(fpath)
no_neg = sp.all(
sp.array([x, y, h, w]) > 0) ## ignore a bad data in sql table
if frame != None and no_neg:
face = fr.get_patch(frame, y, x, (h, w))
face_r, good_example = Datasets.sample_resize(face, k)
if good_example:
print('face:', fpath)
vec = fr.frame_to_vect(face_r)
X.append(vec)
face_flip = fr.flip_frame(face)
face_flip_r = fr.resize_frame(face_flip, (k, k))
vec = fr.frame_to_vect(face_flip_r)
X.append(vec)
#fr.write_frame('F:\\1\\'+'flip'+str(file_id),face_flip)
#fr.write_frame('F:\\1\\'+str(file_id),face)
X = sp.array(X)
y = sp.ones(len(X))
return X, y
def get_aflw_face_data(k = 12, on_drive = False):
dbpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW'
dbpath = join(dbpath,'aflw.sqlite')
rfpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW\\img'
conn = sqlite3.connect(dbpath)
X = []
c = 0
for file_id,x,y,ra,rb,theta in conn.execute('SELECT file_id,x,y,ra,rb,theta FROM Faces NATURAL JOIN FaceEllipse'):
fpath = join(rfpath,file_id)
frame = fr.get_frame(fpath)
x1,y1,x2,y2 = util.ellipse2bbox(a = ra, b = rb, angle = theta, cx = x, cy = y)
x = x1
y = y1
h = abs(y2-y1)
w = abs(x2-x1)
no_neg = sp.all(sp.array([x,y,h,w]) > 0) ## ignore a bad data in sql table
if frame != None and no_neg:
y,x,w,h = [int(e) for e in (y,x,w,h)]
face = fr.get_patch(frame,y,x,(w,h))
face_r,good_example = Datasets.sample_resize(face,k,k)
if good_example:
print('face:',fpath)
vec = fr.frame_to_vect(face_r)
if not on_drive:
X.append(vec)
face_flip_r = fr.flip_frame(face_r)
vec = fr.frame_to_vect(face_flip_r)
X.append(vec)
else:
for item in Datasets.data_augmentation(frame,y,x,w,h):
fr.write_frame('F:\\train_data\\pos\\' + str(c) + '_' + str(file_id)[:-4] + '_' + 'pos',item)
c +=1
X = sp.array(X)
y = sp.ones(len(X))
return X,y
def data_augmentation(frame,y,x,w,h):
face = fr.get_patch(frame,y,x,(w,h))
face_flip = fr.flip_frame(face)
t1 = sp.random.randint(0,3)
t2 = sp.random.randint(0,3)
face_narrow = fr.get_patch(frame,y,x,(w-t2,h-t2))
face_wide = fr.get_patch(frame,y,x,(w+t2,h+t2))
t1 = sp.random.randint(0,3)
t2 = sp.random.randint(0,3)
face_shift1 = fr.get_patch(frame,y+t1,x+t1,(w+t2,h+t2))
face_shift2 = fr.get_patch(frame,y-t1,x-t1,(w-t2,h-t2))
th = float((1 if sp.random.randint(0,2) % 2 == 0 else -1) * sp.random.randint(45,90))
face_rot = fr.frame_rotate(face,theta = th)
faces_list = filter(lambda x: x != None,[face,face_flip,face_narrow,face_wide,face_shift1,face_shift2,face_rot])
return faces_list
def apply_12_net(image, iname, nn, p_level=16, k=1.18, debug=1):
py = fr.get_frame_pyramid(image, k=k, t=p_level)
suff = nn.nn_name[:2]
patch_size = int(suff)
rows, cols, d = image.shape
s = time()
rbb = []
## check last k levels
k = 1
t_level = p_level - k
for m, frame in enumerate(py):
if m < t_level:
continue
H, W, dim = frame.shape
X_test = []
pnt = []
for u in range(0, H - patch_size + 1, 4):
for v in range(0, W - patch_size + 1, 4):
subframe = fr.get_patch(frame, u, v, (patch_size, patch_size))
#subframe = frame[y:y+h,x:x+w]
X_test.append(fr.frame_to_vect(subframe))
pnt.append((v, u))
pnt = sp.array(pnt)
X_test = sp.array(X_test)
pred = nn.predict(X_test)
pnt = pnt[pred == 1]
if len(pred[pred == 1]) == 0:
print('no detection on', str(m))
continue
else:
print('detection on', str(m))
bb = []
for p in pnt:
i = p[0]
j = p[1]
if debug:
cv2.rectangle(frame, (i, j), (i + patch_size, j + patch_size),
(255, 0, 255), 1)
bb.append([i, j, i + patch_size, j + patch_size])
for e in bb:
x1, y1, x2, y2 = e
ox1 = math.floor(image.shape[1] * (x1 / frame.shape[1]))
oy1 = math.floor(image.shape[0] * (y1 / frame.shape[0]))
ox2 = math.floor(image.shape[1] * (x2 / frame.shape[1]))
oy2 = math.floor(image.shape[0] * (y2 / frame.shape[0]))
if m >= p_level - k:
rbb.append([ox1, oy1, ox2, oy2])
if debug:
cv2.imwrite(
iname[:-4] + '_pipeline_' + 'lev' + str(m) + '_' + nn.nn_name +
'_.jpg', frame)
f = time()
print('time is:', f - s)
return sp.array(rbb)
def get_train_wider_calib_data(n=None, k=12):
'''
for calibration net
return X - features
y - labels
cnt - count of examples
'''
X, y = [], []
sn = (0.83, 0.91, 1.0, 1.10, 1.21)
xn = (-0.17, 0.0, 0.17)
yn = (-0.17, 0.0, 0.17)
prod = [e for e in itertools.product(sn, xn, yn)]
inv_calib = lambda i, j, h, w, n: [
round(i - (-prod[n][1] * w / (prod[n][0]**-1))),
round(j - (-prod[n][2] * h / (prod[n][0]**-1))),
round(h / prod[n][0]**-1),
round(w / prod[n][0]**-1)
]
suff = str(k)
X_name = 'train_data_icalib_' + suff + '.npy'
y_name = 'labels_icalib_' + suff + '.npy'
root = 'F:\\Datasets\\image_data_sets\\faces\\WIDERFace\\'
pattern = "*.jpg"
bboxs = Datasets.load_wider_face(
os.path.join(root, 'wider_face_split', 'wider_face_train_v7.mat'))
for path, subdirs, files in os.walk(root, 'WIDER_train'):
for iname in files:
if fnmatch(iname, pattern):
ipath = os.path.join(path, iname)
img = fr.get_frame(ipath)
H, W = img.shape[:2]
bbox_list = bboxs[iname[:-4]]
for bbox in bbox_list:
label = sp.random.randint(0, 45)
i, j, h, w = [
int(e) for e in inv_calib(bbox[1], bbox[0],
bbox[2], bbox[3], label)
]
face = fr.get_patch(img, i, j, (h, w))
face_r, good_example = Datasets.sample_resize(face, k)
if good_example:
#print('orig:',bbox[1],bbox[0],bbox[2],bbox[3])
#print('inv_calib:',i,j,h,w)
vec_icalib = fr.frame_to_vect(face_r)
X.append(vec_icalib)
y.append(label)
print('face calib:', label, ipath)
y = sp.array(y)
sp.save(y_name, y)
X = sp.array(X)
sp.save(X_name, X)
return X, y
def apply_12_net(image,iname,nn,p_level = 16,k = 1.18,debug = 1):
py = fr.get_frame_pyramid(image,k=k,t=p_level)
suff = nn.nn_name[:2]
patch_size = int(suff)
rows,cols,d = image.shape
s = time()
rbb = []
## check last k levels
k = 1
t_level = p_level-k
for m,frame in enumerate(py):
if m < t_level:
continue
H,W,dim = frame.shape
X_test = []
pnt = []
for u in range(0,H - patch_size + 1,4):
for v in range(0,W - patch_size + 1,4):
subframe = fr.get_patch(frame,u,v,(patch_size,patch_size))
#subframe = frame[y:y+h,x:x+w]
X_test.append(fr.frame_to_vect(subframe))
pnt.append((v,u))
pnt = sp.array(pnt)
X_test = sp.array(X_test)
pred = nn.predict(X_test)
pnt = pnt[pred==1]
if len(pred[pred==1]) == 0:
print('no detection on',str(m))
continue
else:
print('detection on',str(m))
bb = []
for p in pnt:
i = p[0]
j = p[1]
if debug:
cv2.rectangle(frame, (i, j), (i+patch_size, j+patch_size), (255,0,255), 1)
bb.append([i,j,i+patch_size,j+patch_size])
for e in bb:
x1,y1,x2,y2 = e
ox1 = math.floor(image.shape[1] * (x1/frame.shape[1]))
oy1 = math.floor(image.shape[0] * (y1/frame.shape[0]))
ox2 = math.floor(image.shape[1] * (x2/frame.shape[1]))
oy2 = math.floor(image.shape[0] * (y2/frame.shape[0]))
if m >= p_level - k:
rbb.append([ox1,oy1,ox2,oy2])
if debug:
cv2.imwrite(iname[:-4]+'_pipeline_' + 'lev'+str(m) + '_' + nn.nn_name + '_.jpg',frame)
f = time()
print('time is:',f-s)
return sp.array(rbb)
def data_augmentation(frame, y, x, w, h):
face = fr.get_patch(frame, y, x, (w, h))
face_flip = fr.flip_frame(face)
t1 = sp.random.randint(0, 3)
t2 = sp.random.randint(0, 3)
face_narrow = fr.get_patch(frame, y, x, (w - t2, h - t2))
face_wide = fr.get_patch(frame, y, x, (w + t2, h + t2))
t1 = sp.random.randint(0, 3)
t2 = sp.random.randint(0, 3)
face_shift1 = fr.get_patch(frame, y + t1, x + t1, (w + t2, h + t2))
face_shift2 = fr.get_patch(frame, y - t1, x - t1, (w - t2, h - t2))
th = float((1 if sp.random.randint(0, 2) % 2 == 0 else -1) *
sp.random.randint(45, 90))
face_rot = fr.frame_rotate(face, theta=th)
faces_list = filter(lambda x: x != None, [
face, face_flip, face_narrow, face_wide, face_shift1, face_shift2,
face_rot
])
return faces_list
def apply_12_net(image,frame,iname,m,nn,debug = 0):
suff = nn.nn_name[:2]
patch_size = int(suff)
if debug:
s = time()
rbb = []
H,W,dim = frame.shape
X_test = []
pnt = []
for u in range(0,H - patch_size + 1,4):
for v in range(0,W - patch_size + 1,4):
subframe = fr.get_patch(frame,u,v,(patch_size,patch_size))
X_test.append(fr.frame_to_vect(subframe))
pnt.append((v,u))
pnt = sp.array(pnt)
X_test = sp.array(X_test)
proba = nn.predict_proba(X_test)
pred = nn.predict(X_test)
pnt = pnt[pred==1]
pos_proba = proba[pred==1][:,1]
if debug:
if len(pnt) == 0:
print('no detection on',str(m))
else:
print('detection on',str(m),frame.shape)
bb = []
for p in pnt:
i = p[0]
j = p[1]
bb.append([i,j,i+patch_size,j+patch_size])
bb = sp.array(bb)
for e in bb:
x1,y1,x2,y2 = e
ratio_x = image.shape[1]/frame.shape[1]
ratio_y = image.shape[0]/frame.shape[0]
ox1 = int(round(x1 * ratio_x))
oy1 = int(round(y1 * ratio_y))
ox2 = int(round(x2 * ratio_x))
oy2 = int(round(y2 * ratio_y))
rbb.append([ox1,oy1,ox2,oy2])
if debug:
f = time()
print('Finish... time is:',f-s)
return (sp.array(rbb),pos_proba)
def get_aflw_face_data(k=12, on_drive=False):
dbpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW'
dbpath = join(dbpath, 'aflw.sqlite')
rfpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW\\img'
conn = sqlite3.connect(dbpath)
X = []
c = 0
for file_id, x, y, ra, rb, theta in conn.execute(
'SELECT file_id,x,y,ra,rb,theta FROM Faces NATURAL JOIN FaceEllipse'
):
fpath = join(rfpath, file_id)
frame = fr.get_frame(fpath)
x1, y1, x2, y2 = util.ellipse2bbox(a=ra,
b=rb,
angle=theta,
cx=x,
cy=y)
x = x1
y = y1
h = abs(y2 - y1)
w = abs(x2 - x1)
no_neg = sp.all(
sp.array([x, y, h, w]) > 0) ## ignore a bad data in sql table
if frame != None and no_neg:
y, x, w, h = [int(e) for e in (y, x, w, h)]
face = fr.get_patch(frame, y, x, (w, h))
face_r, good_example = Datasets.sample_resize(face, k, k)
if good_example:
print('face:', fpath)
vec = fr.frame_to_vect(face_r)
if not on_drive:
X.append(vec)
face_flip_r = fr.flip_frame(face_r)
vec = fr.frame_to_vect(face_flip_r)
X.append(vec)
else:
for item in Datasets.data_augmentation(
frame, y, x, w, h):
fr.write_frame(
'F:\\train_data\\pos\\' + str(c) + '_' +
str(file_id)[:-4] + '_' + 'pos', item)
c += 1
X = sp.array(X)
y = sp.ones(len(X))
return X, y
def get_train_face_wider_data(k=12, write_to_disk=False):
'''
cut faces (positive examples) by bboxes from all images in dataset
return X - features
y - labels
cnt - count of examples
'''
X, y = [], []
root = 'F:\\Datasets\\image_data_sets\\faces\\WIDERFace\\'
pattern = "*.jpg"
bboxs = Datasets.load_wider_face(
os.path.join(root, 'wider_face_split', 'wider_face_train_v7.mat'))
for path, subdirs, files in os.walk(root, 'WIDER_train'):
for indx, iname in enumerate(files):
if fnmatch(iname, pattern):
ipath = os.path.join(path, iname)
print('face:', ipath)
img = fr.get_frame(ipath)
H, W, dim = img.shape
bbox_list = bboxs[iname[:-4]]
for bbox in bbox_list:
face = fr.get_patch(img, bbox[1], bbox[0],
(bbox[2], bbox[3]))
#fr.write_frame('F:\\1\\' + str(c),face)
face_r, good_example = Datasets.sample_resize(
face, k, k)
if good_example:
vec = fr.frame_to_vect(face_r)
X.append(vec)
y.append(1)
face_r_flip = fr.flip_frame(face_r)
vec = fr.frame_to_vect(face_r_flip)
X.append(vec)
y.append(1)
X = sp.array(X)
y = sp.array(y)
#y = sp.ones(len(X))
return X, y
def get_train_wider_calib_data(n = None,k = 12):
'''
for calibration net
return X - features
y - labels
cnt - count of examples
'''
X,y = [],[]
sn = (0.83, 0.91, 1.0, 1.10, 1.21)
xn = (-0.17, 0.0, 0.17)
yn = (-0.17, 0.0, 0.17)
prod = [e for e in itertools.product(sn,xn,yn)]
inv_calib = lambda i,j,h,w,n: [ round(i-(-prod[n][1]*w/(prod[n][0]**-1))),round(j-(-prod[n][2]*h/(prod[n][0]**-1))),round(h/prod[n][0]**-1),round(w/prod[n][0]**-1) ]
suff = str(k)
X_name = 'train_data_icalib_'+ suff + '.npy'
y_name = 'labels_icalib_'+ suff + '.npy'
root = 'F:\\Datasets\\image_data_sets\\faces\\WIDERFace\\'
pattern = "*.jpg"
bboxs = Datasets.load_wider_face(os.path.join(root,'wider_face_split','wider_face_train_v7.mat'))
for path, subdirs, files in os.walk(root,'WIDER_train'):
for iname in files:
if fnmatch(iname, pattern):
ipath = os.path.join(path, iname)
img = fr.get_frame(ipath)
H,W = img.shape[:2]
bbox_list = bboxs[iname[:-4]]
for bbox in bbox_list:
label = sp.random.randint(0,45)
i,j,h,w = [int(e) for e in inv_calib(bbox[1],bbox[0],bbox[2],bbox[3],label)]
face = fr.get_patch(img,i,j,(h,w))
face_r,good_example = Datasets.sample_resize(face,k)
if good_example:
#print('orig:',bbox[1],bbox[0],bbox[2],bbox[3])
#print('inv_calib:',i,j,h,w)
vec_icalib = fr.frame_to_vect(face_r)
X.append(vec_icalib)
y.append(label)
print('face calib:',label,ipath)
y = sp.array(y)
sp.save(y_name,y)
X = sp.array(X)
sp.save(X_name,X)
return X,y
def get_train_face_wider_data(k = 12,write_to_disk = False):
'''
cut faces (positive examples) by bboxes from all images in dataset
return X - features
y - labels
cnt - count of examples
'''
X,y = [],[]
root = 'F:\\Datasets\\image_data_sets\\faces\\WIDERFace\\'
pattern = "*.jpg"
bboxs = Datasets.load_wider_face(os.path.join(root,'wider_face_split','wider_face_train_v7.mat'))
for path, subdirs, files in os.walk(root,'WIDER_train'):
for indx,iname in enumerate(files):
if fnmatch(iname, pattern):
ipath = os.path.join(path, iname)
print('face:',ipath)
img = fr.get_frame(ipath)
H,W,dim = img.shape
bbox_list = bboxs[iname[:-4]]
for bbox in bbox_list:
face = fr.get_patch(img,bbox[1],bbox[0],(bbox[2],bbox[3]))
#fr.write_frame('F:\\1\\' + str(c),face)
face_r,good_example = Datasets.sample_resize(face,k,k)
if good_example:
vec = fr.frame_to_vect(face_r)
X.append(vec)
y.append(1)
face_r_flip = fr.flip_frame(face_r)
vec = fr.frame_to_vect(face_r_flip)
X.append(vec)
y.append(1)
X = sp.array(X)
y = sp.array(y)
#y = sp.ones(len(X))
return X,y
def get_train_calib_data(k=12):
'''
for calibration net
return X - features
y - labels
cnt - count of examples
'''
sp.random.seed(42)
X_data, y_data = [], []
suff = str(k)
c = 0
X_name = 'train_data_icalib_' + suff + '.npz'
y_name = 'labels_icalib_' + suff + '.npz'
label = -1
dbpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW'
dbpath = join(dbpath, 'aflw.sqlite')
rfpath = 'F:\\datasets\\image_data_sets\\faces\\AFLW\\img'
conn = sqlite3.connect(dbpath)
c = 0
for file_id, x, y, ra, rb, theta in conn.execute(
'SELECT file_id,x,y,ra,rb,theta FROM Faces NATURAL JOIN FaceEllipse'
):
fpath = join(rfpath, file_id)
frame = fr.get_frame(fpath)
x1, y1, x2, y2 = util.ellipse2bbox(a=ra,
b=rb,
angle=theta,
cx=x,
cy=y)
x = x1
y = y1
h = abs(y2 - y1)
w = abs(x2 - x1)
no_neg = sp.all(
sp.array([x, y, h, w]) > 0) ## ignore a bad data in sql table
if frame != None and no_neg:
y, x, w, h = [int(e) for e in (y, x, w, h)]
face = fr.get_patch(frame, y, x, (w, h))
#fr.write_frame('F:\\1\\' + str(c) + 'orig',face)
c += 1
for ((new_y, new_x, new_w, new_h),
label) in [(util.calib(y, x, w, h, k), k)
for k in sp.random.randint(0, 45, 5)]:
face = fr.get_patch(frame, new_y, new_x, (new_w, new_h))
no_neg_calib = sp.all(
sp.array([new_x, new_y, new_h, new_w]) > 0)
face_r, good_example = Datasets.sample_resize(face, k, k)
if good_example and no_neg_calib:
#fr.write_frame('F:\\1\\' + str(c) + 'calib_'+str(label) ,face)
print('face:', fpath, label)
vec = fr.frame_to_vect(face_r)
X_data.append(vec)
y_data.append(label)
y_data = sp.array(y_data)
sp.savez(y_name, y_data)
X_data = sp.array(X_data)
sp.savez(X_name, X_data)
return X_data, y_data
