def __init__(self, model_path):
self._model_path = model_path
self._model_files = {
'shape_predictor':
os.path.join(model_path, 'shape_predictor_68_face_landmarks.dat'),
'face_template':
os.path.join(model_path, 'face_template.npy'),
'mean':
os.path.join(model_path, 'mean.npy'),
'stddev':
os.path.join(model_path, 'stddev.npy'),
'cnn_weights':
os.path.join(model_path, 'weights_cnn.h5'),
'tpe_weights':
os.path.join(model_path, 'weights_tpe.h5'),
}
for model_file in self._model_files.values():
if not os.path.exists(model_file):
raise FileNotFoundError(model_file)
self._mean = np.load(self._model_files['mean'])
self._stddev = np.load(self._model_files['stddev'])
self._fd = FaceDetector()
self._fa = FaceAligner(self._model_files['shape_predictor'],
self._model_files['face_template'])
cnn = build_cnn(227, 266)
cnn.load_weights(self._model_files['cnn_weights'])
self._cnn = Bottleneck(cnn, ~1)
_, tpe = build_tpe(256, 256)
tpe.load_weights(self._model_files['tpe_weights'])
self._tpe = tpe
def initialize_model(self):
self._mean = np.load(self._model_files['mean'])
self._stddev = np.load(self._model_files['stddev'])
self._fd = FaceDetector()
self._fa = FaceAligner(self._model_files['shape_predictor'],
self._model_files['face_template'])
cnn = build_cnn(227, 266)
cnn.load_weights(self._model_files['cnn_weights'])
self._cnn = Bottleneck(cnn, ~1)
_, tpe = build_tpe(256, 256)
tpe.load_weights(self._model_files['tpe_weights'])
self._tpe = tpe
def __init__(self):
self.resize_shape = (100, 100)
shape_predictor_path = 'data/shape_predictor_68_face_landmarks.dat'
self.shape_predictor = dlib.shape_predictor(shape_predictor_path)
self.eye_and_mouth_indices = [39, 42, 57]
self.template_landmarks = get_template_landmarks(
self.eye_and_mouth_indices, self.resize_shape[0])
npload = np.load('data/mean_std2.npz')
mean, std = npload['mean'], npload['std']
self.scaler = Scaler(mean=mean, std=std)
# model_emb_path = 'data/epoch_17_test_eer0.191621.hdf5'
model_path = 'data/cnn_model/epoch_66_val_loss1.206078.hdf5'
model_emb_path = 'data/emb_model/model_10_epoch_10_test_eer0.169731_test2_err0.204908.hdf5'
# model_path = 'data/cnn_model/epoch_16_val_loss1.231896.hdf5'
# model_emb_path = 'data/emb_model/model_8_epoch_15_test_eer0.127431_test2_err0.218662.hdf5'
# model_emb_path = 'data/emb_model/model_8_epoch_1_test_eer0.133520_test2_err0.216839.hdf5'
# model_emb_path = 'data/emb_model/model_9_epoch_5_test_eer0.127574_test2_err0.229637.hdf5'
# model_path = 'data/cnn_model/epoch_232_val_loss1.351451.hdf5'
# model_emb_path = 'data/emb_model/model_1_epoch_0_test_eer0.114874.hdf5'
#
# model_path = 'data/cnn_model/epoch_57_val_loss1.699622.hdf5'
# model_emb_path = 'data/emb_model/model_2_epoch_25_test_eer0.106689.hdf5'
# model_path = 'data/cnn_model/epoch_29_val_loss1.441430.hdf5'
# model_emb_path = 'data/emb_model/model_5_epoch_2_test_eer0.143211.hdf5'
# model_emb_path = 'data/emb_model/model_6_epoch_6_test_eer_0.135497_test2_err0.254601.hdf5'
# model_emb_path = '../data/Modeltpe2/epoch_0_test_eer0.139840.hdf5'
# model_emb_path = '../data/Modeltpe3/epoch_12_test_eer0.107399.hdf5'
# model_emb_path = 'data/emb_model/model_4_epoch_1_test_eer0.108006.hdf5'
model = keras.models.load_model(model_path)
self.model_emb = keras.models.load_model(model_emb_path)
self.bottleneck = Bottleneck(model)
npload = np.load('data/face_base.npz')
self.x, self.y = npload['x'], npload['y']
print(self.x.shape, self.y.shape)
with open('data/labels_dict.pkl', 'rb') as file:
self.labels_dict = pickle.load(file)
self.knn = KNeighborsClassifier(n_neighbors=1, metric=metric, n_jobs=1)
self.knn.fit(self.x, self.y)
def embed_imgs(x):
npload = np.load('data/mean_std2.npz')
mean, std = npload['mean'], npload['std']
scaler = Scaler(mean=mean, std=std)
# model_path = 'data/cnn_model/epoch_232_val_loss1.351451.hdf5'
# model_emb_path = 'data/emb_model/model_1_epoch_0_test_eer0.114874.hdf5'
# model_path = 'data/cnn_model/epoch_57_val_loss1.699622.hdf5'
# model_emb_path = 'data/emb_model/model_2_epoch_25_test_eer0.106689.hdf5'
# model_path = 'data/cnn_model/epoch_29_val_loss1.441430.hdf5'
# model_emb_path = 'data/emb_model/model_5_epoch_2_test_eer0.143211.hdf5'
# model_emb_path = 'data/emb_model/model_6_epoch_6_test_eer_0.135497_test2_err0.254601.hdf5'
# model_emb_path = '../data/Modeltpe2/epoch_0_test_eer0.139840.hdf5'
# model_emb_path = '../data/Modeltpe3/epoch_12_test_eer0.107399.hdf5'
# model_emb_path = 'data/emb_model/model_4_epoch_1_test_eer0.108006.hdf5'
# model_path = 'data/cnn_model/epoch_16_val_loss1.231896.hdf5'
# model_emb_path = 'data/emb_model/model_8_epoch_15_test_eer0.127431_test2_err0.218662.hdf5'
# model_emb_path = 'data/emb_model/model_8_epoch_1_test_eer0.133520_test2_err0.216839.hdf5'
# model_emb_path = 'data/emb_model/model_9_epoch_5_test_eer0.127574_test2_err0.229637.hdf5'
model_path = 'data/cnn_model/epoch_66_val_loss1.206078.hdf5'
model_emb_path = 'data/emb_model/model_10_epoch_10_test_eer0.169731_test2_err0.204908.hdf5'
model = keras.models.load_model(model_path)
model_emb = keras.models.load_model(model_emb_path)
bottleneck = Bottleneck(model)
x = scaler.transform(x)
x = bottleneck.predict(transpose_matrix(x))
x = model_emb.predict(x)
# model_emb_path = 'data/epoch_17_test_eer0.191621.hdf5'
# model_emb = keras.models.load_model(model_emb_path)
# x = scaler.transform(x)
# x = model_emb.predict(transpose_matrix(x))
return x
def setup(self):
'''
once we have the parameters and random variate generation methods from
__init__, we can proceed to create instances of: people and bottlenecks
'''
self.precompute()
av_locs = []
bottleneck_locs = []
fire_locs = []
r, c = 0, 0
for loc, attrs in self.graph.items():
r = max(r, loc[0])
c = max(c, loc[1])
if attrs['P']: av_locs += [loc]
elif attrs['B']: bottleneck_locs += [loc]
elif attrs['F']: fire_locs += [loc]
assert len(av_locs) > 0, 'ERR: no people placement locations in input'
for i in range(self.numpeople):
p = Person(i, self.rate_generator(), self.strategy_generator(),
self.location_sampler(av_locs))
self.people += [p]
for loc in bottleneck_locs:
b = Bottleneck(loc)
self.bottlenecks[loc] = b
self.fires.update(set(fire_locs))
self.r, self.c = r + 1, c + 1
print(
'=' * 79,
'initialized a {}x{} floor with {} people in {} locations'.format(
self.r, self.c, len(self.people), len(av_locs)),
'initialized {} bottleneck(s)'.format(len(self.bottlenecks)),
'detected {} fire zone(s)'.format(
len([loc for loc in self.graph if self.graph[loc]['F']])),
'\ngood luck escaping!',
'=' * 79,
'LOGS',
sep='\n')
def __init__(self, vib, data_size, p_dropout, latent_size, num_classes):
super(Net, self).__init__()
self.vib = vib
self.net = nn.Sequential(
nn.Conv1d(1, 10, kernel_size=11),
nn.ReLU(),
Flatten(),
nn.Linear((data_size-10)*10, latent_size),
nn.Dropout(p_dropout),
nn.ReLU(),
)
if vib:
self.reg_layer = Bottleneck(latent_size, 256)
else:
self.reg_layer = nn.Linear(latent_size, 256)
self.net2 = nn.Sequential(
nn.ReLU(),
nn.Linear(256, num_classes),
)
def __init__(self,
obs_space,
action_space,
model_type="default",
use_bottleneck=False,
dropout=0,
use_l2a=False,
use_bn=False,
sni_type=None):
super().__init__()
# Decide which components are enabled
self.use_bottleneck = use_bottleneck
self.use_l2a = use_l2a
self.dropout = dropout
self.model_type = model_type
self.sni_type = sni_type
n = obs_space["image"][0]
m = obs_space["image"][1]
print(n, m)
# Define image embedding
if model_type in ['large', 'semi-large']:
self.conv1 = nn.Sequential(
nn.Conv2d(3, 32, (2, 2)),
nn.ReLU(),
)
self.res1 = nn.Sequential(
nn.Conv2d(32, 32, (3, 3), padding=1),
nn.ReLU(),
nn.Conv2d(32, 32, (3, 3), padding=1),
nn.ReLU(),
)
self.conv2 = nn.Sequential(
nn.Conv2d(32, 32, (2, 2)),
nn.ReLU(),
)
self.conv3 = nn.Sequential(
nn.Conv2d(32, 32, (2, 2)),
nn.ReLU(),
)
self.max_pool = nn.MaxPool2d((2, 2))
self.image_embedding_size = ((n - 3) // 2) * ((m - 3) // 2) * 32
if model_type == 'large':
self.res2 = nn.Sequential(
nn.Conv2d(32, 32, (3, 3), padding=1),
nn.ReLU(),
nn.Conv2d(32, 32, (3, 3), padding=1),
nn.ReLU(),
)
self.res3 = nn.Sequential(
nn.Conv2d(32, 32, (3, 3), padding=1),
nn.ReLU(),
nn.Conv2d(32, 32, (3, 3), padding=1),
nn.ReLU(),
)
# Max Pool
elif model_type == "default":
self.image_conv = nn.Sequential(
nn.Conv2d(3, 16, (2, 2)),
nn.ReLU(),
nn.MaxPool2d((2, 2)),
nn.Conv2d(16, 32, (2, 2)),
nn.ReLU(),
nn.Conv2d(32, 64, (2, 2)),
nn.ReLU(),
)
self.image_embedding_size = ((n - 1) // 2 - 2) * (
(m - 1) // 2 - 2) * 64
assert not use_bn
elif model_type == "default2":
if use_bn:
self.image_conv = nn.Sequential(
nn.Conv2d(3, 16, (2, 2)),
nn.ReLU(),
nn.MaxPool2d((2, 2)),
nn.BatchNorm2d(16),
nn.Conv2d(16, 32, (2, 2)),
nn.ReLU(),
nn.BatchNorm2d(32),
nn.Conv2d(32, 32, (2, 2)),
nn.ReLU(),
nn.BatchNorm2d(32),
)
else:
self.image_conv = nn.Sequential(
nn.Conv2d(3, 16, (2, 2)),
nn.ReLU(),
nn.MaxPool2d((2, 2)),
nn.Conv2d(16, 32, (2, 2)),
nn.ReLU(),
nn.Conv2d(32, 32, (2, 2)),
nn.ReLU(),
)
self.image_embedding_size = ((n - 1) // 2 - 2) * (
(m - 1) // 2 - 2) * 32
elif model_type == "double_pooling":
self.image_conv = nn.Sequential(
nn.Conv2d(3, 16, (2, 2)),
nn.ReLU(),
nn.MaxPool2d((2, 2)),
nn.Conv2d(16, 32, (2, 2)),
nn.ReLU(),
nn.MaxPool2d((2, 2)),
nn.Conv2d(32, 64, (2, 2)),
nn.ReLU(),
)
self.image_embedding_size = (((n - 1) // 2 - 1) // 2 - 1) * ((
(m - 1) // 2 - 1) // 2 - 1) * 64
assert not use_bn
print("Image embedding size: ", self.image_embedding_size)
# Not supported with current bottleneck
# Resize image embedding
self.embedding_size = self.semi_memory_size
# Define actor's model
if not isinstance(action_space, gym.spaces.Discrete):
raise ValueError("Unknown action space: " + str(action_space))
if use_bottleneck:
assert self.dropout == 0
self.reg_layer = Bottleneck(self.embedding_size, 64)
else:
self.reg_layer = nn.Linear(self.embedding_size, 64)
self.dropout_layer = nn.Dropout(p=self.dropout, inplace=False)
self.actor = nn.Sequential(
# nn.Linear(self.embedding_size, 64),
nn.Tanh(),
nn.Linear(64, action_space.n))
# Define critic's model
self.critic = nn.Sequential(
# nn.Linear(self.embedding_size, 64),
nn.Tanh(),
nn.Linear(64, 1))
# Initialize parameters correctly
self.apply(initialize_parameters)
def main():
with open('data/meta.json', 'r') as f:
meta = json.load(f)
cnn = build_cnn(227, meta['n_subjects'])
cnn.load_weights('data/weights/weights.best.h5')
bottleneck = Bottleneck(cnn, ~1)
train_x, train_y = np.load('data/train_x.npy'), np.load('data/train_y.npy')
test_x, test_y = np.load('data/test_x.npy'), np.load('data/test_y.npy')
train_x = np.vstack([train_x, test_x])
train_y = np.hstack([train_y, test_y])
dev_x = np.load('data/dev_x.npy')
dev_protocol = np.load('data/dev_protocol.npy')
train_emb = bottleneck.predict(train_x, batch_size=256)
dev_emb = bottleneck.predict(dev_x, batch_size=256)
del train_x
pca = PCA(N_OUT)
pca.fit(train_emb)
W_pca = pca.components_
np.save('data/w_pca', W_pca)
tpe, tpe_pred = build_tpe(N_IN, N_OUT, W_pca.T)
train_y = np.array(train_y)
subjects = list(set(train_y))
anchors_inds = []
positives_inds = []
labels = []
for subj in subjects:
mask = train_y == subj
inds = np.where(mask)[0]
for a, p in itertools.permutations(inds, 2):
anchors_inds.append(a)
positives_inds.append(p)
labels.append(subj)
anchors = train_emb[anchors_inds]
positives = train_emb[positives_inds]
n_anchors = len(anchors_inds)
inds = np.arange(n_anchors)
def get_batch(hard=False):
batch_inds = np.random.choice(inds, size=BIG_BATCH_SIZE, replace=False)
train_emb2 = tpe_pred.predict(train_emb, batch_size=1024)
scores = train_emb2 @ train_emb2.T
negative_inds = []
for i in batch_inds:
label = labels[i]
mask = train_y == label
if hard:
negative_inds.append(
np.ma.array(scores[label], mask=mask).argmax())
else:
negative_inds.append(
np.random.choice(np.where(np.logical_not(mask))[0],
size=1)[0])
return anchors[batch_inds], positives[batch_inds], train_emb[
negative_inds]
def test():
dev_emb2 = tpe_pred.predict(dev_emb)
tsc, isc = get_scores(dev_emb2, dev_protocol)
eer, _, _, _ = calc_metrics(tsc, isc)
return eer
z = np.zeros((BIG_BATCH_SIZE, ))
min_eer = float('inf')
for e in range(NB_EPOCH):
print('epoch: {}'.format(e))
a, p, n = get_batch(e > COLD_START)
tpe.fit([a, p, n], z, batch_size=BATCH_SIZE, nb_epoch=1)
eer = test()
print('EER: {:.2f}'.format(eer * 100))
if eer < min_eer:
min_eer = eer
tpe.save_weights('data/weights/weights.tpe.mineer.h5')
from bottleneck import Bottleneck
from identification import get_scores, calc_metrics
WEIGHTS_DIR = './data/weights/'
BATCH_SIZE = 32
dev_x = np.load('data/dev_x.npy')
model = build_cnn(227, 266)
weights_to_load = WEIGHTS_DIR + 'weights.best.h5'
model.load_weights(weights_to_load)
bottleneck = Bottleneck(model, ~1)
dev_y = bottleneck.predict(dev_x, batch_size=BATCH_SIZE)
protocol = np.load('data/dev_protocol.npy')
tsc, isc = get_scores(dev_y, protocol)
eer, fars, frrs, dists = calc_metrics(tsc, isc)
print('EER: {}'.format(eer * 100))
plt.figure()
plt.hist(tsc, 20, color='g', normed=True, alpha=0.3)
plt.hist(isc, 20, color='r', normed=True, alpha=0.3)
plt.figure()
plt.loglog(fars, frrs)
plt.show()
class FaceVerificator:
def __init__(self, model_dir):
self._model_dir = model_dir
self._model_files = {
'shape_predictor':
os.path.join(model_dir, 'shape_predictor_68_face_landmarks.dat'),
'face_template':
os.path.join(model_dir, 'face_template.npy'),
'mean':
os.path.join(model_dir, 'mean.npy'),
'stddev':
os.path.join(model_dir, 'stddev.npy'),
'cnn_weights':
os.path.join(model_dir, 'weights_cnn.h5'),
'tpe_weights':
os.path.join(model_dir, 'weights_tpe.h5'),
}
for model_file in self._model_files.values():
if not os.path.exists(model_file):
raise FileNotFoundError(model_file)
def initialize_model(self):
self._mean = np.load(self._model_files['mean'])
self._stddev = np.load(self._model_files['stddev'])
self._fd = FaceDetector()
self._fa = FaceAligner(self._model_files['shape_predictor'],
self._model_files['face_template'])
cnn = build_cnn(227, 266)
cnn.load_weights(self._model_files['cnn_weights'])
self._cnn = Bottleneck(cnn, ~1)
_, tpe = build_tpe(256, 256)
tpe.load_weights(self._model_files['tpe_weights'])
self._tpe = tpe
def normalize(self, img):
img = clip_to_range(img)
return (img - self._mean) / self._stddev
def process_image(self, img):
face_rects = self._fd.detect_faces(img,
upscale_factor=2,
greater_than=GREATER_THAN)
if not face_rects:
return []
faces = self._fa.align_faces(img,
face_rects,
dim=IMSIZE,
border=IMBORDER)
faces = list(map(self.normalize, faces))
faces_y = self._cnn.predict(faces, batch_size=BATCH_SIZE)
faces_y = self._tpe.predict(faces_y, batch_size=BATCH_SIZE)
return list(zip(face_rects, faces_y))
def compare_many(self, dist, xs, ys):
xs = np.array(xs)
ys = np.array(ys)
scores = xs @ ys.T
return scores, scores > dist
import numpy as np
from cnn import build_cnn
from tpe import build_tpe
from bottleneck import Bottleneck
from identification import get_scores, calc_metrics
from sklearn.decomposition import PCA
n_in = 256
n_out = 256
cnn = build_cnn(227, 266)
cnn.load_weights('data/weights/weights.best.h5')
bottleneck = Bottleneck(cnn, ~1)
train_x, train_y = np.load('./data/train_x.npy'), np.load('./data/train_y.npy')
test_x, test_y = np.load('./data/test_x.npy'), np.load('./data/test_y.npy')
train_x = np.vstack([train_x, test_x])
train_y = np.hstack([train_y, test_y])
dev_x = np.load('./data/dev_x.npy')
dev_protocol = np.load('./data/dev_protocol.npy')
train_emb = bottleneck.predict(train_x, batch_size=256)
dev_emb = bottleneck.predict(dev_x, batch_size=256)
del train_x
import torch
from bottleneck import Bottleneck
torch.manual_seed(23337)
# use True to print layerwise sum for all outputs in reference code path
DEBUG = False #True
for stride, o_channel in [(1, 32), (1, 128), (2, 32)]:
print("testing stride ==", stride, ", in_channel == 32 , out_channel ==",
o_channel)
a_ = torch.randn(17, 32, 28, 28)
a = a_.cuda().half().to(memory_format=torch.channels_last).requires_grad_()
model = Bottleneck(
32, 8, o_channel,
stride=stride).cuda().half().to(memory_format=torch.channels_last)
# test model
b = model(a)
b.mean().backward()
d_grad = a.grad.float()
a.grad = None
torch.cuda.synchronize()
if DEBUG:
print("[DEBUG] ref dx :", d_grad.sum().item())
# print wgrad. we don't need to reset since later cpp print before accumulation
for i, w in enumerate(model.w_conv):
print("[DEBUG] ref wgrad{} :".format(i + 1), w.grad.sum().item())
wgrads = []
class Recognizer(object):
def __init__(self):
self.resize_shape = (100, 100)
shape_predictor_path = 'data/shape_predictor_68_face_landmarks.dat'
self.shape_predictor = dlib.shape_predictor(shape_predictor_path)
self.eye_and_mouth_indices = [39, 42, 57]
self.template_landmarks = get_template_landmarks(
self.eye_and_mouth_indices, self.resize_shape[0])
npload = np.load('data/mean_std2.npz')
mean, std = npload['mean'], npload['std']
self.scaler = Scaler(mean=mean, std=std)
# model_emb_path = 'data/epoch_17_test_eer0.191621.hdf5'
model_path = 'data/cnn_model/epoch_66_val_loss1.206078.hdf5'
model_emb_path = 'data/emb_model/model_10_epoch_10_test_eer0.169731_test2_err0.204908.hdf5'
# model_path = 'data/cnn_model/epoch_16_val_loss1.231896.hdf5'
# model_emb_path = 'data/emb_model/model_8_epoch_15_test_eer0.127431_test2_err0.218662.hdf5'
# model_emb_path = 'data/emb_model/model_8_epoch_1_test_eer0.133520_test2_err0.216839.hdf5'
# model_emb_path = 'data/emb_model/model_9_epoch_5_test_eer0.127574_test2_err0.229637.hdf5'
# model_path = 'data/cnn_model/epoch_232_val_loss1.351451.hdf5'
# model_emb_path = 'data/emb_model/model_1_epoch_0_test_eer0.114874.hdf5'
#
# model_path = 'data/cnn_model/epoch_57_val_loss1.699622.hdf5'
# model_emb_path = 'data/emb_model/model_2_epoch_25_test_eer0.106689.hdf5'
# model_path = 'data/cnn_model/epoch_29_val_loss1.441430.hdf5'
# model_emb_path = 'data/emb_model/model_5_epoch_2_test_eer0.143211.hdf5'
# model_emb_path = 'data/emb_model/model_6_epoch_6_test_eer_0.135497_test2_err0.254601.hdf5'
# model_emb_path = '../data/Modeltpe2/epoch_0_test_eer0.139840.hdf5'
# model_emb_path = '../data/Modeltpe3/epoch_12_test_eer0.107399.hdf5'
# model_emb_path = 'data/emb_model/model_4_epoch_1_test_eer0.108006.hdf5'
model = keras.models.load_model(model_path)
self.model_emb = keras.models.load_model(model_emb_path)
self.bottleneck = Bottleneck(model)
npload = np.load('data/face_base.npz')
self.x, self.y = npload['x'], npload['y']
print(self.x.shape, self.y.shape)
with open('data/labels_dict.pkl', 'rb') as file:
self.labels_dict = pickle.load(file)
self.knn = KNeighborsClassifier(n_neighbors=1, metric=metric, n_jobs=1)
self.knn.fit(self.x, self.y)
def iterate_similarities(self, emb):
for i, person_emb in enumerate(self.x):
sim = person_emb @ emb.T
yield sim, i
def predict(self, img, img_gray, rect):
img = align_img(img, img_gray, rect, self.shape_predictor,
self.template_landmarks, self.eye_and_mouth_indices,
self.resize_shape)
batch_x = [img]
import matplotlib.pyplot as plt
# plt.imshow(img)
# plt.show()
batch_x = self.scaler.transform(batch_x)
batch_x = self.bottleneck.predict(transpose_matrix(batch_x))
batch_x = self.model_emb.predict(batch_x)
# batch_x = self.model_emb.predict(transpose_matrix(batch_x))
pred_labels = self.knn.predict(batch_x)
neighbors = self.knn.kneighbors(batch_x)
label_neighbors = [
self.labels_dict[self.y[ind]] for ind in neighbors[1][0]
]
print(label_neighbors, neighbors[0])
# label_ind = max(self.iterate_similarities(batch_x[0]), key=lambda x: x[0])[1]
# label = self.y[label_ind]
label = pred_labels[0]
return self.labels_dict[label], label_neighbors
