def __init__(self, model_path): self._model_path = model_path self._model_files = { 'shape_predictor': os.path.join(model_path, 'shape_predictor_68_face_landmarks.dat'), 'face_template': os.path.join(model_path, 'face_template.npy'), 'mean': os.path.join(model_path, 'mean.npy'), 'stddev': os.path.join(model_path, 'stddev.npy'), 'cnn_weights': os.path.join(model_path, 'weights_cnn.h5'), 'tpe_weights': os.path.join(model_path, 'weights_tpe.h5'), } for model_file in self._model_files.values(): if not os.path.exists(model_file): raise FileNotFoundError(model_file) self._mean = np.load(self._model_files['mean']) self._stddev = np.load(self._model_files['stddev']) self._fd = FaceDetector() self._fa = FaceAligner(self._model_files['shape_predictor'], self._model_files['face_template']) cnn = build_cnn(227, 266) cnn.load_weights(self._model_files['cnn_weights']) self._cnn = Bottleneck(cnn, ~1) _, tpe = build_tpe(256, 256) tpe.load_weights(self._model_files['tpe_weights']) self._tpe = tpe
def initialize_model(self): self._mean = np.load(self._model_files['mean']) self._stddev = np.load(self._model_files['stddev']) self._fd = FaceDetector() self._fa = FaceAligner(self._model_files['shape_predictor'], self._model_files['face_template']) cnn = build_cnn(227, 266) cnn.load_weights(self._model_files['cnn_weights']) self._cnn = Bottleneck(cnn, ~1) _, tpe = build_tpe(256, 256) tpe.load_weights(self._model_files['tpe_weights']) self._tpe = tpe
def __init__(self): self.resize_shape = (100, 100) shape_predictor_path = 'data/shape_predictor_68_face_landmarks.dat' self.shape_predictor = dlib.shape_predictor(shape_predictor_path) self.eye_and_mouth_indices = [39, 42, 57] self.template_landmarks = get_template_landmarks( self.eye_and_mouth_indices, self.resize_shape[0]) npload = np.load('data/mean_std2.npz') mean, std = npload['mean'], npload['std'] self.scaler = Scaler(mean=mean, std=std) # model_emb_path = 'data/epoch_17_test_eer0.191621.hdf5' model_path = 'data/cnn_model/epoch_66_val_loss1.206078.hdf5' model_emb_path = 'data/emb_model/model_10_epoch_10_test_eer0.169731_test2_err0.204908.hdf5' # model_path = 'data/cnn_model/epoch_16_val_loss1.231896.hdf5' # model_emb_path = 'data/emb_model/model_8_epoch_15_test_eer0.127431_test2_err0.218662.hdf5' # model_emb_path = 'data/emb_model/model_8_epoch_1_test_eer0.133520_test2_err0.216839.hdf5' # model_emb_path = 'data/emb_model/model_9_epoch_5_test_eer0.127574_test2_err0.229637.hdf5' # model_path = 'data/cnn_model/epoch_232_val_loss1.351451.hdf5' # model_emb_path = 'data/emb_model/model_1_epoch_0_test_eer0.114874.hdf5' # # model_path = 'data/cnn_model/epoch_57_val_loss1.699622.hdf5' # model_emb_path = 'data/emb_model/model_2_epoch_25_test_eer0.106689.hdf5' # model_path = 'data/cnn_model/epoch_29_val_loss1.441430.hdf5' # model_emb_path = 'data/emb_model/model_5_epoch_2_test_eer0.143211.hdf5' # model_emb_path = 'data/emb_model/model_6_epoch_6_test_eer_0.135497_test2_err0.254601.hdf5' # model_emb_path = '../data/Modeltpe2/epoch_0_test_eer0.139840.hdf5' # model_emb_path = '../data/Modeltpe3/epoch_12_test_eer0.107399.hdf5' # model_emb_path = 'data/emb_model/model_4_epoch_1_test_eer0.108006.hdf5' model = keras.models.load_model(model_path) self.model_emb = keras.models.load_model(model_emb_path) self.bottleneck = Bottleneck(model) npload = np.load('data/face_base.npz') self.x, self.y = npload['x'], npload['y'] print(self.x.shape, self.y.shape) with open('data/labels_dict.pkl', 'rb') as file: self.labels_dict = pickle.load(file) self.knn = KNeighborsClassifier(n_neighbors=1, metric=metric, n_jobs=1) self.knn.fit(self.x, self.y)
def embed_imgs(x): npload = np.load('data/mean_std2.npz') mean, std = npload['mean'], npload['std'] scaler = Scaler(mean=mean, std=std) # model_path = 'data/cnn_model/epoch_232_val_loss1.351451.hdf5' # model_emb_path = 'data/emb_model/model_1_epoch_0_test_eer0.114874.hdf5' # model_path = 'data/cnn_model/epoch_57_val_loss1.699622.hdf5' # model_emb_path = 'data/emb_model/model_2_epoch_25_test_eer0.106689.hdf5' # model_path = 'data/cnn_model/epoch_29_val_loss1.441430.hdf5' # model_emb_path = 'data/emb_model/model_5_epoch_2_test_eer0.143211.hdf5' # model_emb_path = 'data/emb_model/model_6_epoch_6_test_eer_0.135497_test2_err0.254601.hdf5' # model_emb_path = '../data/Modeltpe2/epoch_0_test_eer0.139840.hdf5' # model_emb_path = '../data/Modeltpe3/epoch_12_test_eer0.107399.hdf5' # model_emb_path = 'data/emb_model/model_4_epoch_1_test_eer0.108006.hdf5' # model_path = 'data/cnn_model/epoch_16_val_loss1.231896.hdf5' # model_emb_path = 'data/emb_model/model_8_epoch_15_test_eer0.127431_test2_err0.218662.hdf5' # model_emb_path = 'data/emb_model/model_8_epoch_1_test_eer0.133520_test2_err0.216839.hdf5' # model_emb_path = 'data/emb_model/model_9_epoch_5_test_eer0.127574_test2_err0.229637.hdf5' model_path = 'data/cnn_model/epoch_66_val_loss1.206078.hdf5' model_emb_path = 'data/emb_model/model_10_epoch_10_test_eer0.169731_test2_err0.204908.hdf5' model = keras.models.load_model(model_path) model_emb = keras.models.load_model(model_emb_path) bottleneck = Bottleneck(model) x = scaler.transform(x) x = bottleneck.predict(transpose_matrix(x)) x = model_emb.predict(x) # model_emb_path = 'data/epoch_17_test_eer0.191621.hdf5' # model_emb = keras.models.load_model(model_emb_path) # x = scaler.transform(x) # x = model_emb.predict(transpose_matrix(x)) return x
def setup(self): ''' once we have the parameters and random variate generation methods from __init__, we can proceed to create instances of: people and bottlenecks ''' self.precompute() av_locs = [] bottleneck_locs = [] fire_locs = [] r, c = 0, 0 for loc, attrs in self.graph.items(): r = max(r, loc[0]) c = max(c, loc[1]) if attrs['P']: av_locs += [loc] elif attrs['B']: bottleneck_locs += [loc] elif attrs['F']: fire_locs += [loc] assert len(av_locs) > 0, 'ERR: no people placement locations in input' for i in range(self.numpeople): p = Person(i, self.rate_generator(), self.strategy_generator(), self.location_sampler(av_locs)) self.people += [p] for loc in bottleneck_locs: b = Bottleneck(loc) self.bottlenecks[loc] = b self.fires.update(set(fire_locs)) self.r, self.c = r + 1, c + 1 print( '=' * 79, 'initialized a {}x{} floor with {} people in {} locations'.format( self.r, self.c, len(self.people), len(av_locs)), 'initialized {} bottleneck(s)'.format(len(self.bottlenecks)), 'detected {} fire zone(s)'.format( len([loc for loc in self.graph if self.graph[loc]['F']])), '\ngood luck escaping!', '=' * 79, 'LOGS', sep='\n')
def __init__(self, vib, data_size, p_dropout, latent_size, num_classes): super(Net, self).__init__() self.vib = vib self.net = nn.Sequential( nn.Conv1d(1, 10, kernel_size=11), nn.ReLU(), Flatten(), nn.Linear((data_size-10)*10, latent_size), nn.Dropout(p_dropout), nn.ReLU(), ) if vib: self.reg_layer = Bottleneck(latent_size, 256) else: self.reg_layer = nn.Linear(latent_size, 256) self.net2 = nn.Sequential( nn.ReLU(), nn.Linear(256, num_classes), )
def __init__(self, obs_space, action_space, model_type="default", use_bottleneck=False, dropout=0, use_l2a=False, use_bn=False, sni_type=None): super().__init__() # Decide which components are enabled self.use_bottleneck = use_bottleneck self.use_l2a = use_l2a self.dropout = dropout self.model_type = model_type self.sni_type = sni_type n = obs_space["image"][0] m = obs_space["image"][1] print(n, m) # Define image embedding if model_type in ['large', 'semi-large']: self.conv1 = nn.Sequential( nn.Conv2d(3, 32, (2, 2)), nn.ReLU(), ) self.res1 = nn.Sequential( nn.Conv2d(32, 32, (3, 3), padding=1), nn.ReLU(), nn.Conv2d(32, 32, (3, 3), padding=1), nn.ReLU(), ) self.conv2 = nn.Sequential( nn.Conv2d(32, 32, (2, 2)), nn.ReLU(), ) self.conv3 = nn.Sequential( nn.Conv2d(32, 32, (2, 2)), nn.ReLU(), ) self.max_pool = nn.MaxPool2d((2, 2)) self.image_embedding_size = ((n - 3) // 2) * ((m - 3) // 2) * 32 if model_type == 'large': self.res2 = nn.Sequential( nn.Conv2d(32, 32, (3, 3), padding=1), nn.ReLU(), nn.Conv2d(32, 32, (3, 3), padding=1), nn.ReLU(), ) self.res3 = nn.Sequential( nn.Conv2d(32, 32, (3, 3), padding=1), nn.ReLU(), nn.Conv2d(32, 32, (3, 3), padding=1), nn.ReLU(), ) # Max Pool elif model_type == "default": self.image_conv = nn.Sequential( nn.Conv2d(3, 16, (2, 2)), nn.ReLU(), nn.MaxPool2d((2, 2)), nn.Conv2d(16, 32, (2, 2)), nn.ReLU(), nn.Conv2d(32, 64, (2, 2)), nn.ReLU(), ) self.image_embedding_size = ((n - 1) // 2 - 2) * ( (m - 1) // 2 - 2) * 64 assert not use_bn elif model_type == "default2": if use_bn: self.image_conv = nn.Sequential( nn.Conv2d(3, 16, (2, 2)), nn.ReLU(), nn.MaxPool2d((2, 2)), nn.BatchNorm2d(16), nn.Conv2d(16, 32, (2, 2)), nn.ReLU(), nn.BatchNorm2d(32), nn.Conv2d(32, 32, (2, 2)), nn.ReLU(), nn.BatchNorm2d(32), ) else: self.image_conv = nn.Sequential( nn.Conv2d(3, 16, (2, 2)), nn.ReLU(), nn.MaxPool2d((2, 2)), nn.Conv2d(16, 32, (2, 2)), nn.ReLU(), nn.Conv2d(32, 32, (2, 2)), nn.ReLU(), ) self.image_embedding_size = ((n - 1) // 2 - 2) * ( (m - 1) // 2 - 2) * 32 elif model_type == "double_pooling": self.image_conv = nn.Sequential( nn.Conv2d(3, 16, (2, 2)), nn.ReLU(), nn.MaxPool2d((2, 2)), nn.Conv2d(16, 32, (2, 2)), nn.ReLU(), nn.MaxPool2d((2, 2)), nn.Conv2d(32, 64, (2, 2)), nn.ReLU(), ) self.image_embedding_size = (((n - 1) // 2 - 1) // 2 - 1) * (( (m - 1) // 2 - 1) // 2 - 1) * 64 assert not use_bn print("Image embedding size: ", self.image_embedding_size) # Not supported with current bottleneck # Resize image embedding self.embedding_size = self.semi_memory_size # Define actor's model if not isinstance(action_space, gym.spaces.Discrete): raise ValueError("Unknown action space: " + str(action_space)) if use_bottleneck: assert self.dropout == 0 self.reg_layer = Bottleneck(self.embedding_size, 64) else: self.reg_layer = nn.Linear(self.embedding_size, 64) self.dropout_layer = nn.Dropout(p=self.dropout, inplace=False) self.actor = nn.Sequential( # nn.Linear(self.embedding_size, 64), nn.Tanh(), nn.Linear(64, action_space.n)) # Define critic's model self.critic = nn.Sequential( # nn.Linear(self.embedding_size, 64), nn.Tanh(), nn.Linear(64, 1)) # Initialize parameters correctly self.apply(initialize_parameters)
def main(): with open('data/meta.json', 'r') as f: meta = json.load(f) cnn = build_cnn(227, meta['n_subjects']) cnn.load_weights('data/weights/weights.best.h5') bottleneck = Bottleneck(cnn, ~1) train_x, train_y = np.load('data/train_x.npy'), np.load('data/train_y.npy') test_x, test_y = np.load('data/test_x.npy'), np.load('data/test_y.npy') train_x = np.vstack([train_x, test_x]) train_y = np.hstack([train_y, test_y]) dev_x = np.load('data/dev_x.npy') dev_protocol = np.load('data/dev_protocol.npy') train_emb = bottleneck.predict(train_x, batch_size=256) dev_emb = bottleneck.predict(dev_x, batch_size=256) del train_x pca = PCA(N_OUT) pca.fit(train_emb) W_pca = pca.components_ np.save('data/w_pca', W_pca) tpe, tpe_pred = build_tpe(N_IN, N_OUT, W_pca.T) train_y = np.array(train_y) subjects = list(set(train_y)) anchors_inds = [] positives_inds = [] labels = [] for subj in subjects: mask = train_y == subj inds = np.where(mask)[0] for a, p in itertools.permutations(inds, 2): anchors_inds.append(a) positives_inds.append(p) labels.append(subj) anchors = train_emb[anchors_inds] positives = train_emb[positives_inds] n_anchors = len(anchors_inds) inds = np.arange(n_anchors) def get_batch(hard=False): batch_inds = np.random.choice(inds, size=BIG_BATCH_SIZE, replace=False) train_emb2 = tpe_pred.predict(train_emb, batch_size=1024) scores = train_emb2 @ train_emb2.T negative_inds = [] for i in batch_inds: label = labels[i] mask = train_y == label if hard: negative_inds.append( np.ma.array(scores[label], mask=mask).argmax()) else: negative_inds.append( np.random.choice(np.where(np.logical_not(mask))[0], size=1)[0]) return anchors[batch_inds], positives[batch_inds], train_emb[ negative_inds] def test(): dev_emb2 = tpe_pred.predict(dev_emb) tsc, isc = get_scores(dev_emb2, dev_protocol) eer, _, _, _ = calc_metrics(tsc, isc) return eer z = np.zeros((BIG_BATCH_SIZE, )) min_eer = float('inf') for e in range(NB_EPOCH): print('epoch: {}'.format(e)) a, p, n = get_batch(e > COLD_START) tpe.fit([a, p, n], z, batch_size=BATCH_SIZE, nb_epoch=1) eer = test() print('EER: {:.2f}'.format(eer * 100)) if eer < min_eer: min_eer = eer tpe.save_weights('data/weights/weights.tpe.mineer.h5')
from bottleneck import Bottleneck from identification import get_scores, calc_metrics WEIGHTS_DIR = './data/weights/' BATCH_SIZE = 32 dev_x = np.load('data/dev_x.npy') model = build_cnn(227, 266) weights_to_load = WEIGHTS_DIR + 'weights.best.h5' model.load_weights(weights_to_load) bottleneck = Bottleneck(model, ~1) dev_y = bottleneck.predict(dev_x, batch_size=BATCH_SIZE) protocol = np.load('data/dev_protocol.npy') tsc, isc = get_scores(dev_y, protocol) eer, fars, frrs, dists = calc_metrics(tsc, isc) print('EER: {}'.format(eer * 100)) plt.figure() plt.hist(tsc, 20, color='g', normed=True, alpha=0.3) plt.hist(isc, 20, color='r', normed=True, alpha=0.3) plt.figure() plt.loglog(fars, frrs) plt.show()
class FaceVerificator: def __init__(self, model_dir): self._model_dir = model_dir self._model_files = { 'shape_predictor': os.path.join(model_dir, 'shape_predictor_68_face_landmarks.dat'), 'face_template': os.path.join(model_dir, 'face_template.npy'), 'mean': os.path.join(model_dir, 'mean.npy'), 'stddev': os.path.join(model_dir, 'stddev.npy'), 'cnn_weights': os.path.join(model_dir, 'weights_cnn.h5'), 'tpe_weights': os.path.join(model_dir, 'weights_tpe.h5'), } for model_file in self._model_files.values(): if not os.path.exists(model_file): raise FileNotFoundError(model_file) def initialize_model(self): self._mean = np.load(self._model_files['mean']) self._stddev = np.load(self._model_files['stddev']) self._fd = FaceDetector() self._fa = FaceAligner(self._model_files['shape_predictor'], self._model_files['face_template']) cnn = build_cnn(227, 266) cnn.load_weights(self._model_files['cnn_weights']) self._cnn = Bottleneck(cnn, ~1) _, tpe = build_tpe(256, 256) tpe.load_weights(self._model_files['tpe_weights']) self._tpe = tpe def normalize(self, img): img = clip_to_range(img) return (img - self._mean) / self._stddev def process_image(self, img): face_rects = self._fd.detect_faces(img, upscale_factor=2, greater_than=GREATER_THAN) if not face_rects: return [] faces = self._fa.align_faces(img, face_rects, dim=IMSIZE, border=IMBORDER) faces = list(map(self.normalize, faces)) faces_y = self._cnn.predict(faces, batch_size=BATCH_SIZE) faces_y = self._tpe.predict(faces_y, batch_size=BATCH_SIZE) return list(zip(face_rects, faces_y)) def compare_many(self, dist, xs, ys): xs = np.array(xs) ys = np.array(ys) scores = xs @ ys.T return scores, scores > dist
import numpy as np from cnn import build_cnn from tpe import build_tpe from bottleneck import Bottleneck from identification import get_scores, calc_metrics from sklearn.decomposition import PCA n_in = 256 n_out = 256 cnn = build_cnn(227, 266) cnn.load_weights('data/weights/weights.best.h5') bottleneck = Bottleneck(cnn, ~1) train_x, train_y = np.load('./data/train_x.npy'), np.load('./data/train_y.npy') test_x, test_y = np.load('./data/test_x.npy'), np.load('./data/test_y.npy') train_x = np.vstack([train_x, test_x]) train_y = np.hstack([train_y, test_y]) dev_x = np.load('./data/dev_x.npy') dev_protocol = np.load('./data/dev_protocol.npy') train_emb = bottleneck.predict(train_x, batch_size=256) dev_emb = bottleneck.predict(dev_x, batch_size=256) del train_x
import torch from bottleneck import Bottleneck torch.manual_seed(23337) # use True to print layerwise sum for all outputs in reference code path DEBUG = False #True for stride, o_channel in [(1, 32), (1, 128), (2, 32)]: print("testing stride ==", stride, ", in_channel == 32 , out_channel ==", o_channel) a_ = torch.randn(17, 32, 28, 28) a = a_.cuda().half().to(memory_format=torch.channels_last).requires_grad_() model = Bottleneck( 32, 8, o_channel, stride=stride).cuda().half().to(memory_format=torch.channels_last) # test model b = model(a) b.mean().backward() d_grad = a.grad.float() a.grad = None torch.cuda.synchronize() if DEBUG: print("[DEBUG] ref dx :", d_grad.sum().item()) # print wgrad. we don't need to reset since later cpp print before accumulation for i, w in enumerate(model.w_conv): print("[DEBUG] ref wgrad{} :".format(i + 1), w.grad.sum().item()) wgrads = []
class Recognizer(object): def __init__(self): self.resize_shape = (100, 100) shape_predictor_path = 'data/shape_predictor_68_face_landmarks.dat' self.shape_predictor = dlib.shape_predictor(shape_predictor_path) self.eye_and_mouth_indices = [39, 42, 57] self.template_landmarks = get_template_landmarks( self.eye_and_mouth_indices, self.resize_shape[0]) npload = np.load('data/mean_std2.npz') mean, std = npload['mean'], npload['std'] self.scaler = Scaler(mean=mean, std=std) # model_emb_path = 'data/epoch_17_test_eer0.191621.hdf5' model_path = 'data/cnn_model/epoch_66_val_loss1.206078.hdf5' model_emb_path = 'data/emb_model/model_10_epoch_10_test_eer0.169731_test2_err0.204908.hdf5' # model_path = 'data/cnn_model/epoch_16_val_loss1.231896.hdf5' # model_emb_path = 'data/emb_model/model_8_epoch_15_test_eer0.127431_test2_err0.218662.hdf5' # model_emb_path = 'data/emb_model/model_8_epoch_1_test_eer0.133520_test2_err0.216839.hdf5' # model_emb_path = 'data/emb_model/model_9_epoch_5_test_eer0.127574_test2_err0.229637.hdf5' # model_path = 'data/cnn_model/epoch_232_val_loss1.351451.hdf5' # model_emb_path = 'data/emb_model/model_1_epoch_0_test_eer0.114874.hdf5' # # model_path = 'data/cnn_model/epoch_57_val_loss1.699622.hdf5' # model_emb_path = 'data/emb_model/model_2_epoch_25_test_eer0.106689.hdf5' # model_path = 'data/cnn_model/epoch_29_val_loss1.441430.hdf5' # model_emb_path = 'data/emb_model/model_5_epoch_2_test_eer0.143211.hdf5' # model_emb_path = 'data/emb_model/model_6_epoch_6_test_eer_0.135497_test2_err0.254601.hdf5' # model_emb_path = '../data/Modeltpe2/epoch_0_test_eer0.139840.hdf5' # model_emb_path = '../data/Modeltpe3/epoch_12_test_eer0.107399.hdf5' # model_emb_path = 'data/emb_model/model_4_epoch_1_test_eer0.108006.hdf5' model = keras.models.load_model(model_path) self.model_emb = keras.models.load_model(model_emb_path) self.bottleneck = Bottleneck(model) npload = np.load('data/face_base.npz') self.x, self.y = npload['x'], npload['y'] print(self.x.shape, self.y.shape) with open('data/labels_dict.pkl', 'rb') as file: self.labels_dict = pickle.load(file) self.knn = KNeighborsClassifier(n_neighbors=1, metric=metric, n_jobs=1) self.knn.fit(self.x, self.y) def iterate_similarities(self, emb): for i, person_emb in enumerate(self.x): sim = person_emb @ emb.T yield sim, i def predict(self, img, img_gray, rect): img = align_img(img, img_gray, rect, self.shape_predictor, self.template_landmarks, self.eye_and_mouth_indices, self.resize_shape) batch_x = [img] import matplotlib.pyplot as plt # plt.imshow(img) # plt.show() batch_x = self.scaler.transform(batch_x) batch_x = self.bottleneck.predict(transpose_matrix(batch_x)) batch_x = self.model_emb.predict(batch_x) # batch_x = self.model_emb.predict(transpose_matrix(batch_x)) pred_labels = self.knn.predict(batch_x) neighbors = self.knn.kneighbors(batch_x) label_neighbors = [ self.labels_dict[self.y[ind]] for ind in neighbors[1][0] ] print(label_neighbors, neighbors[0]) # label_ind = max(self.iterate_similarities(batch_x[0]), key=lambda x: x[0])[1] # label = self.y[label_ind] label = pred_labels[0] return self.labels_dict[label], label_neighbors