def face_encodings(self, image, model='128D', jitters=3, prewhiten=True, align=False): image = cv2.resize(image, (160, 160)) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) if align is True: image = self.align_face(image) if model == '512D': if prewhiten is True: image = self.prewhiten(image) with tf.Graph().as_default(): with tf.Session() as session: facenet.load_model('models/20180402-114759.pb') img_holder = tf.get_default_graph().get_tensor_by_name( 'input:0') embeddings = tf.get_default_graph().get_tensor_by_name( 'embeddings:0') phase_train = tf.get_default_graph().get_tensor_by_name( 'phase_train:0') feed_dict = {img_holder: [image], phase_train: False} encoding = session.run(embeddings, feed_dict=feed_dict) else: encoding = FR.face_encodings(image, num_jitters=jitters) return encoding
def __init__(self, PATH_TO_CKPT): # self.detection_graph = tf.Graph() # with self.detection_graph.as_default(): # od_graph_def = tf.GraphDef() # with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: # serialized_graph = fid.read() # od_graph_def.ParseFromString(serialized_graph) # tf.import_graph_def(od_graph_def, name='') # with self.detection_graph.as_default(): # config = tf.ConfigProto() # config.gpu_options.allow_growth = True # self.sess = tf.Session(graph=self.detection_graph, config=config) # self.windowNotSet = True with tf.Graph().as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.6) self.sess = tf.Session(config=tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False)) # with self.sess.as_default(): facenet.load_model(PATH_TO_CKPT) self.images_placeholder = tf.get_default_graph( ).get_tensor_by_name("input:0") self.embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") self.phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") embedding_size = self.embeddings.get_shape()[1] self.emb_array = np.zeros((1, embedding_size))
def main(argv): dataset_path = os.path.expanduser(argv.dataset_path) dataset_path = os.path.abspath(dataset_path) model_file = os.path.expanduser(argv.model) model_file = os.path.abspath(model_file) classifier_filename_exp = os.path.expanduser(argv.output_classifier) classifier_filename_exp = os.path.abspath(classifier_filename_exp) assert os.path.exists(dataset_path) is True, 'please correct path...' np.random.seed(seed=777) # TF gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) pnet, rnet, onet = facenet.align.detect_face.create_mtcnn(sess, None) fn.load_model(model_file) images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name( "phase_train:0") embedding_size = embeddings.get_shape()[1] dataset = fn.get_dataset(dataset_path) for cls in dataset: assert len( cls.image_paths ) > 0, 'there must be at least one image for each class in the dataset' paths, labels = fn.get_image_paths_and_labels(dataset) print('Number of classes: %d' % len(dataset)) print('Number of images: %d' % len(paths)) nrof_images = len(paths) nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / 90)) emb_array = np.zeros((nrof_images, embedding_size)) for i in range(nrof_batches_per_epoch): start_index = i * 90 end_index = min((i + 1) * 90, nrof_images) paths_batch = paths[start_index:end_index] images = fn.load_data(paths_batch, False, False, 160) feed_dict = { images_placeholder: images, phase_train_placeholder: False } emb_array[start_index:end_index, :] = sess.run(embeddings, feed_dict=feed_dict) model = SVC(kernel='linear', probability=True) model.fit(emb_array, labels) class_names = [cls.name.replace('_', ' ') for cls in dataset] with open(classifier_filename_exp, 'wb') as outfile: pickle.dump((model, class_names), outfile) print('Saved classifier model to file "%s"' % classifier_filename_exp) exit(0)
def index_faces(self,paths, paths_to_pk, output_dir, video_pk): with tf.Graph().as_default(): config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.15 with tf.Session(config=config) as sess: output_dir = os.path.expanduser(output_dir) if not os.path.isdir(output_dir): os.makedirs(output_dir) logging.info("Loading trained model...\n") meta_file, ckpt_file, model_dir = facenet.get_model_filenames() facenet.load_model(model_dir, meta_file, ckpt_file) images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") image_size = images_placeholder.get_shape()[1] embedding_size = embeddings.get_shape()[1] logging.info('Generating embeddings from images...\n') start_time = time.time() batch_size = 25 nrof_images = len(paths) nrof_batches = int(np.ceil(1.0 * nrof_images / batch_size)) emb_array = np.zeros((nrof_images, embedding_size)) count = 0 path_count = {} entries = [] for i in xrange(nrof_batches): start_index = i * batch_size end_index = min((i + 1) * batch_size, nrof_images) paths_batch = paths[start_index:end_index] for eindex, fname in enumerate(paths_batch): count += 1 entry = { 'path': fname, 'detection_primary_key': paths_to_pk[fname], 'index': eindex, 'type': 'detection', 'video_primary_key': video_pk } entries.append(entry) images = facenet.load_data(paths_batch, do_random_crop=False, do_random_flip=False, image_size=image_size, do_prewhiten=True) feed_dict = {images_placeholder: images, phase_train_placeholder: False} emb_array[start_index:end_index, :] = sess.run(embeddings, feed_dict=feed_dict) if nrof_images: time_avg_forward_pass = (time.time() - start_time) / float(nrof_images) logging.info("Forward pass took avg of %.3f[seconds/image] for %d images\n" % ( time_avg_forward_pass, nrof_images)) logging.info("Finally saving embeddings and gallery to: %s" % (output_dir)) feat_fname = os.path.join(output_dir, "facenet.npy") entries_fname = os.path.join(output_dir, "facenet.json") np.save(feat_fname, emb_array) fh = open(entries_fname, 'w') json.dump(entries, fh) fh.close() return path_count, emb_array, entries, feat_fname, entries_fname
def create_data_points(img_queue, enc_queue): predictor = dlib.shape_predictor('models/sp_68_point.dat') aligner = FaceAligner(predictor, desiredFaceWidth=300) proc_name = current_process().name with tf.Graph().as_default(), tf.Session() as session: graph = tf.get_default_graph() facenet.load_model('models/20180402-114759.pb') img_holder = graph.get_tensor_by_name('input:0') embeddings = graph.get_tensor_by_name('embeddings:0') phase_train = graph.get_tensor_by_name('phase_train:0') while True: try: path = img_queue.get(timeout=300) except queue.Empty: logging.info('Input queue is empty. Shutting down process.') break else: if path == 'END': break logging.debug("{}: Processing {}".format(proc_name, path)) try: image = cv2.imread(path) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) (h, w) = image.shape[:2] if (w, h) > (640, 480): image = cv2.resize(image, (0, 0), fx=0.4, fy=0.4, interpolation=cv2.INTER_AREA) image = _equalize(image) gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) boxes = FR.face_locations(gray, model=MODEL, number_of_times_to_upsample=2) for box in boxes: (t, r, b, l) = box rect = dlib.rectangle(l, t, r, b) face = aligner.align(image, gray, rect) try: y1, x2, y2, x1 = FR.face_locations(face, model=MODEL)[0] face = cv2.resize(face[y1:y2, x1:x2], (160, 160)) except IndexError: face = cv2.resize(image[t:b, l:r], (160, 160)) # if _blur_check(face): # logging.debug(f"{proc_name}: Face too blurry") # continue face = _prewhiten(face) feed_dict = {img_holder: [face], phase_train: False} encoding = session.run(embeddings, feed_dict=feed_dict) if len(encoding) > 0: enc_queue.put({'path': path, 'encoding': encoding[0]}) except Exception as e: enc_queue.put({'path': path, 'error': str(e)})
def main(): with tf.Graph().as_default() as _: with tf.Session() as sess: if get_num_of_files_in_dir(data_dir) == 0: print('No train data!') return dataset = facenet.get_dataset(data_dir) for cls in dataset: assert(len(cls.image_paths)>0, 'There must be at least one image for each class in the dataset') paths, labels = facenet.get_image_paths_and_labels(dataset) print('Number of classes: %d' % len(dataset)) print('Number of images: %d' % len(paths)) # Load the model print('Loading feature extraction model') facenet.load_model(model_path) # Get input and output tensors images_placeholder = sess.graph.get_tensor_by_name("input:0") embeddings = sess.graph.get_tensor_by_name("embeddings:0") phase_train_placeholder = sess.graph.get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] # Run forward pass to calculate embeddings print('Calculating features for images') nrof_images = len(paths) nrof_batches_per_epoch = int(math.ceil(1.0*nrof_images / batch_size)) emb_array = np.zeros((nrof_images, embedding_size)) for i in range(nrof_batches_per_epoch): start_index = i*batch_size end_index = min((i+1)*batch_size, nrof_images) paths_batch = paths[start_index:end_index] images = facenet.load_data(paths_batch, False, False, image_size) feed_dict = { images_placeholder:images, phase_train_placeholder:False } emb_array[start_index:end_index,:] = sess.run(embeddings, feed_dict=feed_dict) classifier_filename_exp = os.path.expanduser(classifier_filename) x, y = SMOTETomek(random_state=4).fit_sample(emb_array, labels) print('Training classifier') model = SVC(kernel='linear', probability=True) model.fit(x, y) # Create a list of class names class_names = [ cls.name.replace('_', ' ') for cls in dataset] # Saving classifier model with open(classifier_filename_exp, 'wb') as outfile: pickle.dump((model, class_names), outfile) print('Saved classifier model to file "%s"' % classifier_filename_exp)
def __init__(self): np.random.seed(seed=777) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5) self.sess = tf.Session(config=tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False)) self.pnet, self.rnet, self.onet = facenet.align.detect_face.create_mtcnn( self.sess, None) try: model_file = rospy.get_param('~model_file') classifier_file = rospy.get_param('~classifier_file') except KeyError: rospy.logerr('set ~model_file and ~classifier_file.') exit(-1) fn.load_model(model_file) self.classifier_filename_exp = os.path.expanduser(classifier_file) self.classifier_filename_exp = os.path.abspath( self.classifier_filename_exp) if not os.path.exists(self.classifier_filename_exp): self.is_have_classifier = False rospy.logwarn('train first.') else: self.is_have_classifier = True with open(self.classifier_filename_exp, 'rb') as infile: (self.model, self.class_names) = pickle.load(infile) self.images_placeholder = tf.get_default_graph().get_tensor_by_name( "input:0") self.embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") self.phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") self.embedding_size = self.embeddings.get_shape()[1] self.bridge = CvBridge() rgb_sub = message_filters.Subscriber('image_raw', Image) points_sub = message_filters.Subscriber('points', PointCloud2) ts = message_filters.ApproximateTimeSynchronizer([rgb_sub, points_sub], 10, 0.5, allow_headerless=True) ts.registerCallback(self.callback_image) self.pub_debug_image = rospy.Publisher('result_image', Image, queue_size=10) self.pub_result = rospy.Publisher('recognized_faces', RecognizedResult, queue_size=10) rospy.loginfo('initialized...')
def load_networks(self): with tf.Graph().as_default(): gpu_options = tf.GPUOptions() sess = tf.Session(config=tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False)) with sess.as_default(): self.pnet, self.rnet, self.onet = align.detect_face.create_mtcnn( sess, None) with tf.Graph().as_default(): gpu_options = tf.GPUOptions() sess = tf.Session(config=tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False)) with sess.as_default(): facenet.load_model(FACE_MODEL) images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") self.fnet = lambda images: sess.run(embeddings, feed_dict={ images_placeholder: images, phase_train_placeholder: False})
def prepare_encodings(self, image): representative = None with tf.Graph().as_default(): with tf.Session() as session: facenet.load_model('models/20180402-114759.pb') img_holder = tf.get_default_graph().get_tensor_by_name( 'input:0') embeddings = tf.get_default_graph().get_tensor_by_name( 'embeddings:0') phase_train = tf.get_default_graph().get_tensor_by_name( 'phase_train:0') #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image = self._equalize(image) faces = FR.face_locations(image, model='hog') if len(faces) == 1: (t, r, b, l) = faces[0] try: face = cv2.resize(image[t:b, l:r], (160, 160)) face = self._prewhiten(face) feed_dict = {img_holder: [face], phase_train: False} encoding = session.run(embeddings, feed_dict=feed_dict) if len(encoding) == 0: pass else: representative = encoding[0] except: print( 'Could not process face; make sure that the face is actually detectable' ) return None #------------- FOR DEBUG ----------------# #with open('representative.pkl', 'wb') as file: # pickle.dump(representative, file) #----------------------------------------# return representative
def profiling(args): mtcnn = {} mtcnn['session'] = tf.Session() #self.__pnet, self.__rnet, self.__onet = FaceDetector.create_mtcnn(self.__mtcnn, None) mtcnn['pnet'], mtcnn['rnet'], mtcnn['onet'] = FaceDetector.create_mtcnn( mtcnn['session'], None) emoc = EmotionClassifier() emoc.build_network(None) #facenet_res = facenet.load_model('../models/facenet/20170512-110547.pb') # InceptionResnet V1 facenet_sqz = facenet.load_model( '../models/facenet/20180204-160909') # squeezenet
def compare(imageList): images, face_area, file_list = load_and_align_data(imageList, 160, 44, 1.0) # images = images[0:100] with tf.Graph().as_default(): with tf.Session() as sess: # Load the model facenet.load_model(model) # Get input and output tensors images_placeholder = tf.get_default_graph().get_tensor_by_name( "input:0") embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") # Run forward pass to calculate embeddings feed_dict = { images_placeholder: images, phase_train_placeholder: False } emb = sess.run(embeddings, feed_dict=feed_dict) return emb, file_list, face_area
def _getModel(): basepath=os.path.split(os.path.realpath(__file__))[0] model_path=os.path.join(basepath,'model') print('load model from %s'%model_path) with sess.as_default(): facenet.load_model(model_path) # Get input and output tensors images_placeholder = sess.graph.get_tensor_by_name("input:0") embeddings = sess.graph.get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") class RetModel(): def predict(self,image_input): ''' :param image_input: should have shape nx160x160x3 :return: nx512 ''' feed_dict={images_placeholder:image_input,phase_train_placeholder:False} emb=sess.run(embeddings,feed_dict=feed_dict) return emb return RetModel()
minsize = 20 # minimum size of face threshold = [0.6, 0.7, 0.8] # three steps's threshold factor = 0.709 # scale factor print('Creating networks and loading parameters') with tf.Graph().as_default(): gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=0.75, allow_growth=True) tf_config = tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False) sess = tf.Session(config=tf_config) with sess.as_default(): pnet, rnet, onet = align.detect_face.create_mtcnn(sess, None) facenet.load_model(os.path.abspath("./models/facenet/20180402-114759/")) images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") # Load emotion emotion = em.Emotion() emotion.load_weights("models/emotion/emotion.h5") # Load Facenet classifier_filename_exp = os.path.abspath("./models/facenet/20180402-114759/lfw_classifier.pkl") with open(classifier_filename_exp, 'rb') as infile: (model, class_names) = pickle.load(infile) def load_facenet_model():
def create_data_points(self): results = list() with tf.Graph().as_default(): with tf.Session() as session: facenet.load_model('models/20180402-114759.pb') img_holder = tf.get_default_graph().get_tensor_by_name( 'input:0') embeddings = tf.get_default_graph().get_tensor_by_name( 'embeddings:0') phase_train = tf.get_default_graph().get_tensor_by_name( 'phase_train:0') for (i, path) in enumerate(self.videos, 1): try: data = list() capture = cv2.VideoCapture(path) print('[INFO] Processing video %d of %d; path : %s' % (i, len(self.videos), path)) total_frames = capture.get(7) print('\t[INFO] Total number of frames : %d' % (total_frames)) self.clusterSize = int(total_frames / 10) if total_frames > 30 else 5 get_next = True for n in range(int(total_frames) - 1): if get_next is True: ret, frame = capture.read() if not ret: print('\tError!') continue get_next = not get_next try: frame = cv2.cvtColor( frame, cv2.COLOR_BGR2RGB) except: pass (h, w) = frame.shape[:2] if (h, w) > (640, 480): frame = cv2.resize(frame, (0, 0), fx=0.4, fy=0.4) if self.equalize is True: frame = self._equalize(frame) gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) boxes = FR.face_locations( gray, model=self.model, number_of_times_to_upsample=2) for box in boxes: (t, r, b, l) = box face = cv2.resize(frame[t:b, l:r], (160, 160)) if self.blur is True: if self._blur_check(face) is True: print( '\t[INFO] Skipping face - too blurry to process' ) continue face = self._prewhiten(face) feed_dict = { img_holder: [face], phase_train: False } encoding = session.run(embeddings, feed_dict=feed_dict) if len(encoding) > 0: d = [{ 'path': path, 'encoding': encoding[0] }] data.extend(d) else: get_next = True net_faces = self.cluster_data_points(data, processed=False) if net_faces is not None: for labelID in net_faces.keys(): d = [{ 'path': net_faces[labelID]['paths'][0], 'encoding': net_faces[labelID]['mean_encoding'] }] results.extend(d) except: print("There was an error. That's all we know.") return False with open('video_data.pkl', 'wb') as file: pickle.dump(results, file, protocol=pickle.HIGHEST_PROTOCOL) return True
print('==> Creating the {} directory...'.format(args['output_dir'])) os.makedirs(args['output_dir']) else: print('==> Skipping create the {} directory...'.format(args['output_dir'])) # Give the configuration and weight files for the model and load the network # using them. """net = cv2.dnn.readNetFromDarknet(args['model_cfg'], args['model_weights']) net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)""" detector = MTCNN() tf.Graph().as_default() sess = tf.Session() model = facenet.load_model( "./model-weights/20180408-102900/20180408-102900.pb") images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name( "phase_train:0") embedding_size = embeddings.get_shape()[1] face_aligner = face_utils.FaceAligner(dlib.shape_predictor( "./model-weights/shape_predictor_5_face_landmarks.dat"), desiredFaceWidth=160, desiredFaceHeight=160, desiredLeftEye=(0.25, 0.2)) celeb_classes = [ "ben_afflek", "elton_john", "jerry_seinfeld", "madonna", "mindy_kaling" ]
def RecognizeFace(frames, model=None, class_names=None): with tf.Graph().as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.6) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) with sess.as_default(): pnet, rnet, onet = detect_face.create_mtcnn(sess, npy) minsize = 20 # minimum size of face threshold = [0.6, 0.7, 0.7] # three steps's threshold factor = 0.709 # scale factor margin = 32 frame_interval = 3 batch_size = 1000 image_size = 160 input_image_size = 160 print('Loading feature extraction model') facenet.load_model(modeldir) images_placeholder = tf.get_default_graph().get_tensor_by_name( "input:0") embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] classifier_filename_exp = os.path.expanduser(classifier_filename) if model == None or class_names == None: with open(classifier_filename_exp, 'rb') as infile: (model, class_names) = pickle.load(infile) # video_capture = cv2.VideoCapture("akshay_mov.mp4") c = 0 HumanNames = class_names print(HumanNames) print('Start Recognition!') prevTime = 0 # ret, frame = video_capture.read() #frame = cv2.imread(img_path,0) #frame = cv2.resize(frame, (0,0), fx=0.5, fy=0.5) #resize frame (optional) total_faces_detected = {} for frame in frames: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) curTime = time.time() + 1 # calc fps timeF = frame_interval if (c % timeF == 0): find_results = [] if frame.ndim == 2: frame = facenet.to_rgb(frame) frame = frame[:, :, 0:3] bounding_boxes, _ = detect_face.detect_face( frame, minsize, pnet, rnet, onet, threshold, factor) nrof_faces = bounding_boxes.shape[0] print('Face Detected: %d' % nrof_faces) if nrof_faces > 0: det = bounding_boxes[:, 0:4] img_size = np.asarray(frame.shape)[0:2] cropped = [] scaled = [] scaled_reshape = [] bb = np.zeros((nrof_faces, 4), dtype=np.int32) for i in range(nrof_faces): emb_array = np.zeros((1, embedding_size)) bb[i][0] = det[i][0] bb[i][1] = det[i][1] bb[i][2] = det[i][2] bb[i][3] = det[i][3] #inner exception if bb[i][0] <= 0 or bb[i][1] <= 0 or bb[i][ 2] >= len(frame[0]) or bb[i][3] >= len( frame): print('face is too close') break cropped.append(frame[bb[i][1]:bb[i][3], bb[i][0]:bb[i][2], :]) cropped[i] = facenet.flip(cropped[i], False) scaled.append( misc.imresize(cropped[i], (image_size, image_size), interp='bilinear')) scaled[i] = cv2.resize( scaled[i], (input_image_size, input_image_size), interpolation=cv2.INTER_CUBIC) scaled[i] = facenet.prewhiten(scaled[i]) scaled_reshape.append(scaled[i].reshape( -1, input_image_size, input_image_size, 3)) feed_dict = { images_placeholder: scaled_reshape[i], phase_train_placeholder: False } emb_array[0, :] = sess.run(embeddings, feed_dict=feed_dict) predictions = model.predict_proba(emb_array) print(predictions) best_class_indices = np.argmax(predictions, axis=1) # print(best_class_indices) best_class_probabilities = predictions[ np.arange(len(best_class_indices)), best_class_indices] #plot result idx under box text_x = bb[i][0] text_y = bb[i][3] + 20 print('Result Indices: ', best_class_indices[0]) print(HumanNames) for H_i in HumanNames: # print(H_i) if HumanNames[best_class_indices[ 0]] == H_i and best_class_probabilities >= 0.4: result_names = HumanNames[ best_class_indices[0]] if result_names in total_faces_detected: if predictions[0][best_class_indices[ 0]] > total_faces_detected[ result_names]: total_faces_detected[ result_names] = predictions[ 0][best_class_indices[ 0]] else: total_faces_detected[ result_names] = predictions[0][ best_class_indices[0]] else: print("BHAKKK") if len(total_faces_detected) == 0: return None else: x = sorted(total_faces_detected.items(), key=operator.itemgetter(1)) return [x[len(x) - 1][0]]
def classifier(data_dir, model_path, classifier_path): with tf.Graph().as_default(): with tf.Session() as sess: if (get_num_of_files_in_dir(data_dir) == 0): print('There is no train data') return 0 dataset = facenet.get_dataset(data_dir) paths, labels = facenet.get_image_paths_and_labels(dataset) # Load pretrained model's train parameter facenet.load_model(model_path) images_placeholder = tf.get_default_graph().get_tensor_by_name( "input:0") embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] # Preprocess images nrof_images = len(paths) nrof_batches_per_epoch = int( math.ceil(1.0 * nrof_images / batch_size)) emb_array = np.zeros((nrof_images, embedding_size)) for i in range(nrof_batches_per_epoch): start_index = i * batch_size end_index = min((i + 1) * batch_size, nrof_images) paths_batch = paths[start_index:end_index] images = facenet.load_data(paths_batch, False, False, image_size) feed_dict = { images_placeholder: images, phase_train_placeholder: False } emb_array[start_index:end_index, :] = sess.run( embeddings, feed_dict=feed_dict) # Load classifier model, class-label dict with open(classifier_path, 'rb') as infile: (model, class_names) = pickle.load(infile) print(class_names) # print('Loaded classifier model from file "%s"' % classifier_path) predictions = model.predict_proba(emb_array) # print(len(paths)) # print(prediction_time_elapsed/len(paths)) best_class_indices = np.argmax(predictions, axis=1) best_class_probabilities = predictions[ np.arange(len(best_class_indices)), best_class_indices] label_to_class_map = create_label_to_class_map(class_names) print(label_to_class_map) data_dir_result_label_map = create_data_dir_result_label_map( paths, best_class_indices, best_class_probabilities) data_dir_result_class_map = {} for dir_name in data_dir_result_label_map: label = data_dir_result_label_map[dir_name] data_dir_result_class_map[dir_name] = label_to_class_map[label] print('dir_name : ' + dir_name + '\t recognition : ' + label_to_class_map[label]) rename_all_files_by_details(best_class_indices, best_class_probabilities, paths, label_to_class_map) return data_dir_result_class_map
# Imports import cv2 import time import numpy as np import tensorflow as tf import facenet.facenet as facenet # Init Network gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) facenet.load_model("facenet/20180402-114759.pb") images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") # Load Image loaded_img = cv2.imread("images/obama-aligned.jpg") resized_img = cv2.resize(loaded_img, (160, 160), interpolation=cv2.INTER_LINEAR) # Resize to 160x160 images = [] images.append(facenet.prewhiten(resized_img)) images = np.stack(images) # Run on blank # Done because some networks do some setup on the first call... feed_dict = {images_placeholder: images, phase_train_placeholder: False} sess.run(embeddings, feed_dict=feed_dict)
def __init__(self, facenet_model_checkpoint): # tf 2 does not have session anymore self.sess = tf.Session() with self.sess.as_default(): facenet.load_model(facenet_model_checkpoint)
def main(args, self): self.update_state(state='RUNNING') with tf.Graph().as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.25, allow_growth = True) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: np.random.seed(seed=args.seed) if args.use_split_dataset: dataset_tmp = facenet.get_dataset(args.data_dir) train_set, test_set = split_dataset(dataset_tmp, args.min_nrof_images_per_class, args.nrof_train_images_per_class) if (args.mode=='TRAIN'): dataset = train_set elif (args.mode=='CLASSIFY'): dataset = test_set else: dataset = facenet.get_dataset(args.data_dir) # Check that there are at least one training image per class for cls in dataset: assert len(cls.image_paths)>0, 'There must be at least one image for each class in the dataset' paths, labels = facenet.get_image_paths_and_labels(dataset) print('Number of classes: %d' % len(dataset)) print('Number of images: %d' % len(paths)) # Load the model print('Loading feature extraction model') facenet.load_model(args.model) # Get input and output tensors images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] # Run forward pass to calculate embeddings print('Calculating features for images') nrof_images = len(paths) nrof_batches_per_epoch = int(math.ceil(1.0*nrof_images / args.batch_size)) emb_array = np.zeros((nrof_images, embedding_size)) for i in range(nrof_batches_per_epoch): start_index = i*args.batch_size end_index = min((i+1)*args.batch_size, nrof_images) paths_batch = paths[start_index:end_index] images = facenet.load_data(paths_batch, False, False, args.image_size) feed_dict = { images_placeholder:images, phase_train_placeholder:False } emb_array[start_index:end_index,:] = sess.run(embeddings, feed_dict=feed_dict) classifier_filename_exp = os.path.expanduser(args.classifier_filename) if (args.mode=='TRAIN'): # Train classifier print('Training classifier') model = SVC(kernel='linear', probability=True) model.fit(emb_array, labels) # Create a list of class names class_names = [ cls.name.replace('_', ' ') for cls in dataset] # Saving classifier model with open(classifier_filename_exp, 'wb') as outfile: pickle.dump((model, class_names), outfile) print('Saved classifier model to file "%s"' % classifier_filename_exp) elif (args.mode=='CLASSIFY'): # Classify images print('Testing classifier') with open(classifier_filename_exp, 'rb') as infile: (model, class_names) = pickle.load(infile) print('Loaded classifier model from file "%s"' % classifier_filename_exp) predictions = model.predict_proba(emb_array) best_class_indices = np.argmax(predictions, axis=1) best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices] for i in range(len(best_class_indices)): print('%4d %s: %.3f' % (i, class_names[best_class_indices[i]], best_class_probabilities[i])) accuracy = np.mean(np.equal(best_class_indices, labels)) print('Accuracy: %.3f' % accuracy) sess.close()
def main(args): with tf.Graph().as_default(): with tf.Session() as sess: np.random.seed(seed=args.seed) # Check that there are at least one training image per class imgs = args.imgs labels = args.labels class_names = args.class_names # Load the model print('Loading feature extraction model') facenet.load_model(args.model) # Get input and output tensors images_placeholder = tf.get_default_graph().get_tensor_by_name( "input:0") embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] # Run forward pass to calculate embeddings print('Calculating features for images') nrof_images = len(imgs) nrof_batches_per_epoch = int( math.ceil(1.0 * nrof_images / args.batch_size)) emb_array = np.zeros((nrof_images, embedding_size)) for i in range(nrof_batches_per_epoch): start_index = i * args.batch_size end_index = min((i + 1) * args.batch_size, nrof_images) images = imgs[start_index:end_index] feed_dict = { images_placeholder: images, phase_train_placeholder: False } emb_array[start_index:end_index, :] = sess.run( embeddings, feed_dict=feed_dict) classifier_filename_exp = os.path.expanduser( args.classifier_filename) if (args.mode == 'TRAIN'): # Train classifier print('Training classifier') model = SVC(kernel="linear", probability=True) #model = KNeighborsClassifier(n_neighbors=9) #model = BernoulliNB() model.fit(emb_array, labels) # Create a list of class names if classifier_filename_exp == "PIPE_OUT_MODEL": print('Built Classifier') return (model, class_names) # Saving classifier model with open(classifier_filename_exp, 'wb') as outfile: pickle.dump((model, class_names), outfile) print('Saved classifier model to file "%s"' % classifier_filename_exp) elif (args.mode == 'CLASSIFY'): # Classify images print('Testing classifier') with open(classifier_filename_exp, 'rb') as infile: (model, class_names) = pickle.load(infile) print('Loaded classifier model from file "%s"' % classifier_filename_exp) predictions = model.predict_proba(emb_array) best_class_indices = np.argmax(predictions, axis=1) best_class_probabilities = predictions[ np.arange(len(best_class_indices)), best_class_indices] for i in range(len(best_class_indices)): print('%4d %s: %.3f' % (i, class_names[best_class_indices[i]], best_class_probabilities[i])) accuracy = np.mean(np.equal(best_class_indices, labels)) print('Accuracy: %.3f' % accuracy)
def create_data_points(self): data = list() with tf.Graph().as_default(): with tf.Session() as session: facenet.load_model('models/20180402-114759.pb') img_holder = tf.get_default_graph().get_tensor_by_name( 'input:0') embeddings = tf.get_default_graph().get_tensor_by_name( 'embeddings:0') phase_train = tf.get_default_graph().get_tensor_by_name( 'phase_train:0') for (i, path) in enumerate(self.images, 1): try: image = cv2.imread(path) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) (h, w) = image.shape[:2] if (w, h) > (640, 480): image = cv2.resize(image, (0, 0), fx=0.4, fy=0.4, interpolation=cv2.INTER_AREA) print('[INFO] Processing image %d of %d; path : %s' % (i, len(self.images), path)) if self.equalize is True: image = self._equalize(image) gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) boxes = FR.face_locations( gray, model=self.model, number_of_times_to_upsample=2) print('\t[INFO] Found %d faces' % (len(boxes))) for box in boxes: (t, r, b, l) = box if self.alignFace is True: rect = dlib.rectangle(l, t, r, b) face = self.aligner.align(image, gray, rect) try: (y1, x2, y2, x1) = FR.face_locations( face, model=self.model)[0] face = cv2.resize(face[y1:y2, x1:x2], (160, 160)) except: pass # Aligning faces introduces unnecessary background elements. # Re-running face detection is slightly more computationally # intensive, but cuts out those background elements. else: face = cv2.resize(image[t:b, l:r], (160, 160)) if self.blur is True: if self._blur_check(face) is True: print( '\t[INFO] Skipping face - too blurry to process' ) continue face = self._prewhiten(face) feed_dict = { img_holder: [face], phase_train: False } encoding = session.run(embeddings, feed_dict=feed_dict) if len(encoding) > 0: d = [{'path': path, 'encoding': encoding[0]}] data.extend(d) except: print("There was an error. That's all we know.") return False with open('data_points.pkl', 'wb') as file: pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) return True
l, a, b = cv2.split(lab) clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(16, 16)) l1 = clahe.apply(l) processed = cv2.merge((l1, a, b)) processed = cv2.cvtColor(processed, cv2.COLOR_LAB2BGR) return processed cap = cv2.VideoCapture(0) dist = Similarity() with tf.Graph().as_default(): with tf.Session() as session: facenet.load_model('20180402-114759.pb') img_holder = tf.get_default_graph().get_tensor_by_name('input:0') embeddings = tf.get_default_graph().get_tensor_by_name('embeddings:0') phase_train = tf.get_default_graph().get_tensor_by_name( 'phase_train:0') test_image = cv2.imread('test2.jpg') #test_image = hist_equalize(test_image) (y1, x2, y2, x1) = FR.face_locations(test_image, model='hog')[0] test_face = cv2.resize(test_image[y1:y2, x1:x2], (160, 160)) #test_face = hist_equalize(test_face) feed_dict = {img_holder: [test_face], phase_train: False} test_enc1 = session.run(embeddings, feed_dict=feed_dict) cv2.imshow('Test 1', test_face) test_face = prewhiten(test_face) feed_dict = {img_holder: [test_face], phase_train: False}
def main(): if ARGS.test == 'train': train(ARGS) elif ARGS.test == 'gen': gen(ARGS) elif ARGS.test == 'predict': predict(ARGS) elif ARGS.test == 'server': server_start(ARGS) elif ARGS.test == 'server_production': server_start(ARGS, ARGS.port) elif ARGS.test == 'hnm': hnm(ARGS) elif ARGS.test == 'val': val(ARGS) elif ARGS.test == 'fer': fer(ARGS) elif ARGS.test == 'profiling': profiling(ARGS) elif ARGS.test == 'facenet': print(facenet) mtcnn = tf.Session() pnet, rnet, onet = FaceDetector.create_mtcnn(mtcnn, None) # Load the model t_ = time.time() print('Loading model...') #fnet = facenet.load_model('../models/facenet/20180204-160909') # squeezenet fnet = facenet.load_model( '../models/facenet/20170512-110547.pb') # InceptionResnet V1 t_ = time.time() - t_ print('done', t_ * 1000) stats = { 'same': { 'd_avg': 0., 'd_max': -9999., 'd_min': 9999., 'sqr_avg': 0., 'count': 0, }, 'diff': { 'd_avg': 0., 'd_max': -9999., 'd_min': 9999., 'sqr_avg': 0., 'count': 0, }, 'precision': {}, 'timing': { 'count': 0, 'forward': 0. } } if True: # Get input and output tensors emb = None names = [] for iteration in range(16): # Load faces from LFW dataset and parse their names from path to group faces images = [] batch_size = 128 fid = 0 for i in range(batch_size): f = DirectoryWalker().get_a_file( directory='../data/face/lfw', filters=['.jpg']) if f is None or not f.path: break n = os.path.split(os.path.split(f.path)[0])[1] #print('name', n) n = abs(hash(n)) % (10**8) img = cv2.imread(f.path, 1) img = img extents, landmarks = FaceDetector.detect_face( img / 255., 120, pnet, rnet, onet, threshold=[0.6, 0.7, 0.9], factor=0.6, interpolation=cv2.INTER_LINEAR) for j, e in enumerate(extents): x1, y1, x2, y2, confidence = e.astype(dtype=np.int) #print(len(landmarks[j])) #cropped = img[int(x1):int(x2), int(y1):int(y2), :] aligned = FaceApplications.align_face(img, landmarks[j], intensity=1., sz=160, ortho=True, expand=1.5) #cv2.imwrite('../data/face/mtcnn_cropped/'+str(fid).zfill(4)+'.jpg', aligned) images.append(aligned / 255.) names.append(n) """debug = aligned.astype(dtype=np.int) print('debug', debug) for p in debug: cv2.circle(img, (p[0], p[1]), 2, (255, 0, 255)) for p in landmarks[j]: cv2.circle(img, (p[0], p[1]), 2, (255, 255, 0))""" fid += 1 #cv2.imwrite('../data/face/mtcnn_cropped/'+str(i).zfill(4)+'-annotated.jpg', img) # Run forward pass to calculate embeddings if len(images): t_ = time.time() if emb is None: emb = fnet(images) else: emb = np.concatenate((emb, fnet(images))) #emb = emb + sess.run(embeddings, feed_dict=feed_dict) t_ = time.time() - t_ stats['timing']['count'] += len(images) stats['timing']['forward'] += t_ * 1000 print('forward', emb.shape, t_ * 1000) print() print() print('avg. forward time:', stats['timing']['forward'] / stats['timing']['count']) # Test distance samples = sklearn.preprocessing.normalize(emb) for i1, s1 in enumerate(samples): for i2, s2 in enumerate(samples): if i1 != i2: d_ = scipy.spatial.distance.cosine(s1, s2) if names[i1] == names[ i2]: # Same person as annotated by LFW cate = 'same' else: # Different person cate = 'diff' c_ = stats[cate]['count'] stats[cate]['d_avg'] = stats[cate]['d_avg'] * c_ / ( c_ + 1) + d_ / (c_ + 1) d_sqr = d_ * d_ stats[cate]['sqr_avg'] = stats[cate][ 'sqr_avg'] * c_ / (c_ + 1) + d_sqr / (c_ + 1) if d_ > stats[cate]['d_max']: stats[cate]['d_max'] = d_ elif d_ < stats[cate]['d_min']: stats[cate]['d_min'] = d_ stats[cate]['count'] += 1 # Get statistics of precision on different thresholds increments = 64 for t_ in range(increments): threshold = 0.2 + t_ * (0.6 / increments) if threshold not in stats['precision']: stats['precision'][threshold] = { 'correct': 0, 'total': 0, 'precision': 0., 'true_pos': 0, 'total_pos': 0, 'recall': 0., } if (cate == 'same' and d_ <= threshold) or ( cate == 'diff' and d_ > threshold): stats['precision'][threshold]['correct'] += 1 if cate == 'same': if d_ <= threshold: stats['precision'][threshold][ 'true_pos'] += 1 stats['precision'][threshold]['total_pos'] += 1 stats['precision'][threshold][ 'recall'] = stats['precision'][threshold][ 'true_pos'] / stats['precision'][ threshold]['total_pos'] stats['precision'][threshold]['total'] += 1 stats['precision'][threshold]['precision'] = stats[ 'precision'][threshold]['correct'] / stats[ 'precision'][threshold]['total'] """tree = scipy.spatial.KDTree(samples) for i, s in enumerate(samples): print(i, tree.query(s))""" for cate in ['same', 'diff']: stats[cate]['stddev'] = stats[cate][ 'sqr_avg'] - stats[cate]['d_avg'] * stats[cate]['d_avg'] print() pp = pprint.PrettyPrinter(indent=4) pp.pprint(stats) # Print precision vs recall print() print('threshold,recall,precision') for t in stats['precision']: t_stat = stats['precision'][t] print( str(t) + ',' + str(t_stat['recall']) + ',' + str(t_stat['precision'])) elif ARGS.test == 'align': face_app = FaceApplications() face_app.align_dataset() elif ARGS.test == 'fxpress': fxpress(ARGS) elif ARGS.test == 'fxpress_train': fxpress_train(ARGS) elif ARGS.test == 'emoc': classifier = EmotionClassifier() classifier.build_network(ARGS) classifier.val(ARGS) elif ARGS.test == 'face_app': face_app = FaceApplications() face_app.detect() elif ARGS.test == 'face_benchmark': # Test different parameters, resolutions, interpolation methods for MTCNN face detection time vs precision interpolations = ['NEAREST', 'LINEAR', 'AREA'] resolutions = [256, 320, 384, 448, 512, 640, 1024, 1280] factors = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95] expectations = [ ['0001.jpg', 4], ['0002.jpg', 1], ['0003.jpg', 1], ['0004.jpg', 0], ['0005.jpg', 1], ['0006.jpg', 1], ['0007.jpg', 59], ['0008.jpg', 6], ['0009.jpg', 1], ['0010.jpg', 5], ['0011.jpg', 4], ['0012.jpg', 17], ['0013.jpg', 20], ['0014.jpg', 48], ['0015.jpg', 22], ] log_file = open('../data/face_benchmark.csv', 'w') log_file.write('time,precision_index,cp,options\n') for interp in interpolations: for res_cap in resolutions: for factor in factors: #res_cap = resolutions[0] #factor = factors[0] #interp = interpolations[0] options = { 'res_cap': res_cap, 'factor': factor, 'interp': interp, } if interp == 'NEAREST': interpolation = cv2.INTER_NEAREST elif interp == 'LINEAR': interpolation = cv2.INTER_LINEAR elif interp == 'AREA': interpolation = cv2.INTER_AREA iterations = 20 file_count = len(expectations) time_sampling = np.zeros(( file_count, iterations, ), dtype=np.float) pi_sampling = np.zeros(( file_count, iterations, ), dtype=np.float) image_dir = '../data/face_benchmark/' for k, item in enumerate(expectations): filename, expected_faces = item inpath = image_dir + filename img = cv2.imread(inpath, 1) if img is None: break img, scaling = ImageUtilities.fit_resize( img, maxsize=(res_cap, res_cap), interpolation=interpolation) retval, bindata = cv2.imencode('.jpg', img) bindata_b64 = base64.b64encode(bindata).decode() requests = { 'requests': [{ 'requestId': str(uuid.uuid1()), 'media': { 'content': bindata_b64 }, 'services': [{ 'type': 'face_', 'model': 'a-emoc', 'options': options }] }], 'timing': { 'client_sent': time.time() } } for i in range(iterations): #url = 'http://10.129.11.4/cgi/predict' requests['timing']['client_sent'] = time.time() url = 'http://192.168.41.41:8080/predict' postdata = json.dumps(requests) #print() #print(postdata) #print() request = Request(url, data=postdata.encode()) response = json.loads( urlopen(request).read().decode()) timing = response['timing'] server_time = timing['server_sent'] - timing[ 'server_rcv'] #print('server time:', server_time) total_time = (time.time() - timing['client_sent']) * 1000 client_time = total_time - server_time print('response time:', total_time) pi = 0. for r_ in response['requests']: for s_ in r_['services']: rects_ = s_['results']['rectangles'] if expected_faces: pi = len(rects_) / expected_faces elif len(rects_): pi = expected_faces / len(rects_) else: pi = 1.0 #print('faces detected:', len(rects_), pi) time_sampling[k][i] = total_time pi_sampling[k][i] = pi #time.sleep(0.5) #print() #print(response) #print() time_mean = np.mean(time_sampling) pi_mean = np.mean(pi_sampling) * 100 cp = pi_mean * pi_mean / time_mean print(time_mean, pi_mean) print() log_file.write(','.join([ str(time_mean), str(pi_mean), str(cp), json.dumps(options) ]) + '\n') log_file.flush() log_file.close()