def handle_requests(self): factor = self.configs.recognition["factor"] print("Face service is started on: ", self.address) while True: result_dict = {} face_locations = [] face_encodings = [] face_names = [] message = "OK" final_results = [] try: # Get image from socket request = self.socket.recv() image = zmq_comm.decode_request(request) # cv2.imwrite("./received.jpg", image) # Resize image to 1/factor size for faster face recognition processing small_img = cv2.resize(image, (0, 0), fx=1 / factor, fy=1 / factor) # Find all the faces and face encodings in the current frame of video face_locations = face_recognition.face_locations(small_img) face_encodings = face_recognition.face_encodings( small_img, face_locations) # print(face_encodings) for face_encoding in face_encodings: # See if the face is a match for the known face(s) match_name = self.recognize_face( face_encoding, self.encodings, self.img_list, self.configs.recognition["knn"], self.configs.recognition["similarity_threshold"]) face_names.append(match_name) predictions = [] # Display the results for (top, right, bottom, left), match_name in zip(face_locations, face_names): if match_name == self.configs.recognition["not_recog_msg"]: continue # Scale back up face locations since the frame we detected in was scaled to 1/4 size top = int(top * factor) bottom = int(bottom * factor) left = int(left * factor) right = int(right * factor) face_dict = {} face_dict['name'] = match_name face_dict['topleft'] = {"x": left, "y": top} face_dict['bottomright'] = {"x": right, "y": bottom} # print(face_dict) predictions.append(face_dict) final_results = predictions except Exception as e: message = str(e) result_dict["result"] = final_results result_dict["message"] = message self.socket.send_json(result_dict)
def handle_requests(self): print("Face service is started on: ", self.address) factor = self.configs.detection["factor"] while True: result_dict = {} message = "OK" final_results = [] try: # Get image from socket request = self.socket.recv() image = zmq_comm.decode_request(request) gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) gray = cv2.resize(gray, None, fx=1.0 / factor, fy=1.0 / factor, interpolation=cv2.INTER_LINEAR) faces = self.face_detector.detect(gray) face_labels = [] if len(faces) == 0: message = "Could not find any faces" for index, face in enumerate(faces): #print(face) face = dlib.rectangle(int(face.left() * factor), int(face.top() * factor), int(face.right() * factor), int(face.bottom() * factor)) aligned_n_cropped = self.face_detector.align(image, face) face_id = self.face_recognizer.recognize(aligned_n_cropped) face_labels.append(face_id) predictions = [] # Display the results for dlib_rect, match_name in zip(faces, face_labels): left, top, right, bottom = self.face_detector.rect_to_tuple( dlib_rect) if match_name == self.configs.recognition["not_recog_msg"]: continue # Scale back up face locations since the frame we detected in was scaled to 1/4 size top = int(top * factor) bottom = int(bottom * factor) left = int(left * factor) right = int(right * factor) face_dict = {} face_dict['name'] = match_name face_dict['topleft'] = {"x": left, "y": top} face_dict['bottomright'] = {"x": right, "y": bottom} predictions.append(face_dict) final_results = predictions except Exception as e: print("EXCEPTION: ", str(e)) raise e result_dict["result"] = final_results result_dict["message"] = message self.socket.send_json(result_dict)
def handle_requests(socket, plate_recognizer): # Load recognition related configs image_width = PlateConfig.recognition["image_width"] image_height = PlateConfig.recognition["image_height"] # Compile regex that matches with invalid TR plates tr_plate_regex = PlateConfig.recognition["tr_plate_regex"] plate_pattern = re.compile(tr_plate_regex) print("Plate recognition is started on: ", tcp_address) net_inp = plate_recognizer.get_layer(name='the_input').input net_out = plate_recognizer.get_layer(name='softmax').output while True: result_dict = {} message = "OK" found_plate = "" topleft = {} bottomright = {} coord_info = {} try: # Get image from socket and perform detection request = socket.recv() image = zmq_comm.decode_request(request) #TODO: check len of shape it must be three # Do not attempt manipulating image if it is already grayscale if image.shape[2] != 1: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) image = cv2.resize(image, (image_width, image_height)) image = image.astype(np.float32) image /= 255 # width and height are backwards from typical Keras convention # because width is the time dimension when it gets fed into the RNN image = image.T # (128, 32) image = np.expand_dims(image, -1) # (128, 32, 1) # Feeding a single image at a time: (1, image_width, image_height, 1) # Otherwise X_data should be of shape: (n, image_width, image_height, 1) X_data = np.ones([1, image_width, image_height, 1]) X_data[0] = image net_out_value = sess.run(net_out, feed_dict={net_inp: X_data}) pred_texts = decode_batch(net_out_value) if len(pred_texts) > 0: found_plate = pred_texts[0] except Exception as e: message = str(e) all_info = {} all_info["plate"] = found_plate all_info["coords"] = coord_info result_dict["result"] = [all_info] result_dict["message"] = message # print(result_dict) socket.send_json(result_dict)
def handle_requests(socket, plate_detector): iteration_inc = PlateConfig.detection["detection_iteration_increase"] strictness = PlateConfig.detection["detection_strictness"] print("Plate detection is started on: ", tcp_address) while True: result_dict = {} message = "OK" all_detected_plates = [] try: # Get image from socket and perform detection request = socket.recv() image = zmq_comm.decode_request(request) #TODO: check len of shape it must be three # Do not attempt manipulating image if it is already grayscale if image.shape[2] != 1: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) plate_coords = detect_plates(plate_detector, iteration_inc, strictness, image) for (x, y, w, h) in plate_coords: topleft = {} bottomright = {} coord_info = {} topleft["x"] = int(x) topleft["y"] = int(y) bottomright["x"] = int(x + w) bottomright["y"] = int(y + h) coord_info["topleft"] = topleft coord_info["bottomright"] = bottomright detected = {} detected["coords"] = coord_info detected["area"] = int(w * h) all_detected_plates.append(detected) except Exception as e: message = str(e) all_detected_plates = sorted(all_detected_plates, key=lambda detected: detected["area"], reverse=True) result_dict["result"] = all_detected_plates result_dict["message"] = message # print(result_dict) socket.send_json(result_dict)
def handle_requests(self): print("Cropper service is started on: ", self.address) while True: result_dict = {} predictions = [] message = "OK" try: request = self.socket.recv() image = zmq_comm.decode_request(request) predictions = self.extract_objects(image) except Exception as e: message = str(e) result_dict["result"] = predictions result_dict["message"] = message self.socket.send_json(result_dict)
def handle_requests(socket): model = load_model() print('The model is loaded without any errors.') while True: try: request = socket.recv() image = zmq_comm.decode_request(request) predictions = extract_objects(model, image) # Build and send the json result_dict = {} result_dict["result"] = predictions result_dict["message"] = "OK" socket.send_json(result_dict) except Exception as e: result_dict = {} result_dict["result"] = [] result_dict["message"] = str(e) socket.send_json(result_dict)
def handle_requests(socket): # Set tensorflow configs tf_config = K.tf.ConfigProto() tf_config.gpu_options.per_process_gpu_memory_fraction = CarConfig.classifier[ "gpu_memory_frac"] K.set_session(K.tf.Session(config=tf_config)) init = K.tf.global_variables_initializer() sess = K.get_session() sess.run(init) # Load model once model, loaded_model_json = load_model_and_json() print("Car classifier is ready to roll.") while True: result_dict = {} predict_list = [] message = "OK" # Receive image and predict classes try: request = socket.recv() image = zmq_comm.decode_request(request) # Preprocess the image image = image * 1. / 255 image = cv2.resize(image, (299, 299)) image = image.reshape((1, ) + image.shape) # Feed image to classifier preds = model.predict(image)[0] predict_list = classifyIndices(loaded_model_json, preds, CarConfig.classifier["n"]) except tf.errors.OpError as e: message = e.message except Exception as e: message = str(e) predictions = [] tags = ["model", "score"] for index, pred in enumerate(predict_list): predictions.append(dict(zip(tags, [pred.name, str(pred.score)]))) result_dict["result"] = predictions result_dict["message"] = message socket.send_json(result_dict)
def handle_requests(self): # Load detection related configs iteration_inc = self.configs.detection["detection_iteration_increase"] strictness = self.configs.detection["detection_strictness"] # Load recognition related configs required_image_width = self.configs.recognition["image_width"] required_image_height = self.configs.recognition["image_height"] # Compile regex that matches with invalid TR plates tr_plate_regex = self.configs.recognition["tr_plate_regex"] plate_pattern = re.compile(tr_plate_regex) net_inp = self.plate_recognizer.get_layer(name='the_input').input net_out = self.plate_recognizer.get_layer(name='softmax').output print("Plate server is started on: ", self.address) while True: result_dict = {} message = "OK" all_detected_plates = [] found_plate = "" try: # Get image from socket and perform detection request = self.socket.recv() image = zmq_comm.decode_request(request) #TODO: check len of shape it must be three # Do not attempt manipulating image if it is already grayscale image_h, image_w, dim = image.shape if dim != 1: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) plate_coords = self.plate_detector.detectMultiScale( image, iteration_inc, strictness) for (x, y, w, h) in plate_coords: topleft = {} bottomright = {} coord_info = {} topleft["x"] = int(x) topleft["y"] = int(y) bottomright["x"] = int(x + w) bottomright["y"] = int(y + h) coord_info["topleft"] = topleft coord_info["bottomright"] = bottomright detected = {} detected["coords"] = coord_info detected["area"] = int(w * h) all_detected_plates.append(detected) all_detected_plates = sorted( all_detected_plates, key=lambda detected: detected["area"], reverse=True) if len(all_detected_plates) > 0: # Feeding a single image at a time: (1, image_width, image_height, 1) # Otherwise X_data should be of shape: (n, image_width, image_height, 1) X_data = np.ones([ len(all_detected_plates), required_image_width, required_image_height, 1 ]) for index, detected_plate in enumerate( all_detected_plates): coord_info = detected_plate['coords'] # If coordinate info is empty, return empty plate since we could not found any plates if not coord_info: print( "Coord info is empty, could not find any plates.." ) break # Crop the plate out of the car image topleft_x = int(coord_info['topleft']['x']) topleft_y = int(coord_info['topleft']['y']) bottomright_x = int(coord_info['bottomright']['x']) bottomright_y = int(coord_info['bottomright']['y']) width = int(bottomright_x - topleft_x) height = int(bottomright_y - topleft_y) margin_width = int(height / 2) margin_height = int(height / 4) # Add margins topleft_x = max(0, topleft_x - margin_width) topleft_y = max(0, topleft_y - margin_height) bottomright_x = min(image_w, bottomright_x + margin_width) bottomright_y = min(image_h, bottomright_y + margin_height) # Crop the detected plate cropped_plate_img = image[topleft_y:bottomright_y, topleft_x:bottomright_x] # Recognize the cropped plate image cropped_plate_img = cv2.resize( cropped_plate_img, (required_image_width, required_image_height)) cropped_plate_img = cropped_plate_img.astype( np.float32) cropped_plate_img /= 255 # width and height are backwards from typical Keras convention # because width is the time dimension when it gets fed into the RNN cropped_plate_img = cropped_plate_img.T # (128, 32) cropped_plate_img = np.expand_dims( cropped_plate_img, -1) # (128, 32, 1) # Populate X_data with cropped and processed plate regions X_data[index] = cropped_plate_img net_out_value = self.sess.run(net_out, feed_dict={net_inp: X_data}) pred_texts = self.decode_batch(net_out_value) filtered_candidates = [] for plate_text in pred_texts: # If our regex matches with a plate, then it is a good candidate if plate_pattern.search(plate_text): filtered_candidates.append(plate_text) if len(filtered_candidates) > 0: found_plate = filtered_candidates[0] except Exception as e: message = str(e) result_dict["result"] = found_plate result_dict["message"] = message # print(result_dict) self.socket.send_json(result_dict)
def handle_requests(socket): global alpr global plate_pattern # Load SSD model ssd_model = load_SSD_model() # Load trained tensorflow car classifier and set tensorflow configs tf_config = K.tf.ConfigProto() tf_config.gpu_options.per_process_gpu_memory_fraction = CarConfig.crcl[ "classifier_gpu_memory_frac"] K.set_session(K.tf.Session(config=tf_config)) init = K.tf.global_variables_initializer() sess = K.get_session() sess.run(init) # Define a concurrent future and an executor to be used in case plate recognition is enabled future1 = None executor = None use_plate_recognition = CarConfig.crcl["enable_plate_recognition"] if use_plate_recognition: print("Plate recognition is on, loading OpenALPR..") # Load configs and Alpr() once country = PlateConfig.recognition['country'] region = PlateConfig.recognition['region'] openalpr_conf_dir = PlateConfig.recognition['openalpr_conf_dir'] openalpr_runtime_data_dir = PlateConfig.recognition[ 'openalpr_runtime_data_dir'] top_n = PlateConfig.recognition['top_n'] # Compile regex that matches with invalid TR plates tr_plate_regex = PlateConfig.recognition["tr_plate_regex"] plate_pattern = re.compile(tr_plate_regex) alpr = Alpr(country, openalpr_conf_dir, openalpr_runtime_data_dir) if not alpr.is_loaded(): print("Error loading OpenALPR") return alpr.set_top_n(top_n) alpr.set_default_region(region) # Initialize process executor once executor = ProcessPoolExecutor(max_workers=1) # Load model once car_classifier_model, car_classifier_loaded_model_json = load_model_and_json( ) print('Loaded both models successfully, ready to roll.') print('Server is started on:', tcp_address) while True: try: request = socket.recv() image = zmq_comm.decode_request(request) found_objects = [] with isess.as_default(): found_objects = extract_objects(ssd_model, image) clasifications = [] with sess.as_default(): # Filter cars, buses and trucks, classify each of them for o in found_objects: # Crop image according to empirical margin values cropped = crop_image(image, o['topleft'], o['bottomright'], float(o['confidence'])) if cropped is None: continue found_plate = "" predictions = [] filtered_candidates = [] # Run plate recognition in parallel while the main thread continues # Note that, if you call 'future.result()' here, it just waits for process to end if use_plate_recognition: future1 = executor.submit(extract_plate, cropped) # Preprocess the image cropped = cropped * 1. / 255 cropped = cv2.resize(cropped, (299, 299)) cropped = cropped.reshape((1, ) + cropped.shape) # Feed image to classifier preds = car_classifier_model.predict(cropped)[0] predict_list = classifyIndices( car_classifier_loaded_model_json, preds, CarConfig.crcl["n"]) predictions = [] tags = ["model", "score"] for index, p in enumerate(predict_list): predictions.append( dict(zip(tags, [p.name, str(p.score)]))) # Wait for plate recognition to finish its job if use_plate_recognition and future1 is not None: found_plate = future1.result() cl = { 'label': o['label'], 'confidence': o['confidence'], 'topleft': o['topleft'], 'bottomright': o['bottomright'], 'predictions': predictions, 'plate': found_plate } clasifications.append(cl) result_dict = {} result_dict["result"] = clasifications result_dict["message"] = "OK" # print(result_dict) socket.send_json(result_dict) except Exception as e: result_dict = {} result_dict["result"] = [] result_dict["message"] = str(e) socket.send_json(result_dict)
def handle_requests(socket): # Load SSD model ssd_model = load_SSD_model() # Load trained tensorflow car classifier and set tensorflow configs tf_config = K.tf.ConfigProto() tf_config.gpu_options.per_process_gpu_memory_fraction = CarConfig.crcl[ "classifier_gpu_memory_frac"] K.set_session(K.tf.Session(config=tf_config)) init = K.tf.global_variables_initializer() sess = K.get_session() sess.run(init) use_plate_recognition = CarConfig.crcl["enable_plate_recognition"] if use_plate_recognition: print("Plate recognition is on, loading OpenALPR..") # Load configs and Alpr() once country = PlateConfig.recognition['country'] region = PlateConfig.recognition['region'] openalpr_conf_dir = PlateConfig.recognition['openalpr_conf_dir'] openalpr_runtime_data_dir = PlateConfig.recognition[ 'openalpr_runtime_data_dir'] top_n = PlateConfig.recognition['top_n'] # Compile regex that matches with invalid TR plates tr_plate_regex = PlateConfig.recognition["tr_plate_regex"] plate_pattern = re.compile(tr_plate_regex) alpr = Alpr(country, openalpr_conf_dir, openalpr_runtime_data_dir) if not alpr.is_loaded(): print("Error loading OpenALPR") return alpr.set_top_n(top_n) alpr.set_default_region(region) # Load model once car_classifier_model, car_classifier_loaded_model_json = load_model_and_json( ) print('Loaded both models successfully, ready to roll.') print('Server is started on:', tcp_address) while True: try: request = socket.recv() image = zmq_comm.decode_request(request) found_objects = [] with isess.as_default(): found_objects = extract_objects(ssd_model, image) clasifications = [] with sess.as_default(): # Filter cars, buses and trucks, classify each of them for o in found_objects: # Crop image according to empirical margin values cropped = crop_image(image, o['topleft'], o['bottomright'], float(o['confidence'])) if cropped is None: continue # Save a copy for plate recognition original_cropped_img = cropped # cv2.imwrite('/home/taylan/Desktop/res/' + str(uuid.uuid4()) + '.jpg', cropped) # Preprocess the image cropped = cropped * 1. / 255 cropped = cv2.resize(cropped, (299, 299)) cropped = cropped.reshape((1, ) + cropped.shape) # Feed image to classifier preds = car_classifier_model.predict(cropped)[0] predict_list = classifyIndices( car_classifier_loaded_model_json, preds, CarConfig.crcl["n"]) predictions = [] tags = ["model", "score"] for index, p in enumerate(predict_list): predictions.append( dict(zip(tags, [p.name, str(p.score)]))) # If we are very sure about the found car's model, try to find its plate as well found_plate = "" if use_plate_recognition and float( predictions[0]["score"]) > 0.75: results = alpr.recognize_array( bytes( cv2.imencode('.jpg', original_cropped_img)[1])) # print("Results: ", results) filtered_candidates = [] for i, plate in enumerate(results['results']): for candidate in plate['candidates']: # print(candidate['plate']) # If our regex does matches with a plate, then it is a good candidate #WARNING: Since our regex expects spaces and openalpr does not put #any spaces in found plate string, it never matches if plate_pattern.search(candidate['plate']): filtered_candidates.append( candidate['plate']) # WARNING: It is assumed that there is only a single plate in the given image # Hence, we break after the first plate, even if there are more plates break # print(filtered_candidates) if len(filtered_candidates) > 0: found_plate = filtered_candidates[0] cl = { 'label': o['label'], 'confidence': o['confidence'], 'topleft': o['topleft'], 'bottomright': o['bottomright'], 'predictions': predictions, 'plate': found_plate } clasifications.append(cl) result_dict = {} result_dict["result"] = clasifications result_dict["message"] = "OK" # print(result_dict) socket.send_json(result_dict) except Exception as e: result_dict = {} result_dict["result"] = [] result_dict["message"] = str(e) socket.send_json(result_dict)
def handle_requests(self): # Load trained tensorflow car classifier and set tensorflow configs K.set_session(K.tf.Session(config=self.tf_conf)) init = K.tf.global_variables_initializer() sess = K.get_session() sess.run(init) is_initialized = False self.classifier_model, self.classifier_json = self.load() print('Loaded both models successfully, ready to roll.') print('Server is started on:', self.address) while True: try: result_dict = {} classifications = [] message = "OK" request = self.socket.recv() image = zmq_comm.decode_request(request) found_objects = [] with self.isess.as_default(): found_objects = self.extract_objects(image) with sess.as_default(): for o in found_objects: # Crop image according to empirical margin values cropped, cropped_wo_margin = self.crop_image( image, o['topleft'], o['bottomright'], float(o['confidence'])) if cropped is None or cropped_wo_margin is None: continue found_plate = "" predictions = [] filtered_candidates = [] # Run plate recognition in parallel while the main thread continues # Note that, if you call 'future.result()' here, it just waits for process to end # Also notice that, we are sending the original cropped image to plate extraction (a.k.a. no margins) if self.use_plate_recognition: self.plate_future = self.executor.submit( extract_plate, cropped_wo_margin, is_initialized) # Preprocess the image cropped = cropped * 1. / 255 cropped = cv2.resize(cropped, (299, 299)) cropped = cropped.reshape((1, ) + cropped.shape) # Feed image to classifier preds = self.classifier_model.predict(cropped)[0] predict_list = self.classifyIndices( self.classifier_json, preds, self.configs.crcl["n"]) predictions = [] tags = ["model", "score"] for index, p in enumerate(predict_list): predictions.append( dict(zip(tags, [p.name, str(p.score)]))) # Wait for plate recognition to finish its job if self.use_plate_recognition and self.plate_future is not None: try: found_plate, is_initialized = self.plate_future.result( timeout=self.configs. crcl["plate_service_timeout"]) except TimeoutError as e: print( "Could not get any respond from plate service. Timeout." ) cl = { 'label': o['label'], 'confidence': o['confidence'], 'topleft': o['topleft'], 'bottomright': o['bottomright'], 'predictions': predictions, 'plate': found_plate } classifications.append(cl) except Exception as e: message = str(e) result_dict["result"] = classifications result_dict["message"] = message self.socket.send_json(result_dict)
def handle_requests(ctx, socket): # Load SSD model ssd_model = load_SSD_model() # Load trained tensorflow car classifier and set tensorflow configs tf_config = K.tf.ConfigProto() tf_config.gpu_options.per_process_gpu_memory_fraction = CarConfig.crcl[ "classifier_gpu_memory_frac"] K.set_session(K.tf.Session(config=tf_config)) init = K.tf.global_variables_initializer() sess = K.get_session() sess.run(init) # Define a concurrent future and an executor to be used in case plate recognition is enabled future1 = None executor = None is_initialized = False use_plate_recognition = CarConfig.crcl["enable_plate_recognition"] if use_plate_recognition: # Initialize process executor once executor = ProcessPoolExecutor(max_workers=1) # Load model once car_classifier_model, car_classifier_loaded_model_json = load_model_and_json( ) print('Loaded both models successfully, ready to roll.') print('Server is started on:', tcp_address) while True: try: request = socket.recv() image = zmq_comm.decode_request(request) found_objects = [] with isess.as_default(): found_objects = extract_objects(ssd_model, image) clasifications = [] with sess.as_default(): for o in found_objects: # Crop image according to empirical margin values cropped, cropped_wo_margin = crop_image( image, o['topleft'], o['bottomright'], float(o['confidence'])) if cropped is None or cropped_wo_margin is None: continue found_plate = "" predictions = [] filtered_candidates = [] # Run plate recognition in parallel while the main thread continues # Note that, if you call 'future.result()' here, it just waits for process to end # Also notice that, we are sending the original cropped image to plate extraction (a.k.a. no margins) if use_plate_recognition: future1 = executor.submit(extract_plate, cropped_wo_margin, is_initialized) # Preprocess the image cropped = cropped * 1. / 255 cropped = cv2.resize(cropped, (299, 299)) cropped = cropped.reshape((1, ) + cropped.shape) # Feed image to classifier preds = car_classifier_model.predict(cropped)[0] predict_list = classifyIndices( car_classifier_loaded_model_json, preds, CarConfig.crcl["n"]) predictions = [] tags = ["model", "score"] for index, p in enumerate(predict_list): predictions.append( dict(zip(tags, [p.name, str(p.score)]))) # Wait for plate recognition to finish its job if use_plate_recognition and future1 is not None: try: found_plate, is_initialized = future1.result( timeout=CarConfig.crcl["plate_service_timeout"] ) except TimeoutError as e: print( "Could not get any respond from plate service. Timeout." ) cl = { 'label': o['label'], 'confidence': o['confidence'], 'topleft': o['topleft'], 'bottomright': o['bottomright'], 'predictions': predictions, 'plate': found_plate } clasifications.append(cl) result_dict = {} result_dict["result"] = clasifications result_dict["message"] = "OK" # print(result_dict) socket.send_json(result_dict) except Exception as e: result_dict = {} result_dict["result"] = [] result_dict["message"] = str(e) socket.send_json(result_dict) finally: #TODO: Implement this pass