class Main(threading.Thread): def __init__(self, hostname): threading.Thread.__init__(self) self.queue = Queue.Queue() self.images_undistorted = Images(CAPTURES_PATH) self.beers = Beers() self.duplicate_filter = Duplicate_Filter() self.web_interface = WebInterface() self.inventory = Inventory() self.network = Network(hostname, self.network_message_handler, self.network_status_handler) self.gdrive_captures_directory = "0BzpNPyJoi6uoSGlhTnN5RWhXRFU" self.light_level = 10 self.camera_capture_delay = 10 self.object_detection_wait_period = 240 self.whole_process_wait_period = 300 self.soonest_run_time = time.time() self.camera_units = Camera_Units(self.network) self.response_accumulator = Response_Accumulator() self.hostnames = [ "supercoolerA0", "supercoolerA1", "supercoolerA2", "supercoolerA3", "supercoolerA4", "supercoolerA5", "supercoolerA6", "supercoolerA7", "supercoolerA8", "supercoolerA9", "supercoolerA10", "supercoolerA11", "supercoolerB0", "supercoolerB1", "supercoolerB2", "supercoolerB3", "supercoolerB4", "supercoolerB5", "supercoolerB6", "supercoolerB7", "supercoolerB8", "supercoolerB9", "supercoolerB10", "supercoolerB11", "supercoolerC0", "supercoolerC1", "supercoolerC2", "supercoolerC3", "supercoolerC4", "supercoolerC5", "supercoolerC6", "supercoolerC7", "supercoolerC8", "supercoolerC9", "supercoolerC10", "supercoolerC11", "supercoolerD0", "supercoolerD1", "supercoolerD2", "supercoolerD3", "supercoolerD4", "supercoolerD5", "supercoolerD6", "supercoolerD7", "supercoolerD8", "supercoolerD9", "supercoolerD10", "supercoolerD11", "supercooler-hardware" ] self.client_monitor_server = Thirtybirds_Client_Monitor_Server( self.network, self.hostnames) self.client_monitor_server.daemon = True self.client_monitor_server.start() self.network.thirtybirds.subscribe_to_topic("door_closed") self.network.thirtybirds.subscribe_to_topic("door_opened") self.network.thirtybirds.subscribe_to_topic("client_monitor_response") self.network.thirtybirds.subscribe_to_topic("receive_image_data") #self.network.subscribe_to_topic("system") # subscribe to all system messages #self.network.subscribe_to_topic("update_complete") #self.network.subscribe_to_topic("image_capture_from_camera_unit") #self.network.subscribe_to_topic("receive_image_overlay") #self.network.subscribe_to_topic("classification_data_to_conductor") def network_message_handler(self, topic_msg): # this method runs in the thread of the caller, not the tread of Main topic, msg = topic_msg # separating just to eval msg. best to do it early. it should be done in TB. if topic not in ["client_monitor_response"]: print "Main.network_message_handler", topic if len(msg) > 0: msg = eval(msg) self.add_to_queue(topic, msg) def network_status_handler(self, topic_msg): # this method runs in the thread of the caller, not the tread of Main print "Main.network_status_handler", topic_msg def add_to_queue(self, topic, msg): self.queue.put((topic, msg)) def run(self): while True: try: topic, msg = self.queue.get(True) if topic not in ["client_monitor_response"]: print "Main.run", topic if topic == "client_monitor_response": self.client_monitor_server.add_to_queue( msg[0], msg[2], msg[1]) if topic == "door_closed": self.web_interface.send_door_close() if time.time() >= self.soonest_run_time: self.soonest_run_time = time.time( ) + self.whole_process_wait_period timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") #dir_captures_now = self.network.make_directory_on_gdrive(self.gdrive_captures_directory, 'captures_' + timestamp) #dir_unprocessed = self.network.make_directory_on_gdrive(dir_captures_now, 'unprocessed') #dir_annotated = self.network.make_directory_on_gdrive(dir_captures_now, 'annotated') #dir_parsed = self.network.make_directory_on_gdrive(dir_captures_now, 'parsed') self.network.thirtybirds.send("set_light_level", self.light_level) time.sleep(1) self.camera_units.capture_image( self.light_level, timestamp) time.sleep(self.camera_capture_delay) self.network.thirtybirds.send("set_light_level", 0) self.response_accumulator.clear_potential_objects() self.images_undistorted.clear() threading.Timer(self.object_detection_wait_period, self.add_to_queue, (("object_detection_complete", ""))) time.sleep(self.camera_capture_delay) self.camera_units.process_images_and_report() else: print "too soon. next available run time:", self.soonest_run_time if topic == "door_opened": self.web_interface.send_door_open() if topic == "receive_image_data": shelf_id = msg["shelf_id"] camera_id = int(msg["camera_id"]) potential_objects = msg["potential_objects"] undistorted_capture_png = msg["undistorted_capture_ocv"] self.response_accumulator.add_potential_objects( shelf_id, camera_id, potential_objects, True) filename = "{}_{}.png".format(shelf_id, camera_id) #file_bytes = np.asarray(bytearray(img_stream.read()), dtype=np.uint8) #undistorted_capture_ndarray = cv2.imdecode(file_bytes, cv2.IMREAD_UNCHANGED) #img_data_cvmat = cv.fromarray(img_data_ndarray) # convert to old cvmat if needed self.images_undistorted.store(filename, undistorted_capture_png) if topic == "object_detection_complete": print "OBJECT DETECTION COMPLETE ( how's my timing? )" print self.response_accumulator.print_response_status() print self.images.get_filenames() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() print e, repr( traceback.format_exception(exc_type, exc_value, exc_traceback))
class Main(threading.Thread): def __init__(self, hostname): threading.Thread.__init__(self) self.queue = Queue.Queue() self.images_undistorted = Images(CAPTURES_PATH) self.products = Products() self.duplicate_filter = Duplicate_Filter(self.products) self.web_interface = WebInterface() self.inventory = Inventory() self.network = Network(hostname, self.network_message_handler, self.network_status_handler) self.gdrive_captures_directory = "0BzpNPyJoi6uoSGlhTnN5RWhXRFU" self.light_level = 10 self.camera_capture_delay = 10 self.object_detection_wait_period = 300 self.whole_process_wait_period = 330 self.soonest_run_time = time.time() self.camera_units = Camera_Units(self.network) self.response_accumulator = Response_Accumulator() self.detected_objects = Detected_Objects(CAPTURES_PATH, PARSED_CAPTURES_PATH, self.products) self.door_open = False self.door_log = [time.time()] # hold timestamps of door closures since last scan self.scan_log = [time.time() - 1800] # hold timestamps of scans since unit was rebooted self.label_lines = [line.rstrip() for line in tf.gfile.GFile("/home/nvidia/supercooler/Roles/jetson/tf_files/retrained_labels.txt")] with tf.gfile.FastGFile("/home/nvidia/supercooler/Roles/jetson/tf_files/retrained_graph.pb", 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) with tf.Graph().as_default() as imported_graph: tf.import_graph_def(graph_def, name='') self.imported_graph = imported_graph self.hostnames = [ "supercoolerA0","supercoolerA1","supercoolerA2","supercoolerA3","supercoolerA4","supercoolerA5","supercoolerA6","supercoolerA7","supercoolerA8","supercoolerA9","supercoolerA10","supercoolerA11", "supercoolerB0","supercoolerB1","supercoolerB2","supercoolerB3","supercoolerB4","supercoolerB5","supercoolerB6","supercoolerB7","supercoolerB8","supercoolerB9","supercoolerB10","supercoolerB11", "supercoolerC0","supercoolerC1","supercoolerC2","supercoolerC3","supercoolerC4","supercoolerC5","supercoolerC6","supercoolerC7","supercoolerC8","supercoolerC9","supercoolerC10","supercoolerC11", "supercoolerD0","supercoolerD1","supercoolerD2","supercoolerD3","supercoolerD4","supercoolerD5","supercoolerD6","supercoolerD7","supercoolerD8","supercoolerD9","supercoolerD10","supercoolerD11", "supercooler-hardware" ] self.client_monitor_server = Thirtybirds_Client_Monitor_Server(self.network, self.hostnames) self.client_monitor_server.daemon = True self.client_monitor_server.start() self.network.thirtybirds.subscribe_to_topic("door_closed") self.network.thirtybirds.subscribe_to_topic("door_opened") self.network.thirtybirds.subscribe_to_topic("client_monitor_response") self.network.thirtybirds.subscribe_to_topic("receive_image_data") #self.network.subscribe_to_topic("system") # subscribe to all system messages #self.network.subscribe_to_topic("update_complete") #self.network.subscribe_to_topic("image_capture_from_camera_unit") #self.network.subscribe_to_topic("receive_image_overlay") #self.network.subscribe_to_topic("classification_data_to_conductor") def network_message_handler(self, topic_msg): # this method runs in the thread of the caller, not the tread of Main topic, msg = topic_msg # separating just to eval msg. best to do it early. it should be done in TB. if topic not in ["client_monitor_response"]: print "Main.network_message_handler", topic if len(msg) > 0: msg = eval(msg) self.add_to_queue(topic, msg) def network_status_handler(self, topic_msg): # this method runs in the thread of the caller, not the tread of Main print "Main.network_status_handler", topic_msg def add_to_queue(self, topic, msg): self.queue.put((topic, msg)) def quick_picture(self): "taking picture" timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") # turn on the lights self.network.thirtybirds.send("set_light_level", self.light_level) time.sleep(1) # send command to camera nodes to capture image self.camera_units.capture_image(self.light_level, timestamp) time.sleep(self.camera_capture_delay) # turn off the lights self.network.thirtybirds.send("set_light_level", 0) def run(self): while True: # check and see if sufficient time has elapsed between inventory scan now = time.time() last_scan = self.scan_log[-1] last_close = self.door_log[-1] # trigger scan if not self.door_open and (((now - last_scan > 1800) and (now - last_close > 300)) or ((now - last_scan > 3600) and (now - last_close > 1))): timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") print "initiating scan:", timestamp # update scan log with current time self.scan_log.append(now) # turn on the lights self.network.thirtybirds.send("set_light_level", self.light_level) time.sleep(1) # send command to camera nodes to capture image self.camera_units.capture_image(self.light_level, timestamp) time.sleep(self.camera_capture_delay) # turn off the lights self.network.thirtybirds.send("set_light_level", 0) # wait for cameras to capture images self.response_accumulator.clear_potential_objects() self.images_undistorted.clear() time.sleep(self.camera_capture_delay) # set a timer, process receieved images object_detection_timer = threading.Timer(self.object_detection_wait_period, self.add_to_queue, ("object_detection_complete","")) object_detection_timer.start() self.camera_units.process_images_and_report() try: topic, msg = self.queue.get(True) if topic not in ["client_monitor_response"]: print "Main.run", topic if topic == "client_monitor_response": self.client_monitor_server.add_to_queue(msg[0],msg[2],msg[1]) if topic == "door_closed": self.door_open = False self.web_interface.send_door_close() self.door_log.append(time.time()) # if time.time() >= self.soonest_run_time: # self.soonest_run_time = time.time() + self.whole_process_wait_period # timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") # #dir_captures_now = self.network.make_directory_on_gdrive(self.gdrive_captures_directory, 'captures_' + timestamp) # #dir_unprocessed = self.network.make_directory_on_gdrive(dir_captures_now, 'unprocessed') # #dir_annotated = self.network.make_directory_on_gdrive(dir_captures_now, 'annotated') # #dir_parsed = self.network.make_directory_on_gdrive(dir_captures_now, 'parsed') # self.network.thirtybirds.send("set_light_level", self.light_level) # time.sleep(1) # self.camera_units.capture_image(self.light_level, timestamp) # time.sleep(self.camera_capture_delay) # self.network.thirtybirds.send("set_light_level", 0) # self.response_accumulator.clear_potential_objects() # self.images_undistorted.clear() # time.sleep(self.camera_capture_delay) # object_detection_timer = threading.Timer(self.object_detection_wait_period, self.add_to_queue, ("object_detection_complete","")) # object_detection_timer.start() # self.camera_units.process_images_and_report() # else: # print "too soon. next available run time:", self.soonest_run_time if topic == "door_opened": self.door_open = True self.web_interface.send_door_open() print "Door is open. Dammit, Fudge!" if topic == "receive_image_data": shelf_id = msg["shelf_id"] camera_id = int(msg["camera_id"]) potential_objects = msg["potential_objects"] print shelf_id, camera_id print potential_objects undistorted_capture_png = msg["undistorted_capture_ocv"] # decode image to test classifier nparr = np.fromstring(undistorted_capture_png, np.uint8) undistorted_capture_ocv = cv2.imdecode(nparr, cv2.CV_LOAD_IMAGE_COLOR) #classifier.classify_images(potential_objects, undistorted_capture_ocv) self.response_accumulator.add_potential_objects(shelf_id, camera_id, potential_objects, True) filename = "{}_{}.png".format(shelf_id, camera_id) self.images_undistorted.store(filename, undistorted_capture_png) if topic == "object_detection_complete": print "OBJECT DETECTION COMPLETE ( how's my timing? )" print self.response_accumulator.print_response_status() print self.images_undistorted.get_filenames() potential_objects = self.response_accumulator.get_potential_objects() self.detected_objects.create_potential_object_images(potential_objects) print "OBJECT DETECTION COMPLETE" with tf.Session(graph=self.imported_graph) as sess: for shelf_id in ['A','B','C','D']: for camera_id in range(12): potential_objects_subset = filter(lambda d: d['shelf_id'] == shelf_id and int(d['camera_id']) == camera_id, potential_objects) #print shelf_id, camera_id, potential_objects_subset # if no objects were detected, skip if len(potential_objects_subset) == 0: continue # get undisotrted image and begin classification. use first object to grab shelf+cam first_object = potential_objects_subset[0] lens_corrected_img = self.images_undistorted.get_as_nparray( "{shelf_id}_{camera_id}.png".format(**first_object)) #with tf.Session() as sess: self.crop_and_classify_images(potential_objects_subset, lens_corrected_img, sess) #print potential_objects_subset self.detected_objects.create_classified_object_images(potential_objects) self.detected_objects.add_product_parameters(potential_objects) confident_objects = self.detected_objects.filter_out_unconfident_objects(potential_objects) confident_objects = self.duplicate_filter.tag_all_duplicates(confident_objects) self.detected_objects.filter_out_duplicate_objects(confident_objects) self.detected_objects.create_confident_object_images() simplest_inventory = self.detected_objects.tabulate_inventory() self.network.push_to_ab_db(simplest_inventory) # ----------- WEB INTERACE --------------------------------------------------- # Filter out duplicates, return list of objects with normalized global coords objects_for_web = self.duplicate_filter.filter_and_transform(self.detected_objects.confident_objects) #print objects_for_web # prep for web interface (scale coordinates and lookup product ids) and send res = self.web_interface.send_report(self.web_interface.prep_for_web(objects_for_web, self.duplicate_filter.x_max, self.duplicate_filter.y_max)) #print res, res.text #print confident_objects except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback)) def reboot_system(self): # send reboot command to camera nodes + hardware controller print "sending reboot command to nodes..." self.camera_units.send_reboot() time.sleep(5) print "rebooting..." os.system("sudo reboot now") def crop_and_classify_images(self, potential_objects, image, sess, threshold=0.6): # if the best guess falls below this threshold, assume no match confidence_threshold = threshold print "crop_and_classify_images" for i, candidate in enumerate(potential_objects): # report progress every ten images if (i%10) == 0: print 'processing %dth image' % i time.sleep(1) # crop image and encode as jpeg (classifier expects jpeg) #print "cropping..." r = candidate['radius'] (img_height, img_width) = image.shape[:2] x1 = max(candidate['camera_x']-r, 0) y1 = max(candidate['camera_y']-r, 0) x2 = min(x1 + r*2, img_width ) y2 = min(y1 + r*2, img_height) img_crop = image[y1:y2, x1:x2] img_jpg = cv2.imencode('.jpg', img_crop)[1].tobytes() #print "cropped image, w,h = ", x2-x1, y2-y1 # get a list of guesses w/ confidence in this format: # guesses = [(best guess, confidence), (next guess, confidence), ...] #print "running classifier..." guesses = self.guess_image(sess, img_jpg) best_guess, confidence = guesses[0] candidate["classification"] = guesses #print guesses def guess_image(self, tf_session, image): # Feed the image_data as input to the graph and get first prediction softmax_tensor = tf_session.graph.get_tensor_by_name('final_result:0') #print "run tf session..." predictions = tf_session.run(softmax_tensor, {'DecodeJpeg/contents:0': image}) # Sort to show labels of first prediction in order of confidence #print "sort labels.." top_k = predictions[0].argsort()[-len(predictions[0]):][::-1] scores = [(self.label_lines[node_id], predictions[0][node_id]) for node_id in top_k] return scores
class Main(): # rules them all def __init__(self, network): self.network = network self.capture_path = "/home/pi/supercooler/Captures/" self.parsed_capture_path = "/home/pi/supercooler/ParsedCaptures/" self.web_interface = WebInterface() self.lights = Lights() self.door = Door(self.door_close_event_handler, self.door_open_event_handler) self.door.daemon = True self.door.start() self.camera_units = Camera_Units(self.network) self.camera_capture_delay = 15 self.classifier = Classifier() self.last_closure = time.time() hostnames = [ "supercoolerA0", "supercoolerA1", "supercoolerA2", "supercoolerA3", "supercoolerA4", "supercoolerA5", "supercoolerA6", "supercoolerA7", "supercoolerA8", "supercoolerA9", "supercoolerA10", "supercoolerA11", "supercoolerB0", "supercoolerB1", "supercoolerB2", "supercoolerB3", "supercoolerB4", "supercoolerB5", "supercoolerB6", "supercoolerB7", "supercoolerB8", "supercoolerB9", "supercoolerB10", "supercoolerB11", "supercoolerC0", "supercoolerC1", "supercoolerC2", "supercoolerC3", "supercoolerC4", "supercoolerC5", "supercoolerC6", "supercoolerC7", "supercoolerC8", "supercoolerC9", "supercoolerC10", "supercoolerC11", "supercoolerD0", "supercoolerD1", "supercoolerD2", "supercoolerD3", "supercoolerD4", "supercoolerD5", "supercoolerD6", "supercoolerD7", "supercoolerD8", "supercoolerD9", "supercoolerD10", "supercoolerD11" ] self.client_monitor_server = Thirtybirds_Client_Monitor_Server( network, hostnames) self.client_monitor_server.daemon = True self.client_monitor_server.start() # initialize inventory -- this will be recalculated on door close events self.inventory = [] # map watson labels to corresponding ints for web interface self.label_lookup = { "bottlebecks": 1, "bottlebudamerica": 2, "bottlebudlight": 3, "bottleplatinum": 4, "bottlecorona": 5, "bottlehoegaarden": 6, "bottleultra": 7, "bottleshocktopraspberry": 8, "bottleshocktoppretzel": 9, "bottlestella": 10, "canbudamerica": 11, "canbudlight": 12, "canbusch": 13, "canbusch": 14, "cannaturallight": 15, "canbudamerica": 16, "canbudice": 17, "canbudlight": 18 } self.classification_accumulator = Classification_Accumulator( self.all_records_received) self.classification_accumulator.daemon = True self.classification_accumulator.start() self.camera_specific_offsets = { 'A': { 0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 4: (0, 0), 5: (0, 0), 6: (0, 0), 7: (0, 0), 8: (0, 0), 9: (0, 0), 10: (0, 0), 11: (0, 0) }, 'B': { 0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 4: (0, 0), 5: (0, 0), 6: (0, 0), 7: (0, 0), 8: (0, 0), 9: (0, 0), 10: (0, 0), 11: (0, 0) }, 'C': { 0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 4: (0, 0), 5: (0, 0), 6: (0, 0), 7: (0, 0), 8: (0, 0), 9: (0, 0), 10: (0, 0), 11: (0, 0) }, 'D': { 0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 4: (0, 0), 5: (0, 0), 6: (0, 0), 7: (0, 0), 8: (0, 0), 9: (0, 0), 10: (0, 0), 11: (0, 0) } } self.camera_resolution = [1280, 720] self.product_specific_confidence_thresholds = { "bottlebecks": 0.99, "bottlebudamerica": 0.99, "bottlebudlight": 0.99, "bottleplatinum": 0.99, "bottlecorona": 0.95, "bottlehoegaarden": 0.99, "bottleultra": 0.98, "bottleshocktopraspberry": 0.99, "bottleshocktoppretzel": 0.98, "bottlestella": 0.99, "canbudamerica": 0.95, "canbudlight": 0.99, "canbusch": 0.94, "cannaturallight": 0.95, "canbudamerica": 0.99, "canbudice": 0.99, "canbudlight": 0.99 } def map_camera_coords_to_shelf_coords(self, shelf_id, camera_id, x, y): # standard x and y distances between camera origins. adjust as necessary delta_x = 1200 delta_y = 600 # start by doing a rough transformation with standard offsets x_prime = x + delta_x * (camera_id // 4) y_prime = y + delta_y * (3 - camera_id % 4) # now apply specific offsets as defined in self.camera_specific_offsets x_prime = float(x_prime + self.camera_specific_offsets[shelf_id][camera_id][0]) y_prime = float(y_prime + self.camera_specific_offsets[shelf_id][camera_id][1]) # full-scale x and y in terms of camera coordinates, for scaling (adjust as necessary) x_full_scale = float(delta_x * 2 + 1280) y_full_scale = float(delta_y * 3 + 720) # scale to web coordinates x_full_scale_web = 492.0 y_full_scale_web = 565.0 x_offset_web = 120 y_offset_web = -80 # scale and swap x and y coordinates x_web = x_full_scale_web - (y_prime / y_full_scale * y_full_scale_web) + x_offset_web y_web = x_prime / x_full_scale * x_full_scale_web + y_offset_web return (x_web, y_web) def all_records_received(self, records): print "all records received" records_with_shelf_coords = self.map_camera_coords_to_shelf_coords( records) print records_with_shelf_coords print "all records received" for shelf in ['A', 'B', 'C', 'D']: for camera_id, camera_data in enumerate(records[shelf]): print shelf, camera_id, camera_data self.add_to_inventory(shelf, camera_id, camera_data) self.filter_duplicates() #print records def add_to_inventory(self, shelf_id, camera_id, camera_data): for (i, data) in camera_data.iteritems(): productname = data['class'] product_confidence_threshold = self.product_specific_confidence_thresholds[ productname] if data['score'] < product_confidence_threshold: continue print "adding data..." try: x_local = float(data['x']) + data['w'] / 2 y_local = float(data['y']) + data['h'] / 2 print "map from camera coords to shelf coords" x_global, y_global = self.map_camera_coords_to_shelf_coords( shelf_id, camera_id, x_local, y_local) self.inventory.append({ "type": self.label_lookup[data['class']], "shelf": shelf_id, "x": x_global, "y": y_global, "camera": camera_id, "duplicate": False, "score": data['score'] }) except Exception as e: print "exception in Main.add_to_inventory", e def filter_duplicates(self): return # until mapping of camera coords to shelf coords is complete # tag duplicates overlap_threshold = 100 for product_outer in self.inventory: for product_inner in self.inventory: if product_outer['camera'] == product_inner[ 'camera']: # there will be no duplicates from the same camera continue distance = math.sqrt( math.pow((product_outer['x'] - product_inner['x']), 2) + math.pow((product_outer['x'] - product_inner['x']), 2)) if distance < overlap_threshold: if product_inner['score'] < product_outer['score']: product_inner['duplicate'] = True else: product_outer['duplicate'] = True # filter out duplicates new_inventory = [] for product in self.inventory: if not product['duplicate']: new_inventory.append(product) self.inventory = new_inventory def client_monitor_add_to_queue(self, hostname, git_pull_date, pickle_version): self.client_monitor_server.add_to_queue(hostname, git_pull_date, pickle_version) def door_open_event_handler(self): print "Main.door_open_event_handler" self.web_interface.send_door_open() def door_close_event_handler(self): print "Main.door_close_event_handler" self.web_interface.send_door_close() self.classification_accumulator.start_timer() images.clear_captures() # clear inventory (will be populated after classification) self.inventory = [] timestamp = time.strftime("%Y-%m-%d-%H-%m-%S") # tell camera units to captures images at each light level for light_level_sequence_position in range(3): self.lights.play_sequence_step(light_level_sequence_position) self.camera_units.capture_image(light_level_sequence_position, timestamp) time.sleep(self.camera_capture_delay) self.lights.all_off() # tell camera units to parse images and send back the data self.camera_units.process_images_and_report() # pause while conductor waits for captures, then start classification print "waiting for captures... " time.sleep(240) # -------------------------------------------------------------------------- # TODO: After the demo, put this back in. For now, we'll have watson clients # on each of the pi zeros #print "begin classification process" #self.classify_images() # -------------------------------------------------------------------------- #if len(self.inventory) == 0: # print "empty... add dummy beer" # self.inventory.append({"type":1,"shelf":"A","x":10,"y":10}) # print "update web interface" self.web_interface.send_report(self.inventory) #for item in self.inventory: # self.web_interface.send_report(item) print "done updating" print "took inventory:" print self.inventory def classify_images(self, threshold=0.6): # for convenience classifier = self.classifier #images = self.images inventory = self.inventory # if the best guess falls below this threshold, assume no match confidence_threshold = threshold print "Main.classify_images images.cropped_captures", images.cropped_captures # start tensorflow session, necessary to run classifier with tf.Session() as sess: for i, cropped_capture in enumerate(images.cropped_captures): # report progress every ten images if (i % 10) == 0: print 'processing %dth image' % i time.sleep(1) # crop image and encode as jpeg (classifier expects jpeg) print "cropping..." x, y, w, h = cropped_capture["bounds"] img_crop = images.captures[cropped_capture["img_index"]][y:y + h, x:x + w] img_jpg = cv2.imencode('.jpg', img_crop)[1].tobytes() print "cropped image, w,h = ", w, h # --------------------------------------------------------------------- # TODO: remove this later -- this is just so we can see what's going on # create filename from img data filename = cropped_capture["shelf_id"]+cropped_capture["camera_id"]+\ "_" + str(x) + "_" + str(y) + ".jpg" filepath = "/home/pi/supercooler/ParsedCaptures/" + filename # write to file with open(filepath, 'wb') as f: f.write(img_jpg) # --------------------------------------------------------------------- # get a list of guesses w/ confidence in this format: # guesses = [(best guess, confidence), (next guess, confidence), ...] print "running classifier..." guesses = classifier.guess_image(sess, img_jpg) best_guess, confidence = guesses[0] # print result from classifier print guesses # if we beat the threshold, then update the inventory accordingly if confidence > confidence_threshold: inventory.append({ "type": self.label_lookup[best_guess], "shelf": cropped_capture["shelf_id"], "x": x + w / 2, "y": y + h / 2, }) # TODO: move the temp sensing out of guess_image and into here def test_classification(self): # read in a test image for parsing/classification with open("/home/pi/supercooler/Roles/conductor/test_img.png", "rb") as f: img = base64.b64encode(f.read()) # clear inventory (will be populated after classification) self.inventory = [] # an example payload -- this is what the camera units send over payload = { "camera_id": "A02", "light_level": 2, "image": img, "bounds": [(0, 0, 100, 200), (250, 250, 200, 100), (0, 200, 150, 150)] } images.receive_image_data(payload) # store image data from payload self.classify_images(threshold=0.1) # classify images def get_raw_images(self): images.clear_captures() timestamp = time.strftime("%Y-%m-%d-%H-%m-%S") # tell camera units to captures images at each light level for light_level_sequence_position in range(3): self.lights.play_sequence_step(light_level_sequence_position) self.camera_units.capture_image(light_level_sequence_position, timestamp) time.sleep(self.camera_capture_delay) self.lights.all_off() # tell camera units to parse images and send back the data self.camera_units.return_raw_images()