def __init__(self):
#TODO load classifier
self.object_classifier = ObjectClassifier()
self.is_site = False
self.color_classifier = ColorClassifier(self.is_site)
def __init__(self, is_site):
# init object classifier (step 1)
rospy.loginfo('TLClassifier start inititialization')
self.object_classifier = ObjectClassifier()
# init traffic light color classifier (step 2)
#john added is_site log output
if is_site:
rospy.loginfo('is_site TRUE using site classifer')
else:
rospy.loginfo('is_site FALSE using sim classifer')
self.color_classifier = ColorClassifier(
is_site) #john ...now sending boolena to color classifier
rospy.loginfo('TLClassifier inititialized')
class TLClassifier(object):
def __init__(self):
# init object classifier (step 1)
self.object_classifier = ObjectClassifier()
# init traffic light color classifier (step 2)
self.color_classifier = ColorClassifier()
def get_classification(self, image, frame_id):
"""Determines the color of the traffic light in the image
Args:
image (cv::Mat): image containing the traffic light
Returns:
int: ID of traffic light color (specified in styx_msgs/TrafficLight)
"""
# step 1
traffic_light_images = self.object_classifier.get_traffic_light_images(image)
# step 2
traffic_light_color = self.color_classifier.predict_images(traffic_light_images)
for idx, image in enumerate(traffic_light_images):
image.save('cropped/image{}-{}-{}.jpg'.format(frame_id, idx, tf_colors[traffic_light_color]))
return traffic_light_color, traffic_light_images
class TLClassifier(object):
#john added boolean is_site to indicate site or sim classifer to load
def __init__(self, is_site):
# init object classifier (step 1)
rospy.loginfo('TLClassifier start inititialization')
self.object_classifier = ObjectClassifier()
# init traffic light color classifier (step 2)
#john added is_site log output
if is_site:
rospy.loginfo('is_site TRUE using site classifer')
else:
rospy.loginfo('is_site FALSE using sim classifer')
self.color_classifier = ColorClassifier(
is_site) #john ...now sending boolena to color classifier
rospy.loginfo('TLClassifier inititialized')
self.RECORD_CROPPED_IMAGES = False
def get_classification(self, image):
"""Determines the color of the traffic light in the image
Args:
image (cv::Mat): image containing the traffic light
Returns:
int: ID of traffic light color (specified in styx_msgs/TrafficLight)
"""
rospy.loginfo('TLClassifier received image')
# convert from bgr 2 rgb
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# step 1
traffic_light_images = self.object_classifier.get_traffic_light_images(
image)
traffic_light_color = self.color_classifier.predict_images(
traffic_light_images)
tf_color = ['RED', 'YELLOW', 'GREEN', 'UNDEFINED',
'UNKNOWN'] # Not so nice ;-( but want to show text
rospy.loginfo('Traffic light detected {}'.format(
tf_color[traffic_light_color]))
if self.RECORD_CROPPED_IMAGES:
dir = './data/cropped/'
if not os.path.exists(dir):
os.makedirs(dir)
for idx, image in enumerate(traffic_light_images):
f_name = "sim_tl_{}_{}_{}.jpg".format(
calendar.timegm(time.gmtime()),
tf_color[traffic_light_color], idx)
image.save(dir + f_name)
return traffic_light_color
class TLClassifier(object):
def __init__(self):
#TODO load classifier
self.object_classifier = ObjectClassifier()
self.is_site = False
self.color_classifier = ColorClassifier(self.is_site)
def get_classification(self, image):
"""Determines the color of the traffic light in the image
Args:
image (cv::Mat): image containing the traffic light
Returns:
int: ID of traffic light color (specified in styx_msgs/TrafficLight)
"""
# implement light color prediction
# convert from bgr 2 rgb
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# step 1
traffic_light_images = self.object_classifier.get_traffic_light_images(
image)
traffic_light_color = self.color_classifier.predict_images(
traffic_light_images)
tf_color = ['RED', 'YELLOW', 'GREEN', 'UNDEFINED', 'UNKNOWN']
if self.RECORD_CROPPED_IMAGES:
dir = './data/cropped/'
if not os.path.exists(dir):
os.makedirs(dir)
for idx, image in enumerate(traffic_light_images):
f_name = "sim_tl_{}_{}_{}.jpg".format(
calendar.timegm(time.gmtime()),
tf_color[traffic_light_color], idx)
image.save(dir + f_name)
return traffic_light_color
class TLClassifier(object):
#john added boolean is_site to indicate site or sim classifer to load
def __init__(self, is_site):
# init object classifier (step 1)
rospy.loginfo('TLClassifier start inititialization')
self.object_classifier = ObjectClassifier()
# init traffic light color classifier (step 2)
#john added is_site log output
if is_site:
rospy.loginfo('is_site TRUE using site classifer')
else:
rospy.loginfo('is_site FALSE using sim classifer')
self.color_classifier = ColorClassifier(
is_site) #john ...now sending boolena to color classifier
rospy.loginfo('TLClassifier inititialized')
def get_classification(self, image):
"""Determines the color of the traffic light in the image
Args:
image (cv::Mat): image containing the traffic light
Returns:
int: ID of traffic light color (specified in styx_msgs/TrafficLight)
"""
rospy.loginfo('TLClassifier received image')
# step 1
traffic_light_images = self.object_classifier.get_traffic_light_images(
image)
#john there might be a probelm here what if there are 1 or 3 predictions how does that effect below?
# eventually this gets passed back to tl_detector and is a singluar state maybe this sia problem?
traffic_light_color = self.color_classifier.predict_images(
traffic_light_images)
tf_color = ['RED', 'YELLOW', 'GREEN', 'UNDEFINED',
'UNKNOWN'] # Not so nice ;-( but want to show text
rospy.loginfo('Traffic light detected {}'.format(
tf_color[traffic_light_color]))
#changed bacue now using styx msg #TrafficLight(traffic_light_color)))
return traffic_light_color
def __init__(self):
# init object classifier (step 1)
self.object_classifier = ObjectClassifier()
# init traffic light color classifier (step 2)
self.color_classifier = ColorClassifier()
def __init__(self,
classifier_codename,
dataset_codename,
classifier_threshold,
object_detection=True,
object_visualization=True,
lane_detection=True,
lane_visualization=True,
diagnostic_mode=True,
monitor_id=1,
window_top_offset=0,
window_left_offset=0,
window_width=None,
window_height=None,
window_scale=1.0):
# Boolean flag for feature-customization
self.object_detection = object_detection
self.object_visualization = object_visualization
self.diagnostic_mode = diagnostic_mode
self.lane_detection = lane_detection
self.lane_visualization = lane_visualization
# Instance of the MSS-API for captureing screenshots
self.window_manager = mss.mss()
# Note:
# - monitor_id = 0 | grab all monitors together
# - monitor_id = n | grab a given monitor (n) : where n > 0
self.target_window = self.window_manager.monitors[monitor_id]
# Update position of the window that will be captured
if window_left_offset:
self.target_window['left'] += window_left_offset
self.target_window['width'] -= window_left_offset
if window_top_offset:
self.target_window['top'] += window_top_offset
self.target_window['height'] -= window_top_offset
if window_width:
self.target_window['width'] = window_width
if window_height:
self.target_window['height'] = window_height
if window_scale:
self.target_window['scale'] = window_scale
print("Activating DeepEye Advanced Co-pilot Mode")
self.object_detector = ObjectClassifier(
classifier_codename=classifier_codename,
dataset_codename=dataset_codename,
classifier_threshold=classifier_threshold,
visualization=object_visualization,
diagnostic_mode=diagnostic_mode,
frame_height=self.target_window['height'],
frame_width=self.target_window['width'])
self.lane_detector = LaneDetector(visualization=lane_visualization)
self.threats = {
"COLLISION": False,
"PEDESTRIAN": False,
"STOP_SIGN": False,
"TRAFFIC_LIGHT": False,
"VEHICLES": False,
"BIKES": False,
"FAR_LEFT": False,
"FAR_RIGHT": False,
"RIGHT": False,
"LEFT": False,
"CENTER": False,
"UNKNOWN": True
}
self.frame_id = 0
self.columns = [
'FRAME_ID', 'PEDESTRIAN', 'VEHICLES', 'BIKES', 'STOP_SIGN',
'TRAFFIC_LIGHT', 'OFF_LANE', 'COLLISION'
]
self.data_frame = pd.DataFrame(columns=self.columns)
class DrivingAssistant:
# Constructor
def __init__(self,
classifier_codename,
dataset_codename,
classifier_threshold,
object_detection=True,
object_visualization=True,
lane_detection=True,
lane_visualization=True,
diagnostic_mode=True,
monitor_id=1,
window_top_offset=0,
window_left_offset=0,
window_width=None,
window_height=None,
window_scale=1.0):
# Boolean flag for feature-customization
self.object_detection = object_detection
self.object_visualization = object_visualization
self.diagnostic_mode = diagnostic_mode
self.lane_detection = lane_detection
self.lane_visualization = lane_visualization
# Instance of the MSS-API for captureing screenshots
self.window_manager = mss.mss()
# Note:
# - monitor_id = 0 | grab all monitors together
# - monitor_id = n | grab a given monitor (n) : where n > 0
self.target_window = self.window_manager.monitors[monitor_id]
# Update position of the window that will be captured
if window_left_offset:
self.target_window['left'] += window_left_offset
self.target_window['width'] -= window_left_offset
if window_top_offset:
self.target_window['top'] += window_top_offset
self.target_window['height'] -= window_top_offset
if window_width:
self.target_window['width'] = window_width
if window_height:
self.target_window['height'] = window_height
if window_scale:
self.target_window['scale'] = window_scale
print("Activating DeepEye Advanced Co-pilot Mode")
self.object_detector = ObjectClassifier(
classifier_codename=classifier_codename,
dataset_codename=dataset_codename,
classifier_threshold=classifier_threshold,
visualization=object_visualization,
diagnostic_mode=diagnostic_mode,
frame_height=self.target_window['height'],
frame_width=self.target_window['width'])
self.lane_detector = LaneDetector(visualization=lane_visualization)
self.threats = {
"COLLISION": False,
"PEDESTRIAN": False,
"STOP_SIGN": False,
"TRAFFIC_LIGHT": False,
"VEHICLES": False,
"BIKES": False,
"FAR_LEFT": False,
"FAR_RIGHT": False,
"RIGHT": False,
"LEFT": False,
"CENTER": False,
"UNKNOWN": True
}
self.frame_id = 0
self.columns = [
'FRAME_ID', 'PEDESTRIAN', 'VEHICLES', 'BIKES', 'STOP_SIGN',
'TRAFFIC_LIGHT', 'OFF_LANE', 'COLLISION'
]
self.data_frame = pd.DataFrame(columns=self.columns)
def run(self):
"""
Capture frames, initiate both objects and lane detectors, and then visualize output.
"""
# Get raw pixels from the screen, save it to a Numpy array
original_frame = np.asarray(
self.window_manager.grab(self.target_window))
# set frame's height & width
frame_height, frame_width = original_frame.shape[:2]
# convert pixels from BGRA to RGB values
original_frame = cv2.cvtColor(original_frame, cv2.COLOR_BGRA2BGR)
if self.diagnostic_mode:
pre = original_frame.copy()
# draw a box around the area scaned for for PEDESTRIAN/VEHICLES detection
visualization_utils.draw_bounding_box_on_image_array(
pre,
self.object_detector.roi["t"] / frame_height,
self.object_detector.roi["l"] / frame_width,
self.object_detector.roi["b"] / frame_height,
self.object_detector.roi["r"] / frame_width,
color=(255, 255, 0), # BGR VALUE
display_str_list=(' ROI ', ))
# draw a box around the area scaned for collision warnings
visualization_utils.draw_bounding_box_on_image_array(
pre,
self.object_detector.roi["ct"] / frame_height,
self.object_detector.roi["cl"] / frame_width,
self.object_detector.roi["b"] / frame_height,
self.object_detector.roi["cr"] / frame_width,
color=(255, 0, 255), # BGR VALUE
display_str_list=(' COLLISION ROI ', ))
# save a screen shot of the current frame before getting processed
if self.frame_id % 10 == 0:
cv2.imwrite("test/pre/" + str(self.frame_id / 10) + ".jpg",
pre)
# only detect objects in the given frame
if self.object_detection and not self.lane_detection:
(frame, self.threats) = self.object_detector.scan_road(
original_frame, self.threats)
if self.object_visualization:
# Display frame with detected objects.
cv2.imshow('DeepEye Dashboard', cv2.resize(frame, (640, 480)))
# only detect lane in the given frame
elif self.lane_detection and not self.object_detection:
(frame, self.threats) = self.lane_detector.detect_lane(
original_frame, self.threats)
if self.lane_visualization:
# Display frame with detected lane.
cv2.imshow('DeepEye Dashboard', cv2.resize(frame, (640, 480)))
# detect both objects and lane
elif self.object_detection and self.lane_detection:
# Visualize object detection ONLY
if self.object_visualization and not self.lane_visualization:
(frame, self.threats) = self.object_detector.scan_road(
original_frame, self.threats)
# Display frame with detected lane.
cv2.imshow('DeepEye Dashboard', cv2.resize(frame, (640, 480)))
(_, self.threats) = self.lane_detector.detect_lane(
original_frame, self.threats)
# Visualize lane detection ONLY
elif self.lane_visualization and not self.object_visualization:
(frame, self.threats) = self.lane_detector.detect_lane(
original_frame, self.threats)
# Display frame with detected lane.
cv2.imshow('DeepEye Dashboard', cv2.resize(frame, (640, 480)))
(_, self.threats) = self.object_detector.scan_road(
original_frame, self.threats)
# Visualize both object & lane detection
elif self.object_visualization and self.lane_visualization:
(frame, self.threats) = self.object_detector.scan_road(
original_frame, self.threats)
(frame, self.threats) = self.lane_detector.detect_lane(
frame, self.threats)
# Display frame with detected lane.
cv2.imshow('DeepEye Dashboard', cv2.resize(frame, (640, 480)))
# skip visualization
else:
(_, self.threats) = self.object_detector.scan_road(
original_frame, self.threats)
(_, self.threats) = self.lane_detector.detect_lane(
original_frame, self.threats)
# skip detection
else:
frame = original_frame
if (self.frame_id % 10 == 0) and (self.diagnostic_mode):
# draw a box around the area scaned for for PEDESTRIAN/VEHICLES detection
visualization_utils.draw_bounding_box_on_image_array(
frame,
self.object_detector.roi["t"] / frame_height,
self.object_detector.roi["l"] / frame_width,
self.object_detector.roi["b"] / frame_height,
self.object_detector.roi["r"] / frame_width,
color=(255, 255, 0), # BGR VALUE
display_str_list=(' ROI ', ))
# draw a box around the area scaned for collision warnings
visualization_utils.draw_bounding_box_on_image_array(
frame,
self.object_detector.roi["ct"] / frame_height,
self.object_detector.roi["cl"] / frame_width,
self.object_detector.roi["b"] / frame_height,
self.object_detector.roi["cr"] / frame_width,
color=(255, 0, 255), # BGR VALUE
display_str_list=(' COLLISION ROI ', ))
# save a screen shot of the current frame after getting processed
cv2.imwrite("test/post/" + str(self.frame_id / 10) + ".jpg", frame)
if self.threats["FAR_LEFT"] or self.threats["FAR_RIGHT"]:
OFF_LANE = 1
else:
OFF_LANE = 0
# append a new row to dataframe
self.data_frame = self.data_frame.append(
{
'FRAME_ID': int(self.frame_id / 10),
'PEDESTRIAN': int(self.threats['PEDESTRIAN']),
'VEHICLES': int(self.threats['VEHICLES']),
'BIKES': int(self.threats['BIKES']),
'STOP_SIGN': int(self.threats['STOP_SIGN']),
'TRAFFIC_LIGHT': int(self.threats['TRAFFIC_LIGHT']),
'OFF_LANE': int(OFF_LANE),
'COLLISION': int(self.threats['COLLISION']),
},
ignore_index=True)
self.frame_id += 1