def open_camera(self):
UITool.msg_window('Info', 'Start face distant protect mode.')
flag = True
vs = WebcamVideoStream(src=0)
vs.start()
time.sleep(2.0)
cv2.startWindowThread()
while flag:
check = True
frame = vs.read()
frame = imutils.resize(frame, width=800)
display_frame = frame.copy()
capt_faces = FaceCapture.cap(frame)
if capt_faces.has_face:
display_frame = ImgTool.add_face_block(display_frame,
capt_faces)
face_data = FacesProc.get_all_faces_with_encode_low_filter(
frame, self.mode)
if face_data:
check = self.check_face_encode(face_data)
print(f"status: {check}")
if not check:
print(
f" WARN - find stranger and no master in the same time !!!"
)
self.alert = True
else:
print('fetched face data is not enough')
if self.display:
display_frame = cv2.flip(display_frame, 1)
if check:
text = 'Safe'
print(text)
display_frame = ImgTool.add_text(display_frame, text)
else:
text = 'Detect Stranger!'
display_frame = ImgTool.add_text(display_frame,
text,
color='red')
cv2.imshow('Frame', display_frame)
key = cv2.waitKey(1)
if self.alert or key == 27 or key == ord('q'):
flag = False
cv2.waitKey(500)
cv2.destroyAllWindows()
cv2.waitKey(500)
vs.stop()
cv2.destroyAllWindows()
def launch(self):
flag = True
vs = WebcamVideoStream(src=0)
vs.start()
time.sleep(2.0)
fetch_face = 0
last_face_ts = 0
cv2.startWindowThread()
while flag:
frame = vs.read()
if frame is None:
continue
frame = imutils.resize(frame, width=800)
display_frame = frame.copy()
capt_faces = FaceCapture.cap(frame)
if capt_faces.face_count == 1:
display_frame = ImgTool.add_face_block(display_frame,
capt_faces)
ts = FileTool.get_curr_ts()
if ts - last_face_ts > RecordMD.INTERVAL:
if self.__record_face(frame, capt_faces, ts):
fetch_face += 1
print(
f'fetch face successfully. ({fetch_face}/{self.fetch_count})'
)
last_face_ts = ts
else:
print(f'failed to record faces')
if self.display:
display_frame = cv2.flip(display_frame, 1)
text = f"{fetch_face} / {self.fetch_count}"
display_frame = ImgTool.add_text(display_frame, text)
cv2.imshow('Frame', display_frame)
key = cv2.waitKey(1)
if fetch_face == self.fetch_count or key == 27 or key == ord('q'):
flag = False
cv2.waitKey(500)
cv2.destroyAllWindows()
cv2.waitKey(500)
vs.stop()
cv2.destroyAllWindows()
# self.__img_meta.to_csv(RecordMD.meta_path, index=False, encoding='utf-8')
msg = 'Finish recoding master data.'
UITool.msg_window(msg=msg)
print('finish saving img meta.')
def from_camera(self, camera_id=0, show=False):
"""
Detects objects in frames from a camera feed
:param camera_id: the ID of the camera in the system
"""
def get_frame(stream):
frame = stream.read()
ret = True
return ret, frame
stream = WebcamVideoStream(src=camera_id)
stream.start()
self._detect_from_stream(get_frame, stream, show)
stream.stop()
def predict():
face_classifier = cv2.CascadeClassifier(
"C:/Users/uvss/AppData/Local/Programs/Python/Python37-32/Lib/site-packages/cv2/data/haarcascade_frontalface_default.xml"
)
face_recognizer = cv2.face.LBPHFaceRecognizer_create()
face_recognizer.read("trainer/training_data.yml")
vs = WebcamVideoStream(
"rtsp://*****:*****@192.168.1.90/axis-media/media.amp")
vs = vs.start()
if vs.grabbed:
while True:
image = vs.read()
img = image.copy()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_classifier.detectMultiScale(gray, 1.2, 5)
if faces is ():
pass
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (102, 255, 102), 2)
roi_gray = gray[y:y + h, x:x + w]
label, conf = face_recognizer.predict(roi_gray)
print(label, int(conf))
if conf > 85:
label = "Unknown"
cv2.putText(img, str(label), (x + 5, y + h - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)
cv2.imshow("frames", img)
if cv2.waitKey(1) == ord("q"):
break
vs.stop()
cv2.destroyAllWindows()
def training_Data(path):
while True:
try:
details = {
"name": input("Provide a name for the Employee:- "),
"depart": input("Department:- "),
"ID": int(input("Office Id (number) :- "))
}
if bool(re.search(r'\d', details["name"])) or bool(
re.search(r'\d', details["depart"])):
raise ValueError(
"[Error] Invalid Input for Name or Department")
break
except Exception as err:
print("[Error] Invalid Input for Office Id")
continue
sampled_faces = []
sampled_labels = []
img_name = None
usr_id, label = 1, 1
img_id = 0
users = os.listdir(path)
users.sort(
key=lambda x: int(x[1:])) # sorting all the user data directories
if users == []:
usr_dir = path + "/s" + str(usr_id) + "/"
os.mkdir(usr_dir)
else:
label = int(users[-1][1:]) + 1
usr_id = str(label) # get [1:] part of last element in the list
usr_dir = path + "/s" + usr_id + "/"
os.mkdir(usr_dir)
vs = WebcamVideoStream(
src="rtsp://*****:*****@192.168.1.90/axis-media/media.amp")
if vs.grabbed:
vs = vs.start()
while (True):
frame = vs.read()
cv2.imshow("frame", frame)
faces = face_Detector(frame)
if cv2.waitKey(1) == ord("q"):
# sys.exit()
break
if faces is None or faces == []:
continue
for face in faces:
img_id += 1
cv2.imshow("face", face)
img_name = usr_dir + str(img_id) + ".jpg", face
cv2.imwrite(img_name, face)
sampled_labels.append(label)
sampled_faces.append(face)
if img_id >= 50:
break
vs.stop()
cv2.destroyAllWindows()
return (sampled_faces, sampled_labels, details)
class WebcamIngestorSource(FrameIngestorSource):
"""Returns frames from the webcam."""
def __init__(self, source_num=0):
"""Initialize object.
Raises:
TypeError: when source_num is not an int
Args:
source_num: video stream number
"""
if not isinstance(source_num, int):
raise TypeError('`source_num` is not an int')
self._source_num = source_num
def start(self):
"""Open the source.
Raises:
RuntimeError: when could not open stream.
"""
self.vs = WebcamVideoStream(src=self._source_num)
if not self.vs.stream.isOpened():
msg = 'Could not open webcam stream {0}'.format(self._source_num)
raise RuntimeError(msg)
self.vs.start()
self.initialized = True
def get_frame(self):
"""Fetch single frame from source.
Returns:
fetched frame.
Raises:
RuntimeError: if source is not initialized.
"""
if not self.initialized:
raise RuntimeError('Source not initialized.')
return self.vs.read()
def stop(self):
"""Stop the source and free all allocated resources."""
self.vs.stop()
class videoStream:
def __init__(self):
print("Initializing video stream")
self.stream = WebcamVideoStream(src=0)
# Start the video stream
self.stream.start()
time.sleep(1.0) # wait for stream to initialize
print("Video stream initialized")
# end init
def grabFrame(self):
return self.stream.read() # grab a frame and return
# end grabFrame
def stop(self):
self.stream.stop()
class Threaded_Frame:
def __init__(self, src):
self.stream = WebcamVideoStream(src=src)
def start(self):
# start the threaded video stream
return self.stream.start()
def read(self):
# return the current frame
return self.stream.read()
def stop(self):
# stop the thread and release any resources
self.stream.stop()
def detect_faces():
print("[INFO] starting video stream...")
vs = WebcamVideoStream(
src="rtsp://*****:*****@192.168.1.90/axis-media/media.amp")
if vs.grabbed:
vs = vs.start()
time.sleep(2.0)
print("------started survillancing------")
id = 0
x = 1
while (True):
x += 1
frame = vs.read()
id += 1
if x % 2 == 0:
gray = face_detector(frame, id)
cv2.imshow('frame', gray)
if cv2.waitKey(10) & 0xFF == ord('q'):
break
vs.stop()
cv2.destroyAllWindows()
class VideoStream:
def __init__(self,
src=0,
usePiCamera=False,
resolution=(320, 240),
framerate=32):
# check to see if the picamera module should be used
if usePiCamera:
# only import the picamera packages unless we are
# explicity told to do so -- this helps remove the
# requirement of `picamera[array]` from desktops or
# laptops that still want to use the `imutils` package
from pivideostream import PiVideoStream
# initialize the picamera stream and allow the camera
# sensor to warmup
self.stream = PiVideoStream(resolution=resolution,
framerate=framerate)
# otherwise, we are using OpenCV so initialize the webcam
# stream
else:
self.stream = WebcamVideoStream(src=src)
def start(self):
# start the threaded video stream
return self.stream.start()
def update(self):
# grab the next frame from the stream
self.stream.update()
def read(self):
# return the current frame
return self.stream.read()
def stop(self):
# stop the thread and release any resources
self.stream.stop()
class Pipeline:
# created a *threaded* video stream, allow the camera sensor to warmup,
# and start the FPS counter
print("[INFO] sampling THREADED frames from webcam...")
#Constructor creates a list
def __init__(self, fixed_size):
self.vs1 = WebcamVideoStream(src=0) #For anylizing
self.vs1.start()
self.vs2 = WebcamVideoStream(src=0) #For anylizing
self.vs2.start()
self.cb = CircularBuffer(fixed_size)
self.stopped = False
self.current_stream = (None, None)
self.disp = None
self.haptic = None
#Start thread
def start(self):
t = Thread(target=self.capture, args=())
t.daemon = True
t.start()
return self
def stop(self):
self.stopped = True
def capture(self):
# keep looping infinitely
# if the thread indicator variable is set, stop the
# threa
while not self.stopped:
left_captured_frame = self.vs1.read()
right_captured_frame = self.vs2.read()
self.cb.enqueue((left_captured_frame, right_captured_frame))
def push(self):
pushed_stereo = self.capture()
self.cb.enqueue(pushed_stereo)
def pull(self):
pulled_stereo = self.cb.dequeue()
return pulled_stereo
def get_disp(self):
plt.clf()
self.disp = i2d(self.stereo, 16)
plt.imshow(self.disp, 'gray')
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return buf.read()
def get_depth(self):
plt.clf()
# Then send the disparity to haptic
self.haptic = np.array(depth_to_haptic(self.disp))
plt.scatter(self.haptic[:, 0], self.haptic[:, 1])
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return buf.read()
def get_frame(self):
self.stereo = self.pull()
# depth = self.process(stereo_feed)
left_feed = self.stereo[0]
right_feed = self.stereo[1]
# We are using Motion JPEG, but OpenCV defaults to capture raw images,
# so we must encode it into JPEG in order to correctly display the
# video stream.
l_ret, l_jpeg = cv2.imencode('.jpg', left_feed)
r_ret, r_jpeg = cv2.imencode('.jpg', right_feed)
return l_jpeg.tobytes(), r_jpeg.tobytes()
def gen(self, dir):
while True:
self.current_stream = self.get_frame()
if dir == 'left':
yield (b'--stream_left\r\n'
b'Content-Type: image/jpeg\r\n\r\n' +
self.current_stream[0] + b'\r\n\r\n')
if dir == 'right':
yield (b'--stream_right\r\n'
b'Content-Type: image/jpeg\r\n\r\n' +
self.current_stream[1] + b'\r\n\r\n')
if dir == 'disp':
self.disp = self.get_disp()
yield (b'--stream_right\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + self.disp +
b'\r\n\r\n')
if dir == 'depth':
yield (b'--stream_depth\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + self.haptic +
b'\r\n\r\n')
def main():
# Initialize background substractor
# Currently CV2 provide 2 types, "createBackgroundSubtractorKNN" and "createBackgroundSubtractorMOG2"
# 1. "createBackgroundSubtractorKNN"
# - subtraction is alot cleaner if background color more consistence
# 2. "createBackgroundSubtractorMOG2"
# - substraction based on Machine Learning (ML), more cleaner if background have too many color for KNN to work.
# - Result of substraction is not as complete compare to KNN
backSub = cv2.createBackgroundSubtractorKNN(history=450,
dist2Threshold=150.0,
detectShadows=True)
# cap = cv2.VideoCapture('fish4.mp4')
#cap = FileVideoStream('fish4.mp4').start() #initialize file video reader
cap = WebcamVideoStream(0)
cap.start()
time.sleep(1.0) # Block for 1 sec, to let "cap" buffer frames
frame = cap.read() # Get current frame
vh, vw = frame.shape[:
2] # Get current frame height and width, [0]: height, [1]: width
vh = int(vh / 2)
vw = int(vw / 2)
videWriter = cv2.VideoWriter('out.avi',
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
30, (vw, vh)) #Initialize video writer
# Set default center line
line_thickness = 5 # default line thickness to 5px
line_center = int(
(vw / 2)) # divide 2 with video width (vw) to get video center pixel
fishes = [] # store fish id & tracker
counter = 0 # fish counter
fps = FPS().start() #Initialize FPS counter
ttQ = Queue()
tt = Process(target=showMe, args=(ttQ, ))
tt.daemon = True
tt.start()
# While cap has more frame continue
while True:
frame = cv2.resize(frame, (vw, vh), cv2.INTER_AREA)
blank_frame = frame.copy(
) # Create a copy of original image for filtering
trackers = multiTracker.update(frame)
if len(trackers) > 0:
for tracker in trackers:
c, bbox = tracker # c = id
#success, bbox = tracker.update(frame) # run tracker update based on current frame
# Should tracker success = false, removed it from array to prevent future processes
#if not success:
# fishes.pop(idx)
# continue
# Calculate & Draw
offset = 0
xy1 = (int(bbox[0]) - offset, int(bbox[1]) - offset
) # staring X and Y bounding box
xy2 = (int(bbox[2]) + offset, int(bbox[3]) + offset
) # Width and Height bounding box
# Note:
# OpenCV Tracker return Region of Interest (ROI) which consist
# 1. starting point X and Y coordinate in pixel
# 2. width and height of the bounding box in pixel
# OpenCV rectangle function however is draw using 2 set of coordinate in pixel
# starting point (Top Left) and end point (Bottom Right)
# "xy1" is the starting coordinate (Top Left), as such using
# ((xy1[0] + xy2[0]), (xy1[1] + xy2[1]))
# we can calculate the 2nd set coordinate (Bottom Right)
# Bottom = starting X point + bounding box width
# Right = starting Y point + bounding box height
cv2.rectangle(frame, xy1,
((xy1[0] + xy2[0]), (xy1[1] + xy2[1])),
(0, 0, 255), 1)
cv2.putText(frame, 'id: {}'.format(c), (xy1[0], xy1[1] - 10),
cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0), 1,
cv2.LINE_AA)
cv2.rectangle(blank_frame, xy1,
((xy1[0] + xy2[0]), (xy1[1] + xy2[1])),
(255, 255, 255), -1)
# Note:
# OpenCV "findContours" function only work with image in gray color
gFrame = cv2.cvtColor(blank_frame,
cv2.COLOR_RGB2GRAY) # Convert image to gray
fMask = backSub.apply(gFrame) # Apply background seperation algorythm
fMask = cv2.morphologyEx(fMask,
cv2.MORPH_OPEN,
np.ones((5, 5), np.uint8),
iterations=2) # Fix deform contour
fMask = cv2.morphologyEx(fMask,
cv2.MORPH_CLOSE,
np.ones((5, 5), np.uint8),
iterations=2) # Fix deform contour
fMask = cv2.bitwise_and(gFrame, gFrame,
mask=fMask) # combine targeted frame with mask
fMask = cv2.GaussianBlur(fMask, (5, 5),
0) # add blur to further reduce pixel deform
ret, thresh = cv2.threshold(
fMask, 50, 255, cv2.THRESH_BINARY) # Create threshold algorythm
contours, hierarchy = cv2.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # find contours
# Loop through all found contours
# Should any contour found and is within "center_line" tag and track it
for contour in contours:
x, y, w, h = cv2.boundingRect(contour)
if x + w >= int(line_center) and x + w <= int(line_center +
line_thickness):
counter += 1
# tracker = cv2.TrackerCSRT_create()
# tracker.init(frame, (int(x), int(y), int(w), int(h)))
# fishes.append((counter, tracker))
multiTracker.add(counter, frame,
(int(x), int(y), int(w), int(h)))
# cv2.rectangle(frame, (int(x), int(y)), (int(x + w), int(y + h)), (0,0,255), 1)
# Draw "line_center"
frame = cv2.rectangle(frame, (line_center, vh),
(line_center + line_thickness, 0), (0, 0, 0), -1)
# Calculate, Generate and Draw "Total Fish" text
# Calculate, Generate and Draw "FPS" text
fps.update()
fps.stop()
tt = (frame, counter, fps, vw, vh, line_thickness)
ttQ.put(tt)
# Display the final combine of the orignal frame including tracked item
# cv2.imshow('frame', frame)
# cv2.imshow('mask', fMask)
# Display frame is being refresh every 30ms, change to 0 if manual forward required
key = cv2.waitKey(30)
# should "Q" is press, stop loop
if key == ord('q'):
break
# videWriter.write(frame) # Write frame to video output
# ok, frame = cap.read()
frame = cap.read()
def main():
height = 300
stream = WebcamVideoStream(0)
stream.start()
# tracker = Tracker()
fps = FPS().start()
print("[INFO] Camera warming up . . .")
time.sleep(2)
cv2.namedWindow("Explorer")
cv2.createTrackbar("HueMax", "Explorer", 255, 255, nothing)
cv2.createTrackbar("SatMax", "Explorer", 255, 255, nothing)
cv2.createTrackbar("ValMax", "Explorer", 255, 255, nothing)
cv2.createTrackbar("HueMin", "Explorer", 0, 255, nothing)
cv2.createTrackbar("SatMin", "Explorer", 0, 255, nothing)
cv2.createTrackbar("ValMin", "Explorer", 0, 255, nothing)
cv2.createTrackbar("GrayMin", "Explorer", 60, 200, nothing)
upperThresh = np.zeros(3)
lowerThresh = np.zeros(3)
cannyLowerThresh = 50
GRAY = False
while not stream.stopped:
# Get new frame
frame = stream.read()
frame = imutils.resize(frame, height=height)
fps.stop()
# Get track bar values
upperThresh[0] = cv2.getTrackbarPos("HueMax", "Explorer")
upperThresh[1] = cv2.getTrackbarPos("SatMax", "Explorer")
upperThresh[2] = cv2.getTrackbarPos("ValMax", "Explorer")
lowerThresh[0] = cv2.getTrackbarPos("HueMin", "Explorer")
lowerThresh[1] = cv2.getTrackbarPos("SatMin", "Explorer")
lowerThresh[2] = cv2.getTrackbarPos("ValMin", "Explorer")
cannyLowerThresh = cv2.getTrackbarPos("GrayMin", "Explorer")
# Gray scale mode and colour mode
if GRAY:
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
gray = cv2.bilateralFilter(gray, 11, 17, 17)
res1 = cv2.Canny(gray, cannyLowerThresh, 200)
else:
# Detect Object
hsv_mask, res1 = hsv_masking(frame, lowerThresh, upperThresh)
# Track Object
track_position = hsv_detection(hsv_mask)
# Calculate node and draw finger
for centroid in track_position:
# Draw and label each finger
cv2.circle(frame, tuple(centroid), 6, (255, 255, 255), -1)
cv2.putText(
frame,
"(%d, %d)" % (*centroid, ),
(abs(centroid[0] - 25), abs(centroid[1] - 25)),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
(255, 255, 255),
2,
)
cv2.putText(
res1,
"Press G to toggle greyscale canny",
(10, 20),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 255, 0),
2,
)
cv2.putText(
res1,
"Drag the slide bars to explore the HSV colour space",
(10, height - 10),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 255, 0),
2,
)
cv2.putText(
frame,
datetime.datetime.now().strftime("%A %d %B %Y %I:%M:%S%p") +
" Frame %d" % fps._numFrames,
(10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 0, 255),
2,
)
# Update the frames
cv2.imshow("Live Feed", frame)
cv2.imshow("Explorer", res1)
# Keyboard OP
k = cv2.waitKey(5) & 0xFF
if k == 27 or k == ord("q"): # Esc
break
elif k == ord("g"): # toggle gray
GRAY = not GRAY
fps.update()
# Stops video stream, calculates FPS and does some clean ups
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
stream.stop()
cv2.destroyAllWindows()
class EyeCanSee(object):
def __init__(self, center=int(cvsettings.CAMERA_WIDTH / 2), debug=False, is_usb_webcam=True, period_s=0.025):
# Our video stream
# If its not a usb webcam then get pi camera
if not is_usb_webcam:
self.vs = PiVideoStream(resolution=(cvsettings.CAMERA_WIDTH, cvsettings.CAMERA_HEIGHT))
# Camera cvsettings
self.vs.camera.shutter_speed = cvsettings.SHUTTER
self.vs.camera.exposure_mode = cvsettings.EXPOSURE_MODE
self.vs.camera.exposure_compensation = cvsettings.EXPOSURE_COMPENSATION
self.vs.camera.awb_gains = cvsettings.AWB_GAINS
self.vs.camera.awb_mode = cvsettings.AWB_MODE
self.vs.camera.saturation = cvsettings.SATURATION
self.vs.camera.rotation = cvsettings.ROTATION
self.vs.camera.video_stabilization = cvsettings.VIDEO_STABALIZATION
self.vs.camera.iso = cvsettings.ISO
self.vs.camera.brightness = cvsettings.BRIGHTNESS
self.vs.camera.contrast = cvsettings.CONTRAST
# Else get the usb camera
else:
self.vs = WebcamVideoStream(src=0)
self.vs.stream.set(cv2.CAP_PROP_FRAME_WIDTH, cvsettings.CAMERA_WIDTH)
self.vs.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, cvsettings.CAMERA_HEIGHT)
# Has camera started
self.camera_started = False
self.start_camera() # Starts our camera
# To calculate our error in positioning
self.center = center
# To determine if we actually detected lane or not
self.detected_lane = False
# debug mode on? (to display processed images)
self.debug = debug
# Time interval between in update (in ms)
# FPS = 1/period_s
self.period_s = period_s
# Starting time
self.start_time = time.time()
# Mouse event handler for get_hsv
def on_mouse(self, event, x, y, flag, param):
if event == cv2.EVENT_LBUTTONDBLCLK:
# Circle to indicate hsv location, and update frame
cv2.circle(self.img_debug, (x, y), 3, (0, 0, 255))
cv2.imshow('hsv_extractor', self.img_debug)
# Print values
values = self.hsv_frame[y, x]
print('H:', values[0], '\tS:', values[1], '\tV:', values[2])
def get_hsv(self):
cv2.namedWindow('hsv_extractor')
while True:
self.grab_frame()
# Bottom ROI
cv2.rectangle(self.img_debug, (0, cvsettings.HEIGHT_PADDING_BOTTOM-2), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_BOTTOM + cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2)
# Top ROI
cv2.rectangle(self.img_debug, (0, cvsettings.HEIGHT_PADDING_TOP-2), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP + cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2)
# Object
cv2.rectangle(self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2)
self.hsv_frame = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV)
# Mouse handler
cv2.setMouseCallback('hsv_extractor', self.on_mouse, 0)
cv2.imshow('hsv_extractor', self.img_debug)
key = cv2.waitKey(0) & 0xFF
if key == ord('q'):
break
self.stop_camera()
cv2.destroyAllWindows()
# Starts camera (needs to be called before run)
def start_camera(self):
self.camera_started = True
self.vs.start()
time.sleep(2.0) # Wait for camera to cool
def stop_camera(self):
self.camera_started = False
self.vs.stop()
# Grabs frame from camera
def grab_frame(self):
# Starts camera if it hasn't been started
if not self.camera_started:
self.start_camera()
self.img = self.vs.read()
self.img_debug = self.img.copy()
# Normalizes our image
def normalize_img(self):
# Crop img and convert to hsv
self.img_roi_bottom = np.copy(self.img[cvsettings.HEIGHT_PADDING_BOTTOM:int(cvsettings.HEIGHT_PADDING_BOTTOM + cvsettings.IMG_ROI_HEIGHT), :])
self.img_roi_top = np.copy(self.img[cvsettings.HEIGHT_PADDING_TOP:int(cvsettings.HEIGHT_PADDING_TOP + cvsettings.IMG_ROI_HEIGHT), :])
self.img_roi_bottom_hsv = cv2.cvtColor(self.img_roi_bottom, cv2.COLOR_BGR2HSV).copy()
self.img_roi_top_hsv = cv2.cvtColor(self.img_roi_top, cv2.COLOR_BGR2HSV).copy()
# Get our ROI's shape
# Doesn't matter because both of them are the same shape
self.roi_height, self.roi_width, self.roi_channels = self.img_roi_bottom.shape
# Smooth image and convert to bianry image (threshold)
# Filter out colors that are not within the RANGE value
def filter_smooth_thres(self, RANGE, color):
for (lower, upper) in RANGE:
lower = np.array(lower, dtype='uint8')
upper = np.array(upper, dtype='uint8')
mask_bottom = cv2.inRange(self.img_roi_bottom_hsv, lower, upper)
mask_top = cv2.inRange(self.img_roi_top_hsv, lower, upper)
blurred_bottom = cv2.medianBlur(mask_bottom, 5)
blurred_top = cv2.medianBlur(mask_top, 5)
# Morphological transformation
kernel = np.ones((2, 2), np.uint8)
smoothen_bottom = blurred_bottom #cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5)
smoothen_top = blurred_top # cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5)
"""
if self.debug:
cv2.imshow('mask bottom ' + color, mask_bottom)
cv2.imshow('blurred bottom' + color, blurred_bottom)
cv2.imshow('mask top ' + color, mask_top)
cv2.imshow('blurred top' + color, blurred_top)
"""
return smoothen_bottom, smoothen_top
# Gets metadata from our contours
def get_contour_metadata(self):
# Metadata (x,y,w,h)for our ROI
contour_metadata = {}
for cur_img in [self.thres_yellow_bottom, self.thres_yellow_top, self.thres_blue_bottom, self.thres_blue_top]:
key = ''
# Blue is left lane, Yellow is right lane
if cur_img is self.thres_yellow_bottom:
key = 'right_bottom'
elif cur_img is self.thres_yellow_top:
key = 'right_top'
elif cur_img is self.thres_blue_bottom:
key = 'left_bottom'
elif cur_img is self.thres_blue_top:
key = 'left_top'
_, contours, hierarchy = cv2.findContours(cur_img.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cur_height, cur_width = cur_img.shape
# Get index of largest contour
try:
areas = [cv2.contourArea(c) for c in contours]
max_index = np.argmax(areas)
cnt = contours[max_index]
# Metadata of contour
x, y, w, h = cv2.boundingRect(cnt)
# Normalize it to the original picture
x += int(cvsettings.WIDTH_PADDING + w / 2)
if 'top' in key:
y += int(cvsettings.HEIGHT_PADDING_TOP +h /2)
else:
y += int(cvsettings.HEIGHT_PADDING_BOTTOM + h / 2)
contour_metadata[key] = (x, y)
self.detected_lane = True
# If it throws an error then it doesn't have a ROI
# Means we're too far off to the left or right
except:
# Blue is left lane, Yellow is right lane
x = int(cvsettings.WIDTH_PADDING) - cvsettings.CAMERA_WIDTH
if 'bottom' in key:
y = int(cur_height / 2) + int(cvsettings.HEIGHT_PADDING_BOTTOM + cur_height / 2)
else:
y = int(cur_height / 2) + int(cvsettings.HEIGHT_PADDING_TOP + cur_height / 2)
if 'right' in key:
x = int(cur_width)*2
contour_metadata[key] = (x, y)
self.detected_lane = False
return contour_metadata
# Gets the centered coord of the detected lines
def get_centered_coord(self):
bottom_centered_coord = None
top_centered_coord = None
left_xy_bottom = self.contour_metadata['left_bottom']
right_xy_bottom = self.contour_metadata['right_bottom']
left_xy_top = self.contour_metadata['left_top']
right_xy_top = self.contour_metadata['right_top']
bottom_xy = (left_xy_bottom[0] + right_xy_bottom[0], left_xy_bottom[1] + right_xy_bottom[1])
bottom_centered_coord = (int(bottom_xy[0] / 2), int(bottom_xy[1] / 2))
top_xy = (left_xy_top[0] + right_xy_top[0], left_xy_top[1] + right_xy_top[1])
top_centered_coord = (int(top_xy[0] / 2), int(top_xy[1] / 2))
# Left can't be greater than right and vice-versa
if left_xy_top > right_xy_top:
top_centered_coord = (0, top_centered_coord[1])
elif right_xy_top < left_xy_top:
top_centered_corrd = (cvsettings.CAMERA_WIDTH, top_centered_coord[1])
if left_xy_bottom > right_xy_bottom:
bottom_centered_coord = (0, bottom_centered_coord[1])
elif right_xy_bottom < left_xy_bottom:
bottom_centered_coord = (cvsettings.CAMERA_WIDTH, top_centered_coord[1])
return bottom_centered_coord, top_centered_coord
# Gets the error of the centered coordinates (x)
# Also normlizes their values
def get_errors(self):
top_error = (float(self.center_coord_top[0]) - float(self.center))/float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING)
bottom_error = (float(self.center_coord_bottom[0]) - float(self.center))/float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING)
relative_error = (float(self.center_coord_top[0]) - float(self.center_coord_bottom[0]))/float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING)
return (top_error + relative_error + bottom_error)/3.0
# Object avoidance
def where_object_be(self):
# We only want to detect objects in our path: center of top region and bottom region
# So to save processing speed, we'll only threshold from center of top region to center of bottom region
left_x = cvsettings.WIDTH_PADDING
right_x = cvsettings.CAMERA_WIDTH
# Image region with objects
img_roi_object = self.img[cvsettings.OBJECT_HEIGHT_PADDING : int(cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), left_x:right_x]
img_roi_object_hsv = cv2.cvtColor(img_roi_object, cv2.COLOR_BGR2HSV).copy()
# Filtering color and blurring
for (lower, upper) in cvsettings.OBJECT_HSV_RANGE:
lower = np.array(lower, dtype='uint8')
upper = np.array(upper, dtype='uint8')
mask_object = cv2.inRange(img_roi_object_hsv, lower, upper)
blurred_object = cv2.medianBlur(mask_object, 25)
# Finding position of object (if its there)
_, contours, hierarchy = cv2.findContours(blurred_object.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
left_x = self.contour_metadata['left_top'][0]
right_x = self.contour_metadata['right_top'][0]
for c in contours:
areas = cv2.contourArea(c)
# Object needs to be larger than a certain area
if areas > cvsettings.OBJECT_AREA:
x, y, w, h = cv2.boundingRect(c)
y += int(cvsettings.OBJECT_HEIGHT_PADDING + h / 2)
# Confusing part - this finds the object and makes it think that
# it is also a line to avoid bumping into it
# It +w and -w to find which line its closest to and then set
# the opposite as to be the new left/right lane
# e.g. line is closest to left lane (x-w), so left lane new x is
# (x+w)
distance_to_left = abs(x - left_x)
distance_to_right = abs(x+w - right_x)
# Make object's left most area the middle of right lane
if distance_to_left > distance_to_right:
self.contour_metadata['right_top'] = (x, self.contour_metadata['right_top'][1])
# Make object's right most area the middle of left lane
elif distance_to_right > distance_to_right:
self.contour_metadata['left_top'] = (x+w, self.contour_metadata['left_top'][1])
if self.debug:
cv2.circle(self.img_debug, (x+(w/2), y+(h/2)), 5, (240, 32, 160), 2)
if self.debug:
cv2.imshow('Blurred object', blurred_object)
# Where are we relative to our lane
def where_lane_be(self):
# Running once every period_ms
while float(time.time() - self.start_time) < self.period_s:
pass
# Update time instance
self.start_time = time.time()
# Camera grab frame and normalize it
self.grab_frame()
self.normalize_img()
# Filter out them colors
self.thres_blue_bottom, self.thres_blue_top = self.filter_smooth_thres(cvsettings.BLUE_HSV_RANGE, 'blue')
self.thres_yellow_bottom, self.thres_yellow_top = self.filter_smooth_thres(cvsettings.YELLOW_HSV_RANGE, 'yellow')
# Get contour meta data
self.contour_metadata = self.get_contour_metadata()
# Finds objects and (and corrects lane position)
# this overwrite contour_metadata
#self.where_object_be()
# Find the center of the lanes (bottom and top) [we wanna be in between]
self.center_coord_bottom, self.center_coord_top = self.get_centered_coord()
# Gets relative error between top center and bottom center
self.relative_error = self.get_errors()
if self.debug:
# Drawing locations
blue_top_xy = self.contour_metadata['left_top']
blue_bottom_xy = self.contour_metadata['left_bottom']
yellow_top_xy = self.contour_metadata['right_top']
yellow_bottom_xy = self.contour_metadata['right_bottom']
# Circles to indicate lanes
cv2.circle(self.img_debug, blue_top_xy, 5, (255, 0, 0), 2)
cv2.circle(self.img_debug, blue_bottom_xy, 5, (255, 0, 0), 2)
cv2.circle(self.img_debug, yellow_top_xy, 5, (0, 255, 255), 2)
cv2.circle(self.img_debug, yellow_bottom_xy, 5, (0, 255, 255), 2)
cv2.circle(self.img_debug, self.center_coord_bottom, 5, (0, 255, 0), 3)
cv2.circle(self.img_debug, self.center_coord_top, 5, (0, 255, 0), 3)
# ROI for object detection
cv2.rectangle(self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2)
# Displaying image
cv2.imshow('img', self.img_debug)
#cv2.imshow('img_roi top', self.img_roi_top)
#cv2.imshow('img_roi bottom', self.img_roi_bottom)
#cv2.imshow('img_hsv', self.img_roi_hsv)
cv2.imshow('thres_blue_bottom', self.thres_blue_bottom)
cv2.imshow('thres_blue_top', self.thres_blue_top)
cv2.imshow('thres_yellow_bottom', self.thres_yellow_bottom)
cv2.imshow('thres_yellow_top', self.thres_yellow_top)
key = cv2.waitKey(0) & 0xFF # Change 1 to 0 to pause between frames
# Use this to calculate fps
def calculate_fps(self, frames_no=100):
fps = FPS().start()
# Don't wanna display window
if self.debug:
self.debug = not self.debug
for i in range(0, frames_no):
self.where_lane_be()
fps.update()
fps.stop()
# Don't wanna display window
if not self.debug:
self.debug = not self.debug
print('Time taken: {:.2f}'.format(fps.elapsed()))
print('~ FPS : {:.2f}'.format(fps.fps()))
# Use this to save images to a location
def save_images(self, dirname='dump'):
import os
img_no = 1
# Makes the directory
if not os.path.exists('./' + dirname):
os.mkdir(dirname)
while True:
self.grab_frame()
if self.debug:
cv2.imshow('frame', self.img)
k = cv2.waitKey(1) & 0xFF
if k == ord('s'):
cv2.imwrite(os.path.join(dirname, 'dump_' + str(img_no) + '.jpg'), self.img)
img_no += 1
elif k == ord('q'):
break
cv2.destroyAllWindows()
# Destructor
def __del__(self):
self.vs.stop()
cv2.destroyAllWindows()
class App:
def __init__(self, logger, src, ROOT_DIR):
self.vs = WebcamVideoStream(src)
self.fps = FPS()
self.logger = logger
self.ROOT_DIR = ROOT_DIR
cv2.namedWindow("Webcam")
cv2.namedWindow("roi")
cv2.namedWindow("stacked")
cv2.createTrackbar('dilate kernel', 'roi', 3, 5, self.none)
cv2.createTrackbar('erode kernel', 'roi', 2, 5, self.none)
cv2.createTrackbar('blackhat kernel', 'roi', 21, 30, self.none)
self.mouse = Mouse(window="Webcam")
self.gt = Graphics()
# self.hist = Hist()
self.msg = "draw a rectangle to continue ..."
self.font_20 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Light.ttf', 20)
self.font_10 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Light.ttf', 15)
self.font_30 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Light.ttf', 30)
self.font_40 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Medium.ttf', 50)
# self.stabilizer = VidStab()
self.predictor = parser_v3.Predictor(model_path=f'{self.ROOT_DIR}/models/model_0.1v7.h5', root_dir=self.ROOT_DIR)
def run(self):
self.vs.start()
self.fps.start()
self.logger.info("app started ...")
frame = self.vs.read()
wh, ww, _ = frame.shape
cv2.moveWindow("Webcam", 0, 0)
cv2.moveWindow("roi", ww, 0)
cv2.moveWindow("stacked", 0, wh + 79)
self.mouse.rect = ((200, 200), (ww - 200, wh - 200))
while True:
frame = self.vs.read()
# frame = frame[:, ::-1] # flip
orig = frame.copy()
# stabilized_frame = self.stabilizer.stabilize_frame(
# input_frame=frame, smoothing_window=30, border_size=50)
if self.mouse.rect:
p1, p2 = self.mouse.rect
roi = self.gt.draw_roi(orig, p1, p2)
if roi.size != 0:
gray_roi = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
dk = cv2.getTrackbarPos("dilate kernel", "roi")
ek = cv2.getTrackbarPos("erode kernel", "roi")
bk = cv2.getTrackbarPos("blackhat kernel", "roi")
thresh = im.thresify(gray_roi, dk, ek, bk)
# print(thresh.shape)
# overlap thresh on original image
mod_thres = cv2.bitwise_not(thresh)
#orig[p1[1]:p2[1], p1[0]:p2[0]] = cv2.merge((mod_thres, mod_thres, mod_thres))
# boxes, digits, cnts = component.get_symbols(
# thresh, im=roi, draw_contours=True)
digits, boxes = component.connect_cnts2(thresh, roi)
stacked_digits, resized_digits = component.stack_digits(digits, im.resize)
res, latex_image, labels = self.predictor.parse(resized_digits, boxes)
# annotate symbols
# for label, (pre_label, x_min, y_min, x_max, y_max) in zip(labels, boxes):
# self.gt.write(orig, label, (x_min, y_min), self.font_20)
# res = self.predictor.parse(resized_digits, boxes)
#print(f"res: {res}")
self.gt.write(orig, res, (10, wh - wh / 8), self.font_10)
# self.gt.draw_boxes(orig, boxes, p1, p2)
# self.hist.draw(gray_roi)
cv2.imshow('roi', thresh)
cv2.imshow('stacked', stacked_digits)
# cv2.imshow('latex_image', latex_image)
# self.gt.write(orig, self.msg, (10, 10), self.font_20)
else:
orig[:, :] = cv2.blur(orig, (100, 100))
windowRect = cv2.getWindowImageRect('Webcam')
wx, wy, ww, wh = windowRect
self.gt.write(orig, self.msg, (100, wh / 2 - 30), self.font_30)
cv2.imshow('Webcam', orig)
# cv2.imshow('stab', stabilized_frame)
self.fps.update()
key = cv2.waitKey(1)
if (key & 0xFF == ord('q')) or (key & 0xFF == 27):
self.logger.info('exiting ...')
break
elif key & 0xFF == ord('c'):
CAPTURED_DIR = f'{self.ROOT_DIR}/screenshots'
imageName = f'{CAPTURED_DIR}/{str(time.strftime("%Y_%m_%d_%H_%M"))}.png'
cv2.imwrite(imageName, orig)
self.logger.info(f'screenshot saved at {imageName}')
self.fps.stop()
self.vs.stop()
cv2.destroyAllWindows()
self.logger.info("elapsed time: {:.2f}".format(self.fps.elapsed()))
self.logger.info("approx. FPS: {:.2f}".format(self.fps.fps()))
def none(self, x):
pass
def demo(args):
n_models = len(models)
model_idx = 0
# content_image = utils.load_image(args.content_image, scale=args.content_scale)
content_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))
])
style_model = TransformerNet()
load_model(style_model, args, model_idx)
fps = FPS().start()
stream = WebcamVideoStream(src=0) # default camera
stream.start() # default camera
time.sleep(1.0)
last_time = time.time()
# start fps timer
# loop over frames from the video file stream
while True:
# Change style
if time.time() - last_time > SECONDS_PER_MODEL:
model_idx = (model_idx + 1) % n_models
load_model(style_model, args, model_idx)
last_time = time.time()
# grab next frame
content_image = stream.read()
content_image = content_transform(content_image)
content_image = content_image.unsqueeze(0)
if args.cuda:
content_image = content_image.cuda()
content_image = Variable(content_image, volatile=True)
# update FPS counter
fps.update()
output = style_model(content_image)
if args.cuda:
output = output.cpu()
output_data = output.data[0].numpy()
output_data = np.clip(output_data, 0, 255).astype(np.uint8)
output_data = np.transpose(output_data, (1, 2, 0))
key = cv2.waitKey(1) & 0xFF
# keybindings for display
if key == ord('p'): # pause
while True:
key2 = cv2.waitKey(1) & 0xff
if key2 == ord('p'): # resume
break
cv2.imshow('Output', output_data)
if key in EXIT_KEYS: # exit
break
elif key in [LEFT_KEY, RIGHT_KEY]:
sign = -1 if key == LEFT_KEY else 1
model_idx += sign
if model_idx >= n_models:
model_idx = 0
elif model_idx < 0:
model_idx = n_models - 1
load_model(style_model, args, model_idx)
last_time = time.time()
stream.stop()
stream.stream.release()
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
if len(sys.argv) - 1 > 0:
CAMERA_ID = sys.argv[1]
CAM_WIDTH = 960
CAM_HEIGHT = 540
THRESHOLD = 40
PORT = 8000
DEBUG = False
cached_image = BytesIO()
last_data = []
capture = WebcamVideoStream(src=0)
capture.stream.set(cv2.CAP_PROP_FRAME_WIDTH, CAM_WIDTH)
capture.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, CAM_HEIGHT)
time.sleep(2)
capture.start()
time.sleep(2)
def get_host_ip():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
ip = s.getsockname()[0]
s.close()
return ip
def init_claim():
claims = [
{"type": "retract", "fact": [["id", get_my_id_str()], ["postfix", ""]]},
{"type": "claim", "fact": [
["id", get_my_id_str()],
["id", "0"],
def main():
# ######################### #
# ARGUMENT DEFINITIONS #
# ######################### #
# ARGUMENT DEFINITION
# Construct the argument parser and parse the arguments.
# This is where the input arguments for this program is read in.
# These arguments are read in through the Python command line. Otherwise,
# if they are not specified, they will be the default values.
ap = argparse.ArgumentParser()
# Number of seconds to record the video for.
# Keep -1 to keep recording indefinitely until user input terminates program.
# -1 will put it in TTL mode, where it will wait for a pulse before starting.
# Otherwise it will just start immediately and record for the provided amount of seconds.
# Specifying a duration that isn't -1 is best used for benchmarking.
# Default: -1
ap.add_argument("-s",
"--seconds",
type=int,
default=-1,
help="Number of seconds to record video for (default: -1)")
# FPS of the output video.
# Avoid values that are too high. Otherwise the output video will
# not be synchronized and will be too short relative to the recording time.
# Default: 5
ap.add_argument("-f",
"--fps",
type=int,
default=5,
help="Output video file's FPS rate (default: 5)")
# Codec used for the output video.
# Testing between different codecs indicate that we should consider between
# YUV (codec: IYUV) or XVID (codec: XVID). YUV will give better FPS performance
# on this computer, but will cause output files to be huge.
# XVID is fairly slower, but files are relatively small and compressed.
# Potential alternatives can also include MJPG, which is the fastest, but highly compressed
# and will affect video quality more so than the other codecs.
ap.add_argument(
"-c",
"--codec",
type=str,
default='MJPG',
help=
"Codec to use; use IYUV for better FPS, XVID for more compression, MJPG for fastest FPS but hurts quality (default: \"MJPG\")"
)
# Path to save the recorded video files.
# Default: outputX.avi, where X is the respective camera the video was recording from.
ap.add_argument(
"-o1",
"--out1",
type=str,
default='output1.avi',
help=
"Name of the output file path for camera 1 (default: \"output1.avi\")")
ap.add_argument(
"-o2",
"--out2",
type=str,
default='output2.avi',
help=
"Name of the output file path for camera 2 (default: \"output2.avi\")")
ap.add_argument(
"-o3",
"--out3",
type=str,
default='output3.avi',
help=
"Name of the output file path for camera 3 (default: \"output3.avi\")")
ap.add_argument(
"-o4",
"--out4",
type=str,
default='output4.avi',
help=
"Name of the output file path for camera 4 (default: \"output4.avi\")")
# Camera indexes. This is the USB index the program will connect to in order to
# read from the cameras.
# Default: 0, 1, 2, 3 for cameras 1, 2, 3, and 4 respectively.
# Highly RECOMMENDED to find out and specify the correct indices for the cameras.
# There is a high probability that the default values are incorrect for most cases.
ap.add_argument("-c1",
"--cam1",
type=int,
default=0,
help="Index value for camera 1 (default: 0)")
ap.add_argument("-c2",
"--cam2",
type=int,
default=1,
help="Index value for camera 2 (default: 1)")
ap.add_argument("-c3",
"--cam3",
type=int,
default=2,
help="Index value for camera 3 (default: 2)")
ap.add_argument("-c4",
"--cam4",
type=int,
default=3,
help="Index value for camera 4 (default: 3)")
# Conclude parsing the arguments.
args = vars(ap.parse_args())
# ##################### #
# PRINT INFO #
# ##################### #
# PRELIMINARY INFORMATION
# Print out the parameter information before starting the stream.
print('[INST] Starting 4K camera program.')
print(
'[INST] Make sure start.py on the rPi is already initiated and ready to start!\n'
)
print('[STAT] Resolution: ' + str(WIDTH) + 'x' + str(HEIGHT))
print('[STAT] camera 1 index: ' + str(args["cam1"]))
print('[STAT] camera 2 index: ' + str(args["cam2"]))
print('[STAT] camera 3 index: ' + str(args["cam3"]))
print('[STAT] camera 4 index: ' + str(args["cam4"]))
print('[STAT] seconds: ' + str(args["seconds"]))
print('[STAT] fps: ' + str(args["fps"]))
print('[STAT] codec: ' + str(args["codec"]))
print('[STAT] output path 1: ' + str(args["out1"]))
print('[STAT] output path 2: ' + str(args["out2"]))
print('[STAT] output path 3: ' + str(args["out3"]))
print('[STAT] output path 4: ' + str(args["out4"]))
print('[STAT] TTL Train data save path: ' + cwd)
# ##################### #
# INITIALIZE CAMERAS #
# ##################### #
# You can manually edit the codec here if needed.
FOURCC_CODEC = cv2.VideoWriter_fourcc(*args["codec"])
# INITIALIZE CAMERA STREAMS
# Creates a *threaded* video stream, allow the camera sensor to warm-up,
# and starts the FPS counter
# Camera 1 setup
print("\n[INFO] Initializing video camera stream 1...")
vs1 = WebcamVideoStream(src=args["cam1"]) # Record from index
vs1.fps = args["fps"] # Set FPS
vs1.num_frames = args[
"seconds"] * vs1.fps # Set total amount of frames to record
vs1.out = cv2.VideoWriter(args["out1"], FOURCC_CODEC, vs1.fps,
vs1.resolution)
vs1.frame_index = 1 / vs1.fps
# Camera 2 setup
print("\n[INFO] Initializing video camera stream 2...")
vs2 = WebcamVideoStream(src=args["cam2"]) # Record from index
vs2.fps = args["fps"] # Set FPS
vs2.num_frames = args[
"seconds"] * vs2.fps # Set total amount of frames to record
vs2.out = cv2.VideoWriter(args["out2"], FOURCC_CODEC, vs2.fps,
vs2.resolution)
vs2.frame_index = 1 / vs2.fps
# Camera 3 setup
print("\n[INFO] Initializing video camera stream 3...")
vs3 = WebcamVideoStream(src=args["cam3"]) # Record from index
vs3.fps = args["fps"] # Set FPS
vs3.num_frames = args[
"seconds"] * vs3.fps # Set total amount of frames to record
vs3.out = cv2.VideoWriter(args["out3"], FOURCC_CODEC, vs3.fps,
vs3.resolution)
vs3.frame_index = 1 / vs3.fps
# Camera 4 setup
print("\n[INFO] Initializing video camera stream 4...")
vs4 = WebcamVideoStream(src=args["cam4"]) # Record from index
vs4.fps = args["fps"] # Set FPS
vs4.num_frames = args[
"seconds"] * vs4.fps # Set total amount of frames to record
vs4.out = cv2.VideoWriter(args["out4"], FOURCC_CODEC, vs4.fps,
vs4.resolution)
vs4.frame_index = 1 / vs4.fps
# ################# #
# READ ON PULSES #
# ################# #
# INITIALIZE TASK for TTL
# At this point we keep reading for pulses from the ON port until we get a signal to actually start.
if args["seconds"] == -1: # If seconds is set to -1, it will be considered in TTL mode and will wait for a pulse before starting.
# Set up serial port to read start pulse
COM_PORT = 'COM4' # USB Serial COM Port
ser = serial.Serial(port=COM_PORT,
baudrate=115200,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS,
timeout=1)
# Wait for pulse before starting
print('[INFO] Waiting for TTL pulse...')
PULSE_START = False
# Keep reading from the port until a valid pulse is detected.
while PULSE_START is False:
# Read input data
pulse = ser.read(1)
# Check if ON pulse occurred, if so, start recording.
if pulse is b'\x00':
PULSE_START = True
TIMESTAMPS.append([str(datetime.datetime.now()), pulse])
print('\n[INFO] START PULSE DETECTED')
# ######################### #
# START VIDEO STREAMS #
# ######################### #
# Start the video streams
vs1.start()
vs2.start()
vs3.start()
vs4.start()
print("\n[INFO] Recording...")
# ######################### #
# WAIT FOR END CONDITION #
# ######################### #
# INITIALIZE IF RECORDING BY TIME
# This only runs if a duration for seconds is provided.
if args["seconds"] != -1:
# Keep recording until the time duration is reached.
print('[INFO] Initialized time-based recording...')
start = datetime.datetime.now()
end = datetime.datetime.now()
while (end - start).total_seconds() < args["seconds"]:
# Do nothing, wait out the time period
end = datetime.datetime.now()
# INITIALIZE IF RECORDING WITH TTL
# STOP NIDAQ TASK for TTL
# Here we keep recording from the off port, waiting for a signal to stop.
# The cameras will keep recording until a valid off pulse is read.
if args["seconds"] == -1:
print('[INFO] Initialized TTL-based recording...')
# Keep reading in TTL pulse to know when to stop
PULSE_STOP = False
# Keep recording until stop pulse is sent
while PULSE_STOP is False:
# Read from serial
pulse = ser.read(1)
# Add to timestamp
if pulse is b'\x00':
TIMESTAMPS.append([str(datetime.datetime.now()), pulse])
# Check if OFF pulse occurred, and if so, stop recording.
if pulse is b'':
PULSE_STOP = True
print('\n[INFO] STOP PULSE DETECTED\n')
# ################# #
# STOP RECORDING #
# ################# #
# Stop recording from all the streams.
# All these streams must be stopped via multithreading due to the five second sleep command
# in the stop function. Otherwise there would be a five second gap between each video file closing.
# The five second gap is required to prevent crashes from premature closing.
print('[INFO] Recording stopping...')
# Create the threads
vs1_thread = Thread(target=vs1.stop, args=())
vs2_thread = Thread(target=vs2.stop, args=())
vs3_thread = Thread(target=vs3.stop, args=())
vs4_thread = Thread(target=vs4.stop, args=())
# Start all the threads
vs1_thread.start()
vs2_thread.start()
vs3_thread.start()
vs4_thread.start()
# Wait for all of the threads to finish
vs1_thread.join()
vs2_thread.join()
vs3_thread.join()
vs4_thread.join()
print('[INFO] All recordings stopped!')
# ######################### #
# PRINT FPS INFORMATION #
# ######################### #
# Print out FPS here.
# It is CRUCIAL to note that the average FPS for each camera must be extremely close to the target
# FPS, otherwise there will be timing and synchronization issues (Videos will be shorter than
# the expected length!)
average_fps1 = vs1.FPS_counter.fps()
print('\nAverage FPS for camera 1: \t' + str(average_fps1))
print('Target FPS: \t' + str(vs1.fps))
if round(average_fps1) < vs1.fps:
print(
'[WARNING]: Average output FPS for camera 1 does not closely match specified FPS!'
)
average_fps2 = vs2.FPS_counter.fps()
print('\nAverage FPS for camera 2: \t' + str(average_fps2))
print('Target FPS: \t' + str(vs2.fps))
if round(average_fps2) < vs2.fps:
print(
'[WARNING]: Average output FPS for camera 2 does not closely match specified FPS!'
)
average_fps3 = vs3.FPS_counter.fps()
print('\nAverage FPS for camera 3: \t' + str(average_fps3))
print('Target FPS: \t' + str(vs3.fps))
if round(average_fps3) < vs3.fps:
print(
'[WARNING]: Average output FPS for camera 3 does not closely match specified FPS!'
)
average_fps4 = vs4.FPS_counter.fps()
print('\nAverage FPS for camera 4: \t' + str(average_fps4))
print('Target FPS: \t' + str(vs4.fps))
if round(average_fps4) < vs4.fps:
print(
'[WARNING]: Average output FPS for camera 4 does not closely match specified FPS!'
)
print('\n[INFO] Saving TTL pulse data to ' + str(cwd))
# Save data to csv file in train_recordings folder
with open(cwd, 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['Timestamp', 'Value'])
for i in range(len(TIMESTAMPS)):
writer.writerow([TIMESTAMPS[i][0], TIMESTAMPS[i][1]])
csvfile.close()
print('\nPROGRAM CONCLUDED!')
return
class OpenCVController:
# PyGame constants (RGB value for white & black and the radius of the eyeballs to draw)
RGB_WHITE = (255, 255, 255)
RGB_BLACK = (0, 0, 0)
EYEBALL_RADIUS = 30
# Font for text on webcam display (put here to simply our writing commands elsewhere and enable uniformity)
font = cv2.FONT_HERSHEY_SIMPLEX
# Incremented after each photo is taken - this allows each photo to have a unique name and not be overwritten
imageNumber = 0
# Define lower & upper boundaries in HSV color space (for object following/tracking)
greenLower = np.array([48, 100, 50])
greenUpper = np.array([85, 255, 255])
# Constructor/initalizer for this class
def __init__(self):
# Connect to the Arduino via serial port connection. If we're testing without the Arduino then set it to
# False in the line below so we don't get stuck waiting/hanging for the serial connection to complete
self.serialPort = self.com_connect() if True else None # True for running - False for testing
# Variables to hold last command sent to Arduino and when it was sent (epoch seconds). Note:
# lastCommandSentViaSerialTime ended up not being utilized - but its purpose was to send duplicate commands
# after some certain amount of time in case the Arduino needed it for some reason (it did not with our
# current design)
self.lastCommandSentViaSerial = None
self.lastCommandSentViaSerialTime = None
# Connect to webcam video source - WebcamVideoStream is a threaded class. By including it here,
# all our functions will be able to use it (since only one is run at a time) and we won't have to restart our
# video streaming each time we want to run a different function/capability.
self.WebcamVideoStreamObject = WebcamVideoStream(src=1)
# Save the original exposure and gain of the webcam for restoring later... (we will modify these parameters
# when we notice that the motion of the robot is impairing the OpenCV/image-processing capabilities (motion
# blur makes it hard to detect april tags, so by changing these parameters we can capture/detect these tags
# even while the robot is moving quite fast). We'd like to restore the camera to its original settings when
# it's not needed to specialize/adjust them because that enables the camera to choose the best settings for
# the environment and change its exposure automatically.
self.originalExposure = self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_EXPOSURE)
self.originalGain = self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_GAIN)
# Note: original exposure is probably like 0.015 or something small (depending on the lightning) and adjusts
# automatically
# Start the webcam streaming operation and set the object attribute to use...
self.vs = self.WebcamVideoStreamObject.start()
# Allow camera to warm up...
time.sleep(2.0)
# subprocess.check_call("v4l2-ctl -d /dev/video1 -c exposure_absolute=40",shell=True)
# print("New Gain") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_GAIN)))
# print("New exposure") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_EXPOSURE)))
# print("Frame width") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_FRAME_WIDTH)))
# print("Frame height") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_FRAME_HEIGHT)))
# Tries to open a specific URL to confirm that we are connected to the internet - this is never used in the
# codebase because some of our startup steps for our code already depended on internet connection.
@staticmethod
def internet_on():
try:
urlopen('http://216.58.192.142', timeout=1)
return True
except urllib2.URLError as err:
return False
# Connected to Arduino for serial communication. This method will hang/wait until it can make the connection,
# so make sure not to call this method if you aren't testing with an Arduino connected (see __init__ above)
@staticmethod
def com_connect():
ser = None
connection_made = False
while not connection_made:
if os.path.exists('/dev/ttyUSB0'):
connection_made = True
ser = serial.Serial('/dev/ttyUSB0', 9600, write_timeout=0)
print("Connected to Serial")
if os.path.exists('/dev/ttyACM1'):
connection_made = True
ser = serial.Serial('/dev/ttyACM1', 115200, write_timeout=0)
return ser
# Send serial command to Arduino. If the last command that we've sent is the same as the one we're trying to send
# now, then ignore it since the Arduino already has the up-to-date command. Note that there's a commented out
# section of this code that made it so that even if the command was a duplicate of the last send one,
# it would still send the command as long as a certain time period had passed. Depending on how the Arduino code
# worked this may have been necessary, but we did not need it.
def send_serial_command(self, direction_enum, dataToSend):
# If this command is different than the last command sent, then we should sent it
# Or if it's the same command but it's been 1 second since we last sent a command, then we should send it
if self.serialPort is not None:
if self.lastCommandSentViaSerial != direction_enum:
self.serialPort.write(dataToSend)
self.lastCommandSentViaSerial = direction_enum
self.lastCommandSentViaSerialTime = time.time()
# elif (time.time() - self.lastCommandSentViaSerialTime > 1): # TODO also need null check here
# self.serialPort.write(dataToSend)
# self.lastCommandSentViaSerialTime = time.time()
else:
pass # Do nothing - same command sent recently
# Call this when closing this openCV process. It will stop the WebcamVideoStream thread, close all openCV
# windows, and close the SerialPort as long as it exists (if we're connected to an Arduino).
def cleanup_resources(self):
# TODO change the order of stream release and stop() - see what happens
self.vs.stop()
# self.vs.stream.release() # TODO this should be called but is throwing errors - it works as-is though
cv2.destroyAllWindows() # Not necessary since I do it at the end of every method
if self.serialPort is not None: # Close serialPort if it exists
self.serialPort.close()
# Send an email with optional attachments specified (see usage in this file)
@staticmethod
def send_mail(send_from: str, subject: str, text: str, send_to: list, files=None):
username = '******'
password = '******'
msg = MIMEMultipart()
msg['From'] = send_from
msg['To'] = ', '.join(send_to)
msg['Subject'] = subject
msg.attach(MIMEText(text))
for f in files or []:
with open(f, "rb") as fil:
ext = f.split('.')[-1:]
attachedfile = MIMEApplication(fil.read(), _subtype=ext)
attachedfile.add_header(
'content-disposition', 'attachment', filename=basename(f))
msg.attach(attachedfile)
smtp = smtplib.SMTP(host="imap.gmail.com", port=587)
smtp.starttls()
smtp.login(username, password)
smtp.sendmail(send_from, send_to, msg.as_string())
smtp.close()
# Countdown timer and then take a photo using a frame from the WebcamVideoStream. The photo is sent to the emails
# of the group members. Note that the quality of the image can likely be increased by tuning the webcam settings
# (I did not because it was not necessary).
def take_photo(self, cvQueue: Queue):
# Creating a window for later use
cv2.namedWindow('result')
cv2.resizeWindow('result', 600, 600)
# Use the below stuff to do the countdown timer
startTime = time.time()
waitTimeSec = 5
strSec = "54321"
while True:
# Grab frame - break if we don't get it (some unknown error occurred)
frame = self.vs.read()
if frame is None:
print("ERROR - frame read a NONE")
break
frame = imutils.resize(frame, width=600)
currTime = time.time()
timeDiff = int(currTime - startTime) # seconds
if timeDiff < waitTimeSec: # Not time to take the photo yet
# Put time countdown on the frame
cv2.putText(frame, strSec[timeDiff], (250, 300), self.font, 5, (200, 255, 155), 4, cv2.LINE_AA)
cv2.imshow("result", frame)
else: # Time's up - take the photo and do post-processing
cv2.imshow("result", frame)
time.sleep(1) # Freeze the screen so that it's obvious that the photo was taken
# Save the image in the current working directory
cv2.imwrite(filename='saved_img_' + str(self.imageNumber) + '.jpg', img=frame)
cv2.destroyAllWindows() # Close the CV windows
# Send email
username = '******'
password = '******'
sendList = ['*****@*****.**',
'*****@*****.**',
'*****@*****.**']
filesToSend = [os.path.abspath('saved_img_' + str(self.imageNumber) + '.jpg')]
# if self.internet_on():
self.send_mail(send_from=username, subject="test", text="text", send_to=sendList, files=filesToSend)
# else:
# print("Internet is not on - did not send email")
self.imageNumber += 1 # Increment for next possible photo
break
# Close application on 'q' key press or if new stuff added to queue
key = cv2.waitKey(1) & 0xFF
if (key == ord("q")) or (not cvQueue.empty()):
# We've been requested to leave ...
# Don't destroy everything - just destroy cv2 windows ... webcam still runs
cv2.destroyAllWindows()
break
# Utilizes the person/eyeball_follow algorithms to move to the requested april tag and reach a desired distance
# from the tag. This function is called multiple times (once for each april tag we want to go to). The variables
# isFirstUse and isLastUse are used to avoid setting and resetting the webcam's gain and exposure rapidly back
# and forth (this creates a problem). By using these variables we set it only on the first april tag and reset it
# back to the original specifications on the last tag.
def april_following(self, desiredTag, desiredDistance, cvQueue: Queue, isFirstUse, isLastUse):
# Fast-fail. If there is something on the cvQueue then that means we need to respond to it. There are
# multiple calls of april_following(...) being made in succession in a for-loop to get the robot to a
# destiantion. We want to quickly exit from each of the calls in this situation.
if not cvQueue.empty():
return
# Tune the webcam to better see april tags while robot is moving
# (compensating for motion blur). Restore settings when done.
# These settings can be played with to create the best effect (along with other settings if you want)
if isFirstUse:
self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_EXPOSURE, 0.5)
self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_GAIN, 1)
# Frame is considered to be 600x600 (after resize) (actually it's like 600x400)
# Below are variables to set what we consider center and in-range (these numbers are in pixels)
radiusInRangeLowerBound, radiusInRangeUpperBound = desiredDistance - 10, desiredDistance + 10
centerRightBound, centerLeftBound = 400, 200
radiusTooCloseLowerLimit = 250
# When turning to search for the desiredTag, we specify time to turn, and time to wait after each semi-turn.
# Note that these two variables are NO LONGER USED! By adjusting the exposure to reduce the effects of motion
# blur, we no longer have to do this turn-and-stop manuever to search for tags around us. Just rotating works
# fine.
searchingTimeToTurn = 0.3 # seconds
searchingTimeToHalt = 1.0 # seconds
# Creating a window for later use
cv2.namedWindow('result')
cv2.resizeWindow('result', 600, 600)
# Variables to 'smarten' the following procedure. See their usage below.
objectSeenOnce = False # Object has never been seen before
leftOrRightLastSent = None # Keep track of whether we sent left or right last
firstTimeObjectNotSeen = None # Holds timestamp (in seconds) of the first time we haven't been able to see
# the tag. We don't want to instantly start freaking out and turning around looking for the tag because it's
# very possible it was lost in some bad frame, so we wait some X number of seconds before looking around (
# this is what this timestamp is used for).
# Initialize apriltag detector
options = apriltag.DetectorOptions(
families='tag36h11',
border=1,
nthreads=1,
quad_decimate=1.0,
quad_blur=0.0,
refine_edges=True,
refine_decode=True,
refine_pose=False,
debug=False,
quad_contours=True)
det = apriltag.Detector(options)
# TODO delete this block when done (not necessary to so we kept it - just thought removing extra details
# would speed up performance)
start = time.time()
num_frames = 0
inPosition = False
numHalts = 0
while True:
# Grab frame - break if we don't get it (some unknown error occurred)
frame = self.vs.read()
if frame is None:
break
# TODO delete this block when done (same as above TODO)
end = time.time()
seconds = end - start
num_frames += 1
fps = 0 if (seconds == 0) else num_frames / seconds
frame = imutils.resize(frame, width=600)
# frame = cv2.filter2D(frame, -1, np.array([[-1,-1,-1], [-1,9,-1], [-1,-1,-1]])) # Sharpen image
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Use grayscale image for detection
res = det.detect(gray) # Run the image through the apriltag detector and get the results
commandString = None # Stores string to print on the screen (the current command to execute)
# Check if the desiredTag is visible
tagObject = None
for r in res:
if r.tag_id == desiredTag:
tagObject = r
if tagObject is None: # We don't see the tag that we're looking for
# Don't see the tag? Possibly just bad frame, lets wait 2 seconds and then start turning
numHalts += 1
# TODO delete all the numHalt tracking stuff (this was to keep track and lessen the
# effects of motion blur ... we kept it since it didn't affect performance).
if firstTimeObjectNotSeen is None:
firstTimeObjectNotSeen = time.time()
self.send_serial_command(Direction.STOP, b'h')
commandString = "STOP"
else:
secondsOfNoTag = time.time() - firstTimeObjectNotSeen
if secondsOfNoTag > 2: # Haven't seen our tag for more than 2 seconds
if leftOrRightLastSent is not None:
if leftOrRightLastSent == Direction.RIGHT:
self.send_serial_command(Direction.RIGHT, b'r')
commandString = "SEARCHING: GO RIGHT"
elif leftOrRightLastSent == Direction.LEFT:
self.send_serial_command(Direction.LEFT, b'l')
commandString = "SEARCHING: GO LEFT"
else: # variable hasn't been set yet (seems unlikely), but default to left
self.send_serial_command(Direction.LEFT, b'r')
commandString = "DEFAULT SEARCHING: GO RIGHT"
# We've sent the command now wait half a second and then send a halt (WE DON"T NEED THIS ANYMORE)
# time.sleep(searchingTimeToTurn)
# self.send_serial_command(Direction.STOP, b'h');
# time.sleep(searchingTimeToHalt)
else: # Keep waiting - 2 seconds haven't elapsed
self.send_serial_command(Direction.STOP, b'h')
commandString = "STOP"
else: # We see the desired tag!
# Reset firstTimeObjectNotSeen to None for the next time we can't find the tag
if firstTimeObjectNotSeen is not None:
firstTimeObjectNotSeen = None
# Set objectSeenOnce to True if isn't already
if not objectSeenOnce:
objectSeenOnce = True
# Get the corners and draw a minimally enclosing circle of it
# and get the x/y/radius information of that circle to use for our navigation
corners = np.array(tagObject.corners, dtype=np.float32).reshape((4, 2, 1))
cornersList = []
for c in corners:
cornersList.append([int(x) for x in c])
cornersList = np.array(cornersList, dtype=np.int32) # Turn the list into a numpy array
((x, y), radius) = cv2.minEnclosingCircle(cornersList)
M = cv2.moments(cornersList)
# Grab the desired information...
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
filteredPtsRadius = [radius]
filteredPtsX = [center[0]]
filteredPtsY = [center[1]]
# Draw circle and center point on the frame
cv2.circle(frame, (int(x), int(y)), int(filteredPtsRadius[0]), (0, 255, 255), 2)
cv2.circle(frame, center, 5, (0, 0, 255), -1)
# Determine what command to send to the Arudino (motors)
if filteredPtsRadius[0] > radiusTooCloseLowerLimit:
commandString = "MOVE BACKWARD - TOO CLOSE TO TURN"
self.send_serial_command(Direction.BACKWARD, b'b')
elif filteredPtsX[0] > centerRightBound:
commandString = "GO RIGHT"
self.send_serial_command(Direction.RIGHT, b'r')
if leftOrRightLastSent != Direction.RIGHT:
leftOrRightLastSent = Direction.RIGHT
elif filteredPtsX[0] < centerLeftBound:
commandString = "GO LEFT"
self.send_serial_command(Direction.LEFT, b'l')
if leftOrRightLastSent != Direction.LEFT:
leftOrRightLastSent = Direction.LEFT
elif filteredPtsRadius[0] < radiusInRangeLowerBound:
commandString = "MOVE FORWARD"
self.send_serial_command(Direction.FORWARD, b'f')
elif filteredPtsRadius[0] > radiusInRangeUpperBound:
commandString = "MOVE BACKWARD"
self.send_serial_command(Direction.BACKWARD, b'b')
elif radiusInRangeLowerBound < filteredPtsRadius[0] < radiusInRangeUpperBound:
commandString = "STOP MOVING - IN RANGE"
self.send_serial_command(Direction.STOP, b'h')
inPosition = True
# Put text on the camera image to display on the screen
cv2.putText(frame, 'center coordinate: (' + str(filteredPtsX[0]) + ',' + str(filteredPtsY[0]) + ')',
(10, 60), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA)
cv2.putText(frame, 'filtered radius: (' + str(filteredPtsRadius[0]) + ')', (10, 90), self.font, 0.5,
(200, 255, 155), 1, cv2.LINE_AA)
# Show FPS and number of halts (this stuff will be on the frame regardless of whether we see our desired
# tag or not) (TODO delete this stuff later if we don't want it)
cv2.putText(frame, commandString, (10, 30), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA)
cv2.putText(frame, 'FPS: (' + str(fps) + ')', (10, 120), self.font, 0.5,
(200, 255, 155), 1, cv2.LINE_AA)
cv2.putText(frame, 'numHalts: (' + str(numHalts) + ')', (10, 150), self.font, 0.5,
(200, 255, 155), 1, cv2.LINE_AA)
# Display frame
cv2.imshow("result", frame)
# Close application on 'q' key press, new stuff on queue, or if we've reached our destination
key = cv2.waitKey(1) & 0xFF
if (key == ord("q")) or (not cvQueue.empty()) or inPosition:
self.send_serial_command(Direction.STOP, b'h');
# Restore webcam settings
if not inPosition or (inPosition and isLastUse):
# Reset the webcam to its original exposure and gain
self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_EXPOSURE, self.originalExposure)
self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_GAIN, self.originalGain)
# Activate auto_exposure (which is what the webcam starts out with by default but we mess it up
# by changing the exposure manually).
subprocess.check_call("v4l2-ctl -d /dev/video1 -c exposure_auto=3", shell=True)
cv2.destroyAllWindows()
break
# Used to priortize the coordinate information provided by closer tags as opposed to the farther aways ones (
# which would likely have less reliable information).
@staticmethod
def calc_weight(p1, p2):
max_weight = 150
dist = np.linalg.norm(p1 - p2)
return max_weight / dist
# Get an estimate of our current x and z coorindates and return them so they can be passed through to the chat
# server and disseminated to the other teams (for the distress signal).
def get_coordinates(self, cvQueue: Queue):
# To get the coordinate, we rotate on our axis some X number of times to form images that compose a complete
# 360 degree view of our surroundings. We use each image (as long as there are april tags in it) to get a (x,
# z) coordinate value, and then we choose which (x,z) coordinate to return based off of which we deem the
# most correct/reliable (this decision is shown in the code below)
# When turning to search for the desiredTag, we specify time to turn, and time to wait after each semi-turn.
# We do this because we want a stable photo/shot at each
searchingTimeToTurn = 0.5 # seconds
searchingTimeToHalt = 2.0 # seconds
# Note that refine_pose is set to True (takes more work/processing but hopefully gets better coordinates)
options = apriltag.DetectorOptions(
families='tag36h11',
border=1,
nthreads=1,
quad_decimate=1.0,
quad_blur=0.0,
refine_edges=True,
refine_decode=True,
refine_pose=True,
debug=False,
quad_contours=True)
det = apriltag.Detector(options)
# Load camera data
with open('cameraParams.json', 'r') as f:
data = json.load(f)
cameraMatrix = np.array(data['cameraMatrix'], dtype=np.float32)
distCoeffs = np.array(data['distCoeffs'], dtype=np.float32)
# Load world points
world_points = {}
with open('worldPoints.json', 'r') as f:
data = json.load(f)
for k, v in data.items():
world_points[int(k)] = np.array(v, dtype=np.float32).reshape((4, 3, 1))
# Variables for final decision
coordinates_list = []
iterationNumber = 1
numIterations = 10
while True:
# Rotate camera by going left by some amount
self.send_serial_command(Direction.LEFT, b'l')
time.sleep(searchingTimeToTurn)
self.send_serial_command(Direction.STOP, b'h')
time.sleep(searchingTimeToHalt)
# Now lets read the frame (while the robot is halted so that image is clean)
frame = self.vs.read()
if frame is None:
print("ERROR - frame read a NONE")
break
# Use grayscale image for detection
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
res = det.detect(gray)
# Check how many tags we see... if it's 0 then ignore this frame and move on to capturing the next frame
numTagsSeen = len(res)
print("\nNumber of tags seen", numTagsSeen) # TODO remove
if numTagsSeen > 0:
poses = [] # Store poses from each tag to average them over
tagRadiusList = [] # Store tag radius' to determine the largest
for r in res: # Iterate over each tag in the frame
corners = r.corners
tag_id = r.tag_id
corners = np.array(corners, dtype=np.float32).reshape((4, 2, 1))
cornersList = []
for c in corners:
cornersList.append([int(x) for x in c])
cornersList = np.array(cornersList, dtype=np.int32) # Turn into numpy array (openCV wants this)
# Draw circle around tag using its corners & get radius of that tag
((x, y), radius) = cv2.minEnclosingCircle(cornersList)
filteredPtsRadius = [radius]
# Solve pose ((x,z) coordinates)
r, rot, t = cv2.solvePnP(world_points[tag_id], corners, cameraMatrix,
distCoeffs) # get rotation and translation vector using solvePnP
rot_mat, _ = cv2.Rodrigues(rot) # convert to rotation matrix
R = rot_mat.transpose() # Use rotation matrix to get pose = -R * t (matrix mul w/ @)
pose = -R @ t
weight = self.calc_weight(pose, world_points[tag_id][0])
poses.append((pose, weight))
tagRadiusList.append(filteredPtsRadius)
# Done iterating over the tags that're seen in the frame...
# Now get the average pose across the tags and get the largest tag radius that we saw.
# We will store the (x,z) coordinate that we calculate, and we'll also
# store the largest radius for a tag that we've seen in this frame.
avgPose = sum([x * y for x, y in poses]) / sum([x[1] for x in poses])
largestTagRadius = max(tagRadiusList)
coordinates = (avgPose[0][0], avgPose[2][0], largestTagRadius)
print(str(coordinates)) # TODO remove this
coordinates_list.append(coordinates)
# Display frame
cv2.imshow('frame', frame)
# If we've completed our numIterations, then choose the coordinate
# and return (do closing operations too)
if iterationNumber == numIterations:
if len(coordinates_list) > 0:
# TODO 2 things we can try here ...
# 1) The coordinate to return is the one with the smallest z-coordinate
# (which essentially means it's closest to those tags that it used)
# BUT this value seems to vary a lot and I don't think it's reliable
# 2) I have saved the largest radius for a tag seen for each of these
# coordinates, so I can use that (which I bet is more reliable)
# I will go with approach number 2
# coordinateToReturn = min(coordinates_list, key=lambda x: x[1]) # Approach (1)
coordinateToReturn = max(coordinates_list, key=lambda x: x[2]) # Approach (2)
coordinateToReturn = (
int(coordinateToReturn[0]), int(coordinateToReturn[1])) # This stays regardless
else:
coordinateToReturn = (0, -1) # TODO set to some default outside the door
# Cleanup and return
cv2.destroyAllWindows()
print("Value to return:") # TODO remove
print(coordinateToReturn) # TODO remove
return coordinateToReturn
else: # Still have iterations to go, increment the value
iterationNumber += 1
# Q to quit
key = cv2.waitKey(1) & 0xFF
if (key == ord("q")) or (not cvQueue.empty()):
self.send_serial_command(Direction.STOP, b'h')
cv2.destroyAllWindows()
break
# Follow a person (represented by a green folder). If the second argument (run_py_eyes) is set to True then we'll
# create eyeballs using PyGame that follow the user (folder) as it moves around.
def person_following(self, run_py_eyes, cvQueue: Queue):
# Frame is considered to be 600x600 (after resize)
# Below are variables to set what we consider center and in-range
radiusInRangeLowerBound, radiusInRangeUpperBound = 80, 120
centerRightBound, centerLeftBound = 400, 200
radiusTooCloseLowerLimit = 250
# Creating a window for later use
cv2.namedWindow('result')
cv2.resizeWindow('result', 600, 600)
# Variables to 'smarten' the following procedure (see usage below)
objectSeenOnce = False # Object has never been seen before
leftOrRightLastSent = None # Keep track of whether we sent left or right last
# TODO delete this block when done
start = time.time()
num_frames = 0
# PyEyes Setup... Note that I've done some performance tinkering with pygame. Instead of redrawing the entire
# frame on each iteration, I only turn the previously drawn pupils of the last frame white (to match the
# background) and draw the new pupils. I also enable some performance enhancements and disable some unneeded
# functionality. This kept out frame rate at a reliable level.
if run_py_eyes:
screen = pygame.display.set_mode((1024, 600), DOUBLEBUF)
screen.set_alpha(None) # Not needed, so set it to this for performance improvement
surface = pygame.display.get_surface()
# Draw the eyeballs (without pupils) and white background that we'll use for the rest of the process
screen.fill(self.RGB_WHITE) # Fill PyGame screen (white background)
pygame.draw.circle(surface, self.RGB_BLACK, (256, 300), 255, 15)
pygame.draw.circle(surface, self.RGB_BLACK, (768, 300), 255, 15)
pygame.display.flip()
rects = []
while True:
# Reset to default pupil coordinates and width (in case no object is found on this iteration)
leftx, lefty, width = 256, 350, 0
# Grab frame - break if we don't get it (some unknown error occurred)
frame = self.vs.read()
if frame is None:
print("ERROR - frame read a NONE")
break
# TODO delete this block when done (if you want)
end = time.time()
seconds = end - start
num_frames += 1
fps = 0 if (seconds == 0) else num_frames / seconds
# Resize the frame, blur it, and convert it to the HSV color space
frame = imutils.resize(frame, width=600)
blurred = cv2.GaussianBlur(frame, (5, 5), 0)
hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV)
# construct a mask for the desired color, then perform a series of dilations and erosions to
# remove any small blobs left in the mask
mask = cv2.inRange(hsv, self.greenLower, self.greenUpper)
mask = cv2.erode(mask, None, iterations=2) # TODO: these were 3 or 5 before (more small blob removal)
mask = cv2.dilate(mask, None, iterations=2)
# find contours in the mask and initialize the current (x, y) center of the ball
cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
commandString = None
# Only proceed if at least one contour was found
# If nothing is found, then look around OR send the STOP command to halt movement (depends on situation)
if len(cnts) == 0:
# If we haven't seen the object before, then we'll stay halted until we see one. If we HAVE seen the
# object before, then we'll move in the direction (left or right) that we did most recently
if not objectSeenOnce:
self.send_serial_command(Direction.STOP, b'h')
commandString = "STOP"
else: # Object has been seen before
if leftOrRightLastSent is not None:
if leftOrRightLastSent == Direction.RIGHT:
self.send_serial_command(Direction.RIGHT, b'r')
commandString = "SEARCHING: GO RIGHT"
elif leftOrRightLastSent == Direction.LEFT:
self.send_serial_command(Direction.LEFT, b'l')
commandString = "SEARCHING: GO LEFT"
else: # variable hasn't been set yet (seems unlikely), but default to left
self.send_serial_command(Direction.LEFT, b'l')
commandString = "DEFAULT SEARCHING: GO LEFT"
elif len(cnts) > 0: # Else if we are seeing some object...
# Find the largest contour in the mask and use it to compute the minimum enclosing circle and centroid
c = max(cnts, key=cv2.contourArea)
((x, y), radius) = cv2.minEnclosingCircle(c)
M = cv2.moments(c)
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
filteredPtsRadius = [radius]
# Only consider it to a valid object if it's big enough - else it could be some other random thing
if filteredPtsRadius[0] <= 25:
# TODO this is the same code as the block above - I should extract these out to a function
# If we haven't seen the object before, then we'll stay halted until we see one
# If we HAVE seen the object before, then we'll move in the direction (left or right) that we did
# most recently
if not objectSeenOnce:
self.send_serial_command(Direction.STOP, b'h');
commandString = "STOP";
else: # Object has been seen before
if leftOrRightLastSent is not None:
if leftOrRightLastSent == Direction.RIGHT:
self.send_serial_command(Direction.RIGHT, b'r');
commandString = "SEARCHING: GO RIGHT"
elif leftOrRightLastSent == Direction.LEFT:
self.send_serial_command(Direction.LEFT, b'l');
commandString = "SEARCHING: GO LEFT"
else: # variable hasn't been set yet (seems unlikely), but default to left
self.send_serial_command(Direction.LEFT, b'l');
commandString = "DEFAULT SEARCHING: GO LEFT"
else: # This object isn't super small ... we should proceed with the tracking
# Set objectSeenOnce to True if isn't already
if not objectSeenOnce:
objectSeenOnce = True
# draw the circle on the frame TODO consider removing this eventually - could speed things up (barely)
cv2.circle(frame, (int(x), int(y)), int(filteredPtsRadius[0]), (0, 255, 255), 2)
cv2.circle(frame, center, 5, (0, 0, 255), -1)
filteredPtsX = [center[0]]
filteredPtsY = [center[1]]
# Update PyGame Values
if run_py_eyes:
lefty = int(filteredPtsY[0] + 100)
leftx = int(abs(filteredPtsX[0] - 600) / 2 + 106)
width = int(filteredPtsRadius[0])
# Check radius and center of the blob to determine robot action
# What actions should take priority?
# 1. Moving Backward (only if it's super close)
# 2. Moving Left/Right
# 3. Moving Forward/Backward
# Why? Because if we're too close any turn would be too extreme. We need to take care of that first
if filteredPtsRadius[0] > radiusTooCloseLowerLimit:
commandString = "MOVE BACKWARD - TOO CLOSE TO TURN"
self.send_serial_command(Direction.BACKWARD, b'b')
elif filteredPtsX[0] > centerRightBound:
commandString = "GO RIGHT"
self.send_serial_command(Direction.RIGHT, b'r')
if leftOrRightLastSent != Direction.RIGHT:
leftOrRightLastSent = Direction.RIGHT
elif filteredPtsX[0] < centerLeftBound:
commandString = "GO LEFT"
self.send_serial_command(Direction.LEFT, b'l')
if leftOrRightLastSent != Direction.LEFT:
leftOrRightLastSent = Direction.LEFT
elif filteredPtsRadius[0] < radiusInRangeLowerBound:
commandString = "MOVE FORWARD"
self.send_serial_command(Direction.FORWARD, b'f')
elif filteredPtsRadius[0] > radiusInRangeUpperBound:
commandString = "MOVE BACKWARD"
self.send_serial_command(Direction.BACKWARD, b'b')
elif radiusInRangeLowerBound < filteredPtsRadius[0] < radiusInRangeUpperBound:
commandString = "STOP MOVING - IN RANGE"
self.send_serial_command(Direction.STOP, b'h')
# Put text on the camera image to display on the screen
cv2.putText(frame, 'center coordinate: (' + str(filteredPtsX[0]) + ',' + str(filteredPtsY[0]) + ')',
(10, 60), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA)
cv2.putText(frame, 'filtered radius: (' + str(filteredPtsRadius[0]) + ')', (10, 90), self.font, 0.5,
(200, 255, 155), 1, cv2.LINE_AA)
# The below steps are run regardless of whether we see a valid object or not ...
# Show FPS (TODO delete this later)
cv2.putText(frame, commandString, (10, 30), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA)
cv2.putText(frame, 'FPS: (' + str(fps) + ')', (10, 120), self.font, 0.5,
(200, 255, 155), 1, cv2.LINE_AA)
# show webcam video with object drawings on the screen
cv2.imshow("result", frame)
if run_py_eyes:
# If our coordinates are out of bounds for the eyes, then cap them at their correct bounds
if leftx < 106: leftx = 106
if leftx > 406: leftx = 406
if lefty < 150: lefty = 150
if lefty > 450: lefty = 450
# rightx = leftx + 400 + 112 - width
# if rightx < 568:
# rightx = 568
# if rightx > 968:
# rightx = 968
# Note that the eyes could be made to get a little crossed eyed (close together) when you get very
# close to the robot. It's not hard to do, but I didn't include it here (that's why the above lines
# are commented out).
# Draw left pupil
rects.append(pygame.draw.circle(surface, self.RGB_BLACK, (leftx, lefty), self.EYEBALL_RADIUS, 0))
# Draw right pupil
rects.append(
pygame.draw.circle(surface, self.RGB_BLACK, (leftx + 500 + 12, lefty), self.EYEBALL_RADIUS, 0))
# Update the display so our changes show up
pygame.display.update(rects)
# Save the left and right pupil circles so that we can remove them on the next iteration (instead of
# clearing the whole display and redrawing it all (which is expensive))
rects = [pygame.draw.circle(surface, self.RGB_WHITE, (leftx, lefty), self.EYEBALL_RADIUS, 0),
pygame.draw.circle(surface, self.RGB_WHITE, (leftx + 500 + 12, lefty), self.EYEBALL_RADIUS, 0)]
# Close application on 'q' key press, or if the queue is not empty (there's some command to respond to).
key = cv2.waitKey(1) & 0xFF
if (key == ord("q")) or (not cvQueue.empty()):
self.send_serial_command(Direction.STOP, b'h')
# We've been requested to leave ...
# Don't destroy everything - just destroy cv2 windows ... webcam still runs
cv2.destroyAllWindows()
if run_py_eyes:
pygame.display.quit()
pygame.quit()
break
# indices 3,4,5,6 store the bounding box coordinates in order [xmin, ymin, xmax, ymax] with values in the range 0-1
bbox = detections[0, 0, i, 3:7] * np.array(
[orig_w, orig_h, orig_w, orig_h]
) # scaling bounding box coordinates back to original frame dimensions
bbox = bbox.astype(np.int) # type casting and rounding to int type
cv2.rectangle(
frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 0, 255),
2) # drawing rectangular bounding boxes around detections
return frame
# setting up input video stream for reading from webcam
webcam_stream = WebcamVideoStream(
0) # opening video stream from primary camera
webcam_stream.start()
fps = FPS() # for computing frames processed per second
# processing video frames
fps.start()
while True:
# reading next frame from input stream
frame = webcam_stream.read()
fps.update()
# detecting faces in the read frame
frame_with_detections = detect_faces(frame)
# displaying the frame
cv2.imshow('Detected Faces', frame_with_detections)
key_pressed = cv2.waitKey(1) # a 1 millisecond delay
with model.as_default():
with tf.Session(graph=model) as sess:
imageTensor = model.get_tensor_by_name("image_tensor:0")
boxesTensor = model.get_tensor_by_name("detection_boxes:0")
# for each bounding box we would like to know the score
# (i.e., probability) and class label
scoresTensor = model.get_tensor_by_name("detection_scores:0")
classesTensor = model.get_tensor_by_name("detection_classes:0")
numDetections = model.get_tensor_by_name("num_detections:0")
drawboxes = []
# cap = cv2.VideoCapture(url)
vs = WebcamVideoStream(src=0)
vs.start()
while True:
frame = vs.read()
if frame is None:
continue
frame = cv2.flip(frame, 1)
image = frame
(H, W) = image.shape[:2]
# print("H,W:", (H, W))
output = image.copy()
img_ff, bin_mask, res = ff.find_hand_old(image.copy())
image = cv2.cvtColor(res, cv2.COLOR_BGR2RGB)
image = np.expand_dims(image, axis=0)
(boxes, scores, labels, N) = sess.run(
[boxesTensor, scoresTensor, classesTensor, numDetections],
class EyeCanSee(object):
def __init__(self,
center=int(cvsettings.CAMERA_WIDTH / 2),
debug=False,
is_usb_webcam=True,
period_s=0.025):
# Our video stream
# If its not a usb webcam then get pi camera
if not is_usb_webcam:
self.vs = PiVideoStream(resolution=(cvsettings.CAMERA_WIDTH,
cvsettings.CAMERA_HEIGHT))
# Camera cvsettings
self.vs.camera.shutter_speed = cvsettings.SHUTTER
self.vs.camera.exposure_mode = cvsettings.EXPOSURE_MODE
self.vs.camera.exposure_compensation = cvsettings.EXPOSURE_COMPENSATION
self.vs.camera.awb_gains = cvsettings.AWB_GAINS
self.vs.camera.awb_mode = cvsettings.AWB_MODE
self.vs.camera.saturation = cvsettings.SATURATION
self.vs.camera.rotation = cvsettings.ROTATION
self.vs.camera.video_stabilization = cvsettings.VIDEO_STABALIZATION
self.vs.camera.iso = cvsettings.ISO
self.vs.camera.brightness = cvsettings.BRIGHTNESS
self.vs.camera.contrast = cvsettings.CONTRAST
# Else get the usb camera
else:
self.vs = WebcamVideoStream(src=0)
self.vs.stream.set(cv2.CAP_PROP_FRAME_WIDTH,
cvsettings.CAMERA_WIDTH)
self.vs.stream.set(cv2.CAP_PROP_FRAME_HEIGHT,
cvsettings.CAMERA_HEIGHT)
# Has camera started
self.camera_started = False
self.start_camera() # Starts our camera
# To calculate our error in positioning
self.center = center
# To determine if we actually detected lane or not
self.detected_lane = False
# debug mode on? (to display processed images)
self.debug = debug
# Time interval between in update (in ms)
# FPS = 1/period_s
self.period_s = period_s
# Starting time
self.start_time = time.time()
# Mouse event handler for get_hsv
def on_mouse(self, event, x, y, flag, param):
if event == cv2.EVENT_LBUTTONDBLCLK:
# Circle to indicate hsv location, and update frame
cv2.circle(self.img_debug, (x, y), 3, (0, 0, 255))
cv2.imshow('hsv_extractor', self.img_debug)
# Print values
values = self.hsv_frame[y, x]
print('H:', values[0], '\tS:', values[1], '\tV:', values[2])
def get_hsv(self):
cv2.namedWindow('hsv_extractor')
while True:
self.grab_frame()
# Bottom ROI
cv2.rectangle(
self.img_debug, (0, cvsettings.HEIGHT_PADDING_BOTTOM - 2),
(cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_BOTTOM +
cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2)
# Top ROI
cv2.rectangle(
self.img_debug, (0, cvsettings.HEIGHT_PADDING_TOP - 2),
(cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP +
cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2)
# Object
cv2.rectangle(
self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING),
(cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP -
cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2)
self.hsv_frame = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV)
# Mouse handler
cv2.setMouseCallback('hsv_extractor', self.on_mouse, 0)
cv2.imshow('hsv_extractor', self.img_debug)
key = cv2.waitKey(0) & 0xFF
if key == ord('q'):
break
self.stop_camera()
cv2.destroyAllWindows()
# Starts camera (needs to be called before run)
def start_camera(self):
self.camera_started = True
self.vs.start()
time.sleep(2.0) # Wait for camera to cool
def stop_camera(self):
self.camera_started = False
self.vs.stop()
# Grabs frame from camera
def grab_frame(self):
# Starts camera if it hasn't been started
if not self.camera_started:
self.start_camera()
self.img = self.vs.read()
self.img_debug = self.img.copy()
# Normalizes our image
def normalize_img(self):
# Crop img and convert to hsv
self.img_roi_bottom = np.copy(self.img[
cvsettings.
HEIGHT_PADDING_BOTTOM:int(cvsettings.HEIGHT_PADDING_BOTTOM +
cvsettings.IMG_ROI_HEIGHT), :])
self.img_roi_top = np.copy(self.img[
cvsettings.HEIGHT_PADDING_TOP:int(cvsettings.HEIGHT_PADDING_TOP +
cvsettings.IMG_ROI_HEIGHT), :])
self.img_roi_bottom_hsv = cv2.cvtColor(self.img_roi_bottom,
cv2.COLOR_BGR2HSV).copy()
self.img_roi_top_hsv = cv2.cvtColor(self.img_roi_top,
cv2.COLOR_BGR2HSV).copy()
# Get our ROI's shape
# Doesn't matter because both of them are the same shape
self.roi_height, self.roi_width, self.roi_channels = self.img_roi_bottom.shape
# Smooth image and convert to bianry image (threshold)
# Filter out colors that are not within the RANGE value
def filter_smooth_thres(self, RANGE, color):
for (lower, upper) in RANGE:
lower = np.array(lower, dtype='uint8')
upper = np.array(upper, dtype='uint8')
mask_bottom = cv2.inRange(self.img_roi_bottom_hsv, lower, upper)
mask_top = cv2.inRange(self.img_roi_top_hsv, lower, upper)
blurred_bottom = cv2.medianBlur(mask_bottom, 5)
blurred_top = cv2.medianBlur(mask_top, 5)
# Morphological transformation
kernel = np.ones((2, 2), np.uint8)
smoothen_bottom = blurred_bottom #cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5)
smoothen_top = blurred_top # cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5)
"""
if self.debug:
cv2.imshow('mask bottom ' + color, mask_bottom)
cv2.imshow('blurred bottom' + color, blurred_bottom)
cv2.imshow('mask top ' + color, mask_top)
cv2.imshow('blurred top' + color, blurred_top)
"""
return smoothen_bottom, smoothen_top
# Gets metadata from our contours
def get_contour_metadata(self):
# Metadata (x,y,w,h)for our ROI
contour_metadata = {}
for cur_img in [
self.thres_yellow_bottom, self.thres_yellow_top,
self.thres_blue_bottom, self.thres_blue_top
]:
key = ''
# Blue is left lane, Yellow is right lane
if cur_img is self.thres_yellow_bottom:
key = 'right_bottom'
elif cur_img is self.thres_yellow_top:
key = 'right_top'
elif cur_img is self.thres_blue_bottom:
key = 'left_bottom'
elif cur_img is self.thres_blue_top:
key = 'left_top'
_, contours, hierarchy = cv2.findContours(cur_img.copy(),
cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
cur_height, cur_width = cur_img.shape
# Get index of largest contour
try:
areas = [cv2.contourArea(c) for c in contours]
max_index = np.argmax(areas)
cnt = contours[max_index]
# Metadata of contour
x, y, w, h = cv2.boundingRect(cnt)
# Normalize it to the original picture
x += int(cvsettings.WIDTH_PADDING + w / 2)
if 'top' in key:
y += int(cvsettings.HEIGHT_PADDING_TOP + h / 2)
else:
y += int(cvsettings.HEIGHT_PADDING_BOTTOM + h / 2)
contour_metadata[key] = (x, y)
self.detected_lane = True
# If it throws an error then it doesn't have a ROI
# Means we're too far off to the left or right
except:
# Blue is left lane, Yellow is right lane
x = int(cvsettings.WIDTH_PADDING) - cvsettings.CAMERA_WIDTH
if 'bottom' in key:
y = int(cur_height /
2) + int(cvsettings.HEIGHT_PADDING_BOTTOM +
cur_height / 2)
else:
y = int(
cur_height / 2) + int(cvsettings.HEIGHT_PADDING_TOP +
cur_height / 2)
if 'right' in key:
x = int(cur_width) * 2
contour_metadata[key] = (x, y)
self.detected_lane = False
return contour_metadata
# Gets the centered coord of the detected lines
def get_centered_coord(self):
bottom_centered_coord = None
top_centered_coord = None
left_xy_bottom = self.contour_metadata['left_bottom']
right_xy_bottom = self.contour_metadata['right_bottom']
left_xy_top = self.contour_metadata['left_top']
right_xy_top = self.contour_metadata['right_top']
bottom_xy = (left_xy_bottom[0] + right_xy_bottom[0],
left_xy_bottom[1] + right_xy_bottom[1])
bottom_centered_coord = (int(bottom_xy[0] / 2), int(bottom_xy[1] / 2))
top_xy = (left_xy_top[0] + right_xy_top[0],
left_xy_top[1] + right_xy_top[1])
top_centered_coord = (int(top_xy[0] / 2), int(top_xy[1] / 2))
# Left can't be greater than right and vice-versa
if left_xy_top > right_xy_top:
top_centered_coord = (0, top_centered_coord[1])
elif right_xy_top < left_xy_top:
top_centered_corrd = (cvsettings.CAMERA_WIDTH,
top_centered_coord[1])
if left_xy_bottom > right_xy_bottom:
bottom_centered_coord = (0, bottom_centered_coord[1])
elif right_xy_bottom < left_xy_bottom:
bottom_centered_coord = (cvsettings.CAMERA_WIDTH,
top_centered_coord[1])
return bottom_centered_coord, top_centered_coord
# Gets the error of the centered coordinates (x)
# Also normlizes their values
def get_errors(self):
top_error = (float(self.center_coord_top[0]) -
float(self.center)) / float(cvsettings.CAMERA_WIDTH +
cvsettings.WIDTH_PADDING)
bottom_error = (float(self.center_coord_bottom[0]) -
float(self.center)) / float(cvsettings.CAMERA_WIDTH +
cvsettings.WIDTH_PADDING)
relative_error = (float(self.center_coord_top[0]) - float(
self.center_coord_bottom[0])) / float(cvsettings.CAMERA_WIDTH +
cvsettings.WIDTH_PADDING)
return (top_error + relative_error + bottom_error) / 3.0
# Object avoidance
def where_object_be(self):
# We only want to detect objects in our path: center of top region and bottom region
# So to save processing speed, we'll only threshold from center of top region to center of bottom region
left_x = cvsettings.WIDTH_PADDING
right_x = cvsettings.CAMERA_WIDTH
# Image region with objects
img_roi_object = self.img[cvsettings.OBJECT_HEIGHT_PADDING:int(
cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING),
left_x:right_x]
img_roi_object_hsv = cv2.cvtColor(img_roi_object,
cv2.COLOR_BGR2HSV).copy()
# Filtering color and blurring
for (lower, upper) in cvsettings.OBJECT_HSV_RANGE:
lower = np.array(lower, dtype='uint8')
upper = np.array(upper, dtype='uint8')
mask_object = cv2.inRange(img_roi_object_hsv, lower, upper)
blurred_object = cv2.medianBlur(mask_object, 25)
# Finding position of object (if its there)
_, contours, hierarchy = cv2.findContours(blurred_object.copy(),
cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
left_x = self.contour_metadata['left_top'][0]
right_x = self.contour_metadata['right_top'][0]
for c in contours:
areas = cv2.contourArea(c)
# Object needs to be larger than a certain area
if areas > cvsettings.OBJECT_AREA:
x, y, w, h = cv2.boundingRect(c)
y += int(cvsettings.OBJECT_HEIGHT_PADDING + h / 2)
# Confusing part - this finds the object and makes it think that
# it is also a line to avoid bumping into it
# It +w and -w to find which line its closest to and then set
# the opposite as to be the new left/right lane
# e.g. line is closest to left lane (x-w), so left lane new x is
# (x+w)
distance_to_left = abs(x - left_x)
distance_to_right = abs(x + w - right_x)
# Make object's left most area the middle of right lane
if distance_to_left > distance_to_right:
self.contour_metadata['right_top'] = (
x, self.contour_metadata['right_top'][1])
# Make object's right most area the middle of left lane
elif distance_to_right > distance_to_right:
self.contour_metadata['left_top'] = (
x + w, self.contour_metadata['left_top'][1])
if self.debug:
cv2.circle(self.img_debug, (x + (w / 2), y + (h / 2)), 5,
(240, 32, 160), 2)
if self.debug:
cv2.imshow('Blurred object', blurred_object)
# Where are we relative to our lane
def where_lane_be(self):
# Running once every period_ms
while float(time.time() - self.start_time) < self.period_s:
pass
# Update time instance
self.start_time = time.time()
# Camera grab frame and normalize it
self.grab_frame()
self.normalize_img()
# Filter out them colors
self.thres_blue_bottom, self.thres_blue_top = self.filter_smooth_thres(
cvsettings.BLUE_HSV_RANGE, 'blue')
self.thres_yellow_bottom, self.thres_yellow_top = self.filter_smooth_thres(
cvsettings.YELLOW_HSV_RANGE, 'yellow')
# Get contour meta data
self.contour_metadata = self.get_contour_metadata()
# Finds objects and (and corrects lane position)
# this overwrite contour_metadata
#self.where_object_be()
# Find the center of the lanes (bottom and top) [we wanna be in between]
self.center_coord_bottom, self.center_coord_top = self.get_centered_coord(
)
# Gets relative error between top center and bottom center
self.relative_error = self.get_errors()
if self.debug:
# Drawing locations
blue_top_xy = self.contour_metadata['left_top']
blue_bottom_xy = self.contour_metadata['left_bottom']
yellow_top_xy = self.contour_metadata['right_top']
yellow_bottom_xy = self.contour_metadata['right_bottom']
# Circles to indicate lanes
cv2.circle(self.img_debug, blue_top_xy, 5, (255, 0, 0), 2)
cv2.circle(self.img_debug, blue_bottom_xy, 5, (255, 0, 0), 2)
cv2.circle(self.img_debug, yellow_top_xy, 5, (0, 255, 255), 2)
cv2.circle(self.img_debug, yellow_bottom_xy, 5, (0, 255, 255), 2)
cv2.circle(self.img_debug, self.center_coord_bottom, 5,
(0, 255, 0), 3)
cv2.circle(self.img_debug, self.center_coord_top, 5, (0, 255, 0),
3)
# ROI for object detection
cv2.rectangle(
self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING),
(cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP -
cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2)
# Displaying image
cv2.imshow('img', self.img_debug)
#cv2.imshow('img_roi top', self.img_roi_top)
#cv2.imshow('img_roi bottom', self.img_roi_bottom)
#cv2.imshow('img_hsv', self.img_roi_hsv)
cv2.imshow('thres_blue_bottom', self.thres_blue_bottom)
cv2.imshow('thres_blue_top', self.thres_blue_top)
cv2.imshow('thres_yellow_bottom', self.thres_yellow_bottom)
cv2.imshow('thres_yellow_top', self.thres_yellow_top)
key = cv2.waitKey(
0) & 0xFF # Change 1 to 0 to pause between frames
# Use this to calculate fps
def calculate_fps(self, frames_no=100):
fps = FPS().start()
# Don't wanna display window
if self.debug:
self.debug = not self.debug
for i in range(0, frames_no):
self.where_lane_be()
fps.update()
fps.stop()
# Don't wanna display window
if not self.debug:
self.debug = not self.debug
print('Time taken: {:.2f}'.format(fps.elapsed()))
print('~ FPS : {:.2f}'.format(fps.fps()))
# Use this to save images to a location
def save_images(self, dirname='dump'):
import os
img_no = 1
# Makes the directory
if not os.path.exists('./' + dirname):
os.mkdir(dirname)
while True:
self.grab_frame()
if self.debug:
cv2.imshow('frame', self.img)
k = cv2.waitKey(1) & 0xFF
if k == ord('s'):
cv2.imwrite(
os.path.join(dirname, 'dump_' + str(img_no) + '.jpg'),
self.img)
img_no += 1
elif k == ord('q'):
break
cv2.destroyAllWindows()
# Destructor
def __del__(self):
self.vs.stop()
cv2.destroyAllWindows()
class VideoStreamer:
outputFrame = None
lock = None
app = None
vs = None
PORT = '8000'
app = Flask('video_stream')
def __init__(self):
self.stream = False
self.app.add_url_rule('/video_on', 'video_on', self.start_new_stream)
self.app.add_url_rule('/video_off', 'video_off', self.stop_stream)
self.app.add_url_rule('/', 'index', index)
self.app.add_url_rule('/video', 'video', self.video_feed)
self.app.run(host='0.0.0.0', port=self.PORT, threaded=True, use_reloader=False)
def start_new_stream(self):
# initialize the output frame and a lock used to ensure thread-safe
# exchanges of the output frames (useful when multiple browsers/tabs
# are viewing the stream)
print('starting new stream')
self.outputFrame = None
self.lock = threading.Lock()
# initialize a flask object
# initialize the video stream and allow the camera sensor to warmup
print('pre initialize')
try:
self.vs = WebcamVideoStream(src=0)
except:
self.vs = WebcamVideoStream(src=-1)
print(self.vs)
self.vs.start()
time.sleep(2.0)
self.t = threading.Thread(target=self.get_frames, args=(32,))
self.t.daemon = True
self.t.start()
# start the flask app
self.stream = True
print('new_stream started')
return 'video streaming'
def stop_stream(self):
# release the video stream pointer
self.stream = False
self.vs.stream.release()
self.vs.stop()
print('stream stopped')
return 'video stopped'
def get_frames(self,frameCount):
# grab global references to the video stream, output frame, and
# lock variables
total = 0
print('entered thread')
# loop over frames from the video stream
while self.stream:
print(total)
# read the next frame from the video stream, resize it,
# convert the frame to grayscale, and blur it
frame = self.vs.read()
frame = imutils.resize(frame, width=400)
frame = cv2.flip(frame, 0)
# frame = imutils.resize(frame, width=400)
# grab the current timestamp and draw it on the frame
timestamp = datetime.datetime.now()
if frame is not None:
cv2.putText(frame, timestamp.strftime(
"%A %d %B %Y %I:%M:%S%p"), (10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1)
total += 1
with self.lock:
self.outputFrame = frame.copy()
def generate(self):
# grab global references to the output frame and lock variables
# loop over frames from the output stream
while True:
# wait until the lock is acquired
with self.lock:
# check if the output frame is available, otherwise skip
# the iteration of the loop
if self.outputFrame is None:
continue
# encode the frame in JPEG format
(flag, encodedImage) = cv2.imencode(".jpg", self.outputFrame)
# ensure the frame was successfully encoded
if not flag:
continue
# yield the output frame in the byte format
yield (b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + bytearray(encodedImage) + b'\r\n')
def video_feed(self):
# return the response generated along with the specific media
# type (mime type)
if self.stream:
return Response(self.generate(), mimetype="multipart/x-mixed-replace; boundary=frame")
else:
return 'no video available'
# The following was used for troubleshooting
# cv2.imshow("Calibration Image",image)
# Marker = the number of pixels that equate to the width
marker = find_marker(edged)
focalLength = (marker[1][0] * KNOWN_DISTANCE) / KNOWN_WIDTH
print("Pixcel Width", marker[1][0])
print("Focal Length", focalLength)
inches = distance_to_camera(KNOWN_WIDTH, focalLength, marker[1][0])
print("Distance", inches)
print("\nInitalizing camera in multi-thread mode")
print("Initalizing FPS stats collector in multi-thread mode\n")
camera = WebcamVideoStream(src=0).start()
camera.start()
sleep(2)
print("Camera Active: ", camera.grabbed)
while not camera.grabbed:
print("Restarting Camera...")
camera = WebcamVideoStream(src=0).start()
sleep(5)
fps = FPS().start()
camera.start()
getFrame = camera.read()
print(" Cam Width:", getFrame.shape[1])
print("cam Height:", getFrame.shape[0])
green_value_max = 174
green_hsv_max = np.array(
[green_hue_max, green_saturation_max, green_value_max])
mm.append(
MM.M_and_M(color_id, "Green", max_age, greenCircleColor, green_hsv_min,
green_hsv_max,
greenTextLocation)) # Create Green Tracking Object
##########
# Create video camera object and start streaming to it.
##########
#cap = cv2.VideoCapture(0) # NONTHREAD 0 == Continuous stream
capvs = WebcamVideoStream(
src=0) # THREAD Create video capture in separte thread
capvs.start()
# set up visual imaging for the trigger line that is the count line when crossed
cnt_down = 0
y_trigger = 160
trigger_line_color = (255, 0, 0)
trigger_line = np.array([[0, y_trigger], [720, y_trigger]])
counter = 0
# Debug variables used to see how long it takes to process a frame.
millis1 = 0
millis2 = 0
milli_total = 0
fps_count = 1
video = int(video) if str.isdigit(video) else video
# initialize the video stream
video_stream = WebcamVideoStream(src=video)
# 4:3=1280x960, 640x480 / 16:9=1280x720, 640x360
video_stream.stream.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
video_stream.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
video_stream.stream.set(cv2.CAP_PROP_FPS, 24)
print('\nVideo_stream.stream.get')
print(video_stream.stream.get(cv2.CAP_PROP_FRAME_WIDTH))
print(video_stream.stream.get(cv2.CAP_PROP_FRAME_HEIGHT))
print(video_stream.stream.get(cv2.CAP_PROP_FPS), '\n')
video_stream.start()
# a camera warmup time of 2.0 seconds
frame = video_stream.read()
# crop_y to 360
if frame.shape == (480, 640, 3):
frame = frame[60:-60, :]
print('Video Size is', frame.shape)
print('Streaming frames to the server...')
while True:
# read the frame from the camera and send it to the server
frame = video_stream.read()
# crop_y to 360
if frame.shape == (480, 640, 3):
frame = frame[60:-60, :]
