def track(video_name):
ct = CentroidTracker()
(H, W) = (None, None)
net = cv2.dnn.readNetFromCaffe("deploy.prototxt",
"res10_300x300_ssd_iter_140000.caffemodel")
print("[INFO] starting video stream...")
stream = cv2.VideoCapture(video_name)
fps = FPS().start()
while True:
(grabbed, frame) = stream.read()
if not grabbed:
break
# frame = imutils.resize(frame, width=400)
if W is None or H is None:
(H, W) = frame.shape[:2]
blob = cv2.dnn.blobFromImage(frame, 1.0, (W, H), (104.0, 177.0, 123.0))
net.setInput(blob)
detections = net.forward()
print(detections.shape[2])
rects = []
for i in range(0, detections.shape[2]):
if detections[0, 0, i, 2] > 0.85:
box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
rects.append(box.astype("int"))
(startX, startY, endX, endY) = box.astype("int")
det = frame[startY:endY, startX:endX]
# cv2.rectangle(frame, (startX, startY), (endX, endY),
# (0, 255, 0), 2)
# update our centroid tracker using the computed set of bounding
# box rectangles
objects = ct.update(rects, frame)
# loop over the tracked objects
if len(rects) != 0:
for (objectID, centroid) in objects.items():
text = "ID {}".format(objectID)
cv2.putText(frame, text,
(centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4,
(0, 255, 0), -1)
cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF
fps.update()
if key == ord("q"):
break
cv2.destroyAllWindows()
fps.stop()
return ct.pics, ct.objects
def ObjectTracking():
ct = CentroidTracker()
camera = cv2.VideoCapture(gstreamer_pipeline(flip_method=0), cv2.CAP_GSTREAMER)
if not camera.isOpened():
raise RuntimeError('Could not start camera.')
try:
while True:
_, img = camera.read()
boxes, confs, clss = detector.prediction(img, conf_class=[1])
img = detector.draw_boxes(img, boxes, confs, clss)
objects = ct.update(boxes)
if len(boxes) > 0 and 1 in clss:
print("detected {} {} {}".format(confs, objects, boxes))
#print("conf", confs)
#print('clss', clss)
#print('boxes', boxes)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
text = "ID {}".format(objectID)
cv2.putText(img, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(img, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
day = datetime.now().strftime("%Y%m%d")
directory = os.path.join(IMAGE_FOLDER, 'pi', day)
if not os.path.exists(directory):
os.makedirs(directory)
hour = datetime.now().strftime("%H%M%S")
filename_output = os.path.join(
directory, "{}_{}_.jpg".format(hour, "person")
)
cv2.imwrite(filename_output, img)
except KeyboardInterrupt:
print('interrupted!')
camera.release()
print(type(objects))
print(objects)
except Exception as e:
print('interrupted! by:')
print(e)
camera.release()
print(type(objects))
print(objects)
def display_wrapper(out, FLAGS, in_queue: Queue, in2_queue: Queue):
print("display_wrapper: {}".format(threading.current_thread()))
global stop_threads
class_names = [c.strip() for c in open(FLAGS.classes).readlines()]
data_log = {}
frame_count = 0
ct = CentroidTracker()
while True:
data = in_queue.get()
img = in2_queue.get()
if data is None or img is None:
break
boxes, scores, classes, nums, fps = data['boxes'], data[
'scores'], data['classes'], data['nums'], data['fps']
with TimeMeasure('Display frame:' + str(frame_count)):
img, rects, log = draw_outputs(img, (boxes, scores, classes, nums),
class_names)
img = cv2.putText(img, "FPS: {:.2f}".format(fps), (0, 30),
cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255),
2)
objects = ct.update(rects)
if FLAGS.output:
out.write(img)
data_log['frame{}'.format(str(frame_count))] = log
frame_count += 1
cv2.imshow('output', img)
if cv2.waitKey(1) == ord('q'):
stop_threads = True
break
with open(FLAGS.logs, 'w') as f:
json.dump(data_log, f)
cv2.destroyAllWindows()
for arg in contour_arg:
cs.append(cnts[arg])
for c in cs:
((x, y), radius) = cv2.minEnclosingCircle(c)
M = cv2.moments(c)
center.append((int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])))
# only proceed if the radius meets a minimum size
if radius > args["radius"]:
# draw the circle and centroid on the frame,
# then update the list of tracked points
cv2.circle(frame, (int(x), int(y)), int(radius), (0, 255, 255),
2)
cv2.circle(frame, center[-1], 5, (0, 0, 255), -1)
rects.append((x - radius, y - radius, x + radius, y + radius))
# use centroidtracker to associate detected objects with objects in previous frame
objects = ct.update(rects)
# use trackable object to plot object trail
for (objectID, centroid) in objects.items():
to = trackableObjects.get(objectID, None)
if to is None:
to = TrackableObject(objectID, centroid, args["buffer"])
to.deque.appendleft(centroid)
else:
to.centroids.append(centroid)
to.deque.appendleft(centroid)
trackableObjects[objectID] = to
for j in range(1, len(to.deque)):
if to.deque[j - 1] is None or to.deque[j] is None:
continue
def Stream():
st.title("Customer Tracker")
st.text(
"This application will track how many customer enter & exit your premise"
)
st.markdown("\n", unsafe_allow_html=True)
camera = st.text_input("Enter Camera/Webcam Path")
col1, col2 = st.beta_columns(2)
if col1.button('Start ▶️') and not col2.button("Stop ⏹️"):
if camera.isnumeric():
camera = int(camera)
st.info("Live Streaming")
elif camera is not None:
st.error("Please Enter the Correct Camera Path")
image_placeholder = st.empty()
#confidenceValue = 0.4
#frameValue = 30
# initialize the list of class labels MobileNet SSD was trained to
# detect
CLASSES = [
"background", "aeroplane", "bicycle", "bird", "boat", "bottle",
"bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse",
"motorbike", "person", "pottedplant", "sheep", "sofa", "train",
"tvmonitor"
]
# load our serialized model from disk
net = cv2.dnn.readNetFromCaffe("MobileNetSSD_deploy.prototxt",
"MobileNetSSD_deploy.caffemodel")
print("[INFO] Starting the video..")
vs = cv2.VideoCapture(camera)
# initialize the frame dimensions (we'll set them as soon as we read
# the first frame from the video)
W = None
H = None
# instantiate our centroid tracker, then initialize a list to store
# each of our dlib correlation trackers, followed by a dictionary to
# map each unique object ID to a TrackableObject
ct = CentroidTracker(maxDisappeared=80, maxDistance=50)
trackers = []
trackableObjects = {}
# initialize the total number of frames processed thus far, along
# with the total number of objects that have moved either up or down
totalFrames = 0
totalDown = 0
totalUp = 0
x = []
empty = []
empty1 = []
# start the frames per second throughput estimator
fps = FPS().start()
# loop over frames from the video stream
while True:
# grab the next frame and handle if we are reading from either
ret, frame = vs.read()
# resize the frame to have a maximum width of 500 pixels (the
# less data we have, the faster we can process it), then convert
# the frame from BGR to RGB for dlib
frame = imutils.resize(frame, width=700) # Default width = 500
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# if the frame dimensions are empty, set them
if W is None or H is None:
(H, W) = frame.shape[:2]
# initialize the current status along with our list of bounding
# box rectangles returned by either (1) our object detector or
# (2) the correlation trackers
status = "Waiting"
rects = []
# check to see if we should run a more computationally expensive
# object detection method to aid our tracker
if totalFrames % 30 == 0:
# set the status and initialize our new set of object trackers
status = "Detecting"
trackers = []
# convert the frame to a blob and pass the blob through the
# network and obtain the detections
blob = cv2.dnn.blobFromImage(frame, 0.007843, (W, H), 127.5)
net.setInput(blob)
detections = net.forward()
# loop over the detections
for i in np.arange(0, detections.shape[2]):
# extract the confidence (i.e., probability) associated
# with the prediction
confidence = detections[0, 0, i, 2]
# filter out weak detections by requiring a minimum
# confidence
if confidence > 0.4:
# extract the index of the class label from the
# detections list
idx = int(detections[0, 0, i, 1])
# if the class label is not a person, ignore it
if CLASSES[idx] != "person":
continue
# compute the (x, y)-coordinates of the bounding box
# for the object
box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
(startX, startY, endX, endY) = box.astype("int")
# construct a dlib rectangle object from the bounding
# box coordinates and then start the dlib correlation
# tracker
tracker = dlib.correlation_tracker()
rect = dlib.rectangle(startX, startY, endX, endY)
tracker.start_track(rgb, rect)
# add the tracker to our list of trackers so we can
# utilize it during skip frames
trackers.append(tracker)
# otherwise, we should utilize our object *trackers* rather than
# object *detectors* to obtain a higher frame processing throughput
else:
# loop over the trackers
for tracker in trackers:
# set the status of our system to be 'tracking' rather
# than 'waiting' or 'detecting'
status = "Tracking"
# update the tracker and grab the updated position
tracker.update(rgb)
pos = tracker.get_position()
# unpack the position object
startX = int(pos.left())
startY = int(pos.top())
endX = int(pos.right())
endY = int(pos.bottom())
# add the bounding box coordinates to the rectangles list
rects.append((startX, startY, endX, endY))
# draw a horizontal line in the center of the frame -- once an
# object crosses this line we will determine whether they were
# moving 'up' or 'down'
cv2.line(frame, (0, H // 2), (W, H // 2), (0, 0, 255), 3)
cv2.putText(frame, "Prediction border", (10, H - ((i * 20) + 200)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
# use the centroid tracker to associate the (1) old object
# centroids with (2) the newly computed object centroids
objects = ct.update(rects)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
# check to see if a trackable object exists for the current
# object ID
to = trackableObjects.get(objectID, None)
# if there is no existing trackable object, create one
if to is None:
to = TrackableObject(objectID, centroid)
# otherwise, there is a trackable object so we can utilize it
# to determine direction
else:
# the difference between the y-coordinate of the *current*
# centroid and the mean of *previous* centroids will tell
# us in which direction the object is moving (negative for
# 'up' and positive for 'down')
y = [c[1] for c in to.centroids]
direction = centroid[1] - np.mean(y)
to.centroids.append(centroid)
# check to see if the object has been counted or not
if not to.counted:
# if the direction is negative (indicating the object
# is moving up) AND the centroid is above the center
# line, count the object
if direction < 0 and centroid[1] < H // 2:
totalUp += 1
empty.append(totalUp)
to.counted = True
# if the direction is positive (indicating the object
# is moving down) AND the centroid is below the
# center line, count the object
elif direction > 0 and centroid[1] > H // 2:
totalDown += 1
empty1.append(totalDown)
#print(empty1[-1])
x = []
# compute the sum of total people inside
x.append(len(empty1) - len(empty))
#print("Total people inside:", x)
# if the people limit exceeds over threshold, send an email alert
if sum(x) >= config.Threshold:
cv2.putText(frame,
"-ALERT: People limit exceeded-",
(10, frame.shape[0] - 80),
cv2.FONT_HERSHEY_COMPLEX, 0.5,
(0, 0, 255), 2)
if config.ALERT:
print("[INFO] Sending email alert..")
Mailer().send(config.MAIL)
print("[INFO] Alert sent")
to.counted = True
# store the trackable object in our dictionary
trackableObjects[objectID] = to
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4,
(255, 255, 255), -1)
# construct a tuple of information we will be displaying on the
info = [
("Exit", totalUp),
("Enter", totalDown),
("Status", status),
]
info2 = [
("Total people inside", x),
]
# Display the output
for (i, (k, v)) in enumerate(info):
text = "{}: {}".format(k, v)
cv2.putText(frame, text, (10, H - ((i * 20) + 20)),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 250), 2)
for (i, (k, v)) in enumerate(info2):
text = "{}: {}".format(k, v)
cv2.putText(frame, text, (265, H - ((i * 20) + 60)),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
#Logs.csv
# Initiate a simple log to save data at end of the day
# if config.Log:
# datetimee = [datetime.datetime.now()]
# d = [datetimee, empty1, empty, x]
# export_data = zip_longest(*d, fillvalue = '')
# with open('Log.csv', 'w', newline='') as myfile:
# wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
# wr.writerow(("End Time", "In", "Out", "Total Inside"))
# wr.writerows(export_data)
#cv2.imshow("Real-Time Monitoring/Analysis Window", frame)
# show the output frame
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
image_placeholder.image(frame)
#key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
#if key == ord("q"):
# break
# increment the total number of frames processed thus far and
# then update the FPS counter
totalFrames += 1
fps.update()
if config.Timer:
# Automatic timer to stop the live stream. Set to 8 hours (28800s).
t1 = time.time()
num_seconds = (t1 - t0)
if num_seconds > 28800:
break
# stop the timer and display FPS information
fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
# # if we are not using a video file, stop the camera video stream
# if not args.get("input", False):
# vs.stop()
#
# # otherwise, release the video file pointer
# else:
# vs.release()
# close any open windows
cv2.destroyAllWindows()
score = float(detection[2])
if score > 0.5:
# Subtract 1 since LABEL list is 0 indexed while DNN output is 1 indexed
objID = int(detection[1]) - 1
# print("Found a " + LABELS[objID] + " with confidence " + str(detection[2]))
left = int(detection[3] * cols)
top = int(detection[4] * rows)
right = int(detection[5] * cols)
bottom = int(detection[6] * rows)
data = objData((left, bottom, right, top),
datetime.datetime.now().strftime("%H:%M:%S.%f"),
LABELS[objID], detection[2], int(COLORS[objID][0]))
# centroid tracker takes format smallerX, smallerY, largerX, largerY
# and a data object/tuple/structure/etc.
rects.append([left, bottom, right, top, data])
# Now, update the centroid tracker with the newly found bounding boxes
(objects, data) = ct.update(rects)
# Draw the objects being tracked
ct.drawObjects(img)
# Show the image with a rectagle surrounding the detected objects
cv2.imshow('Image', img)
# Press Q on keyboard to exit
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cv2.destroyAllWindows()
class PersonDetector():
def __init__(self, usePiCamera, width, height, camPort):
self.ct = CentroidTracker()
self.targetID = -1000000
self.targetCentroid = []
self.radius = 20
self.cap = VideoStream(usePiCamera, width, height, camPort).start()
self.haar_cascade = cv2.CascadeClassifier("data/HS.xml")
self.frameCount = 0
self.frameCaptureNumber = 7
self.thresholdX = int(width / 2)
self.thresholdY = (int)(height / 2)
self.width = width
self.height = height
def frameAdd(self):
self.frameCount += 1
def checkFrameCount(self):
if self.frameCount % self.frameCaptureNumber == 0:
return True
return False
def robotMoveDirection(self, centroid, center):
print(centroid, " ", center)
cX = center[0]
cY = center[1]
targetX = centroid[0]
targetY = centroid[1]
diffX = cX - targetX
diffY = cY - targetY
hors = "Move: "
verts = " and "
if targetX < cX - self.radius:
hors += "Left " + str(np.abs(diffX)) + " pixels"
elif targetX > cX + self.radius:
hors += "Right " + str(np.abs(diffX)) + " pixels"
else:
hors = ""
verts = "Move: "
if targetY < cY - self.radius:
verts += "Up " + str(np.abs(diffY)) + " pixels"
elif targetY > cY + self.radius:
verts += "Down " + str(np.abs(diffY)) + " pixels"
else:
verts = ""
return hors + verts
def updateObjectIDs(self, upper_body):
return self.ct.update(upper_body)
def detectPerson(self):
capturedframe = self.cap.getFrame()
frame = capturedframe[0:self.height, 0:int(self.width / 2)]
centerX = -10000
centerY = -100000
cColor = (0, 0, 255)
if self.checkFrameCount():
try:
# using a greyscale picture, also for faster detection
gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
upper_body = self.haar_cascade.detectMultiScale(
gray,
scaleFactor=1.1,
minNeighbors=12,
# Min size for valid detection, changes according to video size or body size in the video.
minSize=(50, 100),
flags=cv2.CASCADE_SCALE_IMAGE)
if len(upper_body) > 0:
(x, y, w, h) = upper_body[0]
# creates green color rectangle with a thickness size of 1
centerX = (int)(x + w / 2)
centerY = (int)(y + h / 2)
cv2.rectangle(capturedframe, (x, y), (x + w, y + h),
(0, 255, 0), 1)
cv2.line(capturedframe, (centerX, y), (centerX, y + h),
(255, 0, 0), 1)
cv2.line(capturedframe, (x, centerY), (x + w, centerY),
(255, 0, 0), 1)
cv2.circle(capturedframe, (centerX, centerY), 1,
(0, 0, 255), 1)
# creates green color text with text size of 0.5 & thickness size of 2
cv2.putText(capturedframe, "Head and Shoulders Detected",
(x + 5, y + 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
if np.abs(centerX - self.thresholdX) <= 20 and np.abs(
centerY - self.thresholdY) <= 20:
cColor = (0, 255, 0)
else:
cColor = (0, 0, 255)
objects = self.updateObjectIDs(upper_body)
# checking if the id is the same as the one detected
if len(list(objects.items())) > 0:
targetID = list(objects.items())[0][0]
targetCentroid = list(objects.items())[0][1]
if targetCentroid[0] == centerX and targetCentroid[
1] == centerY:
cv2.putText(capturedframe, str(targetID),
(targetCentroid[0], targetCentroid[1]),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2)
cv2.circle(capturedframe,
(targetCentroid[0], targetCentroid[1]), 4,
(0, 255, 0), -1)
else:
targetID = -100000
targetCentroid = []
print(
self.robotMoveDirection(targetCentroid,
(self.width / 2, self.height / 2)))
except Exception as e:
print(str(e))
return capturedframe
#threshold center lines
cv2.circle(capturedframe, (self.thresholdX, self.thresholdY),
self.radius, cColor, 3)
return capturedframe
def release(self):
self.cap.release()
def main():
model = load_model(tiny=False)
arduino_on = True
try:
arduino = serial.Serial("COM11", 9600)
print("Comunicacion con el arduino exitosa")
except:
arduino_on = False
print("No se puede comunicar con arduino")
n_frames_comunicacion = 12
iterador_comunicacion = 0
posicion_omni = (582, 216)
# Dibujo
img_dibujo = cv2.imread("imagenes_videos/dibujo_cancha.png", 1)
img_dibujo_copy = img_dibujo.copy()
shape_dibujo = img_dibujo.shape
#Variables para enviar a arduino
orientacion = 0
elevacion = 0
velocidad = 0
save_video = False
if save_video:
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
out = cv2.VideoWriter('mapeo.mp4', fourcc, 20.0,
(2 * shape_dibujo[1], shape_dibujo[0]))
#Video volley
cap = cv2.VideoCapture('imagenes_videos/video_volley2.mp4')
#Tracker
ct = CentroidTracker(maxDisappeared=60)
#Trackbar para la altura sobre la malla
cv2.namedWindow('Personas')
cv2.createTrackbar('H sobre malla [cm]', 'Personas', 100, 200,
pass_function)
cv2.namedWindow('Camera')
cv2.setMouseCallback('Camera', select)
calculate_detections = True
detections_aux = None
while (True):
#time1 = time.time()
ret, image = cap.read()
if ret == False:
print("Video terminado")
break
#Resize por hardcodeo de puntos
image = cv2.resize(image, (inputw, inputh)) #1920,1080
#a veces falla por el video mal grabado uwu
try:
transformation_matrix = getCourtTransformMatrix(image, img_dibujo)
except:
continue
#same
if (transformation_matrix is None):
continue
altura_sobre_malla = cv2.getTrackbarPos('H sobre malla [cm]',
'Personas')
#detecciones
if calculate_detections:
detections = model(image)
detections_aux = detections
else:
detections = detections_aux
calculate_detections = not calculate_detections
global Persons_pos, Person_des
if detections[0] is not None:
rects = detections[0].cpu().detach().numpy()[:, :4]
objects = ct.update(rects)
for obj_id, (x1, y1, x2, y2) in objects.items():
x1 = int(x1.item())
y1 = int(y1.item())
x2 = int(x2.item())
y2 = int(y2.item())
color = (148.0, 81.0, 165.0)
color_selection = (82.0, 162.0, 140.0)
Persons_pos.append([str(obj_id), x1, y1, x2, y2])
punto_cancha = np.array([x1, y2])
punto_proyectado = transform_point(punto_cancha,
transformation_matrix)
if len(Person_des) > 0 and str(int(obj_id)) == Person_des[0]:
color = color_selection
orientacion, elevacion, velocidad = calculo_orientacion_elevacion_velocidad(
punto_proyectado,
altura_sobre_malla=altura_sobre_malla)
angulo_servo_1, angulo_servo_2 = angulos_motores(
orientacion, elevacion)
string_para_arduino = "{:.2f}#{:.2f}#{:.2f}".format(
angulo_servo_1, angulo_servo_2, velocidad)
string_2 = "orientacion: {} elevacion: {}".format(
orientacion, elevacion)
print(string_2)
if arduino_on:
if (iterador_comunicacion %
n_frames_comunicacion == 0):
arduino.write(string_para_arduino.encode())
iterador_comunicacion += 1
# Dibujar en imagen
cv2.rectangle(image, (x1, y1), (x2, y2), color, 2)
text = str(obj_id)
cv2.rectangle(image, (x1 - 2, y1 - 25), (x1 + 10, y1), color,
-1)
cv2.putText(image, text, (x1, y1 - 5), FONT, 0.5,
(255, 255, 255), 1, cv2.LINE_AA)
#Dibujar puntos a dibujo
cv2.circle(
img_dibujo,
(int(punto_proyectado[0]), int(punto_proyectado[1])), 10,
color, -1)
cv2.circle(img_dibujo, posicion_omni, 10, (0, 0, 255), -1)
out_frame = img_dibujo
cv2.imshow('Personas', out_frame) #out_frame
cv2.imshow('Camera', cv2.resize(image, (outputw, outputh)))
# print(time.time()-time1)
# cv2.namedWindow( "court", cv2.WINDOW_NORMAL )
# cv2.imshow('court', img_dibujo)
#Para no sobreescribir dibujo
img_dibujo = img_dibujo_copy.copy()
if save_video:
out.write(out_frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
#arduino.close()
cap.release()
if save_video:
out.release()
centroids = []
componentConnect = cv2.connectedComponentsWithStats(frame,
connectivity=8)
_, _, _, centers = componentConnect
centerList = centers[1:].astype(int).tolist()
for txt in centerList:
c_x = txt[0]
c_y = txt[1]
centroids.append(txt)
row = []
objects = ct.update(centroids)
row.append(count_frame)
row.append("{}".format(imagePath.strip(".tiff").split('/')[-1]))
print(imagePath.strip(".tiff").split('/')[-1])
#row.append(len(objects))
for (objectID, centroid) in objects.items():
# row.append(objectID is "")
text = "{}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 5, centroid[1] - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1)
cv2.circle(frame, (centroid[0], centroid[1]), 2, (255, 0, 0), -1)
row.append(objectID)
sTello.takeoffSearching(False) multiTracker = CentroidTracker(args["maxframelost"]) infos["Class"] = [str(alvo.category),(255,119,0)] objectsSameClass = [] for (i,obj) in enumerate(objects_array): if obj.check_category(alvo.category): objectsSameClass.append(obj) if obj.check_centroid(mouse) is True: alvo.id = i cv2.destroyAllWindows() imt.createMovRulesTrackers(obj.area) multiTracker.update(objectsSameClass) particleFilter = pf.ParticleFilter(args["particles"],mouse, args["maxframelost"],args["deltat"],args["velmax"]) centroid_predicted = particleFilter.filter_steps(mouse) targetAcquired = True falhas +=1 else: #TargetAcquired objectsSameClass = [] for obj in objects_array: if (obj.check_category(alvo.category) is True): objectsSameClass.append(obj)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-m',
'--model',
type=str,
required=True,
help='File path of .tflite file.')
parser.add_argument('-l',
'--labelmap',
type=str,
required=True,
help='File path of labels file.')
parser.add_argument('-v',
'--video_path',
type=str,
default='',
help='Path to video. If None camera will be used')
parser.add_argument('-t',
'--threshold',
type=float,
default=0.5,
help='Detection threshold')
parser.add_argument('-roi',
'--roi_position',
type=float,
default=0.6,
help='ROI Position (0-1)')
parser.add_argument('-la',
'--labels',
nargs='+',
type=str,
help='Label names to detect (default="all-labels")')
parser.add_argument('-a',
'--axis',
default=True,
action="store_false",
help='Axis for cumulative counting (default=x axis)')
parser.add_argument('-e',
'--use_edgetpu',
action='store_true',
default=False,
help='Use EdgeTPU')
parser.add_argument(
'-s',
'--skip_frames',
type=int,
default=20,
help='Number of frames to skip between using object detection model')
parser.add_argument('-sh',
'--show',
default=True,
action="store_false",
help='Show output')
parser.add_argument(
'-sp',
'--save_path',
type=str,
default='',
help='Path to save the output. If None output won\'t be saved')
parser.add_argument(
'--type',
choices=['tensorflow', 'yolo', 'yolov3-tiny'],
default='tensorflow',
help='Whether the original model was a Tensorflow or YOLO model')
args = parser.parse_args()
labelmap = load_labels(args.labelmap)
interpreter = make_interpreter(args.model, args.use_edgetpu)
interpreter.allocate_tensors()
_, input_height, input_width, _ = interpreter.get_input_details(
)[0]['shape']
if args.video_path != '':
cap = cv2.VideoCapture(args.video_path)
else:
cap = cv2.VideoCapture(0)
if not cap.isOpened():
print("Error opening video stream or file")
if args.save_path:
width = int(cap.get(3))
height = int(cap.get(4))
fps = cap.get(cv2.CAP_PROP_FPS)
out = cv2.VideoWriter(args.save_path,
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), fps,
(width, height))
counter = [0, 0, 0, 0] # left, right, up, down
total_frames = 0
ct = CentroidTracker(maxDisappeared=40, maxDistance=50)
trackers = []
trackableObjects = {}
while cap.isOpened():
ret, image_np = cap.read()
if not ret:
break
height, width, _ = image_np.shape
rgb = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
status = "Waiting"
rects = []
if total_frames % args.skip_frames == 0:
status = "Detecting"
trackers = []
image_pred = cv2.resize(image_np, (input_width, input_height))
# Perform inference
results = detect_objects(interpreter, image_pred, args.threshold,
args.type)
for obj in results:
y_min, x_min, y_max, x_max = obj['bounding_box']
if obj['score'] > args.threshold and (
args.labels == None
or labelmap[obj['class_id']] in args.labels):
tracker = dlib.correlation_tracker()
rect = dlib.rectangle(int(x_min * width),
int(y_min * height),
int(x_max * width),
int(y_max * height))
tracker.start_track(rgb, rect)
trackers.append(tracker)
else:
status = "Tracking"
for tracker in trackers:
# update the tracker and grab the updated position
tracker.update(rgb)
pos = tracker.get_position()
# unpack the position object
x_min, y_min, x_max, y_max = int(pos.left()), int(
pos.top()), int(pos.right()), int(pos.bottom())
if x_min < width and x_max < width and y_min < height and y_max < height and x_min > 0 and x_max > 0 and y_min > 0 and y_max > 0:
# add the bounding box coordinates to the rectangles list
rects.append((x_min, y_min, x_max, y_max))
objects = ct.update(rects)
for (objectID, centroid) in objects.items():
to = trackableObjects.get(objectID, None)
if to is None:
to = TrackableObject(objectID, centroid)
else:
if args.axis and not to.counted:
x = [c[0] for c in to.centroids]
direction = centroid[0] - np.mean(x)
if centroid[
0] > args.roi_position * width and direction > 0:
counter[1] += 1
to.counted = True
elif centroid[
0] < args.roi_position * width and direction < 0:
counter[0] += 1
to.counted = True
elif not args.axis and not to.counted:
y = [c[1] for c in to.centroids]
direction = centroid[1] - np.mean(y)
if centroid[
1] > args.roi_position * height and direction > 0:
counter[3] += 1
to.counted = True
elif centroid[
1] < args.roi_position * height and direction < 0:
counter[2] += 1
to.counted = True
to.centroids.append(centroid)
trackableObjects[objectID] = to
text = "ID {}".format(objectID)
cv2.putText(image_np, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
cv2.circle(image_np, (centroid[0], centroid[1]), 4,
(255, 255, 255), -1)
# Draw ROI line
if args.axis:
cv2.line(image_np, (int(args.roi_position * width), 0),
(int(args.roi_position * width), height), (0xFF, 0, 0), 5)
else:
cv2.line(image_np, (0, int(args.roi_position * height)),
(width, int(args.roi_position * height)), (0xFF, 0, 0), 5)
# display count and status
font = cv2.FONT_HERSHEY_SIMPLEX
if args.axis:
cv2.putText(image_np, f'Left: {counter[0]}; Right: {counter[1]}',
(10, 35), font, 0.8, (0, 0xFF, 0xFF), 2,
cv2.FONT_HERSHEY_SIMPLEX)
else:
cv2.putText(image_np, f'Up: {counter[2]}; Down: {counter[3]}',
(10, 35), font, 0.8, (0, 0xFF, 0xFF), 2,
cv2.FONT_HERSHEY_SIMPLEX)
cv2.putText(image_np, 'Status: ' + status, (10, 70), font, 0.8,
(0, 0xFF, 0xFF), 2, cv2.FONT_HERSHEY_SIMPLEX)
if args.show:
cv2.imshow('cumulative_object_counting', image_np)
if cv2.waitKey(25) & 0xFF == ord('q'):
break
if args.save_path:
out.write(image_np)
total_frames += 1
cap.release()
if args.save_path:
out.release()
cv2.destroyAllWindows()
class computerVision:
def __init__(self, args):
if (args['method'] == 'HOG'):
self.method = Hog(args['confidence'])
elif (args['method'] == 'CAFFE'):
self.method = Caffe(args['confidence'])
elif (args['method'] == 'YOLO'):
self.method = Yolo(args['confidence'])
else:
print(
'Fail, unknown algorithm. Try with -m HOG, -m CAFFE, -m YOLO')
rospy.signal_shutdown('Quit')
# Initialize the bridge to transform the image detect by topic's ROS
self.bridge = CvBridge()
# initialize our centroid tracker and frame dimensions
self.ct = CentroidTracker()
# Initialize the subscriber and publisher
self.image_sub = rospy.Subscriber("/ardrone/image_raw", Image,
self.callback)
self.image_pub = rospy.Publisher("image_processed",
Image,
queue_size=10)
self.controller = DroneController(args)
self.droneStatus = DroneStatus()
self.previous_position = None
self.actual_position = None
self.detection_status = DETECTION_STATUS['SearchingPerson']
self.last_action = {
'roll': 0,
'pitch': 0,
'yaw_velocity': 0,
'z_velocity': 0
}
self.shutdown = 500
def callback(self, data):
try:
cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
if self:
if self.controller:
self.actual_position = self.controller.Odometry()
if self.actual_position:
explicit_quat = [
self.actual_position.pose.pose.orientation.x,
self.actual_position.pose.pose.orientation.y,
self.actual_position.pose.pose.orientation.z,
self.actual_position.pose.pose.orientation.w
]
roll, pitch, actual_yaw = euler_from_quaternion(
explicit_quat)
actual_yaw = (actual_yaw * 180) / math.pi
self.controller.TakeOff()
# If we are shutting down the drone, send land command
rospy.on_shutdown(self.controller.Land)
except CvBridgeError as e:
print(e)
try:
if (self.controller.Status() == self.droneStatus.Hovering
or self.controller.Status() == self.droneStatus.Flying):
# Obtain the detection and a few params
pick, img_width, img_height, cv_image = self.method.detection(
cv_image)
area_image = img_height * img_width
# update our centroid tracker using the computed set of bounding box rectangles
objects = self.ct.update(pick)
# Tracking
# loop over the tracked objects
minObjectID = None
if len(objects.items()) > 0:
minObjectID = objects.items()[0]
pickToFollow = []
minDistance = None
continue_move = True
# Calculate the pick to follow with the min distance for the centroid with min ID and draw the squares
for (x, y, w, h) in pick:
(objectID, centroid) = minObjectID
if not pickToFollow:
pickToFollow = [(x, y, w, h)]
minDistance = abs(centroid[0] -
((float(x) + float(w)) / 2) +
centroid[1] -
((float(y) + float(h)) / 2))
else:
if (abs(centroid[0] -
((float(x) + float(w)) / 2) + centroid[1] -
((float(y) + float(h)) / 2)) < minDistance):
pickToFollow = [(x, y, w, h)]
minDistance = abs(centroid[0] -
((float(x) + float(w)) / 2) +
centroid[1] -
((float(y) + float(h)) / 2))
cv2.rectangle(cv_image, (x, y), (w, h), (0, 255, 0), 2)
# Avoid people nearly of drone detected
area_rectangle_person = (w - x) * (h - y)
if ((area_rectangle_person >= area_image / 3)
and len(pick) > 1):
continue_move = False
print('There is a person too near')
for (index, (id, centroid)) in enumerate(objects.items()):
# draw both the ID of the object and the centroid of the
# object on the output frame
if (index == 0):
text = "Target"
else:
text = "Possible target"
cv2.putText(cv_image, text,
(centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(cv_image, (centroid[0], centroid[1]), 4,
(0, 255, 0), -1)
MOVEMENTS['pitch'] = 0 # (+)left and (-)right
MOVEMENTS['roll'] = 0 # (+)forward and (-)backward
MOVEMENTS['z_velocity'] = 0 # (+)up and (-)down
MOVEMENTS['yaw_velocity'] = 0 # The rotational velocity
# We left the drone stabilized, without moving
self.controller.SetCommand(MOVEMENTS['roll'],
MOVEMENTS['pitch'],
MOVEMENTS['yaw_velocity'],
MOVEMENTS['z_velocity'])
self.shutdown = self.shutdown - 1
if continue_move:
for (x, y, w, h) in pickToFollow:
area_rectangle = (w - x) * (h - y)
self.detection_status = DETECTION_STATUS[
'CenteringPerson']
print(
'----------------------------------------------------'
)
if ((float(x) / float(img_width)) >= 0.40):
print('Turn right')
MOVEMENTS['yaw_velocity'] = -0.20
elif ((float(x) / float(img_width)) <= 0.30):
print('Turn left')
MOVEMENTS['yaw_velocity'] = 0.05
else:
print("Don't turn")
MOVEMENTS['yaw_velocity'] = 0
if ((float(y) / float(img_height)) >= 0.15):
print('Down')
MOVEMENTS['z_velocity'] = -0.10
elif ((float(y) / float(img_height)) <= 0.08):
print('Up')
MOVEMENTS['z_velocity'] = 0.10
else:
print("Don't up don't down")
MOVEMENTS['z_velocity'] = 0
if (area_rectangle <= (float(area_image) / 5)):
print('Zoom in')
MOVEMENTS['roll'] = 0.10
elif (area_rectangle > (float(area_image) / 4)):
print('Zoom out')
MOVEMENTS['roll'] = -0.05
else:
MOVEMENTS['roll'] = 0
print("Don't zoom in don't zoom out")
self.last_action = {
'roll': MOVEMENTS['roll'],
'pitch': MOVEMENTS['pitch'],
'yaw_velocity': MOVEMENTS['yaw_velocity'],
'z_velocity': MOVEMENTS['z_velocity']
}
# Send the movement corresponding to the drone
self.controller.SetCommand(MOVEMENTS['roll'],
MOVEMENTS['pitch'],
MOVEMENTS['yaw_velocity'],
MOVEMENTS['z_velocity'])
self.previous_position = None
self.shutdown = 500
# If the drone lost the person detected, try moving in a manner contrary to the last movement made
if (len(pick) == 0
and (self.detection_status
== DETECTION_STATUS['CenteringPerson']
or self.detection_status
== DETECTION_STATUS['SearchingLastPerson'])
and self.shutdown < 450):
self.detection_status = DETECTION_STATUS[
'SearchingLastPerson']
print('Search the last person detected')
# Do the last movement but inverted
self.controller.SetCommand(
(self.last_action['roll'] * -1),
(self.last_action['pitch'] * -1),
(self.last_action['yaw_velocity'] * -1),
(self.last_action['z_velocity'] * -1))
# If dont detect people, the drone start to rotate on the left with the objective of finding a person
if self.shutdown <= 250:
if self.previous_position is None:
self.previous_position = self.controller.Odometry()
number = float(self.shutdown) / float(100)
int_part = int(number)
decimal_part = number - int_part
if (decimal_part == 0):
rospy.loginfo(
"Warning: The drone is searching people")
self.controller.SetCommand(0.0, 0.0, 0.15, 0.0)
self.detection_status = DETECTION_STATUS[
'SearchingPerson']
# Obtain the actual position of drone to calculate the rotation
if self.previous_position:
explicit_quat2 = [
self.previous_position.pose.pose.orientation.x,
self.previous_position.pose.pose.orientation.y,
self.previous_position.pose.pose.orientation.z,
self.previous_position.pose.pose.orientation.w
]
roll, pitch, previous_yaw = euler_from_quaternion(
explicit_quat2)
previous_yaw = (previous_yaw * 180) / math.pi
# If the actual yaw is very nearly to the beginning yaw when the drone started to rotate
# land the drone and finish the detection and the system
if abs(actual_yaw - previous_yaw
) <= THRESHOLD and self.shutdown < 200:
rospy.loginfo(
"Warning: The drone turn on and will be landed"
)
self.controller.SetCommand(0.0, 0.0, 0.0, 0.0)
rospy.sleep(2.)
self.controller.Land()
rospy.signal_shutdown('Quit')
else:
print(
'Warning: The drone stop because there is a person detected nearby'
)
else:
print(
'Warning: The drone is in process to flying, hovering or landing, and during this time not detect'
)
print('State: ' + str(self.controller.Status()))
# Convert the image with opencv format to format message ROS topic
try:
self.image_pub.publish(
self.bridge.cv2_to_imgmsg(cv_image, "bgr8"))
except CvBridgeError as e:
print(e)
except KeyboardInterrupt:
print("Shutting down node with KeyboardInterrupt")
def analyze_gcamp(movie,
model,
num_frames,
num_ch,
dim_y,
dim_x,
threshold=0.4):
"""Segments and collects fluorescence data for a single movie.
Parameters:
movie: the movie to be analyzed
model: the pretrained stardist model used for segementation
num_frames: dimension in time
num_ch: number of fluorescence channels to analyze
dim_y: number of rows in image
dim_x: number of columns in image
threshold: confidence threshold for including an object detected by stardist
Returns:
mask: an 8-bit tiff mask, with each cell labeled, for every frame
data_list: a Pandas DataFrame containing fluorescence values for each
cell in each frame
"""
# Assumes GCaMP is second channel if there are two channels
if num_ch == 1:
primary_ch = 0
else:
primary_ch = 1
secondary_ch = 0
# create an empty mask image with size equal to the movie
mask_img = np.zeros((num_frames, dim_y, dim_x))
# initiates tracker for object tracking between frames
tracker = CentroidTracker(maxDisappeared=num_frames)
data_list = []
for frame in range(num_frames):
# prepare GCaMP frame for stardist
if 'secondary_ch' in locals():
img = normalize(movie[frame, primary_ch, :, :],
1,
99.8,
axis=(0, 1))
else:
img = normalize(movie[frame, :, :], 1, 99.8, axis=(0, 1))
# predict labels using stardist
labels, details = model.predict_instances(img, prob_thresh=threshold)
# get connected components in each frame
label_props = measure.regionprops(labels)
# find center position for each region
centroids = [props.centroid for props in label_props]
# convert to centroid: property pair for each label
label_props = {
centroid: props
for centroid, props in zip(centroids, label_props)
}
# track the objects!
objects = tracker.update(centroids)
# collect data for each object
for obj, centroid in objects.items():
# this try-except statement handles objects that
# are missing in some frames
try:
cur_label = label_props[centroid].label
label_mask = labels == cur_label
mask_img[frame, label_mask] = obj + 1
area = np.sum(label_mask)
if 'secondary_ch' in locals():
primary_mean = np.mean(movie[frame, primary_ch,
label_mask])
primary_intden = np.sum(movie[frame, primary_ch,
label_mask])
secondary_mean = np.mean(movie[frame, secondary_ch,
label_mask])
secondary_intden = np.sum(movie[frame, secondary_ch,
label_mask])
data = pd.Series({
'frame': frame,
'cell': obj + 1,
'area': area,
'primary_mean': primary_mean,
'primary_intden': primary_intden,
'secondary_mean': secondary_mean,
'secondary_intden': secondary_intden
})
else:
primary_mean = np.mean(movie[frame, label_mask])
primary_intden = np.sum(movie[frame, label_mask])
data = pd.Series({
'frame': frame,
'cell': obj + 1,
'area': area,
'primary_mean': primary_mean,
'primary_intden': primary_intden
})
data_list.append(data)
except KeyError:
pass
return mask_img, pd.DataFrame(data_list)
(rects, weights) = hog.detectMultiScale(image,
winStride=(4, 4),
padding=(8, 8),
scale=1.05)
# this helps in the combination of multiple boxes detected on a single pedestrian into a single boundary box
rects = np.array([[x, y, x + w, y + h] for (x, y, w, h) in rects])
pick = non_max_suppression(rects, probs=None, overlapThresh=0.65)
# draw the final bounding boxes
for (xA, yA, xB, yB) in pick:
cv2.rectangle(image, (xA, yA), (xB, yB), (0, 255, 0), 2)
####################################################################
#########################--CENTROID TRACKER & ARCS OF CUSTOMERS--##############################
objects = ct.update(pick)
# centre of our subsequent arcs
x = image.shape[1] // 2
y = image.shape[0]
major_axis = 40
minor_axis = 20
# loop over the tracked objects
for (OID, centroid) in objects.items():
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(OID)
# for i in range(len(OID)):
# cv2.ellipse(image, (x, y), (major_axis + (10*i) , minor_axis + (10*i)), angle=0, startAngle=180, endAngle=360, color=(0, 0, 255), thickness=3)
cv2.putText(image, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
gd = gender[int(predict_gender + 0.5)]
rects.append((x, y, w, h, gd))
txt = "{}".format(gd)
if (i % 2 == 0):
cv2.putText(frame, txt, (int(x), int(y)),
cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2)
else:
cv2.putText(frame, txt, (int(x), int(y + h)),
cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2)
i += 1
objects, nMan, nWomen = ct.update(rects)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
# draw both the ID of the object and the centroid of the
# object on the output frame
#text = "ID {}".format(objectID)
if (objectID > person):
person = person + 1
#cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
#cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
frameID += 1
fps = frameID / (time.time() - start_time)
def evaluate(stream=None):
mixer.init()
mixer.music.load('support/alert.wav')
playing = False
args = dict(
prototxt='support/model.prototxt',
model='support/model.caffemodel',
input=stream,
output=None,
confidence=0.4,
skip_frames=30)
# initialize the list of class labels MobileNet SSD was trained to
# detect
CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
"bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
"dog", "horse", "motorbike", "person", "pottedplant", "sheep",
"sofa", "train", "tvmonitor"]
# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(args["prototxt"], args["model"])
# if a video path was not supplied, grab a reference to the webcam
if not args.get("input", False):
print("[INFO] starting video stream...")
vs = VideoStream(src=0).start()
time.sleep(2.0)
# otherwise, grab a reference to the video file
else:
print("[INFO] opening video file...")
vs = cv2.VideoCapture(args["input"])
# initialize the video writer (we'll instantiate later if need be)
writer = None
# initialize the frame dimensions (we'll set them as soon as we read
# the first frame from the video)
W = None
H = None
# instantiate our centroid tracker, then initialize a list to store
# each of our dlib correlation trackers, followed by a dictionary to
# map each unique object ID to a TrackableObject
ct = CentroidTracker(maxDisappeared=40, maxDistance=50)
trackers = []
trackableObjects = {}
# initialize the total number of frames processed thus far, along
# with the total number of objects that have moved either up or down
totalFrames = 0
total = 0
# start the frames per second throughput estimator
fps = FPS().start()
# loop over frames from the video stream
while True:
# grab the next frame and handle if we are reading from either
# VideoCapture or VideoStream
frame = vs.read()
frame = frame[1] if args.get("input", False) else frame
# if we are viewing a video and we did not grab a frame then we
# have reached the end of the video
if args["input"] is not None and frame is None:
break
# resize the frame to have a maximum width of 500 pixels (the
# less data we have, the faster we can process it), then convert
# the frame from BGR to RGB for dlib
frame = imutils.resize(frame, width=500)
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# if the frame dimensions are empty, set them
if W is None or H is None:
(H, W) = frame.shape[:2]
# if we are supposed to be writing a video to disk, initialize
# the writer
if args["output"] is not None and writer is None:
fourcc = cv2.VideoWriter_fourcc(*"MJPG")
writer = cv2.VideoWriter(args["output"], fourcc, 30,
(W, H), True)
# initialize the current status along with our list of bounding
# box rectangles returned by either (1) our object detector or
# (2) the correlation trackers
status = "Waiting"
rects = []
# check to see if we should run a more computationally expensive
# object detection method to aid our tracker
if totalFrames % args["skip_frames"] == 0:
# set the status and initialize our new set of object trackers
status = "Detecting"
trackers = []
# convert the frame to a blob and pass the blob through the
# network and obtain the detections
blob = cv2.dnn.blobFromImage(frame, 0.007843, (W, H), 127.5)
net.setInput(blob)
detections = net.forward()
# loop over the detections
for i in np.arange(0, detections.shape[2]):
# extract the confidence (i.e., probability) associated
# with the prediction
confidence = detections[0, 0, i, 2]
# filter out weak detections by requiring a minimum
# confidence
if confidence > args["confidence"]:
# extract the index of the class label from the
# detections list
idx = int(detections[0, 0, i, 1])
# if the class label is not a person, ignore it
if CLASSES[idx] != "person":
continue
# compute the (x, y)-coordinates of the bounding box
# for the object
box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
(startX, startY, endX, endY) = box.astype("int")
# construct a dlib rectangle object from the bounding
# box coordinates and then start the dlib correlation
# tracker
tracker = dlib.correlation_tracker()
rect = dlib.rectangle(startX, startY, endX, endY)
tracker.start_track(rgb, rect)
# add the tracker to our list of trackers so we can
# utilize it during skip frames
trackers.append(tracker)
# otherwise, we should utilize our object *trackers* rather than
# object *detectors* to obtain a higher frame processing throughput
else:
# loop over the trackers
for tracker in trackers:
# set the status of our system to be 'tracking' rather
# than 'waiting' or 'detecting'
status = "Tracking"
# update the tracker and grab the updated position
tracker.update(rgb)
pos = tracker.get_position()
# unpack the position object
startX = int(pos.left())
startY = int(pos.top())
endX = int(pos.right())
endY = int(pos.bottom())
# add the bounding box coordinates to the rectangles list
rects.append((startX, startY, endX, endY))
# draw a horizontal line in the center of the frame -- once an
# object crosses this line we will determine whether they were
# moving 'up' or 'down'
# cv2.line(frame, (0, H // 2), (W, H // 2), (0, 255, 255), 2)
# use the centroid tracker to associate the (1) old object
# centroids with (2) the newly computed object centroids
objects = ct.update(rects)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
# check to see if a trackable object exists for the current
# object ID
to = trackableObjects.get(objectID, None)
# if there is no existing trackable object, create one
if to is None:
to = TrackableObject(objectID, centroid)
# otherwise, there is a trackable object so we can utilize it
# to determine direction
else:
# the difference between the y-coordinate of the *current*
# centroid and the mean of *previous* centroids will tell
# us in which direction the object is moving (negative for
# 'up' and positive for 'down')
y = [c[1] for c in to.centroids]
direction = centroid[1] - np.mean(y)
to.centroids.append(centroid)
# check to see if the object has been counted or not
if not to.counted:
# if the direction is negative (indicating the object
# is moving up) AND the centroid is above the center
# line, count the object
if direction < 0 and centroid[1] < H // 2:
total += 1
to.counted = True
# if the direction is positive (indicating the object
# is moving down) AND the centroid is below the
# center line, count the object
elif direction > 0 and centroid[1] > H // 2:
total += 1
to.counted = True
# store the trackable object in our dictionary
trackableObjects[objectID] = to
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
corona_danger_count = 0
current_id_centroid = []
for (objectID, centroid) in objects.items():
current_id_centroid.append(centroid)
t = np.array(current_id_centroid)
already_played = []
if(t.ndim == 2):
D = dist.cdist(current_id_centroid, current_id_centroid,metric ="euclidean")
(row,col) = D.shape
for i in range(row):
for j in range(col):
if(i==j):
continue
if(D[i][j]< 120):
corona_danger_count+=1
cv2.circle(frame,(current_id_centroid[i][0],current_id_centroid[i][1]), 8, (0, 0 , 255), -1)
danger = "DANGER"
if(mixer.music.get_busy() or (i,j) in already_played):
pass
else:
mixer.music.play()
already_played.append((i,j))
already_played.append((j,i))
cv2.putText(frame, danger, (current_id_centroid[i][0] - 10, current_id_centroid[i][1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 3)
# construct a tuple of information we will be displaying on the
# frame
info = [
("Distinguishable Objects", total),
("Voilation Count", corona_danger_count),
("Status", status),
]
# loop over the info tuples and draw them on our frame
for (i, (k, v)) in enumerate(info):
text = "{}: {}".format(k, v)
cv2.putText(frame, text, (10, H - ((i * 20) + 20)),
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 255), 2)
# check to see if we should write the frame to disk
if writer is not None:
writer.write(frame)
# show the output frame
cv2.imshow("Social Distancing Enforcer", frame)
key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
if key == ord("q"):
break
# increment the total number of frames processed thus far and
# then update the FPS counter
totalFrames += 1
fps.update()
# stop the timer and display FPS information
fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
# check to see if we need to release the video writer pointer
if writer is not None:
writer.release()
# if we are not using a video file, stop the camera video stream
if not args.get("input", False):
vs.stop()
# otherwise, release the video file pointer
else:
vs.release()
# close any open windows
cv2.destroyAllWindows()
tracker.init(frame,tuple(box))
trackers.append(tracker)
spent = time() - lasttime
print("Spent {} second(s) on registering trackers".format(spent))
else:
status = 'TRACKING'
lasttime = time()
centroids = [[] for _ in range(len(trackers))]
for i,tracker in enumerate(trackers):
success, box = tracker.update(frame)
cent = [box[0]+box[2]/2,box[1]+box[3]/2]
centroids[i] = cent
spent = time() - lasttime
print("Spent {} second(s) on updating trackers".format(spent))
lasttime = time()
objects = ct.update(np.array(centroids,dtype=np.int32))
spent = time() - lasttime
print("Spent {} second(s) on updating cent trackers".format(spent))
cv2.polylines(frame,roicoords,isClosed=False,color=[255,0,0],thickness=2)
lasttime = time()
for objid,centroid in objects.items():
x,y = centroid
x,y = check_and_correct_boundaries(x,y,0,0,W,H)
locations[y,x] = 1
trackedobject = trackedobjects.get(objid,None)
if trackedobject is None:
trackedobject = TrackableObject(objid,centroid)
else:
trackedobject.centroids = centroid
trackedobjects[objid] = trackedobject
text = "ID {}".format(objid)
def inference():
# prob_model = probnet()
counter = 0
sess = tf.Session()
detection_graph = load_model(args['model'], sess).graph
labels = load_labels(args['labels'], int(args['num_classes']))
cap = cv2.VideoCapture(int(args['input']) + cv2.CAP_DSHOW)
cap.set(cv2.CAP_PROP_SETTINGS, 1)
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
h, w = cap.get(cv2.CAP_PROP_FRAME_HEIGHT), cap.get(
cv2.CAP_PROP_FRAME_WIDTH)
print(h, w)
if args['tracking']:
ct = CentroidTracker()
(H, W) = (None, None)
disapear_delay = 2 # seconds
maxDisappeared = None
wrong_direction = 0
while True:
start = time.time()
_, frame = cap.read()
# frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
if W is None or H is None:
(H, W) = frame.shape[:2]
start = time.time()
output_dict = run_inference_for_single_image(frame, detection_graph,
sess)
print('Inference time: %s' % (time.time() - start))
if args['direction']:
obj = [
output_dict['detection_boxes'][i]
for i in range(output_dict['num_detections'])
if output_dict['detection_scores'][i] > args['threshold']
]
# if obj:
# obj = obj[0]
# center = (int((obj[3] - (obj[3] - obj[1]) / 2) * w) ,int((obj[2] - (obj[2] - obj[0]) / 2) * h))
# # print(objects)
# pts.appendleft(center)
# counter = find_direction(frame, pts, counter, args['direction_buffer'])
vis_util.visualize_boxes_and_labels_on_image_array(
frame,
output_dict['detection_boxes'],
output_dict['detection_classes'],
output_dict['detection_scores'],
labels,
instance_masks=output_dict.get('detection_masks'),
use_normalized_coordinates=True,
line_thickness=8,
min_score_thresh=args['threshold'])
if args['tracking']:
rects = []
# print(output_dict["detection_scores"].shape[0])
for i in range(0, output_dict["detection_scores"].shape[0]):
# filter out weak out by ensuring the predicted
# probability is greater than a minimum threshold
if output_dict["detection_scores"][i] < args["threshold"]:
break
# compute the (x, y)-coordinates of the bounding box for
# the object, then update the bounding box rectangles list
box = (output_dict['detection_boxes'][i] *
np.array([H, W, H, W])).astype("int"), labels[
output_dict['detection_classes'][i]]['name']
rects.append(box)
objects = ct.update(rects)
# loop over the tracked objects
y_pad = 20
for (objectID, centroid) in objects.items():
# draw both the ID of the object and the centroid of the
# object on the output frame
direction = "ID {}: {}".format(objectID,
ct.objectDirection[objectID])
cv2.putText(frame, direction, (10, y_pad),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
y_pad += 15
if ct.preds[
objectID] == 'hand_with_somethinh' and ct.objectDirection[
objectID] == "to basket":
wrong_direction += 1
text = "ID {}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0),
-1)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
cv2.imshow('reco', frame)
end = time.time()
ct.maxDisappeared = int(disapear_delay / (end - start))
# print(ct.maxDisappeared)
key = cv2.waitKey(30) & 0xFF
if key == 27:
break
print("Hand disapeared %s times" % ct.deregistered)
print("Hand moves with product to the basket %s times" % wrong_direction)
prob_score = probnet_inference(prob_model, ct.deregistered,
wrong_direction, random.randint(1, 10))
print("Probability model result: %s" % prob_score)
class objectTracker():
def __init__(self, maxDisappeared=50, maxDistance=10, classify=False):
self.ct = CentroidTracker(maxDisappeared=maxDisappeared,
maxDistance=maxDistance)
self.trackers = []
self.trackableObjects = {}
self.frameID = 0
self.lastObjects = []
self.classify = classify
#ToDo Implement file writing??
def update(self, frameID, frameDetections, image):
self.frameID = frameID
rects = []
for detection in frameDetections:
# compute the (x, y)-coordinates of the bounding box
# for the object [detection[2], detection[3], detection[4], detection[5]]
startY = int(detection[3])
startX = int(detection[2])
endX = int(detection[4])
endY = int(detection[5])
# add the bounding box coordinates to the rectangles list
rects.append((startX, startY, endX, endY))
(objects, bboxes) = self.ct.update(rects)
det_impr = []
if len(frameDetections) > 0:
for (trackID, bbox) in zip(objects.keys(), bboxes.values()):
print('FrameDetections', frameDetections)
det_impr.append([
frameDetections[0][0], trackID, bbox[0], bbox[1], bbox[2],
bbox[3], 0
])
saveData(
PROJECT_ROOT + '/data/experiment/Frames/' +
str(frameDetections[0][0]) + '.json', det_impr)
# loop over the tracked objects
newObjects = []
for (objectID, centroid) in objects.items():
detection = bboxes[objectID]
# check to see if a trackable object exists for the current
# object ID
to = self.trackableObjects.get(objectID, None)
imgCrop = image[int(detection[1]):int(detection[3]),
int(detection[0]):int(detection[2])]
# if there is no existing trackable object, create one
result = [0]
characteristics = []
if self.classify:
frame_detection = [
detection[0], detection[1], detection[2], detection[3]
]
characteristics = extractCharacteristics(
image, frame_detection)
#print(self.clf)
result = self.clf.predict(characteristics)
if to is None:
frame_detection = [
detection[0], detection[1], detection[2], detection[3]
]
to = trackableobject.TrackableObject(objectID, centroid,
self.frameID, imgCrop,
result[0],
frame_detection,
characteristics, image)
# otherwise, there is a trackable object so we can utilize it
# to determine direction
else:
frame_detection = [
detection[0], detection[1], detection[2], detection[3]
]
to.update(frameID, centroid, imgCrop, result[0],
frame_detection, characteristics, image)
# store the trackable object in our dictionary
self.trackableObjects[objectID] = to
newObjects.append(
[objectID, centroid[0], centroid[1],
int(result[0])])
""""
# Write bbox to file
xmin = bboxes[objectID][0]
ymin = bboxes[objectID][1]
xmax = bboxes[objectID][2]
ymax = bboxes[objectID][3]
fout_pos.write('{} {} {} {} {} {}\n'.format(frameID, objectID, int(round(xmin, 0)),
int(round(ymin, 0)), int(round(xmax - xmin, 0)),
int(round(ymax - ymin, 0))))
"""
self.lastObjects = newObjects
def plotTracks(self, image):
if len(image.shape) == 2:
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
for element in self.lastObjects:
# draw both the ID of the object and the centroid of the
# object on the output frame
objectID = element[0]
centroid = element[1:3]
result = element[3]
text = "ID {}".format(objectID)
if result == 0: #0--> False Positive
color = (0, 255, 0)
else: # 1 --> HotSpot
color = (0, 0, 255)
cv2.putText(image, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
cv2.circle(image, (centroid[0], centroid[1]), 4, color, -1)
return image
def saveTracks(self, filename):
# write tracks
trackFile = open(filename, 'w')
trackID = 0
for objectID in sorted(self.trackableObjects.keys()):
startFrame = self.trackableObjects[objectID].startFrame
endFrame = self.trackableObjects[objectID].endFrame
conf = endFrame - startFrame + 1
out_str = '{} {:06d} {:06d} {:06d} {:06d} {}\n'.format(
1, startFrame, endFrame, trackID, objectID, conf)
trackFile.write(out_str)
trackID += 1
trackFile.close()
def saveData(self, filename="test.txt"):
trackFile = open(filename, 'w')
for objectID in sorted(self.trackableObjects.keys()):
trackData = self.trackableObjects[objectID].histDist
for frameData in trackData:
frameID = frameData[0]
histDist = frameData[1]
out_str = '{} {:06d} {:06.4f} {:06d} \n'.format(
1, frameID, histDist, objectID)
trackFile.write(out_str)
trackFile.close()
def saveResults(self, filename="test.txt"):
trackFile = open(filename, 'w')
for objectID in sorted(self.trackableObjects.keys()):
percentage = self.trackableObjects[objectID].countResults()
out_str = ' {:06d} {:03.4f} \n'.format(objectID, percentage)
trackFile.write(out_str)
trackFile.close()
def generateOutput(self, filename="output.txt"):
trackFile = open(filename, 'w')
for objectID in sorted(self.trackableObjects.keys()):
percentage = self.trackableObjects[objectID].countResults()
initFrame = self.trackableObjects[objectID].startFrame
finalFrame = self.trackableObjects[objectID].endFrame
deltaFrames = finalFrame - initFrame
centroids = np.asarray(self.trackableObjects[objectID].centroids)
xCentroid = np.mean(centroids[:, 0])
yCentroid = np.mean(centroids[:, 1])
trackData = self.trackableObjects[objectID].histDist
count = 0
threshold = 5
for frameData in trackData:
histDist = frameData[1]
if histDist > threshold:
count = count + 1
out_str = ' {:06d} {:03.4f} {:06d} {:06d} {:06d} {:06f} {:06f} {:06d} \n' \
.format(objectID,
percentage,
initFrame,
finalFrame,
deltaFrames,
xCentroid,
yCentroid,
count)
trackFile.write(out_str)
trackFile.close()
ymax = int(obj[6] * initial_h)
person_rect.append((xmin, ymin, xmax, ymax))
class_id = 21
# Draw box and label\class_id
color = (min(class_id * 12.5,
255), min(class_id * 7, 255), min(class_id * 5, 255))
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2)
det_label = 'person'
cv2.putText(frame,
det_label + ' ' + str(round(obj[2] * 100, 1)) + ' %',
(xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color,
1)
objects = ct.update(person_rect)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
# Face detection
res = exec_net_face.infer(inputs={input_blob_face: face_in_frame})
output_node_name = list(res.keys())[0]
res = res[output_node_name]
for obj in res[0][0]:
# Draw only objects when probability more than specified threshold
# boxes
idxs = cv2.dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold)
# ensure at least one detection exists
# ensure at least one detection exists
if len(idxs) > 0:
# loop over the indexes we are keeping
for i in idxs.flatten():
# extract the bounding box coordinates
(x, y) = (boxes[i][0], boxes[i][1])
(w, h) = (boxes[i][2], boxes[i][3])
rects.append((x, y, w, h))
#tracker
objects = tracker.update(rects)
print(objects)
objectId_ls = []
for (objectId, bbox) in objects.items():
x1, y1, x2, y2 = bbox
x1 = int(x1)
y1 = int(y1)
x2 = int(x2)
y2 = int(y2)
objectId_ls.append(objectId)
cv2.rectangle(image, (x1, y1), (x1 + x2, y1 + y2), (0, 0, 255), 2)
text = "ID:{}".format(objectId)
cv2.putText(image, text, (x1, y1 - 5), cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 0, 255), 1)
class MainApp:
####################################################################
def __init__(self, master, camera, uart):
"""Create a main application with the root thread, camera, and
UART instances."""
# Member Data
#######################################################
self.tracker = CentroidTracker()
self.controller = Controller()
self.test = "alnilam"
self.status = Status()
self.threshold = 5
self.camera = camera
self.UART = uart
# GUI Status Data
#######################################################
self.exposing = False
self.calibrating = False
self.calibrated = False
self.running = False
# Calibration Data
#######################################################
self.calibration = Calibration()
# Primary GUI Objects
#######################################################
# master root frame
self.master = master
# self.img - cv2 binary image without any markup
# self.gui_img - PIL colored image with markup
self.img, self.gui_img = initial_img_load(self.tracker)
# Holds image frame
self.panel = Label(master, image=self.gui_img)
self.panel.pack(side=LEFT)
# Contains all control and output widgets right of image panel
self.frame = Frame(master)
self.frame.pack(side=RIGHT, expand=TRUE, fill=BOTH)
# data that contains calibration and guiding status data
self.status_txt = StringVar()
self.status_txt.set(self.tracker.status)
# GUI object containing status data
self.text = Label(self.frame, textvariable=self.status_txt)
self.text.config(height=10,
width=32,
justify="left",
bg="grey25",
fg="white")
self.text.pack()
# Secondary GUI Objects (widgets)
#######################################################
# load Astrothoughts Logo
self.logo_img = load_logo()
self.logo = Label(self.frame, image=self.logo_img)
self.logo.pack(side=BOTTOM, anchor=S)
# Binary threshold slider
self.slider = Scale(self.frame,
from_=0,
to=200,
orient=HORIZONTAL,
length=225)
self.slider.config(sliderlength=15,
label="Binary Threshold",
bg="grey25",
fg="white")
self.slider.set(5)
self.slider.pack()
# Loop button
self.expose_img = PhotoImage(file="figures/expose.png")
self.expose_btn = Button(self.frame,
image=self.expose_img,
command=self.expose_button_cb)
self.expose_btn.config(height=51, width=51, bg="white")
self.expose_btn.pack(side="left", anchor=NW)
self.expose_ttp = CreateToolTip(
self.expose_btn, "Begin looping exposures from tracking camera")
# Run button
self.run_img = PhotoImage(file="figures/run_gs.png")
self.run_btn = Button(self.frame,
image=self.run_img,
command=self.run_button_cb)
self.run_btn.config(height=51, width=51, bg="white")
self.run_btn.pack(side="left", anchor=NW)
self.run_ttp = CreateToolTip(self.run_btn, "Start autoguiding program")
# Stop button
self.stop_img = PhotoImage(file="figures/stop_gs.png")
self.stop_btn = Button(self.frame,
image=self.stop_img,
command=self.stop_button_cb)
self.stop_btn.config(height=51, width=51, bg="white")
self.stop_btn.pack(side="left", anchor=NW)
self.stop_ttp = CreateToolTip(self.stop_btn,
"Stop looping and guiding")
# Calibration button
self.cal_img = PhotoImage(file="figures/cal_gs.png")
self.cal_btn = Button(self.frame,
image=self.cal_img,
command=self.cal_button_cb)
self.cal_btn.config(height=51, width=51, bg="white")
self.cal_btn.pack(side="left", anchor=NW)
self.cal_ttp = CreateToolTip(self.cal_btn,
"Begin calibration sequence")
####################################################################
def update(self):
"""Implement the desired appropriate function and update the GUI
objects depending on the desired outcome. Repeat every second
(1000 ms).
Possible configurations:
1) idle
2) exposing
3) exposing and calibrating
4) exposing and running
"""
# Take camera captures and find guide star if exposing
if self.exposing:
self.expose()
# Calibrate motors
if self.calibrating:
self.calibrate()
# Or Implement guiding algorithm with transmission of motor rates
elif self.running:
self.run()
# Reset img_num if it reaches 10 --> temporary
if self.tracker.status.img_num is 10:
self.tracker.status.clear()
# self.tracker.clear()
# Update panel image, threshold slider, and status text after exposure and run
self.panel.config(image=self.gui_img)
self.threshold = self.slider.get()
self.status_txt.set(self.tracker.status)
# Don't do anything if not exposing
elif not self.exposing:
pass
# Update color of calibration button given tracking status
# can't calibrate if currently calibrating or already calibrated
if self.calibrated or self.calibrating:
self.cal_img = PhotoImage(file="figures/cal_gs.png")
else:
# can calibrate if our tracker is LOCKED
if self.tracker.status.mode is self.tracker.LOCKED:
self.cal_img = PhotoImage(file="figures/cal.png")
# don't calibrate if looking for a guide star
else:
self.cal_img = PhotoImage(file="figures/cal_gs.png")
self.cal_btn.config(image=self.cal_img)
# Rerun update every second
self.master.after(1000, self.update)
####################################################################
def expose(self):
"""Load an image either from a test directory or the actual USB
Camera, and autoselect the guide star closest to the center."""
# Either A) load image from test directory
self.img = load_image(self.test, self.tracker.status.img_num)
# or B) capture frame from USB Camera
# initial_img = self.camera.capture()
# locate the centroids as a list of (x, y) tuples and get binary thresholded image
centroids, colored_img = find_centroids(self.img,
lower_thresh=self.threshold)
# update the Tracker object for the next list of input centroids
dX, dY = self.tracker.update(centroids)
# if the mode is SEARCHING, autoselect a guide star
if self.tracker.status.mode is self.tracker.SEARCHING:
# get a guide star and return image with smallest bounding rectangle
autosel_img = self.tracker.autoselect(colored_img)
else:
# autoselection not activated, autoselected image is the filtered image
autosel_img = colored_img
# show camera circle, orthogonal axes, and tracking box
marked_img = markup_img(autosel_img, self.tracker)
pil_img = Image.fromarray(marked_img)
self.gui_img = ImageTk.PhotoImage(pil_img)
# Update status object incremented image number, mode, and displacement
self.tracker.status.set(self.tracker.status.img_num + 1,
self.tracker.status.mode, (dX, dY))
####################################################################
def run(self):
"""Run the autoguiding program.
More specifically, plug in pixel error into the conversion matrix
to get motor error, plug into the controller, and transmit the final
motor rates to the MCU. Note: this is only accessible after a
successful calibration."""
# Fetch distance from origin
dX, dY = self.tracker.status.COM
# Plug into conversion matrix
calRARate, calDECRate = self.calibration.calculate_rates((dX, dY))
# Get rates from PI Controller
raRate, decRate = self.controller.calculate(calRARate, calDECRate)
# Transmit calculated motor rates over UART
self.UART.transmit(raRate, decRate)
# Update status object motor rates
self.tracker.status.set_rates(raRate, decRate)
####################################################################
def calibrate(self):
"""Calibrate the program. More specifically, individually move each
motor to get sample points along each axis, and calculate a conversion
matrix that will map (x, y) pixel error into error along each motor axis.
This is necessary because at different declinations, orthogonal pixel
error doesn't directly correspond to motor error. Adjustments to the
rotator angle will correspond to different angles relative to the
camera frame."""
# tell motors what to do and record data samples if necessary
self.calibration.execute(self.UART, self.tracker.status)
# next state logic based on calibration state
self.calibration.next_state()
# update status object for current calibration rates
self.tracker.status.set_rates(self.calibration.RARate,
self.calibration.DECRate)
# After calibration finishes...
if self.calibration.state is self.calibration.DONE:
self.calibration.least_squares(
) # run least squares to get conversion matrix
self.calibrating = False # stop calibrating
self.calibrated = True # calibration has finished
self.stop_button_cb() # stop all processes
print(self.calibration) # print result
# GUI Operational state codes
####################################################################
# CI - calibrating
# CD - calibrated
# E - exposing
# R - running
# S - stop
####################################################################
def expose_button_cb(self):
if self.exposing:
print("<E: no action>")
elif not self.exposing:
print("<start exposing>")
self.exposing = True
# if calibrated, user can now run
if self.calibrated:
self.run_img = PhotoImage(file="figures/run.png")
self.cal_img = PhotoImage(file="figures/cal_gs.png")
# user can't run if not calibrated
elif not self.calibrated:
self.run_img = PhotoImage(file="figures/run_gs.png")
# can't expose if we're already exposing
self.expose_img = PhotoImage(file="figures/expose_gs.png")
self.stop_img = PhotoImage(file="figures/stop.png")
# update GUI object colors
self.run_btn.config(image=self.run_img)
self.expose_btn.config(image=self.expose_img)
self.stop_btn.config(image=self.stop_img)
self.cal_btn.config(image=self.cal_img)
####################################################################
def stop_button_cb(self):
self.exposing = False
self.running = False
# Warn user about incomplete calibration
if self.calibrating:
print("<WARNING: !CD>")
self.calibrating = False
self.calibrated = False
# can't do anything until we expose again
self.expose_img = PhotoImage(file="figures/expose.png")
self.run_img = PhotoImage(file="figures/run_gs.png")
self.stop_img = PhotoImage(file="figures/stop_gs.png")
self.cal_img = PhotoImage(file="figures/cal_gs.png")
# update GUI object colors
self.expose_btn.config(image=self.expose_img)
self.run_btn.config(image=self.run_img)
self.stop_btn.config(image=self.stop_img)
self.cal_btn.config(image=self.cal_img)
print("<stop all>")
####################################################################
def run_button_cb(self):
# Only run if currently exposing and calibration done
if self.exposing and self.calibrated:
if self.running:
print("<R; no action>")
elif not self.running:
self.run_img = PhotoImage(file="figures/run_gs.png")
self.running = True
print("<CD, E, and !R; set R>")
self.run_btn.config(image=self.run_img)
# Do nothing if not currently running
elif not self.exposing:
print("<!E; no action>")
elif not self.calibrated:
print("<!CD; no action>")
####################################################################
def cal_button_cb(self):
# only calibrate if we haven't already done so
if not self.calibrated:
if not self.calibrating:
if self.exposing and self.tracker.status.mode is self.tracker.LOCKED:
print("<start calibrating>")
self.cal_img = PhotoImage(file="figures/cal_gs.png")
self.calibrating = True
self.cal_btn.config(image=self.cal_img)
elif not self.exposing:
print("<!CD but !E; no action>")
elif self.tracker.status.mode is not self.tracker.LOCKED:
print("<!LOCKED; can't calibrate>")
# accept calibration while exposing
elif self.calibrating:
print("<CI; no action>")
# don't do anything if already calibrated
else:
print("<CD: do nothing>")
def run_inference(model, category_index, cap, labels, roi_position=0.6, threshold=0.5, x_axis=True, skip_frames=20, save_path='', show=True):
counter = [0, 0, 0, 0] # left, right, up, down
total_frames = 0
ct = CentroidTracker(maxDisappeared=40, maxDistance=50)
trackers = []
trackableObjects = {}
# Check if results should be saved
if save_path:
width = int(cap.get(3))
height = int(cap.get(4))
fps = cap.get(cv2.CAP_PROP_FPS)
out = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc('M','J','P','G'), fps, (width, height))
while cap.isOpened():
ret, image_np = cap.read()
if not ret:
break
height, width, _ = image_np.shape
rgb = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
status = "Waiting"
rects = []
if total_frames % skip_frames == 0:
status = "Detecting"
trackers = []
# Actual detection.
output_dict = run_inference_for_single_image(model, image_np)
for i, (y_min, x_min, y_max, x_max) in enumerate(output_dict['detection_boxes']):
if output_dict['detection_scores'][i] > threshold and (labels == None or category_index[output_dict['detection_classes'][i]]['name'] in labels):
tracker = dlib.correlation_tracker()
rect = dlib.rectangle(int(x_min * width), int(y_min * height), int(x_max * width), int(y_max * height))
tracker.start_track(rgb, rect)
trackers.append(tracker)
else:
status = "Tracking"
for tracker in trackers:
# update the tracker and grab the updated position
tracker.update(rgb)
pos = tracker.get_position()
# unpack the position object
x_min, y_min, x_max, y_max = int(pos.left()), int(pos.top()), int(pos.right()), int(pos.bottom())
# add the bounding box coordinates to the rectangles list
rects.append((x_min, y_min, x_max, y_max))
objects = ct.update(rects)
for (objectID, centroid) in objects.items():
to = trackableObjects.get(objectID, None)
if to is None:
to = TrackableObject(objectID, centroid)
else:
if x_axis and not to.counted:
x = [c[0] for c in to.centroids]
direction = centroid[0] - np.mean(x)
if centroid[0] > roi_position*width and direction > 0:
counter[1] += 1
to.counted = True
elif centroid[0] < roi_position*width and direction < 0:
counter[0] += 1
to.counted = True
elif not x_axis and not to.counted:
y = [c[1] for c in to.centroids]
direction = centroid[1] - np.mean(y)
if centroid[1] > roi_position*height and direction > 0:
counter[3] += 1
to.counted = True
elif centroid[1] < roi_position*height and direction < 0:
counter[2] += 1
to.counted = True
to.centroids.append(centroid)
trackableObjects[objectID] = to
text = "ID {}".format(objectID)
cv2.putText(image_np, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
cv2.circle(image_np, (centroid[0], centroid[1]), 4, (255, 255, 255), -1)
# Draw ROI line
if x_axis:
cv2.line(image_np, (int(roi_position*width), 0), (int(roi_position*width), height), (0xFF, 0, 0), 5)
else:
cv2.line(image_np, (0, int(roi_position*height)), (width, int(roi_position*height)), (0xFF, 0, 0), 5)
# display count and status
font = cv2.FONT_HERSHEY_SIMPLEX
if x_axis:
cv2.putText(image_np, f'Left: {counter[0]}; Right: {counter[1]}', (10, 35), font, 0.8, (0, 0xFF, 0xFF), 2, cv2.FONT_HERSHEY_SIMPLEX)
else:
cv2.putText(image_np, f'Up: {counter[2]}; Down: {counter[3]}', (10, 35), font, 0.8, (0, 0xFF, 0xFF), 2, cv2.FONT_HERSHEY_SIMPLEX)
cv2.putText(image_np, 'Status: ' + status, (10, 70), font, 0.8, (0, 0xFF, 0xFF), 2, cv2.FONT_HERSHEY_SIMPLEX)
if show:
cv2.imshow('cumulative_object_counting', image_np)
if cv2.waitKey(25) & 0xFF == ord('q'):
break
if save_path:
out.write(image_np)
total_frames += 1
cap.release()
if save_path:
out.release()
cv2.destroyAllWindows()
def run_face_tracker(args):
# initialize our centroid tracker and frame dimensions
ct = CentroidTracker()
(H, W) = (None, None)
# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(args["prototxt"], args["model"])
# initialize the video stream and allow the camera sensor to warmup
print("[INFO] starting video stream...")
vs = VideoStream(src=0).start()
time.sleep(2.0)
# loop over the frames from the video stream
while True:
# read the next frame from the video stream and resize it
frame = vs.read()
frame = imutils.resize(frame, width=600)
# if the frame dimensions are None, grab them
if W is None or H is None:
(H, W) = frame.shape[:2]
# construct a blob from the frame, pass it through the network,
# obtain our output predictions, and initialize the list of
# bounding box rectangles
blob = cv2.dnn.blobFromImage(frame, 1.0, (W, H), (104.0, 177.0, 123.0))
net.setInput(blob)
detections = net.forward()
rects = []
# loop over the detections
for i in range(0, detections.shape[2]):
# filter out weak detections by ensuring the predicted
# probability is greater than a minimum threshold
if detections[0, 0, i, 2] > args["confidence"]:
# compute the (x, y)-coordinates of the bounding box for
# the object, then update the bounding box rectangles list
box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
rects.append(box.astype("int"))
# draw a bounding box surrounding the object so we can
# visualize it
(startX, startY, endX, endY) = box.astype("int")
cv2.rectangle(frame, (startX, startY), (endX, endY),
(0, 255, 0), 2)
# update our centroid tracker using the computed set of bounding
# box rectangles
objects = ct.update(rects)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
# show the output frame
cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
if key == ord("q"):
break
# do a bit of cleanup
cv2.destroyAllWindows()
vs.stop()
class FetchData:
def __init__(self, fullPath, enableImshow=0, enableGPU=0, printData=0, printProgress=1, exportJSON=0, exportCSV=1):
self.printData = printData
self.fullPath = fullPath
self.enableImshow = enableImshow
self.exportJSON = exportJSON
self.exportCSV = exportCSV
self.printProgress = printProgress
self.basepath, self.filename, self.videoStartDatetime = self.getStartDatetime(self.fullPath)
self.tracker = CentroidTracker(maxDisappeared=20, maxDistance=90)
self.net, self.faceNet, self.ageNet, self.genderNet = self.modelsInit(enableGPU)
self.timestamp = self.videoStartDatetime
self.DATADICT = {
"timestamp": "",
"screenTime": 0,
"dwellTime": 0,
"totalCount": 0,
"impressions": 0,
"males": 0,
"females": 0,
"young": 0,
"middleAged": 0,
"elderly": 0
}
if (exportJSON):
data = {self.filename:[]}
self.jsonPathName = os.path.join(self.basepath, self.filename+".json")
self.write_json(data)
if (exportCSV):
data = ['timestamp','screenTime','dwellTime','totalCount','impressions','males','females','young','middleAged','elderly']
self.csvPathName = os.path.join(self.basepath,self.filename+".csv")
self.write_csv(data, 'w')
self.run()
if (exportJSON):
print("JSON- ", self.jsonPathName)
if (exportCSV):
print("CSV- ", self.csvPathName)
def write_csv(self, data, m):
with open(self.csvPathName, mode=m) as csv_file:
csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(data)
def write_json(self, data):
with open(self.jsonPathName,'w') as f:
json.dump(data, f, indent=2)
def read_json(self):
with open(self.jsonPathName) as f:
return json.load(f)
def run(self):
cap = cv2.VideoCapture(self.fullPath)
video_framerate = cap.get(cv2.CAP_PROP_FPS)
video_frametime = timedelta(seconds=(1/video_framerate))
totalFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
ln = self.net.getLayerNames()
ln = [ln[i[0] - 1] for i in self.net.getUnconnectedOutLayers()]
object_id_list = []
impressionsObjectId = []
genderDecidedConfidence = {}
ageDecidedConfidence = {}
gender = {}
age = {}
total_count = 0
screenTime_Time = dict()
dwellTime_Time = dict()
AccumulatedScreenTime = 0
AccumulatedDwellTime = 0
fps = 0
frameNo = 0
while True:
start_time = time.time()
ret, image = cap.read()
if not ret:
break
image_cpy = copy.deepcopy(image)
unique_count = 0
current_count = 0
h, w = image.shape[:2]
DwellYLevel = h//2
cv2.line(image, (0, DwellYLevel), (w, DwellYLevel), (255, 128, 0), 2)
cv2.putText(image, "Dwell Area", (w - 250, DwellYLevel + 50), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 128, 0), 1)
blob = cv2.dnn.blobFromImage(image_cpy, 1 / 255.0, (416, 416), swapRB=True, crop=False)
self.net.setInput(blob)
layer_outputs = self.net.forward(ln)
outputs = np.vstack(layer_outputs)
boxes, confidences, class_ids, rects = [], [], [], []
for output in outputs:
scores = output[5:]
class_id = np.argmax(scores)
confidence = scores[class_id]
if class_id == 0 and confidence > 0.5:
box = output[:4] * np.array([w, h, w, h])
(centerX, centerY, width, height) = box.astype("int")
x = int(centerX - (width / 2))
y = int(centerY - (height / 2))
boxes.append([x, y, int(width), int(height)])
confidences.append(float(confidence))
class_ids.append(class_id)
indices = cv2.dnn.NMSBoxes(boxes, confidences, 0.6, 0.3)
if len(indices) > 0:
for i in indices.flatten():
(x, y) = (boxes[i][0], boxes[i][1])
(w, h) = (boxes[i][2], boxes[i][3])
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 1)
person_box = [x, y, x + w, y + h]
rects.append(person_box)
boundingboxes = np.array(rects)
boundingboxes = boundingboxes.astype(int)
objects, dereg = self.tracker.update(rects)
for (objectId, bbox) in objects.items():
x1, y1, x2, y2 = bbox
x1 = abs(int(x1))
y1 = abs(int(y1))
x2 = abs(int(x2))
y2 = abs(int(y2))
centroidX = int((x1 + x2) / 2)
centroidY = int((y1 + y2) / 2)
cv2.circle(image, (centroidX, centroidY), 5, (255, 255, 255), thickness=-1)
cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2)
if objectId not in object_id_list: # NEW PERSON
object_id_list.append(objectId) # Append in ObjectID list - list of ID of people who have come before
unique_count += 1
screenTime_Time[objectId] = 0 # Initialize
dwellTime_Time[objectId] = 0
genderDecidedConfidence[objectId] = 0
gender[objectId] = ""
ageDecidedConfidence[objectId] = 0
age[objectId] = ""
else: # EXISTING PERSON
screenTime_Time[objectId] += 1 / video_framerate
AccumulatedScreenTime += 1 / video_framerate
if centroidY > DwellYLevel:
dwellTime_Time[objectId] += 1 / video_framerate
AccumulatedDwellTime += 1 / video_framerate
if (objectId not in impressionsObjectId):
impressionsObjectId.append(objectId)
self.DATADICT["impressions"]+=1
personCrop = image_cpy[y1:y2,x1:x2]
frapersonCropEnlarge = cv2.resize(personCrop, (0, 0), fx=2, fy=2)
# cv2.imshow("person",frapersonCropEnlarge)
ageGenderResult = self.detect_and_predict_age(frapersonCropEnlarge)
if (ageGenderResult):
if(ageGenderResult[0]["gender"][1]>genderDecidedConfidence[objectId]):
gender[objectId] = ageGenderResult[0]["gender"][0]
genderDecidedConfidence[objectId] = ageGenderResult[0]["gender"][1]
if(ageGenderResult[0]["age"][1]>ageDecidedConfidence[objectId]):
age[objectId] = ageGenderResult[0]["age"][0]
ageDecidedConfidence[objectId] = ageGenderResult[0]["age"][1]
##################################################
# cv2.rectangle(image, (x1, y1), (x2, y2), (0, 0, 255), 2)
ID_text = "Person:" + str(objectId)
ScreenTimeText = "DwellTime: {:.2f}".format(screenTime_Time[objectId]) + str("s") # INTERCHANGED
DwellTimeText = "ScreenTime: {:.2f}".format(dwellTime_Time[objectId]) + str("s") # INTERCHANGED
cv2.putText(
image,
ID_text,
(x1, y1 - 7),
cv2.FONT_HERSHEY_COMPLEX_SMALL,
1.5,
(0, 0, 255),
2,
)
cv2.putText(
image,
ScreenTimeText,
(x1, y1 - 30),
cv2.FONT_HERSHEY_COMPLEX_SMALL,
1.5,
(0, 0, 255),
2,
)
cv2.putText(
image,
DwellTimeText,
(x1, y1 - 60),
cv2.FONT_HERSHEY_COMPLEX_SMALL,
1.5,
(0, 0, 255),
2,
)
current_count += 1
for l in list(dereg.items()):
# print(l)
objectID = l[0]
if (age[objectID] and gender[objectID]):
if (age[objectID] == "Young"):
self.DATADICT["young"]+=1
elif (age[objectID] == "Middle"):
self.DATADICT["middleAged"]+=1
elif (age[objectID] == "Elderly"):
self.DATADICT["elderly"]+=1
if (gender[objectId] == "MALE"):
self.DATADICT["males"]+=1
elif (gender[objectId] == "FEMALE"):
self.DATADICT["females"]+=1
self.tracker.deleteDereg(l[0])
total_count += unique_count
self.DATADICT["totalCount"]=total_count
total_count_text = "Total Count:" + str(total_count)
cv2.putText(
image,
total_count_text,
(5, 400),
cv2.FONT_HERSHEY_SIMPLEX,
1.5,
(0, 0, 255),
2,
)
current_count_text = "Current Count:" + str(current_count)
cv2.putText(
image,
current_count_text,
(5, 450),
cv2.FONT_HERSHEY_SIMPLEX,
1.5,
(0, 0, 255),
2,
)
self.DATADICT["screenTime"] = round(AccumulatedScreenTime / 60, 2)
AccumulatedScreenTime_Text = "Total DwellTime: {:.2f}".format(AccumulatedScreenTime / 60) + str("m")
cv2.putText(
image,
AccumulatedScreenTime_Text,
(5, 550),
cv2.FONT_HERSHEY_SIMPLEX,
1.5,
(0, 0, 255),
2,
) # INTERCHANGED
self.DATADICT["dwellTime"] = round(AccumulatedDwellTime / 60,2)
AccumulatedDwellTime_Text = "Total ScreenTime: {:.2f}".format(AccumulatedDwellTime / 60) + str("m")
cv2.putText(
image,
AccumulatedDwellTime_Text,
(5, 600),
cv2.FONT_HERSHEY_SIMPLEX,
1.5,
(0, 0, 255),
2,
) # INTERCHANGED
end_time = time.time()
frameTime = (end_time - start_time)
frameTimeDatetime = timedelta(seconds=frameTime)
self.timestamp+=video_frametime
self.DATADICT["timestamp"]=self.timestamp.strftime("%m/%d/%Y, %H:%M:%S")
fps = 1 / frameTime
fps_text = "FPS: {:.2f}".format(fps)
cv2.putText(image, fps_text, (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
frameNo += 1
if (self.enableImshow):
cv2.imshow("Application", image)
key = cv2.waitKey(1)
if key == ord("q"):
break
###################### GLOBAL VARIABLES ######################
global DATADICT, FRAME
DATADICT = self.DATADICT.copy()
FRAME = copy.deepcopy(image)
if(self.printData):
print(DATADICT)
if(self.printProgress):
print("Processed", frameNo, "of", totalFrames, "Frames |", round((frameNo/totalFrames)*100,2), "% Completion |", fps_text)
if(self.exportJSON):
data = self.read_json()
data[self.filename].append(DATADICT)
self.write_json(data)
if(self.exportCSV):
data = []
for key in DATADICT:
data.append(DATADICT[key])
self.write_csv(data, 'a')
cap.release()
cv2.destroyAllWindows()
return
def detect_and_predict_age(self, frame):
faceNet = self.faceNet
ageNet = self.ageNet
genderNet = self.genderNet
minConf=0.5
AGE_BUCKETS = [
"(0-2)",
"(4-6)",
"(8-12)",
"(15-20)",
"(25-32)",
"(38-43)",
"(48-53)",
"(60-100)",
]
AGE_CATEGORY = {
"Young":["(0-2)","(4-6)","(8-12)","(15-20)"],
"Middle":["(25-32)","(38-43)"],
"Elderly":["(48-53)","(60-100)"]
}
GENDER_BUCKETS = ["MALE", "FEMALE"]
results = []
(h, w) = frame.shape[:2]
blob = cv2.dnn.blobFromImage(frame, 1.0, (300, 300), (104.0, 177.0, 123.0))
faceNet.setInput(blob)
detections = faceNet.forward()
for i in range(0, detections.shape[2]):
confidence = detections[0, 0, i, 2]
if confidence > minConf:
box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
(startX, startY, endX, endY) = box.astype("int")
face = frame[startY:endY, startX:endX]
# ensure the face ROI is sufficiently large
if face.shape[0] < 20 or face.shape[1] < 20:
continue
faceBlob = cv2.dnn.blobFromImage(
face,
1.0,
(227, 227),
(78.4263377603, 87.7689143744, 114.895847746),
swapRB=False,
)
ageNet.setInput(faceBlob)
preds = ageNet.forward()
i = preds[0].argmax()
age = AGE_BUCKETS[i]
for key in AGE_CATEGORY:
if (age in AGE_CATEGORY[key]):
ageCategory = key
ageConfidence = preds[0][i]
genderNet.setInput(faceBlob)
genderPred = genderNet.forward()
j = genderPred[0].argmax()
gender = GENDER_BUCKETS[j]
genderConfidence = genderPred[0][j]
d = {
"loc": (startX, startY, endX, endY),
"age": (ageCategory, ageConfidence),
"gender": (gender, genderConfidence),
}
results.append(d)
return results
def getStartDatetime(self, path):
directory = os.path.split(path)[0]
basename = os.path.basename(path)
filename = os.path.splitext(basename)[0]
try:
year = int(filename[1:5])
month = int(filename[5:7])
day = int(filename[7:9])
hours = int(filename[9:11])
minutes = int(filename[11:13])
seconds = int(filename[13:15])
videoStartDatetime = datetime(year, month, day, hours, minutes, seconds, 0)
except:
videoStartDatetime = datetime.now()
print("Invalid Video Name for DateTime extraction. Setting start time to ", videoStartDatetime)
return directory, filename, videoStartDatetime
def modelsInit(self, enableGPU):
labels = open("ModelsAndWeights/data/coco.names").read().strip().split("\n")
net = cv2.dnn.readNetFromDarknet("ModelsAndWeights/cfg/yolov4.cfg", "ModelsAndWeights/yolov4.weights")
faceNet = cv2.dnn.readNet(
"ModelsAndWeights/face_detector/deploy.prototxt",
"ModelsAndWeights/face_detector/res10_300x300_ssd_iter_140000.caffemodel",
)
ageNet = cv2.dnn.readNet(
"./ModelsAndWeights/age_detector/age_deploy.prototxt",
"./ModelsAndWeights/age_detector/age_net.caffemodel",
)
genderNet = cv2.dnn.readNet(
"./ModelsAndWeights/gender_detector/gender_deploy.prototxt",
"./ModelsAndWeights/gender_detector/gender_net.caffemodel",
)
if (enableGPU):
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
faceNet.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
faceNet.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
ageNet.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
ageNet.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
genderNet.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
genderNet.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
return net, faceNet, ageNet, genderNet
def main():
model = load_model()
# Dibujo
img_dibujo = cv2.imread("imagenes_videos/dibujo_cancha.png",1)
img_dibujo_copy = img_dibujo.copy()
shape_dibujo = img_dibujo.shape
save_video = False
if save_video:
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
out = cv2.VideoWriter('mapeo.mp4', fourcc, 20.0, (2*shape_dibujo[1],shape_dibujo[0]))
ct = CentroidTracker()
#Video volley
cap = cv2.VideoCapture('imagenes_videos/video_volley2.mp4')
while(True):
time1= time.time()
ret, image = cap.read()
#Resize por hardcodeo de puntos
#image = cv2.resize(image,(1920,1080))
#detecciones
detections = model(image)
#a veces falla por el video mal grabado uwu
#try:
# transformation_matrix = getCourtTransformMatrix(image, img_dibujo)
#except:
# print('error (?)')
# continue
#same
#if(transformation_matrix is None):
# continue
rects = []
#si hay detecciones
if len(detections) > 0:
for i, (x1, y1, x2, y2, obj_conf, cls_conf, cls_pred) in enumerate(detections[0]):
x = round(x1.item())
y = round(y1.item())
w = round(x2.item() - x1.item())
h = round(y2.item() - y1.item())
rects.append((x1, y1, x2, y2))
#Dibujarlas
cv2.rectangle(image,(x,y),(x+w,y+h),(0,255,0),3)
#Dibujar puntos a dibujo
punto_cancha = np.array([x, y+h])
#punto_proyectado = transform_point(punto_cancha, transformation_matrix)
#cv2.circle(img_dibujo,(int(punto_proyectado[0]),int(punto_proyectado[1])), 10, (255,0,0), -1)
#Realizar transformacion
#img_transformed = cv2.warpPerspective(image, transformation_matrix, (shape_dibujo[1],shape_dibujo[0]))
objects = ct.update(rects)
for (objectID, centroid) in objects.items():
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(objectID)
cv2.putText(image, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(image, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
#Mostrar
cv2.namedWindow( "Personas", cv2.WINDOW_NORMAL)
#out_frame = np.hstack((img_transformed, img_dibujo))
cv2.imshow('Personas', image)
if save_video:
out.write(out_frame)
# cv2.namedWindow( "court", cv2.WINDOW_NORMAL )
# cv2.imshow('court', img_dibujo)
#Para no sobreescribir dibujo
img_dibujo = img_dibujo_copy.copy()
if cv2.waitKey(1) & 0xFF == ord('q'):
break
print(time.time()-time1)
cap.release()
if save_video:
out.release()
def detect(opt, save_img=False):
ct = CentroidTracker()
out, source, weights, view_img, save_txt, imgsz = \
opt.output, opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size
webcam = source == '0' or source.startswith(
'rtsp') or source.startswith('http') or source.endswith('.txt')
# initialize deepsort
cfg = get_config()
cfg.merge_from_file(opt.config_deepsort)
deepsort = DeepSort(cfg.DEEPSORT.REID_CKPT,
max_dist=cfg.DEEPSORT.MAX_DIST, min_confidence=cfg.DEEPSORT.MIN_CONFIDENCE,
nms_max_overlap=cfg.DEEPSORT.NMS_MAX_OVERLAP, max_iou_distance=cfg.DEEPSORT.MAX_IOU_DISTANCE,
max_age=cfg.DEEPSORT.MAX_AGE, n_init=cfg.DEEPSORT.N_INIT, nn_budget=cfg.DEEPSORT.NN_BUDGET,
use_cuda=True)
# Initialize
device = select_device(opt.device)
if os.path.exists(out):
shutil.rmtree(out) # delete output folder
os.makedirs(out) # make new output folder
half = device.type != 'cpu' # half precision only supported on CUDA
now = datetime.datetime.now().strftime("%Y/%m/%d/%H:%M:%S") # current time
# Load model
model = torch.load(weights, map_location=device)[
'model'].float() # load to FP32
model.to(device).eval()
# =============================================================================
filepath_mask = 'D:/Internship Crime Detection/YOLOv5 person detection/AjnaTask/Mytracker/yolov5/weights/mask.pt'
model_mask = torch.load(filepath_mask, map_location = device)['model'].float()
model_mask.to(device).eval()
if half:
model_mask.half()
names_m = model_mask.module.names if hasattr(model_mask, 'module') else model_mask.names
# =============================================================================
if half:
model.half() # to FP16
# Set Dataloader
vid_path, vid_writer = None, None
if webcam:
view_img = True
cudnn.benchmark = True # set True to speed up constant image size inference
dataset = LoadStreams(source, img_size=imgsz)
else:
view_img = False
save_img = True
dataset = LoadImages(source, img_size=imgsz)
# Get names and colors
names = model.module.names if hasattr(model, 'module') else model.names
# Run inference
t0 = time.time()
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
# run once
_ = model(img.half() if half else img) if device.type != 'cpu' else None
save_path = str(Path(out))
txt_path = str(Path(out)) + '/results.txt'
memory = {}
people_counter = 0
in_people = 0
out_people = 0
people_mask = 0
people_none = 0
time_sum = 0
# now_time = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S')
colors = [[random.randint(0, 255) for _ in range(3)] for _ in names]
for frame_idx, (path, img, im0s, vid_cap) in enumerate(dataset):
img = torch.from_numpy(img).to(device)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
# Inference
t1 = time_synchronized()
pred = model(img, augment=opt.augment)[0]
# =============================================================================
pred_mask = model_mask(img)[0]
# =============================================================================
# Apply NMS
pred = non_max_suppression(
pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms)
# =============================================================================
pred_mask = non_max_suppression(pred_mask, 0.4, 0.5, classes = [0, 1, 2], agnostic = None)
if pred_mask is None:
continue
classification = torch.cat(pred_mask)[:, -1]
if len(classification) == 0:
print("----",None)
continue
index = int(classification[0])
mask_class = names_m[index]
print("MASK CLASS>>>>>>> \n", mask_class)
# =============================================================================
# Create the haar cascade
# cascPath = "D:/Internship Crime Detection/YOLOv5 person detection/AjnaTask/Mytracker/haarcascade_frontalface_alt2.xml"
# faceCascade = cv2.CascadeClassifier(cascPath)
t2 = time_synchronized()
# Process detections
for i, det in enumerate(pred): # detections per image
if webcam: # batch_size >= 1
p, s, im0 = path[i], '%g: ' % i, im0s[i].copy()
else:
p, s, im0 = path, '', im0s
s += '%gx%g ' % img.shape[2:] # print string
save_path = str(Path(out) / Path(p).name)
img_center_y = int(im0.shape[0]//2)
# line = [(int(im0.shape[1]*0.258),int(img_center_y*1.3)),(int(im0.shape[1]*0.55),int(img_center_y*1.3))]
# print("LINE>>>>>>>>>", line,"------------")
# line = [(990, 672), (1072, 24)]
line = [(1272, 892), (1800, 203)]
# [(330, 468), (704, 468)]
print("LINE>>>>>>>>>", line,"------------")
cv2.line(im0,line[0],line[1],(0,0,255),5)
# =============================================================================
# gray = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
# # Detect faces in the image
# faces = faceCascade.detectMultiScale(
# gray,
# scaleFactor=1.1,
# minNeighbors=5,
# minSize=(30, 30)
# )
# # Draw a rectangle around the faces
# for (x, y, w, h) in faces:
# cv2.rectangle(im0, (x, y), (x+w, y+h), (0, 255, 0), 2)
# text_x = x
# text_y = y+h
# cv2.putText(im0, mask_class, (text_x, text_y), cv2.FONT_HERSHEY_COMPLEX_SMALL,
# 1, (0, 0, 255), thickness=1, lineType=2)
# =============================================================================
if det is not None and len(det):
# Rescale boxes from img_size to im0 size
det[:, :4] = scale_coords(
img.shape[2:], det[:, :4], im0.shape).round()
# Print results
for c in det[:, -1].unique():
n = (det[:, -1] == c).sum() # detections per class
s += '%g %ss, ' % (n, names[int(c)]) # add to string
bbox_xywh = []
confs = []
bbox_xyxy = []
rects = [] # Is it correct?
# Adapt detections to deep sort input format
for *xyxy, conf, cls in det:
x_c, y_c, bbox_w, bbox_h = bbox_rel(*xyxy)
# label = f'{names[int(cls)]}'
xyxy_list = torch.tensor(xyxy).view(1,4).view(-1).tolist()
plot_one_box(xyxy, im0, label='person', color=colors[int(cls)], line_thickness=3)
rects.append(xyxy_list)
obj = [x_c, y_c, bbox_w, bbox_h,int(cls)]
#cv2.circle(im0,(int(x_c),int(y_c)),color=(0,255,255),radius=12,thickness = 10)
bbox_xywh.append(obj)
# bbox_xyxy.append(rec)
confs.append([conf.item()])
xywhs = torch.Tensor(bbox_xywh)
confss = torch.Tensor(confs)
# Pass detections to deepsort
outputs = ct.update(rects) # xyxy
# outputs = deepsort.update(xywhs, confss, im0) # deepsort
index_id = []
previous = memory.copy()
memory = {}
boxes = []
names_ls = []
# draw boxes for visualization
if len(outputs) > 0:
# print('output len',len(outputs))
for id_,centroid in outputs.items():
# boxes.append([output[0],output[1],output[2],output[3]])
# index_id.append('{}-{}'.format(names_ls[-1],output[-2]))
index_id.append(id_)
boxes.append(centroid)
memory[index_id[-1]] = boxes[-1]
i = int(0)
print(">>>>>>>",boxes)
for box in boxes:
# extract the bounding box coordinates
# (x, y) = (int(box[0]), int(box[1]))
# (w, h) = (int(box[2]), int(box[3]))
x = int(box[0])
y = int(box[1])
# GGG
if index_id[i] in previous:
previous_box = previous[index_id[i]]
(x2, y2) = (int(previous_box[0]), int(previous_box[1]))
# (w2, h2) = (int(previous_box[2]), int(previous_box[3]))
p0 = (x,y)
p1 = (x2,y2)
cv2.line(im0, p0, p1, (0,255,0), 3) # current frame obj center point - before frame obj center point
if intersect(p0, p1, line[0], line[1]):
people_counter += 1
print('==============================')
print(p0,"---------------------------",p0[1])
print('==============================')
print(line[1][1],'------------------',line[0][0],'-----------------', line[1][0],'-------------',line[0][1])
# if p0[1] <= line[1][1]:
# in_people +=1
# else:
# # if mask_class == 'mask':
# # print("COUNTING MASK..", mask_class)
# # people_mask += 1
# # if mask_class == 'none':
# # people_none += 1
# out_people +=1
if p0[1] >= line[1][1]:
in_people += 1
if mask_class == 'mask':
people_mask += 1
else:
people_none += 1
else:
out_people += 1
i += 1
# Write MOT compliant results to file
if save_txt and len(outputs) != 0:
for j, output in enumerate(outputs):
bbox_left = output[0]
bbox_top = output[1]
bbox_w = output[2]
bbox_h = output[3]
identity = output[-1]
with open(txt_path, 'a') as f:
f.write(('%g ' * 10 + '\n') % (frame_idx, identity, bbox_left,
bbox_top, bbox_w, bbox_h, -1, -1, -1, -1)) # label format
else:
deepsort.increment_ages()
cv2.putText(im0, 'Person [down][up] : [{}][{}]'.format(out_people,in_people),(130,50),cv2.FONT_HERSHEY_COMPLEX,1.0,(0,0,255),3)
cv2.putText(im0, 'Person [mask][no_mask] : [{}][{}]'.format(people_mask, people_none), (130,100),cv2.FONT_HERSHEY_COMPLEX,1.0,(0,0,255),3)
# Print time (inference + NMS)
print('%sDone. (%.3fs)' % (s, t2 - t1))
time_sum += t2-t1
# Stream results
if view_img:
cv2.imshow(p, im0)
if cv2.waitKey(1) == ord('q'): # q to quit
raise StopIteration
# Save results (image with detections)
if save_img:
if dataset.mode == 'images':
# im0= cv2.resize(im0,(0,0),fx=0.5,fy=0.5,interpolation=cv2.INTER_LINEAR)
cv2.imwrite(save_path, im0)
else:
if vid_path != save_path: # new video
vid_path = save_path
if isinstance(vid_writer, cv2.VideoWriter):
vid_writer.release() # release previous video writer
fps = vid_cap.get(cv2.CAP_PROP_FPS)
w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
vid_writer = cv2.VideoWriter(
save_path, cv2.VideoWriter_fourcc(*opt.fourcc), fps, (w, h))
vid_writer.write(im0)
if save_txt or save_img:
print('Results saved to %s' % os.getcwd() + os.sep + out)
if platform == 'darwin': # MacOS
os.system('open ' + save_path)
print('Done. (%.3fs)' % (time.time() - t0))
classIDs.append(classID)
text = "{}: {:.4f}".format(LABELS[classIDs[counter]],
confidences[counter])
cv2.putText(frame, text, (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX,
0.5, color, 2)
#update counter
counter = counter + 1
# apply non-maxima suppression to suppress weak, overlapping
# bounding boxes
idxs = cv2.dnn.NMSBoxes(boxes, confidences, args["confidence"],
args["threshold"])
filteredObjects, objects = ct.update(detections, frameTimestamp)
# loop over the filtered tracked objects
counter = 0
for (objectID, detection) in filteredObjects.items():
text = "ID {}".format(objectID)
# draw both the ID of the object and the centroid of the
# object on the output frame
cx = int(detection[0])
cy = int(detection[1])
width = int(detection[2])
height = int(detection[3])
x = int(cx - (width / 2))
y = int(cy - (height / 2))
def processVideo(prototxt, model, filepath):
print("[INFO] Filepath: " + filepath)
print("[INFO] model: " + model)
print("[INFO] prototxt: " + prototxt)
outputPath = "./userapp/output.avi"
skipframes = 30
conf = 0.4
# construct the argument parse and parse the arguments
# ap = argparse.ArgumentParser()
# ap.add_argument(prototxt)
# ap.add_argument(model)
# ap.add_argument(filepath)
# # ap.add_argument("-o", "--output", type=str,
# # help="path to optional output video file")
# ap.add_argument("-c", "--confidence", type=float, default=0.4,
# help="minimum probability to filter weak detections")
# ap.add_argument("-s", "--skip-frames", type=int, default=30,
# help="# of skip frames between detections")
# args = vars(ap.parse_args())
# print("[INFO] Starting2.....")
# initialize the list of class labels MobileNet SSD was trained to
# detect
CLASSES = [
"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus",
"car", "cat", "chair", "cow", "diningtable", "dog", "horse",
"motorbike", "person", "pottedplant", "sheep", "sofa", "train",
"tvmonitor"
]
# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(prototxt, model)
# if a video path was not supplied, grab a reference to the webcam
# if not args.get("input", False):
# print("[INFO] starting video stream...")
# vs = VideoStream(src=0).start()
# time.sleep(2.0)
# otherwise, grab a reference to the video file
# else:
# print("[INFO] opening video file...")
# vs = cv2.VideoCapture(args["input"])
vs = cv2.VideoCapture(filepath)
# initialize the video writer (we'll instantiate later if need be)
writer = None
# initialize the frame dimensions (we'll set them as soon as we read
# the first frame from the video)
W = None
H = None
# instantiate our centroid tracker, then initialize a list to store
# each of our dlib correlation trackers, followed by a dictionary to
# map each unique object ID to a TrackableObject
ct = CentroidTracker(maxDisappeared=40, maxDistance=50)
trackers = []
trackableObjects = {}
# initialize the total number of frames processed thus far, along
# with the total number of objects that have moved either up or down
totalFrames = 0
totalDown = 0
totalUp = 0
# start the frames per second throughput estimator
fps = FPS().start()
# loop over frames from the video stream
while True:
# grab the next frame and handle if we are reading from either
# VideoCapture or VideoStream
print("[Info] Filepath is" + filepath)
frame = vs.read()
frame = frame[1] if filepath else frame
# if we are viewing a video and we did not grab a frame then we
# have reached the end of the video
if filepath is not None and frame is None:
break
# resize the frame to have a maximum width of 500 pixels (the
# less data we have, the faster we can process it), then convert
# the frame from BGR to RGB for dlib
frame = imutils.resize(frame, width=500)
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# if the frame dimensions are empty, set them
if W is None or H is None:
(H, W) = frame.shape[:2]
# if we are supposed to be writing a video to disk, initialize
# the writer
if outputPath is not None and writer is None:
print("[INFO] loading model...after vs1")
fourcc = cv2.VideoWriter_fourcc(*"MJPG")
print("[INFO] loading model...after vs2 :" + outputPath)
writer = cv2.VideoWriter(outputPath, fourcc, 30, (W, H), True)
print("[INFO] loading model...after vs3")
# initialize the current status along with our list of bounding
# box rectangles returned by either (1) our object detector or
# (2) the correlation trackers
status = "Waiting"
print("[INFO] loading model...after vs")
rects = []
print("[INFO] loading model...after cv2")
# check to see if we should run a more computationally expensive
# object detection method to aid our tracker
if totalFrames % skipframes == 0:
# set the status and initialize our new set of object trackers
status = "Detecting"
trackers = []
# convert the frame to a blob and pass the blob through the
# network and obtain the detections
blob = cv2.dnn.blobFromImage(frame, 0.007843, (W, H), 127.5)
net.setInput(blob)
detections = net.forward()
# loop over the detections
for i in np.arange(0, detections.shape[2]):
# extract the confidence (i.e., probability) associated
# with the prediction
confidence = detections[0, 0, i, 2]
# filter out weak detections by requiring a minimum
# confidence
if confidence > conf:
# extract the index of the class label from the
# detections list
idx = int(detections[0, 0, i, 1])
# if the class label is not a person, ignore it
if CLASSES[idx] != "person":
continue
# compute the (x, y)-coordinates of the bounding box
# for the object
box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
(startX, startY, endX, endY) = box.astype("int")
# construct a dlib rectangle object from the bounding
# box coordinates and then start the dlib correlation
# tracker
tracker = dlib.correlation_tracker()
rect = dlib.rectangle(startX, startY, endX, endY)
tracker.start_track(rgb, rect)
# add the tracker to our list of trackers so we can
# utilize it during skip frames
trackers.append(tracker)
# otherwise, we should utilize our object *trackers* rather than
# object *detectors* to obtain a higher frame processing throughput
else:
# loop over the trackers
for tracker in trackers:
# set the status of our system to be 'tracking' rather
# than 'waiting' or 'detecting'
status = "Tracking"
# update the tracker and grab the updated position
tracker.update(rgb)
pos = tracker.get_position()
# unpack the position object
startX = int(pos.left())
startY = int(pos.top())
endX = int(pos.right())
endY = int(pos.bottom())
# add the bounding box coordinates to the rectangles list
rects.append((startX, startY, endX, endY))
# draw a horizontal line in the center of the frame -- once an
# object crosses this line we will determine whether they were
# moving 'up' or 'down'
cv2.line(frame, (0, H // 2), (W, H // 2), (0, 255, 255), 2)
# use the centroid tracker to associate the (1) old object
# centroids with (2) the newly computed object centroids
objects = ct.update(rects)
# loop over the tracked objects
for (objectID, centroid) in objects.items():
# check to see if a trackable object exists for the current
# object ID
to = trackableObjects.get(objectID, None)
# if there is no existing trackable object, create one
if to is None:
to = TrackableObject(objectID, centroid)
# otherwise, there is a trackable object so we can utilize it
# to determine direction
else:
# the difference between the y-coordinate of the *current*
# centroid and the mean of *previous* centroids will tell
# us in which direction the object is moving (negative for
# 'up' and positive for 'down')
y = [c[1] for c in to.centroids]
direction = centroid[1] - np.mean(y)
to.centroids.append(centroid)
# check to see if the object has been counted or not
if not to.counted:
# if the direction is negative (indicating the object
# is moving up) AND the centroid is above the center
# line, count the object
if direction < 0 and centroid[1] < H // 2:
totalUp += 1
to.counted = True
# if the direction is positive (indicating the object
# is moving down) AND the centroid is below the
# center line, count the object
elif direction > 0 and centroid[1] > H // 2:
totalDown += 1
to.counted = True
# store the trackable object in our dictionary
trackableObjects[objectID] = to
# draw both the ID of the object and the centroid of the
# object on the output frame
text = "ID {}".format(objectID)
cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
# construct a tuple of information we will be displaying on the
# frame
info = [
("Up", totalUp),
("Down", totalDown),
("Status", status),
]
# loop over the info tuples and draw them on our frame
for (i, (k, v)) in enumerate(info):
text = "{}: {}".format(k, v)
cv2.putText(frame, text, (10, H - ((i * 20) + 20)),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)
# check to see if we should write the frame to disk
if writer is not None:
writer.write(frame)
# show the output frame
# cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
if key == ord("q"):
break
# increment the total number of frames processed thus far and
# then update the FPS counter
totalFrames += 1
fps.update()
# stop the timer and display FPS information
fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
# check to see if we need to release the video writer pointer
if writer is not None:
writer.release()
print("[INFO] Total Up Count: " + str(totalUp))
# if we are not using a video file, stop the camera video stream
# if filepath is not None:
# vs.stop()
# # otherwise, release the video file pointer
# else:
# vs.release()
# close any open windows
cv2.destroyAllWindows()
class ROSTensorFlow(object):
def __init__(self):
# Setup tensorflow (v1.14 / v2.0 compatible)
#tf.compat.v1.disable_eager_execution()
#np.set_printoptions(threshold=sys.maxsize)
# Init CV bridge
self.cv_bridge1 = CvBridge()
# Setup raytracing and transform
cam_info = rospy.wait_for_message("/door_kinect/rgb/camera_info",
CameraInfo,
timeout=None)
self.img_proc = PinholeCameraModel()
self.img_proc.fromCameraInfo(cam_info)
self.tf_broadcaster = tf.TransformBroadcaster()
self.camera_frame = 'door_kinect_rgb_optical_frame'
# Subscribe to RGB and D data topics
self.sub_rgb = message_filters.Subscriber(
"/door_kinect/rgb/image_color", Image)
self.sub_d = message_filters.Subscriber(
"/door_kinect/depth_registered/sw_registered/image_rect", Image)
# Setup publishing topics
self.pub_rgb = rospy.Publisher("/humandetect_imagergb",
Image,
queue_size=1)
self.pub_depth = rospy.Publisher("/humandetect_imagedepth",
Image,
queue_size=1)
self.pub_pose = rospy.Publisher("/humandetect_poses",
PoseArray,
queue_size=1)
# Synchronisation
self.ts = message_filters.ApproximateTimeSynchronizer(
[self.sub_rgb, self.sub_d], 10, 0.1, allow_headerless=False)
# Setup Tensorflow object detection
# Tensorflow path setup
path_add = os.path.join(
rospkg.RosPack().get_path("yeetbot_humantracker"),
"models/research")
sys.path.append(path_add)
os.path.join(path_add, "slim")
sys.path.append(path_add)
objectdetect_path = os.path.join(
rospkg.RosPack().get_path("yeetbot_humantracker"),
"models/research/object_detection")
sys.path.append(objectdetect_path)
print(sys.path)
# Import object detection API utils
from utils import label_map_util
from utils import visualization_utils as viz_utils
self.label_map_util1 = label_map_util
self.viz_utils1 = viz_utils
# Tensorflow model setup
MODEL_NAME = 'ssdlite_mobilenet_v2_coco_2018_05_09'
CWD_PATH = os.getcwd()
PATH_TO_CKPT = os.path.join(objectdetect_path, MODEL_NAME,
'frozen_inference_graph.pb')
PATH_TO_LABELS = os.path.join(objectdetect_path, 'data',
'mscoco_label_map.pbtxt')
NUM_CLASSES = 90
self.label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
self.categories = label_map_util.convert_label_map_to_categories(
self.label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
self.category_index = label_map_util.create_category_index(
self.categories)
print(self.category_index)
# Tensorflow session setup
self.sess = None
detection_graph = tensorflow.Graph()
with detection_graph.as_default():
od_graph_def = tensorflow.GraphDef()
with tensorflow.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tensorflow.import_graph_def(od_graph_def, name='')
self.sess = tensorflow.Session(graph=detection_graph)
# Tensorflow detection output setup
self.image_tensor = detection_graph.get_tensor_by_name(
'image_tensor:0')
self.detection_boxes = detection_graph.get_tensor_by_name(
'detection_boxes:0')
self.detection_scores = detection_graph.get_tensor_by_name(
'detection_scores:0')
self.detection_classes = detection_graph.get_tensor_by_name(
'detection_classes:0')
self.num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
self.frame_rate_calc = 1
self.freq = cv2.getTickFrequency()
self.font = cv2.FONT_HERSHEY_SIMPLEX
## Setup KCF tracker
#self.kcf_tracker = cv2.TrackerKCF_create()
self.ct = CentroidTracker()
# Callback register
self.ts.registerCallback(self.callback)
print('Init done')
def callback(self, color_msg, depth_msg):
t1 = cv2.getTickCount()
print(t1)
# Receive camera data
frame_color = self.cv_bridge1.imgmsg_to_cv2(
color_msg, desired_encoding="passthrough").copy()
frame_depth = self.cv_bridge1.imgmsg_to_cv2(
depth_msg, desired_encoding="passthrough").copy()
# Convert to format
frame_rgb = cv2.cvtColor(frame_color, cv2.COLOR_BGR2RGB)
frame_expanded = np.expand_dims(frame_rgb, axis=0)
# Get image geometry
rgb_height, rgb_width, rgb_channels = frame_rgb.shape
depth_height, depth_width = frame_depth.shape
print("RGB INFO")
print("{0} {1} {2}".format(rgb_height, rgb_width, rgb_channels))
print("DEPTH INFO")
print("{0} {1}".format(depth_height, depth_width))
# Find people on RGB image with Tensorflow
with self.sess.graph.as_default():
(boxes, scores, classes, num) = self.sess.run(
[
self.detection_boxes, self.detection_scores,
self.detection_classes, self.num_detections
],
feed_dict={self.image_tensor: frame_expanded})
#print("CLASSES")
#print(classes)
#print("NUM")
#print(int(num[0]))
#print("BOXES")
# Find people on image from Tensorflow results
boxes_coords = list()
center_points = list()
center_pointsd = list()
depth_coords = list()
depth_frames = list()
depths = list()
for i in range(int(num[0])):
if (int(classes[0][i]) == 1
and scores[0][i] >= 0.70): # if human detected
box = [None] * 2
box_depth = [None] * 2
min_y = int(boxes[0][i][0] * rgb_height)
min_x = int(boxes[0][i][1] * rgb_width)
max_y = int(boxes[0][i][2] * rgb_height)
max_x = int(boxes[0][i][3] * rgb_width)
# ok = tracker.init(frame, min_x, min_y, max_x, max_y)
box[0] = (min_x, min_y)
box[1] = (max_x, max_y)
center_x = int((min_x + max_x) / 2)
center_y = int((min_y + max_y) / 2)
center = (center_x, center_y)
box_w = max_x - min_x
box_h = max_y - min_y
depth_min_y = int(center_y - box_h / 4)
depth_max_y = int(center_y + box_h / 4)
depth_min_x = int(center_x - box_w / 4)
depth_max_x = int(center_x + box_w / 4)
box_depth[0] = (depth_min_x, depth_min_y)
box_depth[1] = (depth_max_x, depth_max_y)
boxes_coords.append(box)
depth_coords.append(box_depth)
center_points.append(center)
depth_frame = frame_depth[depth_min_y:depth_max_y,
depth_min_x:depth_max_x]
#np.nan_to_num(depth_frame, False, 10)
depth_frame = depth_frame
depth_median = np.nanmedian(depth_frame)
#print("TYPE")
#print(type(depth_frame))
#print(depth_frame)
depth_frames.append(depth_frame)
depths.append(depth_median)
center_d = (center_x, center_y, depth_median)
center_pointsd.append(center_d)
#print("OBJECT {0} HAS DEPTH {1}".format(i, depth_median))
#print(depth_frame[0])
#print("BOXES COORDS")
#print(boxes_coords)
#self.viz_utils1.visualize_boxes_and_labels_on_image_array(
# frame_color,
# np.squeeze(boxes),
# np.squeeze(classes).astype(np.int32),
# np.squeeze(scores),
# self.category_index,
# use_normalized_coordinates=True,
# line_thickness=8,
# min_score_thresh=0.70)
#cv2.imshow('Object detector', frame_color)
t2 = cv2.getTickCount()
time1 = (t2 - t1) / self.freq
frame_rate_calc = 1 / time1
print("FPS: {0:.2f}".format(frame_rate_calc))
objects = self.ct.update(center_pointsd.copy())
for (objectID, centroid) in objects.items():
text = "ID {}".format(objectID)
centroidx, centroidy, centroidz = centroid
centroidxy = (centroidx, centroidy)
cv2.putText(frame_color, text, (centroidx - 10, centroidy - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.circle(frame_color, (centroidx, centroidy), 4, (0, 255, 0), -1)
#print(boxes_coords)
#print(depth_coords)
for box in boxes_coords:
cv2.rectangle(frame_color, box[0], box[1], (255, 0, 0), 2)
for depth_box in depth_coords:
cv2.rectangle(frame_color, depth_box[0], depth_box[1], (0, 255, 0),
2)
for point in center_points:
print(point)
cv2.circle(frame_color, point, 5, (0, 0, 255), 2)
#boxes_coords = list()
#center_points = list()
#depth_coords = list()
#depth_frames = list()
#depths = list()
out_pose = PoseArray()
out_pose.header.stamp = color_msg.header.stamp
out_pose.header.frame_id = self.camera_frame
poses1 = list()
for (objectID, centroid) in objects.items():
centroidx, centroidy, centroidz = centroid
print("OBJECT {0} AT {1} DEPTH {2}".format(objectID,
(centroidx, centroidy),
centroidz))
x, y, _ = self.img_proc.projectPixelTo3dRay((centroidx, centroidy))
print(x)
print(y)
z = (centroidz)
print(z)
self.tf_broadcaster.sendTransform(
[x, y, z], tf.transformations.quaternion_from_euler(0, 0, 0),
color_msg.header.stamp, '/humanpos_{0}'.format(i),
self.camera_frame)
quat = tf.transformations.quaternion_from_euler(0, 1.5, 0)
pose1 = Pose()
pose1.position.x = x
pose1.position.y = y
pose1.position.z = z
pose1.orientation.x = quat[0]
pose1.orientation.y = quat[1]
pose1.orientation.z = quat[2]
pose1.orientation.w = quat[3]
poses1.append(pose1)
out_pose.poses = poses1
self.pub_pose.publish(out_pose)
out_message = self.cv_bridge1.cv2_to_imgmsg(frame_color)
#print("TYPE2")
#print(type(depth_frame[0]))
#print("TYPE3")
#print(type(frame_color))
#out_message2 = self.cv_bridge1.cv2_to_imgmsg(depth_frames[0])
out_message.header.stamp = color_msg.header.stamp
#out_message2.header.stamp = depth_msg.header.stamp
try:
self.pub_rgb.publish(out_message)
#self.pub_depth.publish(out_message2)
except rospy.ROSException as e:
raise e
