class CameraReader(QtCore.QThread):
signal = QtCore.pyqtSignal(QtGui.QImage, object)
def __init__(self):
super(CameraReader, self).__init__()
# self.cam = cv2.VideoCapture(0)
self.cam = cv2.VideoCapture("/home/dek/my_video-1.mkv")
self.width = int(self.cam.get(cv2.CAP_PROP_FRAME_WIDTH))
self.height = int(self.cam.get(cv2.CAP_PROP_FRAME_HEIGHT))
self.objdet = ObjectDetector()
def run(self):
while True:
ret, img = self.cam.read()
expand = np.expand_dims(img, axis=0)
result, category_index = self.objdet.detect(expand)
image = QtGui.QImage(img.data, self.width, self.height,
QtGui.QImage.Format_RGB888).rgbSwapped()
w, h = self.width, self.height
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.4:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = box[0] * h, box[1] * w
y2, x2 = box[2] * h, box[3] * w
boxes.append((category_index[class_]['name'], score, x1,
y1, x2, y2))
self.signal.emit(image, boxes)
class CameraReader(QtCore.QThread):
signal = QtCore.Signal(QtGui.QImage, object)
def __init__(self):
super(CameraReader, self).__init__()
self.cam = cv2.VideoCapture(0)
self.width = long(self.cam.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH))
self.height = long(self.cam.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))
self.objdet = ObjectDetector()
def run(self):
while True:
ret, img = self.cam.read()
expand = np.expand_dims(img, axis=0)
result = self.objdet.detect(expand)
image = QtGui.QImage(img.data, self.width, self.height,
QtGui.QImage.Format_RGB888).rgbSwapped()
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.5:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
y1, x1 = box[0] * self.width, box[1] * self.height
y2, x2 = box[2] * self.width, box[3] * self.height
boxes.append((class_, x1, y1, x2, y2))
self.signal.emit(image, boxes)
def get_blur_experiment_results(video, blur):
cam = cv2.VideoCapture(args.movie)
width = int(cam.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cam.get(cv2.CAP_PROP_FRAME_HEIGHT))
objdet = ObjectDetector(args.path_to_ckpt, args.path_to_labels,
args.num_classes)
detection_counts = [] # Just the raw number of detections
weighted_detection_counts = [] # Sum the scores
ret, frame = cam.read()
while ret == True:
img = frame # Aliased, but lets us turn off as necessary.
img = cv2.GaussianBlur(img, (blur, blur), 0) # Magic happens here
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h, w, _ = img.shape
expand = np.expand_dims(img, axis=0)
result = objdet.detect(expand)
boxes = []
detection_counts.append(0)
weighted_detection_counts.append(0)
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.6:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = int(box[0] * h), int(box[1] * w)
y2, x2 = int(box[2] * h), int(box[3] * w)
boxes.append((class_, score, x1, y1, x2, y2))
# Less efficient, but keeps it all in the same place.
weighted_detection_counts[-1] += score
detection_counts[-1] += 1
for box in boxes:
class_, score, x1, y1, x2, y2 = box
w1 = x2 - x1
h1 = y2 - y1
cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2)
cv2.putText(img, "%s: %5.2f" % (layers[class_ - 1], score),
(x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imshow('image', img)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
ret, frame = cam.read()
return np.array(detection_counts), np.array(weighted_detection_counts)
config_file,
classes,
conf_thresh=0.01,
net=net)
while True:
path = raw_input("Image path (press 0 to exit):")
if path == '0':
break
if not os.path.isfile(path):
print("Image not exists!")
continue
img = cv2.imread(path)
# detect text regions
start = time.time()
all_regions = detector.detect(img)
end = time.time()
print("Cost time {}".format(end - start))
for key in all_regions.keys():
boxes = all_regions[key]
for i in range(len(boxes)):
print("Boxes {}: {}".format(key, boxes[i]))
cv2.rectangle(img, (boxes[i][0], boxes[i][1]),
(boxes[i][2], boxes[i][3]),
color=(255, 255, 0),
thickness=1)
show(img)
class MainWindow(QtWidgets.QGraphicsView):
def __init__(self):
super(MainWindow, self).__init__()
self.scene = QtWidgets.QGraphicsScene(self)
self.scene.setBackgroundBrush(QtGui.QBrush(QtGui.QColor(131, 213, 247)))
self.scene.setSceneRect(0, 0, WIDTH, HEIGHT)
self.machine = QtSvg.QGraphicsSvgItem('../static/assets/burger_chute.svg')
self.scene.addItem(self.machine)
self.machine.setScale(2)
self.machine.setPos(1000,0)
self.setFixedSize(WIDTH,HEIGHT)
self.setScene(self.scene)
self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.cam = cv2.VideoCapture(0)
self.cam.set(cv2.CAP_PROP_FRAME_WIDTH, CAMERA_WIDTH)
self.cam.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_HEIGHT)
# # self.cam = cv2.VideoCapture(filename)
self.width = int(self.cam.get(cv2.CAP_PROP_FRAME_WIDTH))
self.height = int(self.cam.get(cv2.CAP_PROP_FRAME_HEIGHT))
self.cam_pixmap = None
self.aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_6X6_1000)
self.parameters = cv2.aruco.DetectorParameters_create()
rms, self.mtx, self.dist, rvecs, tvecs = pickle.load(open("calib.pkl","rb"))
self.objdet = ObjectDetector()
self.burger_pixmap = None
self.rectified_pixmap = None
font = QtGui.QFont('Arial', 75)
self.label_text = self.scene.addText("", font)
self.label_text.setDefaultTextColor(QtCore.Qt.red)
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.camera)
self.timer.start(0)
self.counter = 0
self.burger_classifier = BurgerClassifier()
font = QtGui.QFont('Arial', 100)
self.explanation_text = self.scene.addText("test", font)
self.explanation_text.setHtml("Design your own burger!")
self.explanation_text.setPos(0, 100)
font = QtGui.QFont('Arial', 60)
self.detail_text = self.scene.addText("test", font)
self.detail_text.setHtml("Place burger layers on the white board<br>Then show the board to the camera<br>Can the computer recognize burgers?<br>Can you fool the computer with a bad burger?")
self.detail_text.setPos(0, 250)
def imageTo(self, image):
pixmap = QtGui.QPixmap.fromImage(image)
pixmap = pixmap.transformed(QtGui.QTransform().scale(-1, 1))
if self.cam_pixmap is None:
self.cam_pixmap = self.scene.addPixmap(pixmap)
self.cam_pixmap.setPos(QtCore.QPointF(50, 2160-720-50))
else:
self.cam_pixmap.setPixmap(pixmap)
def imageTo2Clear(self):
if self.rectified_pixmap is not None:
self.scene.removeItem(self.rectified_pixmap)
self.rectified_pixmap = None
self.label_text.setPlainText("")
if self.burger_pixmap is not None:
self.scene.removeItem(self.burger_pixmap)
self.burger_pixmap = None
self.label_text.setPlainText("")
def imageTo2(self, image2, boxes):
pixmap2 = QtGui.QPixmap.fromImage(image2)
boxes = self.analyzeBoxes(boxes)
if boxes is None:
return
p = QtGui.QPainter()
p.begin(pixmap2)
for box in boxes:
p.setPen(QtCore.Qt.red)
class_, score, x1, y1, x2, y2 = box
w = x2-x1
h = y2-y1
p.drawRect(QtCore.QRect(QtCore.QPoint(x1, y1), QtCore.QSize(w, h)))
p.drawText(x1, y1, "%s: %5.2f" % (labels[class_], score))
p.end()
pixmap2 = pixmap2.scaled(pixmap2.width()*2, pixmap2.height()*2)
if self.rectified_pixmap == None:
self.rectified_pixmap = self.scene.addPixmap(pixmap2)
self.rectified_pixmap.setPos(QtCore.QPointF(3840 - pixmap2.width(), 2160-pixmap2.height()))
else:
self.rectified_pixmap.setPixmap(pixmap2)
def analyzeBoxes(self, boxes):
all_intersections = []
for i in range(len(boxes)):
first = boxes[i]
first_rect = QtCore.QRectF(QtCore.QPointF(first[2], first[3]), QtCore.QPointF(first[4], first[5]))
intersections = []
for j in range(i+1, len(boxes)):
second = boxes[j]
second_rect = QtCore.QRectF(QtCore.QPointF(second[2], second[3]), QtCore.QPointF(second[4], second[5]))
if first_rect.intersects(second_rect):
intersections.append((first, second))
if intersections != []:
all_intersections.append(intersections)
nonoverlapping_boxes = set(boxes)
if len(all_intersections):
for intersections in all_intersections:
for intersection in intersections:
first, second = intersection
if first[1] > second[1]:
if second in nonoverlapping_boxes:
nonoverlapping_boxes.remove(second)
else:
if first in nonoverlapping_boxes:
nonoverlapping_boxes.remove(first)
boxes = nonoverlapping_boxes
ordered_boxes = sorted(nonoverlapping_boxes, key=lambda x: boxCenter(x)[1])
ordered_boxes_with_gaps = []
for i in range(len(ordered_boxes)):
box = ordered_boxes[i]
bc = boxCenter(box[2:])
ordered_boxes_with_gaps.append(box)
if i < len(ordered_boxes) - 1:
nextBox = ordered_boxes[i+1]
dy = nextBox[3] - box[5]
nbc = boxCenter(nextBox[2:])
height = 50
if dy > height:
pos = int(dy / height)
width = (((box[4] - box[2]) + (nextBox[4] - nextBox[2]))/2)/2
centerX = (box[4] + nextBox[2]) / 2
for j in range(1, pos):
currPosY = box[5] + ((height+5)*j)
newBox = (0, 1.0, centerX - width, currPosY - height/2, centerX + width, currPosY + height/2)
ordered_boxes_with_gaps.append(newBox)
if len(ordered_boxes_with_gaps) == 6:
break
if len(ordered_boxes_with_gaps) == 6:
break
classes_ = [box[0] for box in ordered_boxes_with_gaps]
while len(classes_) < 6:
classes_.insert(0, 0)
burger_renderer = BurgerRenderer(classes_, 720, 720, renderEmpty=True)
image = burger_renderer.image
image_128 = burger_renderer.image.scaled(128, 128).convertToFormat(QtGui.QImage.Format_RGB888)
bits = image_128.constBits().asstring(128*128*3)
img = np.fromstring(bits, dtype=np.uint8).reshape(128, 128, 3)
pixmap3 = QtGui.QPixmap.fromImage(image)
if self.burger_pixmap is None:
self.burger_pixmap = self.scene.addPixmap(pixmap3)
self.burger_pixmap.setPos(QtCore.QPointF(1370, 190))
else:
self.burger_pixmap.setPixmap(pixmap3)
if len(classes_) != 6:
print("Wrong size:", len(classes_))
return None
result = self.burger_classifier.classify(classes_)[0]
if result[1] > 0.5:
label = 'burger'
self.label_text.setPlainText("BURGER")
self.label_text.setPos(1500,1150)
self.label_text.setDefaultTextColor(QtCore.Qt.green)
else:
label = 'notburger'
self.label_text.setHtml("NOT<br>BURGER")
self.label_text.setPos(1500,1150)
self.label_text.setDefaultTextColor(QtCore.Qt.red)
# self.image3_text.setText("P(burger) = %5.2f P(notburger) = %5.2f (%s" % (result[1], result[0], label))
# label = label_burger(classes_)
return ordered_boxes_with_gaps
def camera(self):
ret, img = self.cam.read()
if ret == True:
h, w, _ = img.shape
newcameramtx, roi=cv2.getOptimalNewCameraMatrix(self.mtx, self.dist, (w,h), 1, (w,h))
dst = cv2.undistort(img, self.mtx, self.dist, None, newcameramtx)
# # crop the image
x, y, w2, h2 = roi
dst = dst[y:y+h2, x:x+w2]
img = np.ascontiguousarray(dst, dst.dtype)
h3, w3, _ = img.shape
img2 = img.copy()
corners, ids, rejectedImgPoints = cv2.aruco.detectMarkers(img2, self.aruco_dict, parameters=self.parameters)
cv2.aruco.drawDetectedMarkers(img2, corners, ids)
for corner in corners:
rvec, tvec, _ = cv2.aruco.estimatePoseSingleMarkers(corner, 0.195, newcameramtx, self.dist)
try:
cv2.aruco.drawAxis(img2, newcameramtx, self.dist, rvec, tvec, 0.1)
except cv2.error:
print( "bad matrix")
d = {}
boxes = []
image2 = QtGui.QImage(self.width, self.height, QtGui.QImage.Format_RGB888)
boxes = []
if ids is None:
image = QtGui.QImage(img2.data, w3, h3, w3*3, QtGui.QImage.Format_RGB888).rgbSwapped()
self.imageTo(image)
self.imageTo2Clear()
return
else:
short_ids = [id_[0] for id_ in ids]
for i, corner in enumerate(corners):
d[short_ids[i]] = corner
if len(d.keys()) != 4:
image = QtGui.QImage(img2.data, w3, h3, w3*3, QtGui.QImage.Format_RGB888).rgbSwapped()
self.imageTo(image)
self.imageTo2Clear()
return
ul = d[0][0][0]
ur = d[1][0][0]
lr = d[2][0][0]
ll = d[3][0][0]
pts = np.array([ul, ur, lr, ll], np.int32)
pts = pts.reshape((-1,1,2))
cv2.polylines(img,[pts],True,(255,0,255))
image = QtGui.QImage(img2.data, w3, h3, w3*3, QtGui.QImage.Format_RGB888).rgbSwapped()
self.imageTo(image)
orig_width = ur[0] - ul[0]
orig_height = ll[1] - ul[1]
# ratio = orig_width / float(orig_height)
destCorners = np.array([ [0, 0], [WARPED_WIDTH, 0], [WARPED_WIDTH, WARPED_HEIGHT], [0, WARPED_HEIGHT]], dtype=np.float32)
srcCorners = np.array([ul, ur, lr, ll], dtype=np.float32)
pt = cv2.getPerspectiveTransform(srcCorners, destCorners)
warped = cv2.warpPerspective(img, pt, (WARPED_WIDTH, WARPED_HEIGHT))
cv2.imwrite("rectified/%05d.png" % self.counter, warped)
expand = np.expand_dims(warped, axis=0)
result = self.objdet.detect(expand)
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.5:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = box[0] * WARPED_HEIGHT, box[1] * WARPED_WIDTH
y2, x2 = box[2] * WARPED_HEIGHT, box[3] * WARPED_WIDTH
boxes.append((class_, score, x1, y1, x2, y2))
warped_image = QtGui.QImage(warped.data, WARPED_WIDTH, WARPED_HEIGHT, 3*WARPED_WIDTH, QtGui.QImage.Format_RGB888).rgbSwapped()
self.imageTo2(warped_image, boxes)
self.counter += 1
class Widget(QtWidgets.QWidget):
def __init__(self):
super(Widget, self).__init__()
self.scene = QtWidgets.QGraphicsScene()
self.scene.setSceneRect(0, 0, WIDTH, HEIGHT)
self.scene.changed.connect(self.changed)
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.classify)
self.timer.start(250)
self.view = QGraphicsView(self.scene)
self.view.setFixedSize(WIDTH,HEIGHT)
self.view.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.view.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.view.setMouseTracking(True)
# pixmap = QtGui.QPixmap("my_photo-1-crop.jpg")
# topbun_webcam = QGraphicsPixmapItem(pixmap)
# topbun_webcam.setFlags(QtWidgets.QGraphicsItem.ItemIsMovable)
# topbun_webcam.setScale(2)
# self.scene.addItem(topbun_webcam)
# lettuce = QGraphicsSvgItem("../../../assets/lettuce.svg")
# lettuce.setFlags(QtWidgets.QGraphicsItem.ItemIsMovable)
# lettuce.setScale(2)
# self.scene.addItem(lettuce)
self.image_widget = QtWidgets.QLabel(self)
self.image_widget.setFixedSize(WIDTH,HEIGHT)
self.icons = QtWidgets.QWidget(self)
self.icons.setFixedSize(128, 6*64)
self.iconsLayout = QtWidgets.QVBoxLayout()
self.icons.setLayout(self.iconsLayout)
self.topbun = QSvgWidget("../../../static/assets/topbun.svg")
self.lettuce = QSvgWidget("../../../static/assets/lettuce.svg")
self.tomato = QSvgWidget("../../../static/assets/tomato.svg")
self.cheese = QSvgWidget("../../../static/assets/cheese.svg")
self.patty = QSvgWidget("../../../static/assets/patty.svg")
self.bottombun = QSvgWidget("../../../static/assets/bottombun.svg")
self.banana = QSvgWidget("../../../static/assets/banana.svg")
self.book = QSvgWidget("../../../static/assets/book.svg")
self.shoe = QSvgWidget("../../../static/assets/shoe.svg")
self.iconsLayout.addWidget(self.topbun)
self.iconsLayout.addWidget(self.lettuce)
self.iconsLayout.addWidget(self.tomato)
self.iconsLayout.addWidget(self.cheese)
self.iconsLayout.addWidget(self.patty)
self.iconsLayout.addWidget(self.bottombun)
self.iconsLayout.addWidget(self.banana)
self.iconsLayout.addWidget(self.book)
self.iconsLayout.addWidget(self.shoe)
self.buttons = QtWidgets.QWidget(self)
self.buttonsLayout = QtWidgets.QHBoxLayout()
self.buttons.setLayout(self.buttonsLayout)
self.buttonA = QtWidgets.QPushButton("Classify")
self.buttonA.clicked.connect(self.buttonAClicked)
self.buttonsLayout.addWidget(self.buttonA)
self.layout = QtWidgets.QHBoxLayout(self)
self.layout.addWidget(self.icons)
# self.layout.addWidget(self.buttons)
self.layout.addWidget(self.view)
self.layout.addWidget(self.image_widget)
self.setLayout(self.layout)
self.objdet = ObjectDetector()
def topbun_clicked(self, *args):
print(args)
def buttonAClicked(self, *args):
self.classify()
def changed(self):
self.classify()
def classify(self):
image = QtGui.QImage(QtCore.QSize(WIDTH, HEIGHT), QtGui.QImage.Format_RGB888)
image.fill(QtCore.Qt.white)
painter = QtGui.QPainter(image)
self.scene.render(painter)
painter.end()
bits = image.constBits().asstring(HEIGHT*WIDTH*3)
img = np.fromstring(bits, dtype=np.uint8).reshape(HEIGHT, WIDTH, 3)
# image = canny(img)
expand = np.expand_dims(img, axis=0)
result = self.objdet.detect(expand)
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.4:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = box[0] * HEIGHT, box[1] * WIDTH
y2, x2 = box[2] * HEIGHT, box[3] * WIDTH
boxes.append((class_, score, x1, y1, x2, y2))
pixmap = QtGui.QPixmap.fromImage(image)
p = QtGui.QPainter()
p.begin(pixmap)
for box in boxes:
p.setPen(QtCore.Qt.red)
class_, score, x1, y1, x2, y2 = box
w = x2-x1
h = y2-y1
p.drawRect(x1, y1, w, h)
p.drawText(x1, y1, "%s: %5.2f" % (labels[class_], score))
p.end ()
self.image_widget.setPixmap(pixmap)
else:
raise
counter = 0
ret, frame = cam.read()
while ret == True:
img = frame.copy(
) # Aliased, but lets us turn off transformations as necessary.
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h, w, _ = img.shape
expand = np.expand_dims(img, axis=0)
# Time how long detections take, roughly.
det_start = time.time()
result = objdet.detect(expand)
det_end = time.time()
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > args.threshold:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = int(box[0] * h), int(box[1] * w)
y2, x2 = int(box[2] * h), int(box[3] * w)
boxes.append((class_, score, x1, y1, x2, y2))
bboxes = [
np.array([x1, y1, x2 - x1, y2 - y1])
for (cls, score, x1, y1, x2, y2) in boxes
def main():
''' input '''
# choose the input stream
#captureSource = "video/video_116.mp4"
#captureSource = "video/video_205.mp4"
captureSource = "video/video_white.mp4"
#captureSource = 0 # webcam
cap = cv2.VideoCapture(captureSource)
''' trackers typology '''
# choose the tracker
trackerName = "CSRT" # "MOSSE" | "KCF" | "CSRT"
tm = TrackerManager(trackerName, maxFailures=20)
''' parameters '''
# try to change these parameters
period = 1 # length of the period: only on the first frame of the period we detect objects (instead, we track them in every frame)
maintainDetected = True # True if in transition frames, in case of overlapping bboxes, we want to keep those of the detector (False if we want to keep those of the tracker)
frameWidth = 512
''' some background subtractor with default params '''
# bgSubtractor = cv2.bgsegm.createBackgroundSubtractorMOG(history=200, nmixtures=5, backgroundRatio=0.7, noiseSigma=0)
# bgSubtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True)
# bgSubtractor = cv2.bgsegm.createBackgroundSubtractorGMG(initializationFrames=120, decisionThreshold=0.8)
# bgSubtractor = cv2.createBackgroundSubtractorKNN(history=500, dist2Threshold=400.0, detectShadows=True)
# bgSubtractor = CompositeBackgroundSubtractor(bgSubtractor1, bgSubtractor2, ...)
''' background subtractor '''
# define a background subtractor to be used for object detection
#bgSubtractor = cv2.createBackgroundSubtractorMOG2(history=200, varThreshold=25, detectShadows=True) # good for video/video_116.mp4
bgSubtractor = cv2.createBackgroundSubtractorKNN(
history=500, dist2Threshold=500.0,
detectShadows=True) # good for video/video_205.mp4
# bgSubtractor = CompositeBackgroundSubtractor(
# cv2.bgsegm.createBackgroundSubtractorMOG(history=600, nmixtures=3, backgroundRatio=0.2, noiseSigma=2.3),
# cv2.createBackgroundSubtractorMOG2(history=200, varThreshold=14, detectShadows=True)) # good for video/video_white.mp4
''' pipeline '''
# define the pipeline of functions to be executed on the b/w image, after the background subtraction and before getting bounding rects of contours
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
pipeline = ProcessPipeline()
pipeline \
.add(cv2.medianBlur, ksize=5) \
.add(cv2.dilate, kernel=kernel) \
.add(cv2.dilate, kernel=kernel) \
.add(cv2.dilate, kernel=kernel) \
.add(cv2.dilate, kernel=kernel) \
.add(cv2.dilate, kernel=kernel) \
.add(fillHoles) \
.add(cv2.erode, kernel=kernel) \
.add(cv2.erode, kernel=kernel) \
''' create object detector and face detector '''
od = ObjectDetector(bgSubtractor, pipeline)
fd = FaceDetector()
''' auto-definition of output folder '''
outputDir = "output"
if captureSource == 0:
outputDir = os.path.join(outputDir, "webcam")
else:
outputDir = os.path.join(outputDir,
captureSource[:captureSource.find(".")])
outputDir = os.path.join(outputDir, trackerName)
print("Tracking video '%s' with tracker %s" % (captureSource, trackerName))
''' cycle begins '''
frameNumber = 0
frames = 0
seconds = 0
eta = 0.05
totalTime = 0
show = True
oneSkipOnly = False
while True:
''' handle input: esc to quit; space to pause/start; "n" to go one frame at a time '''
k = cv2.waitKey(30) & 0xff
if k == 27:
break
elif k == ord(' ') or oneSkipOnly:
show = not show
oneSkipOnly = False
elif k == ord('n'):
show = True
oneSkipOnly = True
if not show:
if oneSkipOnly:
show = False
continue
start = timer()
''' reading next frame '''
ret, frameOrig = cap.read()
if not ret:
break
frameOrig = cv2.flip(frameOrig, 1)
frame = imutils.resize(frameOrig, width=frameWidth)
scale = frameOrig.shape[1] / frameWidth
detectedObjects = []
if frameNumber % period == 0:
''' detection by background subtraction '''
detectedObjects = od.detect(frame)
''' objects tracking, faces detection'''
''' tracking '''
success, objects = tm.update(frame,
detectedObjects,
maintainDetected=maintainDetected)
objIDs = tm.getIDs()
tm.removeDeadTrackers()
failed_objects = [obj for suc, obj in zip(success, objects) if not suc]
failed_objIDs = [
objID for suc, objID in zip(success, objIDs) if not suc
]
succ_objIDs = [objID for suc, objID in zip(success, objIDs) if suc]
objects = [obj for suc, obj in zip(success, objects) if suc]
''' detection of faces '''
faces_bboxes = fd.detectFaces(frameOrig, objects, objIDs, scale=scale)
''' images merging and show '''
frameOrig = draw_bboxes(frameOrig,
objects, (255, 0, 0),
succ_objIDs,
scale=scale)
frameOrig = draw_bboxes(frameOrig,
failed_objects, (0, 0, 255),
failed_objIDs,
scale=scale)
frameOrig = draw_bboxes(frameOrig, faces_bboxes, (0, 255, 0))
frameOrig = cv2.resize(frameOrig, (640, 640))
cv2.imshow('frame', frameOrig)
''' some stats '''
frameNumber += 1
end = timer()
frames = eta + (1 - eta) * frames
seconds = eta * (end - start) + (1 - eta) * seconds
print(
"\rFrame: %04d FPS: %03d Active trackers: %02d Failed trackers: %02d "
% (frameNumber, int(
frames // seconds), len(objects), len(failed_objects)),
end="")
totalTime += end - start
cap.release()
cv2.destroyAllWindows()
''' save on disk '''
fd.dump(outputDir)
avgFPS = str(round(frameNumber / totalTime, 2))
print("\rAverage FPS: " + avgFPS)
with open(os.path.join(outputDir, "info.txt"), "w") as file:
bgSubClsName = str(bgSubtractor.__class__)
bgSubClsName = bgSubClsName[bgSubClsName.index("'") +
1:bgSubClsName.rindex("'")]
file.write("tracker: " + trackerName + "\n")
file.write("background subtractor: " + bgSubClsName + "\n")
file.write("average FPS: " + avgFPS + "\n")
class Widget(QtWidgets.QWidget):
def __init__(self):
super(Widget, self).__init__()
self.scene = QtWidgets.QGraphicsScene()
self.scene.setSceneRect(0, 0, WIDTH, HEIGHT)
self.scene.changed.connect(self.changed)
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.classify)
self.timer.start(250)
self.view = QGraphicsView(self.scene)
self.view.setFixedSize(WIDTH, HEIGHT)
self.view.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.view.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.view.setMouseTracking(True)
# pixmap = QtGui.QPixmap("my_photo-1-crop.jpg")
# topbun_webcam = QGraphicsPixmapItem(pixmap)
# topbun_webcam.setFlags(QtWidgets.QGraphicsItem.ItemIsMovable)
# topbun_webcam.setScale(2)
# self.scene.addItem(topbun_webcam)
# lettuce = QGraphicsSvgItem("../../../assets/lettuce.svg")
# lettuce.setFlags(QtWidgets.QGraphicsItem.ItemIsMovable)
# lettuce.setScale(2)
# self.scene.addItem(lettuce)
self.image_widget = QtWidgets.QLabel(self)
self.image_widget.setFixedSize(WIDTH, HEIGHT)
self.icons = QtWidgets.QWidget(self)
self.icons.setFixedSize(128, 6 * 64)
self.iconsLayout = QtWidgets.QVBoxLayout()
self.icons.setLayout(self.iconsLayout)
self.topbun = QSvgWidget("../../../static/assets/topbun.svg")
self.lettuce = QSvgWidget("../../../static/assets/lettuce.svg")
self.tomato = QSvgWidget("../../../static/assets/tomato.svg")
self.cheese = QSvgWidget("../../../static/assets/cheese.svg")
self.patty = QSvgWidget("../../../static/assets/patty.svg")
self.bottombun = QSvgWidget("../../../static/assets/bottombun.svg")
self.banana = QSvgWidget("../../../static/assets/banana.svg")
self.book = QSvgWidget("../../../static/assets/book.svg")
self.shoe = QSvgWidget("../../../static/assets/shoe.svg")
self.iconsLayout.addWidget(self.topbun)
self.iconsLayout.addWidget(self.lettuce)
self.iconsLayout.addWidget(self.tomato)
self.iconsLayout.addWidget(self.cheese)
self.iconsLayout.addWidget(self.patty)
self.iconsLayout.addWidget(self.bottombun)
self.iconsLayout.addWidget(self.banana)
self.iconsLayout.addWidget(self.book)
self.iconsLayout.addWidget(self.shoe)
self.buttons = QtWidgets.QWidget(self)
self.buttonsLayout = QtWidgets.QHBoxLayout()
self.buttons.setLayout(self.buttonsLayout)
self.buttonA = QtWidgets.QPushButton("Classify")
self.buttonA.clicked.connect(self.buttonAClicked)
self.buttonsLayout.addWidget(self.buttonA)
self.layout = QtWidgets.QHBoxLayout(self)
self.layout.addWidget(self.icons)
# self.layout.addWidget(self.buttons)
self.layout.addWidget(self.view)
self.layout.addWidget(self.image_widget)
self.setLayout(self.layout)
self.objdet = ObjectDetector()
def topbun_clicked(self, *args):
print(args)
def buttonAClicked(self, *args):
self.classify()
def changed(self):
self.classify()
def classify(self):
image = QtGui.QImage(QtCore.QSize(WIDTH, HEIGHT),
QtGui.QImage.Format_RGB888)
image.fill(QtCore.Qt.white)
painter = QtGui.QPainter(image)
self.scene.render(painter)
painter.end()
bits = image.constBits().asstring(HEIGHT * WIDTH * 3)
img = np.fromstring(bits, dtype=np.uint8).reshape(HEIGHT, WIDTH, 3)
# image = canny(img)
expand = np.expand_dims(img, axis=0)
result = self.objdet.detect(expand)
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.4:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = box[0] * HEIGHT, box[1] * WIDTH
y2, x2 = box[2] * HEIGHT, box[3] * WIDTH
boxes.append((class_, score, x1, y1, x2, y2))
pixmap = QtGui.QPixmap.fromImage(image)
p = QtGui.QPainter()
p.begin(pixmap)
for box in boxes:
p.setPen(QtCore.Qt.red)
class_, score, x1, y1, x2, y2 = box
w = x2 - x1
h = y2 - y1
p.drawRect(x1, y1, w, h)
p.drawText(x1, y1, "%s: %5.2f" % (labels[class_], score))
p.end()
self.image_widget.setPixmap(pixmap)
def run(model: str, camera_id: int, width: int, height: int, num_threads: int,
enable_edgetpu: bool) -> None:
"""Continuously run inference on images acquired from the camera.
Args:
model: Name of the TFLite object detection model.
camera_id: The camera id to be passed to OpenCV.
width: The width of the frame captured from the camera.
height: The height of the frame captured from the camera.
num_threads: The number of CPU threads to run the model.
enable_edgetpu: True/False whether the model is a EdgeTPU model.
"""
# Variables to calculate FPS
counter, fps = 0, 0
start_time = time.time()
# Start capturing video input from the camera
cap = cv2.VideoCapture(camera_id)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
# Visualization parameters
row_size = 20 # pixels
left_margin = 24 # pixels
text_color = (0, 0, 255) # red
font_size = 1
font_thickness = 1
fps_avg_frame_count = 10
# Initialize the object detection model
options = ObjectDetectorOptions(num_threads=num_threads,
score_threshold=0.3,
max_results=3,
enable_edgetpu=enable_edgetpu)
detector = ObjectDetector(model_path=model, options=options)
# Continuously capture images from the camera and run inference
while cap.isOpened():
success, image = cap.read()
if not success:
sys.exit(
'ERROR: Unable to read from webcam. Please verify your webcam settings.'
)
counter += 1
image = cv2.flip(image, 1)
# Run object detection estimation using the model.
detections = detector.detect(image)
# Draw keypoints and edges on input image
image = utils.visualize(image, detections)
# Calculate the FPS
if counter % fps_avg_frame_count == 0:
end_time = time.time()
fps = fps_avg_frame_count / (end_time - start_time)
start_time = time.time()
# Show the FPS
fps_text = 'FPS = {:.1f}'.format(fps)
text_location = (left_margin, row_size)
cv2.putText(image, fps_text, text_location, cv2.FONT_HERSHEY_PLAIN,
font_size, text_color, font_thickness)
# Stop the program if the ESC key is pressed.
if cv2.waitKey(1) == 27:
break
cv2.imshow('object_detector', image)
cap.release()
cv2.destroyAllWindows()
cam = cv2.VideoCapture(args.movie)
width = int(cam.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cam.get(cv2.CAP_PROP_FRAME_HEIGHT))
objdet = ObjectDetector()
counter = 0
ret, img = cam.read()
while ret == True:
img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h, w, _ = img.shape
expand = np.expand_dims(img, axis=0)
result = objdet.detect(expand)
boxes = []
for i in range(result['num_detections']):
if result['detection_scores'][i] > 0.75:
class_ = result['detection_classes'][i]
box = result['detection_boxes'][i]
score = result['detection_scores'][i]
y1, x1 = int(box[0] * h), int(box[1] * w)
y2, x2 = int(box[2] * h), int(box[3] * w)
boxes.append((class_, score, x1, y1, x2, y2))
for box in boxes:
class_, score, x1, y1, x2, y2 = box
w1 = x2-x1
h1 = y2-y1
cv2.rectangle(img, (x1, y1), (x2, y2), (255,0,0), 2)