def performAnalyse(self, htmlnode):
se = StructureExtractor()
se.drawFeature(htmlnode)
extractor = Extractor(htmlnode)
extractor.process()
detector = Detector()
detector.detect(htmlnode)
(tp, fp, fn) = self.calcAccTitleLine(htmlnode)
cerr = self.calcAccColumn(htmlnode)
return (tp, fp, fn, cerr)
def process(self,root):
se = StructureExtractor()
se.drawFeature(root)
self.extractor = Extractor(root)
self.extractor.process()
#self.crossP = self.extractor.string2sparse(self.htmlnode.attrib['crossP'],self.extractor.totalheight+1)
Config.init()
detector = Detector()#Config.nbTLstr)
detector.detect(root)
self.toolbox.setDetector(detector)
def gen(camera):
"""Video streaming generator function"""
while True:
global referenceFrame
frame = camera.get_frame()
data = np.fromstring(frame, dtype=np.uint8)
image = cv2.imdecode(data, 1)
if referenceFrame is None:
_, _, referenceFrame = Detector.generateReferenceImageItem(image)
referenceFrameDeque = Detector.generateReferenceImageDeque(
referenceFrame, REFERENCE_FRAME_AVERAGE_FRAME_COUNT)
referenceFrameLastUpdate = TimeService.getTimestamp()
if TimeService.getTimestamp(
) > referenceFrameLastUpdate + REFERENCE_FRAME_UPDATE_INTERVAL_MS:
_, _, referenceFrameItem = Detector.generateReferenceImageItem(
image)
referenceFrameDeque.append(referenceFrameItem)
referenceFrame = Detector.generateReferenceImage(
referenceFrameDeque)
referenceFrameLastUpdate = TimeService.getTimestamp()
detectedRegionRectangles, processedImage, deltaImage, thresholdImage = Detector.detect(
MIN_HEIGHT, MIN_WIDTH, referenceFrame, image)
if DRAW_DETECTED_REGIONS and len(detectedRegionRectangles):
for rectangle in detectedRegionRectangles:
ImageManipulator.drawRectangleMutable(processedImage,
rectangle)
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' +
cv2.imencode(".jpeg", processedImage)[1].tostring() + b'\r\n')
detector = Detector(Detector.Params())
# img = cv2.imread("p__3.jpg")
j = 0
calib_img_pnts = []
calib_obj_pnts = []
objects_good = []
if not os.path.isdir(f"dataset_warped"):
os.mkdir("dataset_warped")
for p in os.listdir("dataset_all"):
if not ("big" in p):
t = cv2.imread("dataset_all/" + p)
ts = time.time()
objs = detector.detect(t)
cv2.waitKey(0)
# print("objs", objs)
print("RUN time", (time.time() - ts) * 1000)
points_ob_for_calib = []
points_im_for_calib = []
for o in objs:
if "warp_img" in o.other_props.keys():
# if os.dir
# os.mkdir()
if str(o.type).lower() == "none":
print("FILE ", p)
cv2.imshow('o.other_props["warp_img"]',
o.other_props["warp_img"])
cv2.waitKey(0)
continue
class EnvironmentRealTime:
def __init__(self, image_shape=(720, 1280), step_sizes=[-40, -20, 0, 20, 40], max_guided_eps=1000):
self.current_episode = 0
self.max_guided_eps = max_guided_eps
self.im_width = image_shape[1]
self.im_height = image_shape[0]
self.step_sizes = step_sizes
self.fig = None
self.max_dist = np.linalg.norm([self.im_width, self.im_height])
self.SCALE_IOU = 8.0
self.SCALE_DIST = 1.0
self.min_reward = 0.08
self.current_frame = None
self.current_output = None
self._detector = Detector()
self._connector = MultiRotorConnector()
self._car_connector = CarConnector()
self.old_x = None
self.old_y = None
self.TEST = False
def state_to_array(self, state):
out = np.zeros((2,), dtype='float32')
out[0] = float(state.DELTA_X)/float(self.im_width)
out[1] = float(state.DELTA_Y)/float(self.im_height)
return out
def nearest_to_step(self, dist):
nearest_dist = [abs(step_size - dist) for step_size in self.step_sizes]
return self.step_sizes[nearest_dist.index(min(nearest_dist))]
def reset(self):
self.current_episode += 1
car_pos, car_ort = self._car_connector.reset()
offset = (car_pos.x_val, car_pos.y_val, self._connector.INIT_Z)
self._connector.move_to_position(offset)
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if not output:
raise Exception('Unable to Detect')
POS_X1 = output[0]
POS_Y1 = output[1]
WIDTH1 = output[2]
HEIGHT1 = output[3]
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if not output:
raise Exception('Unable to Detect')
POS_X2 = output[0]
POS_Y2 = output[1]
WIDTH2 = output[2]
HEIGHT2 = output[3]
_state = State()
_state.DELTA_X = POS_X2 - POS_X1
_state.DELTA_Y = POS_Y2 - POS_Y1
print "\n-----Parameters-----"
print "Delta X:", _state.DELTA_X
print "Delta Y:", _state.DELTA_Y
self.old_x = POS_X1
self.old_y = POS_Y1
self.old_gx = POS_X1
self.old_gy = POS_Y1
self.current_frame = frame
self.current_state = _state
self.current_output = output
return self.state_to_array(_state)
def get_iou(self, bb1, bb2):
assert bb1['x1'] < bb1['x2']
assert bb1['y1'] > bb1['y2']
assert bb2['x1'] < bb2['x2']
assert bb2['y1'] > bb2['y2']
x_left = max(bb1['x1'], bb2['x1'])
y_top = min(bb1['y1'], bb2['y1'])
x_right = min(bb1['x2'], bb2['x2'])
y_bottom = max(bb1['y2'], bb2['y2'])
if x_right < x_left or y_bottom > y_top:
return 0.0
intersection_area = (x_right - x_left) * (y_bottom - y_top)
bb1_area = (bb1['x2'] - bb1['x1']) * (bb1['y2'] - bb1['y1'])
bb2_area = (bb2['x2'] - bb2['x1']) * (bb2['y2'] - bb2['y1'])
iou = intersection_area / float(bb1_area + bb2_area - intersection_area)
assert iou >= 0.0
assert iou <= 1.0
return iou
def step(self, action):
# TRACKER
done = 0
reward = 0.0
frame = self.current_frame.copy()
output = self.current_output
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
det_box = {
'x1': POS_X - WIDTH/2,
'x2': POS_X + WIDTH/2,
'y1': POS_Y + HEIGHT/2,
'y2': POS_Y - HEIGHT/2
}
print "\n-----Parameters-----"
print "Delta X:", self.current_state.DELTA_X
print "Delta Y:", self.current_state.DELTA_Y
if action is not None:
# GREEDY (BASELINE) AGENT
step_x = self.nearest_to_step(POS_X - self.old_gx)
step_y = self.nearest_to_step(POS_Y - self.old_gy)
new_gx = self.old_gx + step_x
new_gy = self.old_gy + step_y
baseline_box = {
'x1': new_gx - WIDTH/2,
'x2': new_gx + WIDTH/2,
'y1': new_gy + HEIGHT/2,
'y2': new_gy - HEIGHT/2
}
# CUSTOM AGENT
new_x = self.old_x + action[0]
new_y = self.old_y + action[1]
agent_box = {
'x1': new_x - WIDTH/2,
'x2': new_x + WIDTH/2,
'y1': new_y + HEIGHT/2,
'y2': new_y - HEIGHT/2
}
# vis_util.draw_bounding_boxes_on_image_array( frame,
# np.array([[(float(self.im_height/2 - det_box['y1'])/float(self.im_height)),
# (float(det_box['x1'] + self.im_width/2)/float(self.im_width)),
# (float(self.im_height/2 - det_box['y2'])/float(self.im_height)),
# (float(det_box['x2'] + self.im_width/2)/float(self.im_width))]]),
# color='blue',
# thickness=3)
# vis_util.draw_bounding_boxes_on_image_array( frame,
# np.array([[(float(self.im_height/2 - agent_box['y1'])/float(self.im_height)),
# (float(agent_box['x1'] + self.im_width/2)/float(self.im_width)),
# (float(self.im_height/2 - agent_box['y2'])/float(self.im_height)),
# (float(agent_box['x2'] + self.im_width/2)/float(self.im_width))]]),
# color='yellow',
# thickness=5)
# vis_util.draw_bounding_boxes_on_image_array( frame,
# np.array([[(float(self.im_height/2 - baseline_box['y1'])/float(self.im_height)),
# (float(baseline_box['x1'] + self.im_width/2)/float(self.im_width)),
# (float(self.im_height/2 - baseline_box['y2'])/float(self.im_height)),
# (float(baseline_box['x2'] + self.im_width/2)/float(self.im_width))]]),
# color='red',
# thickness=5)
iou_constrained = self.get_iou(det_box, agent_box)
iou_baseline = self.get_iou(det_box, baseline_box)
print "-----IoU Stats-----"
print "~Cons IoU :", iou_constrained
print "~Grdy IoU :", iou_baseline
if not self.TEST:
dist_x = float(agent_box['x1'] + agent_box['x2'])/2.0 - float(det_box['x1'] + det_box['x2'])/2.0
dist_y = float(agent_box['y1'] + agent_box['y2'])/2.0 - float(det_box['y1'] + det_box['y2'])/2.0
dist = np.linalg.norm([dist_x, dist_y])/self.max_dist
reward = ((1-dist)*self.SCALE_DIST + iou_constrained*self.SCALE_IOU)/(self.SCALE_DIST + self.SCALE_IOU)
print "Distance :", dist, \
"\nIoU :", iou_constrained, \
"\nDist Reward:", (1-dist)*self.SCALE_DIST, \
"\nIoU Reward :", iou_constrained*self.SCALE_IOU
print "Action RL :", action, "pixels"
print "Action BASE:", (step_x, step_y), "pixels"
# cv2.imshow('Simulation', frame)
# cv2.waitKey(2)
self.old_x = new_x
self.old_y = new_y
if (not self.TEST) and self.current_episode<=self.max_guided_eps:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = new_gx
self.old_gy = new_gy
else:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = POS_X
self.old_gy = POS_Y
# NEXT frame
_state = State()
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if (not output) or reward<=self.min_reward:
done = 1
reward = -20.0
else:
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
_state.DELTA_X = POS_X - self.old_x
_state.DELTA_Y = POS_Y - self.old_y
self.current_state = _state
self.current_output = output
self.current_frame = frame
print "Reward :", reward
print "Done :", done
return self.state_to_array(_state), reward, done
class cdc(QtGui.QMainWindow, cdcui.Ui_MainWindow):
def __init__(self, parent=None):
super(cdc, self).__init__(parent)
# video stream is initially paused
self.isPaused = False
# mission is not running initially
self.isRunning = False
# establish a connection the quadcopter
print 'Opening telemetry radio connection...'
self.quad = None
for i in range(0, 5):
tty = '/dev/ttyUSB' + str(i)
try:
self.quad = mavutil.mavlink_connection(device=tty, baud=57600)
except:
print 'No telemetry radio on ' + tty
if self.quad:
print 'Telemetry radio found on ' + tty
break
if self.quad is None:
print 'Could not open telemetry radio connection'
return
# min and max sequence number
# determines when to start and stop capturing images
self.quad.MIN_WP_SEQ = 1
self.quad.MAX_WP_SEQ = 90
# CC3100 connection information
self.host = '192.168.1.1'
self.port = 7277
# set up the GoProController object used for pulling static images
# Ethernet Adapter is wlan0; USB Wi-Fi Adapter is wlan1
self.gpc = GoProController(device_name='wlan0')
self.gpc.connect('SARSGoPro', 'sarsgopro')
# set up the Detector object for testing images pulled from the GoPro
self.det = Detector()
# set up the UI generated by Qt/PyQt
self.setupUi(self)
# create and set up the VLC instance and associated media player
self.instance = libvlc.Instance()
self.mediaplayer = self.instance.media_player_new()
self.setupVLC()
# set up the telemetry window so it can periodically update with diagnostic info
self.setupTelemetry()
# set up the command console so it can process user input
self.setupCmdCon()
def setupVLC(self):
"""Set up the vlc frame, signals & slots"""
# In this widget, the video will be drawn
self.videoframe = QtGui.QFrame()
self.palette = self.videoframe.palette()
self.palette.setColor(QtGui.QPalette.Window, QtGui.QColor(0, 0, 0))
self.videoframe.setPalette(self.palette)
self.videoframe.setAutoFillBackground(True)
self.hbuttonbox = QtGui.QHBoxLayout()
self.playbutton = QtGui.QPushButton('Play')
self.hbuttonbox.addWidget(self.playbutton)
self.connect(self.playbutton, QtCore.SIGNAL('clicked()'),
self.PlayPause)
self.stopbutton = QtGui.QPushButton('Stop')
self.hbuttonbox.addWidget(self.stopbutton)
self.connect(self.stopbutton, QtCore.SIGNAL('clicked()'), self.Stop)
self.hbuttonbox.addStretch(1)
self.volumeslider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.volumeslider.setMaximum(100)
self.volumeslider.setValue(100)
self.volumeslider.setToolTip('Volume')
self.hbuttonbox.addWidget(self.volumeslider)
self.connect(self.volumeslider, QtCore.SIGNAL('valueChanged(int)'),
self.setVolume)
self.vboxlayout = QtGui.QVBoxLayout()
self.vboxlayout.addWidget(self.videoframe)
self.vboxlayout.addLayout(self.hbuttonbox)
self.widgetVideoStream.setLayout(self.vboxlayout)
self.vlcTimer = QtCore.QTimer(self)
self.vlcTimer.setInterval(200)
self.connect(self.vlcTimer, QtCore.SIGNAL('timeout()'), self.updateVLC)
def setupTelemetry(self):
"""Set up the telemetry timer, signals & slots"""
self.telemTimer = QtCore.QTimer(self)
self.telemTimer.setInterval(1000)
self.connect(self.telemTimer, QtCore.SIGNAL('timeout()'),
self.updateTelemetry)
self.telemTimer.start()
def setupCmdCon(self):
self.connect(self.lineEditCommandConsole,
QtCore.SIGNAL('returnPressed()'), self.processCmd)
def PlayPause(self):
"""Toggle play/pause status"""
if self.mediaplayer.is_playing():
self.mediaplayer.pause()
self.playbutton.setText('Play')
self.isPaused = True
else:
if self.mediaplayer.play() == -1:
self.OpenURL('http://10.5.5.9:8080/live/amba.m3u8')
return
while not self.mediaplayer.is_playing():
self.mediaplayer.play()
self.playbutton.setText('Pause')
self.vlcTimer.start()
self.isPaused = False
def Stop(self):
"""Stop player"""
self.mediaplayer.stop()
self.playbutton.setText('Play')
def OpenURL(self, url=None):
"""Open a URL in a MediaPlayer"""
if url is None:
url = QtGui.QInputDialog.getText(self, 'Enter URL', '',
QtGui.QLineEdit.Normal, '')
if url[0] == '' or url[1] == False:
return
else:
url = url[0]
# create the media
self.media = self.instance.media_new(unicode(url))
# put the media in the media player
self.mediaplayer.set_media(self.media)
# parse the metadata of the file
self.media.parse()
# the media player has to be 'connected' to the QFrame
# (otherwise a video would be displayed in it's own window)
self.mediaplayer.set_xwindow(self.videoframe.winId())
self.PlayPause()
def setVolume(self, Volume):
"""Set the volume"""
self.mediaplayer.audio_set_volume(Volume)
def updateVLC(self):
"""updates the GoPro video feed"""
if not self.mediaplayer.is_playing():
# no need to call this function if nothing is played
self.vlcTimer.stop()
if not self.isPaused:
# after the video finished, the play button stills shows
# 'Pause', not the desired behavior of a media player
# this will fix it
self.Stop()
def updateTelemetry(self):
"""updates the quadcopter and rover telemetry information"""
# get quadcopter telemetry
self.quad.gpi = self.quad.recv_match(type='GLOBAL_POSITION_INT',
blocking=False)
if self.quad.gpi is not None:
self.plainTextEditTelemetry.setPlainText('Quadcopter:')
self.plainTextEditTelemetry.appendPlainText('Latitude:\t\t' +
str(self.quad.gpi.lat /
10000000.0))
self.plainTextEditTelemetry.appendPlainText('Longitude:\t\t' +
str(self.quad.gpi.lon /
10000000.0))
self.plainTextEditTelemetry.appendPlainText('Heading:\t\t' +
str(self.quad.gpi.hdg /
100.0))
self.plainTextEditTelemetry.appendPlainText(
'Altitude:\t\t' + str(self.quad.gpi.relative_alt / 1000.0))
self.plainTextEditTelemetry.appendPlainText('Velocity (X):\t' +
str(self.quad.gpi.vx))
self.plainTextEditTelemetry.appendPlainText('Velocity (Y):\t' +
str(self.quad.gpi.vy))
self.plainTextEditTelemetry.appendPlainText('Velocity (Z):\t' +
str(self.quad.gpi.vz))
# get rover telemetry
self.rover_speed = self.sendTCP('s')
self.rover_ping_dist = self.sendTCP('p')
self.rover_wp_dist = self.sendTCP('d')
self.rover_gps_lat = self.sendTCP('a')
self.rover_gps_lon = self.sendTCP('o')
self.rover_heading = self.sendTCP('h')
self.plainTextEditTelemetry.appendPlainText('Rover:')
self.plainTextEditTelemetry.appendPlainText('Speed:\t\t' +
self.rover_speed)
self.plainTextEditTelemetry.appendPlainText('Clearance:\t\t' +
self.rover_ping_dist)
self.plainTextEditTelemetry.appendPlainText('Waypoint Distance:\t' +
self.rover_wp_dist)
self.plainTextEditTelemetry.appendPlainText('Latitude:\t\t' +
self.rover_gps_lat)
self.plainTextEditTelemetry.appendPlainText('Longitude:\t\t' +
self.rover_gps_lon)
self.plainTextEditTelemetry.appendPlainText('Heading:\t\t' +
self.rover_heading)
def processCmd(self):
cmd = self.lineEditCommandConsole.text()
if cmd == 'get-image':
image = self.gpc.getImage('SARSGoPro', 'sarsgopro')
if image:
self.plainTextEditMissionStatus.appendPlainText(
'Image downloaded succeeded!')
else:
self.plainTextEditMissionStatus.appendPlainText(
'Image download failed.')
elif cmd == 'test-image':
test = self.det.detect('image.png')
if test:
self.plainTextEditMissionStatus.appendPlainText(
'Object detected!')
else:
self.plainTextEditMissionStatus.appendPlainText(
'Object not detected.')
elif cmd == 'run':
if not self.isRunning:
self.runMission()
elif cmd == 'sendwp1':
self.plainTextEditMissionStatus.appendPlainText(
'Sending Waypoint 1 to the Rover...')
msg = self.sendTCP('1')
self.plainTextEditMissionStatus.appendPlainText('Rover status: ' +
msg)
elif cmd == 'sendwp2':
self.plainTextEditMissionStatus.appendPlainText(
'Sending Waypoint 2 to the Rover...')
msg = self.sendTCP('2')
self.plainTextEditMissionStatus.appendPlainText('Rover status: ' +
msg)
elif cmd == 'sendwp3':
self.plainTextEditMissionStatus.appendPlainText(
'Sending Waypoint 3 to the Rover...')
msg = self.sendTCP('3')
self.plainTextEditMissionStatus.appendPlainText('Rover status: ' +
msg)
elif cmd == 'sendwp4':
self.plainTextEditMissionStatus.appendPlainText(
'Sending Waypoint 4 to the Rover...')
msg = self.sendTCP('4')
self.plainTextEditMissionStatus.appendPlainText('Rover status: ' +
msg)
elif cmd == 'sendwp5':
self.plainTextEditMissionStatus.appendPlainText(
'Sending Waypoint 5 to the Rover...')
msg = self.sendTCP('5')
self.plainTextEditMissionStatus.appendPlainText('Rover status: ' +
msg)
elif cmd == 'sendwp6':
self.plainTextEditMissionStatus.appendPlainText(
'Sending Waypoint 6 to the Rover...')
msg = self.sendTCP('6')
self.plainTextEditMissionStatus.appendPlainText('Rover status: ' +
msg)
else:
self.plainTextEditMissionStatus.appendPlainText(
'Unrecognized command.')
self.lineEditCommandConsole.setText('')
def runMission(self):
"""start the show"""
self.isRunning = True
self.plainTextEditMissionStatus.appendPlainText(
'Initiating Mission...')
QtGui.QApplication.processEvents()
# create an array to store all pulled images
self.imgs = []
# fetch the initial sequence number
msg = self.quad.recv_match(type='MISSION_CURRENT', blocking=True)
self.quad.current_seq = msg.seq
self.plainTextEditMissionStatus.appendPlainText(
'Starting Sequence Number: ' + str(self.quad.current_seq))
QtGui.QApplication.processEvents()
# loop until we've made it past the takeoff sequence
in_takeoff_seq = True
while in_takeoff_seq:
msg = self.quad.recv_match(type='MISSION_CURRENT', blocking=True)
self.quad.current_seq = msg.seq
if self.quad.current_seq < self.quad.MIN_WP_SEQ:
self.plainTextEditMissionStatus.appendPlainText(
'Waiting until takeoff has finished... Current Sequence: '
+ str(self.quad.current_seq))
QtGui.QApplication.processEvents()
else:
in_takeoff_seq = False
self.plainTextEditMissionStatus.appendPlainText(
'Takeoff finished! Current Sequence: ' +
str(self.quad.current_seq))
QtGui.QApplication.processEvents()
# loop through waypoint sequences until we reach the landing sequence
failed = False
in_waypoint_seq = True
while not failed and in_waypoint_seq:
if self.quad.current_seq == 2 or self.quad.current_seq == 32 or self.quad.current_seq == 62:
# wait until we've reached the waypoint
msg = self.quad.recv_match(type='NAV_CONTROLLER_OUTPUT',
blocking=True)
while msg.wp_dist > 0:
msg = self.quad.recv_match(type='NAV_CONTROLLER_OUTPUT',
blocking=True)
self.plainTextEditMissionStatus.appendPlainText(
'Waypoint Distance: ' + str(msg.wp_dist))
QtGui.QApplication.processEvents()
self.plainTextEditMissionStatus.appendPlainText(
'Waypoint reached!')
QtGui.QApplication.processEvents()
# wait 3 seconds, then capture an image from the GoPro
self.plainTextEditMissionStatus.appendPlainText(
'Waiting 2 seconds...')
for i in range(1, 3):
time.sleep(1)
self.plainTextEditMissionStatus.appendPlainText(str(i))
QtGui.QApplication.processEvents()
self.plainTextEditMissionStatus.appendPlainText(
'Capturing image...')
img = self.gpc.getImage('SARSGoPro', 'sarsgopro')
if img:
im = Image.open('image.png')
pix = im.load()
self.imgs.append(pix)
self.plainTextEditMissionStatus.appendPlainText(
'Image capture succeeded!')
QtGui.QApplication.processEvents()
else:
failed = True
self.plainTextEditMissionStatus.appendPlainText(
'Image capture failed. Abort mission.')
QtGui.QApplication.processEvents()
return
# wait for the sequence number to increment
incremented = False
while not incremented:
msg = self.quad.recv_match(type='MISSION_CURRENT',
blocking=True)
if self.quad.current_seq == msg.seq:
self.plainTextEditMissionStatus.appendPlainText(
'Waiting for sequence number to increment...')
QtGui.QApplication.processEvents()
else:
self.quad.current_seq = msg.seq
incremented = True
self.plainTextEditMissionStatus.appendPlainText(
'Sequence number incremented! Current Sequence Number: '
+ str(self.quad.current_seq))
QtGui.QApplication.processEvents()
if self.quad.current_seq >= self.quad.MAX_WP_SEQ:
in_waypoint_seq = False
self.plainTextEditMissionStatus.appendPlainText(
'All waypoints visited!')
QtGui.QApplication.processEvents()
# run the object detection the images
if not failed:
self.plainTextEditMissionStatus.appendPlainText(
'Running object detection..')
QtGui.QApplication.processEvents()
found = False
for pix in self.imgs:
if (self.det.detect(pix)):
found = True
self.plainTextEditMissionStatus.appendPlainText(
'Object detected at waypoint ' +
str(self.imgs.index(pix) + 1) + '!')
QtGui.QApplication.processEvents()
self.plainTextEditMissionStatus.appendPlainText(
'Sending waypoint ' + str(self.imgs.index(pix) + 1) +
' to the rover...')
QtGui.QApplication.processEvents()
self.sendTCP(str(self.imgs.index(pix) + 1))
break
if not found:
self.plainTextEditMissionStatus.appendPlainText(
'Object not detected.')
QtGui.QApplication.processEvents()
self.isRunning = False
def sendTCP(self, str):
try:
s = socket.socket()
s.connect((self.host, self.port))
s.settimeout(0.5)
s.send(str)
for i in range(0, 500):
time.sleep(0.001)
QtGui.QApplication.processEvents()
msg = s.recv(2**10)
QtGui.QApplication.processEvents()
msg = msg.rstrip()
s.close()
return msg
except Exception as e:
print e
dirimgs = os.path.join('/home/xian/imaxes_politicos/Imaxes')
dircaras = os.path.join('/home/xian/imaxes_politicos/Caras')
factor = 2
detector = Detector('SSD-mobilenet-face', threshold=0.5)
detector.initialize()
people = os.listdir(dirimgs)
for person in people:
imgs = os.listdir(os.path.join(dirimgs, person))
for imgname in imgs:
print(person + ' - ' + imgname)
imaxe = cv2.imread(os.path.join(dirimgs, person, imgname))
result = detector.detect(imaxe)
if len(result) == 0:
continue
elif len(result) == 1:
detection = result[0]
else:
conf = []
for det in result:
conf.append(det.conf)
idx = np.argmax(np.float32(conf))
detection = result[idx]
detection.width *= factor
detection.height *= factor
face = utils.crop_image_from_detection_with_square(imaxe, detection)
newname = os.path.join(dircaras, person, imgname)
cv2.imwrite(newname, face)
class DetectorNode:
def __init__(self):
self.posePublisher_front = rospy.Publisher('/pose_estimator/charger_pose/detection_front', PoseStamped,
queue_size=1)
self.posePublisher_back = rospy.Publisher('/pose_estimator/charger_pose/detection_back', PoseStamped,
queue_size=1)
self.keypointsPublisher = rospy.Publisher('/pose_estimator/keypoints', KeypointsWithCovarianceStamped,
queue_size=1)
self.blackfly_topic = '/blackfly/camera/image_color/compressed'
self.pointgrey_topic = '/pointgrey/camera/image_color/compressed'
# self.camera_topic = '/video_player/compressed'
self.imu_topic = '/xsens/data'
self.gt_pose_topic = '/pose_estimator/charger_pose/location_gps'
rospy.init_node('deep_pose_estimator', log_level=rospy.DEBUG)
# path_to_model_bottom = "/root/share/tf/Keras/09_05_bottom_PP"
path_to_model_bottom = "/root/share/tf/Keras/18_06_PP_4_wo_mask_bigger_head"
# path_to_model_front = "/root/share/tf/Keras/24_08_4pts_1280_ROI560_5plus"
# path_to_model_front = "/root/share/tf/Keras/07_08_4pts_960"
path_to_model_front = "/root/share/tf/Keras/27_08_4pts_960_ROI960_pool_36"
# path_to_model_front = "/root/share/tf/Keras/28_07_aug_960_blob"
path_to_pole_model = os.path.join("/root/share/tf/Faster/pole/model_Inea_3", 'frozen_inference_graph.pb')
self.equalize_histogram = False
self.blackfly_camera_matrix = np.array([[4885.3110509, 0, 2685.5111516],
[0, 4894.72687634, 2024.08742622],
[0, 0, 1]]).astype(np.float64)
# self.blackfly_camera_matrix = np.array([[ 4921.3110509, 0, 2707.5111516],
# [0, 4925.72687634, 1896.08742622],
# [0, 0, 1]]).astype(np.float64)
self.blackfly_camera_distortion = (-0.14178835, 0.09305661, 0.00205776, -0.00133743)
self.pointgrey_camera_matrix = np.array([[671.18360957, 0, 662.70347005],
[0, 667.94555172, 513.26149201],
[0, 0, 1]]).astype(np.float64)
self.pointgrey_camera_distortion = (-0.04002205, 0.04100822, 0.00137423, 0.00464031, 0.0)
# self.pitch = None
# self.frame_pitch = None
self.keypoints = None
self.gt_keypoints = []
# self.gt_keypoints_slice = []
self.blackfly_image_msg = None
self.blackfly_image = None
self.pointgrey_image_msg = None
self.pointgrey_image = None
self.gt_pose = None
# self.gt_mat = None
self.frame_gt = None
# self.frame_scale = None
# Detection performance
self.all_frames = 0
self.frames_sent_to_detector = 0
self.detected_frames = 0
# Uncertainty estimation
self.total_kp = 0
self.prediction_errors = []
# self.cov_matrices = np.empty((10, 10, 0), np.float)
self.cov_matrices = np.empty((8, 8, 0), np.float)
# Initialize detector
self.pointgrey_frame_shape = (5, 5) # self.get_image_shape(self.pointgrey_topic)
# self.frame_shape = self.get_image_shape(self.blackfly_topic)
# self.frame_shape = (1280, 1280)
self.frame_shape = (960, 960)
self.detector = Detector(path_to_model_front,
path_to_pole_model) # , path_to_model_bottom=path_to_model_bottom)
self.detector.init_size(self.frame_shape)
# self.detector.init_size((5000,5000))
self.pose_estimator = PoseEstimator(self.blackfly_camera_matrix)
def get_image_shape(self, camera_topic):
im = rospy.wait_for_message(camera_topic, CompressedImage)
np_arr = np.fromstring(im.data, np.uint8)
image_shape = cv2.imdecode(np_arr, -1).shape[:2]
return list(image_shape)
def start(self):
# rospy.Subscriber(self.blackfly_topic, CompressedImage, self.update_blackfly_image, queue_size=1)
# rospy.Subscriber(self.pointgrey_topic, CompressedImage, self.update_pointgrey_image, queue_size=1)
# rospy.Subscriber(self.imu_topic, Imu, self.get_pitch, queue_size=1)
rospy.Subscriber(self.gt_pose_topic, PoseStamped, self.update_gt, queue_size=1)
# rospy.wait_for_message(self.pointgrey_topic, CompressedImage)
# rospy.wait_for_message(self.blackfly_topic, CompressedImage)
# base_path = '/root/share/tf/dataset/14_08_4pts_960_ROI_448'
base_path = '/root/share/tf/dataset/4_point_aug_960_centered/val/'
# base_path = '/root/share/tf/dataset/4_point_aug_1280_centered/val/'
images = os.listdir(os.path.join(base_path, 'annotations'))
random.shuffle(images)
# images.sort(reverse=False)
images = iter(images)
while not rospy.is_shutdown():
self.gt_keypoints = []
try:
fname = next(images)
# if 'train1' in fname:
img_fname = base_path + 'images/' + fname[:-4] + '.png'
ann_fname = base_path + 'annotations/' + fname
# print(img_fname)
tree = ET.parse(ann_fname)
root = tree.getroot()
if not os.path.exists(img_fname):
continue
self.blackfly_image = cv2.imread(img_fname)
# self.image = cv2.resize(image, None, fx=self.detector.scale, fy=self.detector.scale)
# self.frame_shape = self.blackfly_image.shape[:2]
# self.detector.init_size(self.frame_shape)
# print("fr_sh", self.frame_shape)
# label = cv2.imread(label_fname) * 255
size = root.find('size')
w = int(size.find('width').text)
h = int(size.find('height').text)
offset_x = int(root.find('offset_x').text)
offset_y = int(root.find('offset_y').text)
# print("offsets", offset_y, offset_x)
for object in root.findall('object'):
scale = float(object.find('scale').text)
# print("scale", scale)
bbox = object.find('bndbox')
kps = object.find('keypoints')
# keypoints = []
# print(kps.find('keypoint6'))
for i in range(4):
kp = kps.find('keypoint' + str(i))
self.gt_keypoints.append(
(int(float(kp.find('x').text) * w), int((float(kp.find('y').text) * h))))
# for i, kp in enumerate(keypoints):
# self.gt_keypoints.append(
# ((int(kp[0] * w)), (int(kp[1] * h))))
except StopIteration:
# sum_cov_mtx = np.zeros((10, 10))
sum_cov_mtx = np.zeros((8, 8))
for single_img_pred in self.prediction_errors:
# print(np.average(self.kp_predictions, axis=0))
# single_img_error = single_img_pred - np.average(self.prediction_errors)
# print(single_img_pred.shape)
# print(np.transpose(single_img_pred).shape)
cov_matrix = np.matmul(single_img_pred, np.transpose(single_img_pred))
sum_cov_mtx += cov_matrix
print(sum_cov_mtx / len(self.prediction_errors))
break
if self.blackfly_image is not None: # and self.pointgrey_image is not None:
self.frame_gt = self.gt_pose
# self.frame_pitch = self.pitch
# self.frame_scale = self.detector.scale
k = cv2.waitKey(500)
if k == ord('q') or k == 27:
exit(0)
if k == ord('z'):
# sum_cov_mtx = np.zeros((10, 10))
sum_cov_mtx = np.zeros((8, 8))
for single_img_pred in self.prediction_errors:
# print("avg", np.average(self.prediction_errors, axis=0).shape, "\n", np.average(self.prediction_errors, axis=0))
# single_img_error = single_img_pred - np.average(self.prediction_errors, axis=0)
# print("single_img_pred", single_img_pred.shape)
# print("single_img_error", single_img_error.shape)
# cov_matrices = np.matmul(single_img_error, np.transpose(single_img_error))
cov_matrices = np.matmul(single_img_pred, np.transpose(single_img_pred))
sum_cov_mtx += cov_matrices
print(sum_cov_mtx / len(self.prediction_errors))
if self.detector.bottom:
self.detect(self.pointgrey_image, self.pointgrey_image_msg.header.stamp, self.frame_gt)
else:
self.detect(self.blackfly_image, None, self.frame_gt)
# self.detect(self.blackfly_image, self.blackfly_image_msg.header.stamp, self.frame_gt)
rospy.spin()
# def get_pitch(self, imu_msg):
# quat = imu_msg.orientation
# quat = np.array([quat.x, quat.y, quat.z, quat.w], dtype=np.float64)
# r = Rotation.from_quat(quat)
# euler = r.as_euler('xyz')
# self.pitch = euler[1]
def update_gt(self, gt_msg):
self.gt_pose = gt_msg
# self.gt_mat = np.identity(4)
# self.gt_mat[0, 3] = self.gt_pose.pose.position.x
# self.gt_mat[1, 3] = self.gt_pose.pose.position.y
# self.gt_mat[2, 3] = self.gt_pose.pose.position.z
def update_blackfly_image(self, image_msg):
self.all_frames += 1
self.blackfly_image_msg = image_msg
self.frame_gt = self.gt_pose
# self.frame_pitch = self.pitch
np_arr = np.fromstring(image_msg.data, np.uint8)
image = cv2.imdecode(np_arr, -1)
image = cv2.undistort(image, self.blackfly_camera_matrix, self.blackfly_camera_distortion)
# cv2.imwrite("/root/share/tf/image.png", image)
# self.frame_shape = list(image.shape[:2])
if self.equalize_histogram:
img_yuv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
img_yuv[:, :, 2] = cv2.equalizeHist(img_yuv[:, :, 2])
image = cv2.cvtColor(img_yuv, cv2.COLOR_HSV2BGR)
self.blackfly_image = image
# self.blackfly_image = cv2.resize(image, None, fx=self.detector.scale, fy=self.detector.scale)
def update_pointgrey_image(self, image_msg):
self.pointgrey_image_msg = image_msg
self.frame_gt = self.gt_pose
# self.frame_pitch = self.pitch
np_arr = np.fromstring(image_msg.data, np.uint8)
image = cv2.imdecode(np_arr, -1)
# print(image.shape)
image = cv2.undistort(image, self.pointgrey_camera_matrix, self.pointgrey_camera_distortion)
# self.pointgrey_frame_shape = list(image.shape[:2])
self.pointgrey_image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE) # cv2.resize(image, None, fx=0.7, fy=0.7)
def detect(self, frame, stamp, gt_pose):
start_time = time.time()
disp = np.copy(frame)
working_copy = np.copy(frame)
self.detector.detect(working_copy, gt_pose, self.gt_keypoints)
self.frames_sent_to_detector += 1
if self.detector.best_detection is not None:
self.keypoints = self.detector.best_detection['keypoints']
# color = (255, 255, 255)
for i, pt in enumerate(self.gt_keypoints):
# if i == 0:
# color = (255, 255, 255)
# elif i == 1:
# color = (255, 0, 0)
# elif i == 2:
# color = (0, 255, 0)
# elif i == 3:
# color = (0, 255, 255)
# else:
# color = (255, 0, 255)
cv2.circle(disp, (int(pt[0]), int(pt[1])), 10, (255, 255, 255), -1)
# cv2.imshow("GT", disp)
cv2.waitKey(10)
self.detected_frames += 1
if self.detector.bottom:
camera_matrix = self.pointgrey_camera_matrix
else:
camera_matrix = self.blackfly_camera_matrix
if self.detector.best_detection['score'] > 0.5:
# self.keypoints = np.multiply(self.keypoints, 1 / self.detector.scale)
self.publish_keyponts(stamp)
# self.publish_pose(stamp, camera_matrix)
# self.detector.scale = 1.0
self.blackfly_image = None
print("detection time:", time.time() - start_time)
def publish_pose(self, stamp, camera_matrix):
tvec, rvec = self.pose_estimator.calc_PnP_pose(self.keypoints, camera_matrix)
rot = Rotation.from_rotvec(np.squeeze(rvec))
out_msg = PoseStamped()
out_msg.header.stamp = stamp
out_msg.header.frame_id = "camera"
out_msg.pose.position.x = tvec[0]
out_msg.pose.position.y = tvec[1]
out_msg.pose.position.z = tvec[2]
np_quat = rot.as_quat()
ros_quat = Quaternion(np_quat[0], np_quat[1], np_quat[2], np_quat[3])
out_msg.pose.orientation = ros_quat
if self.detector.bottom:
self.posePublisher_back.publish(out_msg)
else:
self.posePublisher_front.publish(out_msg)
def publish_keyponts(self, stamp):
# out_msg = KeypointsWithCovarianceStamped()
# out_msg.header.stamp = stamp
# out_msg.keypoints = []
# out_msg.covariance = []
# for idx, kp in enumerate(self.keypoints):
# if idx == 2:
# out_msg.keypoints.append(self.keypoints[4][0])
# out_msg.keypoints.append(self.keypoints[4][1])
# if idx == 4:
# continue
# out_msg.keypoints.append(kp[0])
# out_msg.keypoints.append(kp[1])
# cov_mx = np.array(self.detector.best_detection["uncertainty"])
# for idx, cm in enumerate(cov_mx):
# if idx == 2:
# for itm in cm.flatten():
# out_msg.covariance.append(itm)
# if idx == 4:
# continue
# for itm in cm.flatten():
# out_msg.covariance.append(itm)
# print(out_msg.keypoints)
# self.keypointsPublisher.publish(out_msg)
# out_msg = Float64MultiArray()
# for idx, kp in enumerate(self.keypoints):
# if idx == 2:
# out_msg.data.append(self.keypoints[4][0])
# out_msg.data.append(self.keypoints[4][1])
# if idx == 4:
# continue
# out_msg.data.append(kp[0])
# out_msg.data.append(kp[1])
# cov_mx = np.array(self.detector.best_detection["uncertainty"])
# for idx, cm in enumerate(cov_mx):
# if idx == 2:
# for itm in cm.flatten():
# out_msg.data.append(itm)
# if idx == 4:
# continue
# for itm in cm.flatten():
# out_msg.data.append(itm)
# print(out_msg.data)
# self.keypointsPublisher.publish(out_msg)
def calc_dist(x, z):
# return math.sqrt((x[0]-z[0]) ** 2 + (x[1]-z[1]) ** 2)
return abs(x[0] - z[0]), abs(x[1] - z[1])
single_img_pred = []
for idx, kp in enumerate(self.keypoints):
# print(len(self.gt_keypoints))
# print("kp", len(self.keypoints))
if calc_dist(kp, self.gt_keypoints[idx])[1] > 150:
return
single_img_pred.append(calc_dist(kp, self.gt_keypoints[idx]))
self.prediction_errors.append(
np.array(single_img_pred).reshape((8, 1))) # calc_dist(kp, self.gt_keypoints[idx]))
if __name__ == '__main__':
det_node = DetectorNode()
det_node.start()
class Environment:
def __init__(self, gt_box=None):
self.gt_box = gt_box
self._connector = MultiRotorConnector()
self._car_connector = CarConnector()
self._detector = Detector()
def state_to_array(self, state):
out = np.zeros((8,), dtype='float32')
out[0] = state.VEL_X
out[1] = state.VEL_Y
out[2] = state.VEL_Z
out[3] = state.POS_X
out[4] = state.POS_Y
out[5] = state.WIDTH
out[6] = state.HEIGHT
out[7] = state.ALTITUDE
return out
def update(self, state):
velocity = self._connector.get_velocity()
state.VEL_X = velocity.x_val
state.VEL_Y = velocity.y_val
state.VEL_Z = velocity.z_val
position = self._connector.get_position()
state.ALTITUDE = position.z_val
frame = self._connector.get_frame()
output = self._detector.detect(frame, self.gt_box)
if not output:
return None
state.POS_X = output[0]
state.POS_Y = output[1]
state.WIDTH = output[2]
state.HEIGHT = output[3]
print "\n-----Parameters-----"
print "Velocity X:", state.VEL_X
print "Velocity Y:", state.VEL_Y
print "Velocity Z:", state.VEL_Z
print "Altitude :", state.ALTITUDE
print "Position X:", state.POS_X
print "Position Y:", state.POS_Y
print "Width :", state.WIDTH
print "Height :", state.HEIGHT
return 1
def reset(self):
car_pos, car_ort = self._car_connector.reset()
offset = (car_pos.x_val, car_pos.y_val, self._connector.INIT_Z)
self._connector.move_to_position(offset)
# self._connector.move_to_position(offset, yaw_mode=YawMode(is_rate= False, yaw_or_rate=0.0))
# time.sleep(3)
# offset = (car_ort.x_val, car_ort.y_val, car_ort.z_val)
# self._connector.move_by_angle(offset, self._connector.INIT_Z)
# time.sleep(3)
_state = State()
_ = self.update(_state)
return self.state_to_array(_state)
def step(self, action, duration=5):
_state = State()
self._connector.move_by_velocity(action, duration=duration)
collision_info = self._connector.get_collision_info()
flag = self.update(_state)
if not flag:
raise Exception('Unable to detect any Object')
return self.state_to_array(_state), collision_info
class EnvironmentSeqRT:
def __init__(self,
image_shape=(720, 1280),
step_sizes=[-40, -20, 0, 20, 40]):
self.ncols = 45
self.nrows = 45
self.im_width = image_shape[1]
self.im_height = image_shape[0]
self.step_sizes = step_sizes
self.fig = None
self.current_frame = None
self.current_output = None
self._detector = Detector()
self._connector = MultiRotorConnector()
self.old_x = None
self.old_y = None
def state_to_array(self, state):
out = np.zeros((2, ), dtype='float32')
out[0] = float(state.DELTA_X) / float(self.im_width)
out[1] = float(state.DELTA_Y) / float(self.im_height)
return out
def nearest_to_step(self, dist):
nearest_dist = [abs(step_size - dist) for step_size in self.step_sizes]
return self.step_sizes[nearest_dist.index(min(nearest_dist))]
def reset(self):
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if not output:
raise Exception('Unable to Detect')
POS_X1 = output[0]
POS_Y1 = output[1]
WIDTH1 = output[2]
HEIGHT1 = output[3]
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if not output:
raise Exception('Unable to Detect')
POS_X2 = output[0]
POS_Y2 = output[1]
WIDTH2 = output[2]
HEIGHT2 = output[3]
_state = State()
_state.DELTA_X = POS_X2 - POS_X1
_state.DELTA_Y = POS_Y2 - POS_Y1
print "\n-----Parameters-----"
print "Delta X:", _state.DELTA_X
print "Delta Y:", _state.DELTA_Y
self.old_x = POS_X1
self.old_y = POS_Y1
self.old_gx = POS_X1
self.old_gy = POS_Y1
self.current_frame = frame
self.current_state = _state
self.current_output = output
return self.state_to_array(_state)
def get_iou(self, bb1, bb2):
assert bb1['x1'] < bb1['x2']
assert bb1['y1'] > bb1['y2']
assert bb2['x1'] < bb2['x2']
assert bb2['y1'] > bb2['y2']
x_left = max(bb1['x1'], bb2['x1'])
y_top = min(bb1['y1'], bb2['y1'])
x_right = min(bb1['x2'], bb2['x2'])
y_bottom = max(bb1['y2'], bb2['y2'])
if x_right < x_left or y_bottom > y_top:
return 0.0
intersection_area = (x_right - x_left) * (y_bottom - y_top)
bb1_area = (bb1['x2'] - bb1['x1']) * (bb1['y2'] - bb1['y1'])
bb2_area = (bb2['x2'] - bb2['x1']) * (bb2['y2'] - bb2['y1'])
iou = intersection_area / float(bb1_area + bb2_area -
intersection_area)
assert iou >= 0.0
assert iou <= 1.0
return iou
def step(self, action):
# TRACKER
frame = self.current_frame.copy()
output = self.current_output
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
det_box = {
'x1': POS_X - WIDTH / 2,
'x2': POS_X + WIDTH / 2,
'y1': POS_Y + HEIGHT / 2,
'y2': POS_Y - HEIGHT / 2
}
print "\n-----Parameters-----"
print "Delta X:", self.current_state.DELTA_X
print "Delta Y:", self.current_state.DELTA_Y
if action is not None:
# GREEDY (BASELINE) AGENT
step_x = self.nearest_to_step(POS_X - self.old_gx)
step_y = self.nearest_to_step(POS_Y - self.old_gy)
new_gx = self.old_gx + step_x
new_gy = self.old_gy + step_y
baseline_box = {
'x1': new_gx - WIDTH / 2,
'x2': new_gx + WIDTH / 2,
'y1': new_gy + HEIGHT / 2,
'y2': new_gy - HEIGHT / 2
}
# CUSTOM AGENT
new_x = self.old_x + action[0]
new_y = self.old_y + action[1]
agent_box = {
'x1': new_x - WIDTH / 2,
'x2': new_x + WIDTH / 2,
'y1': new_y + HEIGHT / 2,
'y2': new_y - HEIGHT / 2
}
print "[DEBUG]:", det_box
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - det_box['y1']) /
float(self.im_height)),
(float(det_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - det_box['y2']) /
float(self.im_height)),
(float(det_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='blue',
thickness=3)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - agent_box['y1']) /
float(self.im_height)),
(float(agent_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - agent_box['y2']) /
float(self.im_height)),
(float(agent_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='yellow',
thickness=5)
vis_util.draw_bounding_boxes_on_image_array(
frame,
np.array([[(float(self.im_height / 2 - baseline_box['y1']) /
float(self.im_height)),
(float(baseline_box['x1'] + self.im_width / 2) /
float(self.im_width)),
(float(self.im_height / 2 - baseline_box['y2']) /
float(self.im_height)),
(float(baseline_box['x2'] + self.im_width / 2) /
float(self.im_width))]]),
color='red',
thickness=5)
print "Action :", action, "pixels"
# cv2.imshow('Simulation', frame)
# cv2.waitKey(10)
self.old_x = new_x
self.old_y = new_y
self.old_gx = new_gx
self.old_gy = new_gy
else:
self.old_x = POS_X
self.old_y = POS_Y
self.old_gx = POS_X
self.old_gy = POS_Y
# NEXT frame
_state = State()
frame = self._connector.get_frame()
output = self._detector.detect(frame)
if not output:
raise Exception('Unable to Detect')
POS_X = output[0]
POS_Y = output[1]
WIDTH = output[2]
HEIGHT = output[3]
_state.DELTA_X = POS_X - self.old_x
_state.DELTA_Y = POS_Y - self.old_y
self.current_state = _state
self.current_output = output
self.current_frame = frame
return self.state_to_array(_state)
from GoProController import GoProController
from Detector import Detector
from PIL import Image
gpc = GoProController(device_name='wlan1')
gpc.connect('SARSGoPro', 'sarsgopro')
det = Detector()
imgs = []
for i in range(0, 25):
if (gpc.getImage('SARSGoPro', 'sarsgopro')):
print('Image download successful!')
img = Image.open('image.png')
pix = img.load()
imgs.append(pix)
if (len(imgs) == 9):
break
else:
print('Image download failed. Trying again...')
if (len(imgs) > 0):
for pix in imgs:
if (det.detect(pix)):
print('Object detected in Image ' + str(imgs.index(pix)) + '!')
else:
print('Object not detected in Image ' + str(imgs.index(pix)) + '.')
else:
print('No images downloaded.')
class VideoProcessor:
def __init__(self, keep_history=False):
self.keep_history = keep_history
self.tracker = Tracker()
self.tracks_history = []
self.frames = []
self.detections = []
def initialize(self):
#self.detector = Detector('YOLO')
# self.detector = Detector('SSD-mobilenet-face')
self.detector = Detector('SSD-brainlab')
self.detector.initialize()
def process_frame(self, frame):
print('')
# Detect in the current frame:
self.image = frame
self.detections = self.detector.detect(frame)
# Update tracker:
self.tracker.update_tracks(self.detections)
# Update history:
if self.keep_history:
self.frames.append(frame)
self.update_history()
def get_last_frame_with_detections(self):
img_cp = self.image.copy()
print(str(len(self.tracker.get_tracks())) + ' tracks')
for tr in self.tracker.get_tracks():
detection = tr.get_last_detection()
x = int(detection.x_center)
y = int(detection.y_center)
w = int(detection.width)//2
h = int(detection.height)//2
if tr.get_nframes_missing() > 0:
print('missing frame for track ' + str(tr.get_id()) + ' - nframes_missing: ' + str(tr.get_nframes_missing()))
color = (255, 0, 0)
else:
color = (0, 255, 0)
cv2.rectangle(img_cp, (x-w,y-h), (x+w,y+h), color, 2)
cv2.rectangle(img_cp, (x-w,y-h-20), (x+w,y-h), (125,125,125), -1)
cv2.putText(img_cp, detection.class_name + ' ' + str(tr.get_id()) + ' : %.2f' % detection.conf, (x-w+5,y-h-7), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,0), 1)
return img_cp
def clear(self):
self.tracker.clear()
self.tracks_history = []
self.frames = []
self.detections = []
def update_history(self):
for track in self.tracker.tracks:
exists = False
for i in range(len(self.tracks_history)):
if self.tracks_history[i].id == track.id:
exists = True
self.tracks_history[i].update(track)
break
if not exists:
self.tracks_history.append(TrackHistory(track))
import cv2
cap = cv2.VideoCapture(0)
cd = Detector()
# initialize the FourCC, video writer, dimensions of the frame, and
# zeros array
fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
writer = None
(h, w) = (None, None)
while (True):
ret, frame = cap.read()
output = cd.detect(frame.copy())
cv2.imshow("output", output)
if writer is None:
(h, w) = output.shape[:2]
writer = cv2.VideoWriter("example.avi", fourcc, 10, (w, h), True)
#writer.write(output)
if (cv2.waitKey(1) & 0xFF) == ord('q'):
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
class Analyser:
def __init__(self):
Config.init()
self.detector = Detector()#Config.nbTLstr)
def analyse(self, xmlfilelist):
parser = etree.XMLParser(recover=True)
total_tp = 0
total_fp = 0
total_fn = 0
fnfid = {}
fpfid = {}
cerrs = dict.fromkeys(['AUTHOR', 'COUNT_READ', 'COUNT_REPLY', 'TITLE', 'TM_POST', 'TM_REPLY'],0)
cerrfiles = {}
num = 0
for xmlfile in xmlfilelist:
#print(xmlfile)
if num % 200 == 0 :
print(num/len(xmlfilelist))
print('REV.150')
print('total_tp:', total_tp)
print('total_fp:', total_fp)
print('total_fn:', total_fn)
print('fnfid:', fnfid)
print('fpfid:', fpfid)
print('cerrs:',cerrs)
print('cerrfiles:', cerrfiles)
num += 1
root = etree.parse(xmlfile,parser).getroot()
htmlnode = root[0]
se = StructureExtractor()
se.drawFeature(htmlnode)
extractor = Extractor(htmlnode)
extractor.process()
self.detector.detect(htmlnode)
(tp, fp, fn) = self.calcAccTitleLine(htmlnode)
total_tp += tp
total_fp += fp
total_fn += fn
if fn == 0:
cerr = self.calcAccColumn(htmlnode)
for k in cerr:
cerrs[k] += 1
if len(cerr)>0:
cerrfiles[path.basename(xmlfile)] = (cerr, root.attrib['fid'])
if fp > 0:
#print(xmlfile)
fpfid[root.attrib['fid']] = fpfid.get(root.attrib['fid'],0) + 1
if fn > 0:
print(xmlfile)
fnfid[root.attrib['fid']] = fnfid.get(root.attrib['fid'],0) + 1
print('REV.150')
print('total_tp:', total_tp)
print('total_fp:', total_fp)
print('total_fn:', total_fn)
print('fnfid:', fnfid)
print('fpfid:', fpfid)
print('cerrs:',cerrs)
print('cerrfiles:', cerrfiles)
def analyseOnce(self, xmlfile):
parser = etree.XMLParser(recover=True)
root = etree.parse(xmlfile,parser).getroot()
htmlnode = root[0]
return self.performAnalyse(htmlnode)
def performAnalyse(self, htmlnode):
se = StructureExtractor()
se.drawFeature(htmlnode)
extractor = Extractor(htmlnode)
extractor.process()
detector = Detector()
detector.detect(htmlnode)
(tp, fp, fn) = self.calcAccTitleLine(htmlnode)
cerr = self.calcAccColumn(htmlnode)
return (tp, fp, fn, cerr)
def calcAccTitleLine(self, root):
ptls = set(root.findall('.//*[@predict="{}"]'.format(LABEL['TITLE_LINE'])))
ttls = set(root.findall('.//*[@label="{}"]'.format(LABEL['TITLE_LINE'])))
tp = len(ptls & ttls)
fp = len(ptls) - tp
fn = len(ttls) - tp
return (tp, fp, fn)
def calcAccColumn(self, root):
k = 1
columns = {}
while True:
cs = root.findall('.//*[@column="{}"]'.format(k))
if len(cs) == 0:
break
columns[k] = set(cs)
k += 1
dls = ['AUTHOR', 'COUNT_READ', 'COUNT_REPLY', 'TITLE', 'TM_POST', 'TM_REPLY']
err = set()
for key in dls:
ts = set(root.findall('.//*[@predict="{}"]//*[@label="{}"]'.format(LABEL['TITLE_LINE'],LABEL[key])))
cs = set()
for tmpnode in ts:
if tmpnode.tag == 'text':
cs.add(tmpnode.getparent())
else:
cs.add(tmpnode)
if len(cs) == 0:
continue
for k in columns:
if len(columns[k] - cs)/len(columns[k]) < 0.03 or len(cs - columns[k])/len(cs) < 0.03:
break
else:
err.add(key)
return err