class WebsocketClientModule(Thread):
def __init__(self, baseConfig, pInBoundEventQueue, pOutBoundEventQueue,
loggingQueue):
super(WebsocketClientModule, self).__init__()
self.alive = True
self.config = baseConfig
self.inQueue = pInBoundEventQueue # inQueue are messages from the main process to websocket clients
self.outQueue = pOutBoundEventQueue # outQueue are messages from clients to main process
self.websocketClient = None
self.loggingQueue = loggingQueue
self.threadProcessQueue = None
# Constants
self._port = self.config['WebsocketPort']
self._host = self.config['WebsocketHost']
# logging setup
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def run(self):
""" Main thread entry point.
Sets up websocket server and event callbacks.
Starts thread to monitor inbound message queue.
"""
self.logger.info("Starting websocket %s" % __name__)
self.connect()
def listen(self):
self.threadProcessQueue = Thread(target=self.processQueue)
self.threadProcessQueue.setDaemon(True)
self.threadProcessQueue.start()
def connect(self):
#websocket.enableTrace(True)
ws = websocket.WebSocketApp("ws://%s:%s" % (self._host, self._port),
on_message=self.onMessage,
on_error=self.onError,
on_close=self.onClose)
ws.on_open = self.onOpen
ws.run_forever()
def onError(self, ws, message):
self.logger.error("Error from websocket client: %s" % message)
def onClose(self, ws):
if self.alive:
self.logger.warn("Closed")
self.alive = False
# TODO: reconnect timer
else:
self.logger.info("Closed")
def onMessage(self, ws, message):
self.logger.info("Message from websocket server: %s" % message)
def onOpen(self, ws):
self.alive = True
self.websocketClient = ws
self.listen()
def shutdown(self):
""" Handle shutdown message.
Close and shutdown websocket server.
Join queue processing thread.
"""
self.logger.info("Shutting down websocket server %s" %
(multiprocessing.current_process().name))
try:
self.logger.info("Closing websocket")
self.websocketClient.close()
except Exception as e:
self.logger.error("Websocket close error : %s " % e)
self.alive = False
self.threadProcessQueue.join()
time.sleep(1)
self.exit = True
def sendOutMessage(self, message):
""" Send message to server """
self.websocketClient.send(json.dumps(message.__dict__))
def processQueue(self):
""" Monitor queue of messages from main process to this thread. """
while self.alive:
if (self.inQueue.empty() == False):
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None:
if message == "SHUTDOWN":
self.logger.debug("SHUTDOWN handled")
self.shutdown()
else:
self.sendOutMessage(message)
except Exception as e:
self.logger.error("Websocket unable to read queue : %s " %
e)
else:
time.sleep(.25)
class BtleThreadCollectionPoint(object):
def __init__(self,
clientEventHandler,
btleConfig,
loggingQueue,
debugMode=False):
# Logger
self.loggingQueue = loggingQueue
self.logger = ThreadsafeLogger(loggingQueue, __name__)
self.btleConfig = btleConfig
self.clientEventHandler = clientEventHandler
self.debug = debugMode
# define basic BGAPI parser
self.bgapi_rx_buffer = []
self.bgapi_rx_expected_length = 0
def start(self):
packet_mode = False
# create BGLib object
self.ble = BGLib()
self.ble.packet_mode = packet_mode
self.ble.debug = self.debug
# add handler for BGAPI timeout condition (hopefully won't happen)
self.ble.on_timeout += self.my_timeout
# on busy hander
self.ble.on_busy = self.on_busy
# add handler for the gap_scan_response event
self.ble.ble_evt_gap_scan_response += self.clientEventHandler
# create serial port object and flush buffers
self.logger.info(
"Establishing serial connection to BLED112 on com port %s at baud rate %s"
% (self.btleConfig['BtleDeviceId'],
self.btleConfig['BtleDeviceBaudRate']))
self.serial = Serial(port=self.btleConfig['BtleDeviceId'],
baudrate=self.btleConfig['BtleDeviceBaudRate'],
timeout=1)
self.serial.flushInput()
self.serial.flushOutput()
# disconnect if we are connected already
self.ble.send_command(self.serial,
self.ble.ble_cmd_connection_disconnect(0))
self.ble.check_activity(self.serial, 1)
# stop advertising if we are advertising already
self.ble.send_command(self.serial, self.ble.ble_cmd_gap_set_mode(0, 0))
self.ble.check_activity(self.serial, 1)
# stop scanning if we are scanning already
self.ble.send_command(self.serial,
self.ble.ble_cmd_gap_end_procedure())
self.ble.check_activity(self.serial, 1)
# set the TX
# range 0 to 15 (real TX power from -23 to +3dBm)
#self.ble.send_command(self.serial, self.ble.ble_cmd_hardware_set_txpower(self.btleConfig['btleDeviceTxPower']))
#self.ble.check_activity(self.serial,1)
#ble_cmd_connection_update connection: 0 (0x00) interval_min: 30 (0x001e) interval_max: 46 (0x002e) latency: 0 (0x0000) timeout: 100 (0x0064)
#interval_min 6-3200
#interval_man 6-3200
#latency 0-500
#timeout 10-3200
self.ble.send_command(
self.serial,
self.ble.ble_cmd_connection_update(0x00, 0x001e, 0x002e, 0x0000,
0x0064))
self.ble.check_activity(self.serial, 1)
# set scan parameters
#scan_interval 0x4 - 0x4000
#Scan interval defines the interval when scanning is re-started in units of 625us
# Range: 0x4 - 0x4000
# Default: 0x4B (75ms)
# After every scan interval the scanner will change the frequency it operates at
# at it will cycle through all the three advertisements channels in a round robin
# fashion. According to the Bluetooth specification all three channels must be
# used by a scanner.
#
#scan_window 0x4 - 0x4000
# Scan Window defines how long time the scanner will listen on a certain
# frequency and try to pick up advertisement packets. Scan window is defined
# as units of 625us
# Range: 0x4 - 0x4000
# Default: 0x32 (50 ms)
# Scan windows must be equal or smaller than scan interval
# If scan window is equal to the scan interval value, then the Bluetooth module
# will be scanning at a 100% duty cycle.
# If scan window is half of the scan interval value, then the Bluetooth module
# will be scanning at a 50% duty cycle.
#
#active 1=active 0=passive
# 1: Active scanning is used. When an advertisement packet is received the
# Bluetooth stack will send a scan request packet to the advertiser to try and
# read the scan response data.
# 0: Passive scanning is used. No scan request is made.
#self.ble.send_command(self.serial, self.ble.ble_cmd_gap_set_scan_parameters(0x4B,0x32,1))
self.ble.send_command(
self.serial,
self.ble.ble_cmd_gap_set_scan_parameters(0xC8, 0xC8, 0))
self.ble.check_activity(self.serial, 1)
# start scanning now
self.ble.send_command(self.serial, self.ble.ble_cmd_gap_discover(1))
self.ble.check_activity(self.serial, 1)
# handler to notify of an API parser timeout condition
def my_timeout(self, sender, args):
self.logger.error(
"BGAPI timed out. Make sure the BLE device is in a known/idle state."
)
# might want to try the following lines to reset, though it probably
# wouldn't work at this point if it's already timed out:
self.ble.send_command(self.serial, self.ble.ble_cmd_system_reset(0))
self.ble.check_activity(self.serial, 1)
self.ble.send_command(self.serial, self.ble.ble_cmd_gap_discover(1))
self.ble.check_activity(self.serial, 1)
def on_busy(self, sender, args):
self.logger.warn("BGAPI device is busy.")
def scan(self):
# check for all incoming data (no timeout, non-blocking)
self.ble.check_activity(self.serial)
class BlueGigaBtleCollectionPointThread(Thread):
def __init__(self, queue, btleConfig, loggingQueue, debugMode=False):
Thread.__init__(self)
# Logger
self.loggingQueue = loggingQueue
self.logger = ThreadsafeLogger(loggingQueue, __name__)
self.alive = True
self.btleConfig = btleConfig
self.queue = queue
self.btleCollectionPoint = BtleThreadCollectionPoint(self.eventScanResponse,self.btleConfig,self.loggingQueue)
def bleDetect(self,__name__,repeatcount=10):
try:
self.btleCollectionPoint.start()
except Exception as e:
self.logger.error("[btleThread] Unable to connect to BTLE device: %s"%e)
self.sendFailureNotice("Unable to connect to BTLE device")
quit()
while self.alive:
try:
self.btleCollectionPoint.scan()
except Exception as e:
self.logger.error("[btleThread] Unable to scan BTLE device: %s"%e)
self.sendFailureNotice("Unable to connect to BTLE device to perform a scan")
quit()
# don't burden the CPU
time.sleep(0.01)
# handler to print scan responses with a timestamp
def eventScanResponse(self,sender,args):
#check to make sure there is enough data to be a beacon
if len(args["data"]) > 15:
# self.logger.debug("=============================== eventScanResponse START ===============================")
try:
majorNumber = args["data"][26] | (args["data"][25] << 8)
# self.logger.debug("majorNumber=%i"%majorNumber)
except:
majorNumber = 0
try:
minorNumber = args["data"][28] | (args["data"][27] << 8)
# self.logger.debug("minorNumber=%i"%minorNumber)
except:
minorNumber = 0
if self.btleConfig['BtleAdvertisingMajor'] == majorNumber and self.btleConfig['BtleAdvertisingMinor'] == minorNumber:
self.logger.debug("self.btleConfig['BtleAdvertisingMinor'] == %i and self.btleConfig['BtleAdvertisingMinor'] == %i "%(majorNumber,minorNumber))
self.logger.debug("yep, we care about this major and minor so lets create a detected client and pass it to the event manager")
udid = "%s" % ''.join(['%02X' % b for b in args["data"][9:25]])
self.logger.debug("UDID=%s"%udid)
rssi = args["rssi"]
self.logger.debug("rssi=%s"%rssi)
beaconMac = "%s" % ''.join(['%02X' % b for b in args["sender"][::-1]])
self.logger.debug("beaconMac=%s"%beaconMac)
rawTxPower = args["data"][29]
self.logger.debug("rawTxPower=%i"%rawTxPower)
if rawTxPower <= 127:
txPower = rawTxPower
else:
txPower = rawTxPower - 256
self.logger.debug("txPower=%i"%txPower)
arrayDetectedClients = [] #we send an array to the event queue, we used to process bacthes of responses
#package it up for sending to the queue
detectedClient = DetectedClient('btle',udid=udid,beaconMac=beaconMac,majorNumber=majorNumber,minorNumber=minorNumber,tx=txPower,rssi=rssi)
arrayDetectedClients.append(detectedClient)
#put it on the queue for the event manager to pick up
self.queue.put(arrayDetectedClients)
self.logger.debug("================================= eventScanResponse END =================================")
def stop(self):
self.alive = False
def sendFailureNotice(self,msg):
if len(self.btleConfig['SlackChannelWebhookUrl']) > 10:
myMsg = 'Help I have fallen and can not get back up! \n %s. \nSent from %s'%(msg,platform.node())
payload = {'text': myMsg}
r = requests.post(self.btleConfig['SlackChannelWebhookUrl'], data = json.dumps(payload))
class MultiTracker(object):
def __init__(self, kind="KCF", moduleConfig=None, loggingQueue=None):
""" Create an initialize new MultiTracker. Set up constants and parameters. """
self.config = moduleConfig
self.trackers = [] # List of trackers
self.kind = kind
self.focus = None
self.loggingQueue = loggingQueue
# Constants
self._useVelocity = self.config['UseVelocity']
self._closestThreshold = self.config["ClosestThreshold"]
self._primaryTarget = self.config['PrimaryTarget']
# Setup logging queue
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def add(self, bbox, frame, kind="KCF"):
""" Add new tracker with default type KCF. """
aTracker = Tracker(bbox, frame, kind, self.config, self.loggingQueue)
self.trackers.append(aTracker)
def removeAt(self, i):
""" Remove Tracker at index i. """
self.trackers.pop(i)
def remove(self, aTracker):
""" Remove tracker provided as parameter. """
self.trackers.remove(aTracker)
def update(self, frame):
""" Loop through each tracker updating bounding box, keep track of failures. """
bboxes = []
ind = 0
failed = []
for aTracker in self.trackers:
ok, bbox = aTracker.update(frame)
if not ok:
failed.append(ind)
else:
bboxes.append(bbox)
ind += 1
if len(failed) == 0:
return True, bboxes, None
else:
self.logger.error('Failed to update all trackers')
return False, bboxes, failed
def clear(self):
""" Remove all trackers. """
self.trackers.clear()
self.focus = None
def bboxContainsPt(self, bbox, pt, vBuffer):
""" Check if bbox contains pt.
Optionally provide velocity buffer to spread containing space.
"""
if ((bbox['x'] - vBuffer[0] <= pt[0] <=
(bbox['x'] + bbox['w'] + vBuffer[0]))
and (bbox['y'] - vBuffer[1] <= pt[1] <=
(bbox['y'] + bbox['h'] + vBuffer[1]))):
return True
else:
return False
def projectedLocationMatches(self, tracker, bbox):
""" Check if the velocity of the tracker could put it in the same spot as the bbox. """
if tracker.velocity:
return self.bboxContainsPt(bbox,
tracker.getProjectedLocation(time()),
tracker.getVelocityBuffer())
else:
return False
def intersects(self, tracker, bbox):
""" Check if the bbox and the trackers bounds intersect. """
if (tracker.right() < bbox['x']
or bbox['x'] + bbox['w'] < tracker.left()
or tracker.top() < bbox['y']
or bbox['y'] + bbox['h'] < tracker.bottom()):
return False # intersection is empty
else:
return True # intersection is not empty
def contains(self, bbox):
""" Check if the MultiTracker already has a tracker for the object detected.
Uses intersections and projected locations to determine if the tracker overlaps others.
This means objects that overlap when first detected will not _both_ be added to the MultiTracker.
"""
for aTracker in self.trackers:
if self._useVelocity:
if self.intersects(aTracker,
bbox) and self.projectedLocationMatches(
aTracker, bbox):
return True
elif self.intersects(aTracker, bbox):
return True
return False
def length(self):
""" Get number of Trackers in the MultiTracker. """
return len(self.trackers)
def getFocus(self):
""" Get focal object based on primaryTarget configuration.
Currently only closest is supported - checks whether there is a tracker
that is larger than the previous closest tracker by the configured threshold.
"""
if self._primaryTarget == "closest":
focusChanged = False
if self.focus:
# area = self.focus.area()
area = self.focus.area()
else:
area = None
for aTracker in self.trackers:
# If there's no focus or aTracker is larger than focus, and they aren't the same tracker
if not self.focus or (
aTracker.area() > area *
(1 + (self._closestThreshold / 100))
and self.focus.getCreated() != aTracker.getCreated()):
focusChanged = True
self.focus = aTracker
area = aTracker.area()
if focusChanged:
return self.focus
else:
return None
elif self._primaryTarget == "closest_engaged":
#TODO
self.logger.error('Primary Target %s is not implemented.' %
self._primaryTarget)
return None
else:
self.logger.error('Primary Target %s is not implemented.' %
self._primaryTarget)
return None
def checkFocus(self):
""" Check if focal Tracker has changed by updating the focus. """
focus = self.getFocus()
if focus:
return True, focus.bbox
else:
return False, None
class MQTTClientModule(Thread):
""" Threaded MQTT client for processing and publishing outbound messages"""
def __init__(self, baseConfig, pInBoundEventQueue, pOutBoundEventQueue,
loggingQueue):
super(MQTTClientModule, self).__init__()
self.config = baseConfig
self.alive = True
self.inQueue = pInBoundEventQueue
# Constants
self._keepAlive = self.config['MqttKeepAlive']
self._feedName = self.config['MqttFeedName']
self._username = self.config['MqttUsername']
self._key = self.config['MqttKey']
self._host = self.config['MqttHost']
self._port = self.config['MqttPort']
self._publishJson = self.config['MqttPublishJson']
self._publishFaceValues = self.config['MqttPublishFaceValues']
# MQTT setup
self._client = mqtt.Client()
self._client.username_pw_set(self._username, self._key)
self._client.on_connect = self.onConnect
self._client.on_disconnect = self.onDisconnect
self._client.on_message = self.onMessage
self.mqttConnected = False
# Logging setup
self.logger = ThreadsafeLogger(loggingQueue, "MQTT")
def onConnect(self, client, userdata, flags, rc):
self.logger.debug('MQTT onConnect called')
# Result code 0 is success
if rc == 0:
self.mqttConnected = True
# Subscribe to feed here
else:
self.logger.error('MQTT failed to connect: %s' % rc)
raise RuntimeError('MQTT failed to connect: %s' % rc)
def onDisconnect(self, client, userdata, rc):
self.logger.debug('MQTT onDisconnect called')
self.mqttConnected = False
if rc != 0:
self.logger.debug('MQTT disconnected unexpectedly: %s' % rc)
self.handleReconnect(rc)
def onMessage(self, client, userdata, msg):
self.logger.debug('MQTT onMessage called for client: %s' % client)
def connect(self):
""" Connect to MQTT broker
Skip calling connect if already connected.
"""
if self.mqttConnected:
return
self._client.connect(self._host,
port=self._port,
keepalive=self._keepAlive)
def disconnect(self):
""" Check if connected"""
if self.mqttConnected:
self._client.disconnect()
def subscribe(self, feed=False):
"""Subscribe to feed, defaults to feed specified in config"""
if not feed: feed = _feedName
self._client.subscribe('{0}/feeds/{1}'.format(self._username, feed))
def publish(self, value, feed=False):
"""Publish a value to a feed"""
if not feed: feed = _feedName
self._client.publish('{0}/feeds/{1}'.format(self._username, feed),
payload=value)
def publishFaceValues(self, message):
""" Publish face detection values to individual MQTT feeds
Parses _extendedData.predictions.faceAttributes property
Works with Azure face API responses and
"""
try:
for face in message._extendedData['predictions']:
faceAttrs = face['faceAttributes']
for key in faceAttrs:
if type(faceAttrs[key]) is dict:
val = self.flattenDict(faceAttrs[key])
print('val: ', val)
else:
val = faceAttrs[key]
self.publish(val, key)
except Exception as e:
self.logger.error('Error publishing values: %s' % e)
def flattenDict(self, aDict):
""" Get average of simple dictionary of numerical values """
try:
val = float(sum(aDict[key] for key in aDict)) / len(aDict)
except Exception as e:
self.logger.error('Error flattening dict, returning 0: %s' % e)
return val or 0
def publishJsonMessage(self, message):
msg_str = self.stringifyMessage(message)
self.publish(msg_str)
def stringifyMessage(self, message):
""" Dump into JSON string """
return json.dumps(message.__dict__).encode('utf8')
def processQueue(self):
self.logger.info('Processing queue')
while self.alive:
# Pump the loop
self._client.loop(timeout=1)
if (self.inQueue.empty() == False):
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None and self.mqttConnected:
if message == "SHUTDOWN":
self.logger.debug("SHUTDOWN command handled")
self.shutdown()
else:
# Send message as string or split into channels
if self._publishJson:
self.publishJsonMessage(message)
elif self._publishFaceData:
self.publishFaceValues(message)
else:
self.publishValues(message)
except Exception as e:
self.logger.error("MQTT unable to read queue : %s " % e)
else:
time.sleep(.25)
def shutdown(self):
self.logger.info("Shutting down MQTT %s" % (mp.current_process().name))
self.alive = False
time.sleep(1)
self.exit = True
def run(self):
""" Thread start method"""
self.logger.info("Running MQTT")
self.connect()
self.alive = True
# Start queue loop
self.processQueue()
class BtleCollectionPoint(Thread):
def __init__(self, baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue):
""" Initialize new CamCollectionPoint instance.
Setup queues, variables, configs, constants and loggers.
"""
super(BtleCollectionPoint, self).__init__()
# Queues
self.outQueue = pOutBoundQueue #messages from this thread to the main process
self.inQueue = pInBoundQueue
self.loggingQueue = loggingQueue
self.queueBLE = mp.Queue()
# Configs
self.moduleConfig = configLoader.load(
self.loggingQueue) #Get the config for this module
self.config = baseConfig
# Logger
self.logger = ThreadsafeLogger(loggingQueue, __name__)
# Variables
self.registeredClientRegistry = RegisteredClientRegistry(
self.moduleConfig, self.loggingQueue)
self.eventManager = EventManager(self.moduleConfig, pOutBoundQueue,
self.registeredClientRegistry,
self.loggingQueue)
self.alive = True
self.btleThread = None
self.BLEThread = None
self.repeatTimerSweepClients = None
# main start method
def run(self):
###Pausing Startup to wait for things to start after a system restart
self.logger.info(
"Pausing execution 15 seconds waiting for other system services to start"
)
time.sleep(15)
self.logger.info(
"Done with our nap. Time to start looking for clients")
######### setup global client registry start #########
# self.registeredClientRegistry = RegisteredClientRegistry(self.moduleConfig, self.loggingQueue)
######### setup global client registry end #########
self.logger.info('here 1')
self.btleThread = BlueGigaBtleCollectionPointThread(
self.queueBLE, self.moduleConfig, self.loggingQueue)
self.BLEThread = Thread(target=self.btleThread.bleDetect,
args=(__name__, 10))
self.BLEThread.daemon = True
self.BLEThread.start()
self.logger.info('here 2')
#Setup repeat task to run the sweep every X interval
self.repeatTimerSweepClients = RepeatedTimer(
(self.moduleConfig['AbandonedClientCleanupIntervalInMilliseconds']
/ 1000), self.registeredClientRegistry.sweepOldClients)
# Process queue from main thread for shutdown messages
self.threadProcessQueue = Thread(target=self.processQueue)
self.threadProcessQueue.setDaemon(True)
self.threadProcessQueue.start()
self.logger.info('here 3')
#read the queue
while self.alive:
if not self.queueBLE.empty():
self.logger.info(
'got a thing here herhehrehfhve!~ ~ ~@~@~!#~ ~ #~ #@@~ ~@# @~#'
)
result = self.queueBLE.get(block=False, timeout=1)
self.__handleBtleClientEvents(result)
def processQueue(self):
self.logger.info(
"Starting to watch collection point inbound message queue")
while self.alive:
if not self.inQueue.empty():
self.logger.info("Queue size is %s" % self.inQueue.qsize())
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None:
if message == "SHUTDOWN":
self.logger.info("SHUTDOWN command handled on %s" %
__name__)
self.shutdown()
else:
self.sendOutMessage(message)
except Exception as e:
self.logger.error("Unable to read queue, error: %s " % e)
self.shutdown()
self.logger.info("Queue size is %s after" %
self.inQueue.qsize())
else:
time.sleep(.25)
#handle btle reads
def __handleBtleClientEvents(self, detectedClients):
self.logger.debug("doing handleBtleClientEvents: %s" % detectedClients)
for client in detectedClients:
self.logger.debug("--- Found client ---")
self.logger.debug(vars(client))
self.logger.debug("--- Found client end ---")
self.eventManager.registerDetectedClient(client)
def shutdown(self):
self.logger.info("Shutting down")
# self.threadProcessQueue.join()
self.repeatTimerSweepClients.stop()
self.btleThread.stop()
self.alive = False
time.sleep(1)
self.exit = True
class TVCollectionPoint(Thread):
def __init__(self, baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue):
""" Initialize new TVCollectionPoint instance.
Setup queues, variables, configs, constants and loggers.
"""
super(TVCollectionPoint, self).__init__()
if not self.check_opencv_version("3.", cv2):
print(
"OpenCV version {0} is not supported. Use 3.x for best results."
.format(self.get_opencv_version()))
# Queues
self.outQueue = pOutBoundQueue #messages from this thread to the main process
self.inQueue = pInBoundQueue
self.loggingQueue = loggingQueue
# Variables
self.video = None
self.alive = True
self.ix = -1
self.iy = -1
self.fx = -1
self.fy = -1
self.clicking = False
self.boundSet = False
self.x1, self.x2, self.y1, self.y2 = 0, 0, 0, 0
# Configs
#self.moduleConfig = camConfigLoader.load(self.loggingQueue) #Get the config for this module
self.config = baseConfig
# Constants
self._captureWidth = 1600
self._captureHeight = 900
self._numLEDs = 60
self._collectionPointId = "tvcam1"
self._collectionPointType = "ambiLED"
self._showVideoStream = True
self._delimiter = ';'
self._colorMode = 'edgeDominant'
# self._colorMode = 'edgeMean'
self._perimeterDepth = 20
self._topSegments = 3
self._sideSegments = 2
# Logger
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def run(self):
""" Main thread method, run when the thread's start() function is called.
Controls flow of detected faces and the MultiTracker.
Sends color data in string format, like "#fffff;#f1f1f1;..."
"""
# Monitor inbound queue on own thread
self.threadProcessQueue = Thread(target=self.processQueue)
self.threadProcessQueue.setDaemon(True)
self.threadProcessQueue.start()
self.initializeCamera()
# Setup timer for FPS calculations
start = time.time()
frameCounter = 1
fps = 0
# Start timer for collection events
self.collectionStart = time.time()
ok, frame = self.video.read()
if not ok:
self.logger.error('Cannot read video file')
self.shutdown()
else:
framecopy = frame.copy()
cont = True
while cont or not self.boundSet:
cv2.imshow('Set ROI', framecopy)
cv2.setMouseCallback('Set ROI', self.getROI, frame)
k = cv2.waitKey(0)
if k == 32 and self.boundSet:
# on space, user wants to finalize bounds, only allow them to exit if bounds set
cont = False
# elif k != 27:
# any other key clears rectangles
# framecopy = frame.copy()
#ok, frame = self.video.read()
# cv2.imshow('Set ROI', framecopy)
# cv2.setMouseCallback('Set ROI', self.getROI, framecopy)
cv2.destroyWindow('Set ROI')
self.initKMeans()
# Set up for all modes
top_length_pixels = self.fx - self.ix
side_length_pixels = self.fy - self.iy
perimeter_length_pixels = top_length_pixels * 2 + side_length_pixels * 2
# mode specific setup
if self._colorMode == 'dominant':
pass
if self._colorMode == 'edgeDominant' or self._colorMode == 'edgeMean':
perimeter_depth = 0
if self._perimeterDepth < side_length_pixels / 2 and self._perimeterDepth < top_length_pixels / 2:
perimeter_depth = self._perimeterDepth
else:
perimeter_depth = min(side_length_pixels / 2,
top_length_pixels / 2)
while self.alive:
ok, ogframe = self.video.read()
if not ok:
self.logger.error('Error while reading frame')
break
frame = ogframe.copy()
# Dominant color
if self._colorMode == 'dominant':
data = self.getDominantColor(
cv2.resize(frame[:, :, :], (0, 0), fx=0.4, fy=0.4),
self.ix, self.fx, self.iy, self.fy)
#self.putCPMessage(data, 'light-dominant')
#print('data: ',data)
elif self._colorMode == 'edgeMean':
data = self.getEdgeMeanColors(frame, top_length_pixels,
side_length_pixels,
perimeter_length_pixels,
perimeter_depth)
print('data: ', data)
elif self._colorMode == 'edgeDominant':
# this is the most promising
colorData = self.getEdgeDominantColors(
frame, top_length_pixels, side_length_pixels,
perimeter_length_pixels, perimeter_depth)
# assuming LEDs are evenly distributed, find number for each edge of ROI
top_num_leds = self._numLEDs * (top_length_pixels /
perimeter_length_pixels)
side_num_leds = self._numLEDs * (side_length_pixels /
perimeter_length_pixels)
data = self.getColorString(colorData, top_num_leds,
side_num_leds)
self.putCPMessage(data, 'light-edges')
# print('data: ', data)
if self._showVideoStream:
cv2.rectangle(frame, (self.ix, self.iy), (self.fx, self.fy),
(255, 0, 0), 1)
cv2.imshow("output", frame)
cv2.waitKey(1)
def getMeanColor(self, frame):
color = [frame[:, :, i].mean() for i in range(frame.shape[-1])]
return color
def initKMeans(self):
# kmeans vars
self.n_colors = 5
self.criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER,
200, .1)
self.flags = cv2.KMEANS_RANDOM_CENTERS
def getColorString(self, colorData, top_num_leds, side_num_leds):
toReturn = ''
for key in colorData:
if key == 'top' or key == 'bottom':
for i in range(len(colorData[key])):
toReturn += (colorData[key][i] + self._delimiter) * int(
top_num_leds / self._topSegments)
if key == 'right' or key == 'left':
for i in range(len(colorData[key])):
toReturn += (colorData[key][i] + self._delimiter) * int(
side_num_leds / self._sideSegments)
return toReturn
def getDominantSegmentColor(self, segment):
average_color = [
segment[:, :, i].mean() for i in range(segment.shape[-1])
]
arr = np.float32(segment)
pixels = arr.reshape((-1, 3))
# kmeans clustering
_, labels, centroids = cv2.kmeans(pixels, self.n_colors, None,
self.criteria, 10, self.flags)
palette = np.uint8(centroids)
quantized = palette[labels.flatten()]
quantized = quantized.reshape(segment.shape)
dominant_color = palette[np.argmax(itemfreq(labels)[:, -1])]
return dominant_color
def getEdgeMeanColors(self, frame, top_length_pixels, side_length_pixels,
perimeter_length_pixels, perimeter_depth):
# assuming LEDs are evenly distributed, find number for each edge of ROI
top_num_leds = self._numLEDs * (top_length_pixels /
perimeter_length_pixels)
side_num_leds = self._numLEDs * (side_length_pixels /
perimeter_length_pixels)
top_segment_length = top_length_pixels / self._topSegments
side_segment_length = side_length_pixels / self._sideSegments
for i in range(0, self._topSegments):
ix = int(self.ix + i * top_segment_length)
fx = int(self.ix + (i + 1) * top_segment_length)
iy = int(self.iy)
fy = int(self.iy + perimeter_depth)
c = self.getMeanColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['top'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 0, 255), 1)
cv2.rectangle(frame, (ix, iy - (10 + perimeter_depth)),
(fx, fy - perimeter_depth),
(int(c[0]), int(c[1]), int(c[2])), 10)
for i in range(0, self._sideSegments):
ix = int(self.fx - perimeter_depth)
fx = int(self.fx)
iy = int(self.iy + i * side_segment_length)
fy = int(self.iy + (i + 1) * side_segment_length)
c = self.getMeanColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['right'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 255, 0), 1)
cv2.rectangle(frame, (ix + perimeter_depth, iy),
(fx + (10 + perimeter_depth), fy),
(int(c[0]), int(c[1]), int(c[2])), 10)
for i in range(0, self._topSegments):
ix = int(self.fx - (i + 1) * top_segment_length)
fx = int(self.fx - i * top_segment_length)
iy = int(self.fy - perimeter_depth)
fy = int(self.fy)
c = self.getMeanColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['bottom'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 0, 255), 1)
cv2.rectangle(frame, (ix, iy + perimeter_depth),
(fx, fy + (10 + perimeter_depth)),
(int(c[0]), int(c[1]), int(c[2])), 10)
for i in range(0, self._sideSegments):
ix = int(self.ix)
fx = int(self.ix + perimeter_depth)
iy = int(self.fy - (i + 1) * side_segment_length)
fy = int(self.fy - i * side_segment_length)
c = self.getMeanColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['left'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 255, 0), 1)
cv2.rectangle(frame, (ix - (10 + perimeter_depth), iy),
(fx - perimeter_depth, fy),
(int(c[0]), int(c[1]), int(c[2])), 10)
return data
def getEdgeDominantColors(self, frame, top_length_pixels,
side_length_pixels, perimeter_length_pixels,
perimeter_depth):
top_segment_length = top_length_pixels / self._topSegments
side_segment_length = side_length_pixels / self._sideSegments
data = {}
data['top'] = [None] * self._topSegments
data['right'] = [None] * self._sideSegments
data['bottom'] = [None] * self._topSegments
data['left'] = [None] * self._sideSegments
for i in range(0, self._topSegments):
ix = int(self.ix + i * top_segment_length)
fx = int(self.ix + (i + 1) * top_segment_length)
iy = int(self.iy)
fy = int(self.iy + perimeter_depth)
c = self.getDominantSegmentColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['top'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 0, 255), 1)
cv2.rectangle(frame, (ix, iy - (10 + perimeter_depth)),
(fx, fy - perimeter_depth),
(int(c[0]), int(c[1]), int(c[2])), 10)
for i in range(0, self._sideSegments):
ix = int(self.fx - perimeter_depth)
fx = int(self.fx)
iy = int(self.iy + i * side_segment_length)
fy = int(self.iy + (i + 1) * side_segment_length)
c = self.getDominantSegmentColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['right'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 255, 0), 1)
cv2.rectangle(frame, (ix + perimeter_depth, iy),
(fx + (10 + perimeter_depth), fy),
(int(c[0]), int(c[1]), int(c[2])), 10)
for i in range(0, self._topSegments):
ix = int(self.fx - (i + 1) * top_segment_length)
fx = int(self.fx - i * top_segment_length)
iy = int(self.fy - perimeter_depth)
fy = int(self.fy)
c = self.getDominantSegmentColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['bottom'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 0, 255), 1)
cv2.rectangle(frame, (ix, iy + perimeter_depth),
(fx, fy + (10 + perimeter_depth)),
(int(c[0]), int(c[1]), int(c[2])), 10)
for i in range(0, self._sideSegments):
ix = int(self.ix)
fx = int(self.ix + perimeter_depth)
iy = int(self.fy - (i + 1) * side_segment_length)
fy = int(self.fy - i * side_segment_length)
c = self.getDominantSegmentColor(
cv2.resize(frame[iy:fy, ix:fx, :], (0, 0), fx=0.2, fy=0.2))
data['left'][i] = self.getRGBHexString(c)
if self._showVideoStream:
cv2.rectangle(frame, (ix, iy), (fx, fy), (0, 255, 0), 1)
cv2.rectangle(frame, (ix - (10 + perimeter_depth), iy),
(fx - perimeter_depth, fy),
(int(c[0]), int(c[1]), int(c[2])), 10)
return data
def getRGBHexString(self, bgr):
return "%x%x%x" % (bgr[2], bgr[1], bgr[0])
def getDominantColor(self, img, ix, fx, iy, fy):
ix = int(ix)
fx = int(fx)
iy = int(iy)
fy = int(fy)
average_color = [
img[iy:fy, ix:fx, i].mean() for i in range(img.shape[-1])
]
arr = np.float32(img)
pixels = arr.reshape((-1, 3))
# kmeans clustering
_, labels, centroids = cv2.kmeans(pixels, self.n_colors, None,
self.criteria, 10, self.flags)
palette = np.uint8(centroids)
quantized = palette[labels.flatten()]
quantized = quantized.reshape(img.shape)
dominant_color = palette[np.argmax(itemfreq(labels)[:, -1])]
return dominant_color
def initializeCamera(self):
# open first webcam available
self.video = cv2.VideoCapture(0)
if not self.video.isOpened():
self.video.open()
#set the resolution from config
self.video.set(cv2.CAP_PROP_FRAME_WIDTH, self._captureWidth)
self.video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._captureHeight)
def getROI(self, event, x, y, flags, frame):
framecopy = frame.copy()
if event == cv2.EVENT_LBUTTONDOWN:
self.clicking = True
self.ix, self.iy = x, y
elif event == cv2.EVENT_MOUSEMOVE:
if self.clicking:
cv2.rectangle(framecopy, (self.ix, self.iy), (x, y),
(0, 255, 0), -1)
cv2.imshow('Set ROI', framecopy)
elif event == cv2.EVENT_LBUTTONUP:
self.clicking = False
cv2.rectangle(framecopy, (self.ix, self.iy), (x, y), (0, 255, 0),
-1)
cv2.imshow('Set ROI', framecopy)
self.fx, self.fy = x, y
self.boundSet = True
def processQueue(self):
self.logger.info(
"Starting to watch collection point inbound message queue")
while self.alive:
if (self.inQueue.empty() == False):
self.logger.info("Queue size is %s" % self.inQueue.qsize())
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None:
if message == "SHUTDOWN":
self.logger.info("SHUTDOWN command handled on %s" %
__name__)
self.shutdown()
else:
self.handleMessage(message)
except Exception as e:
self.logger.error("Unable to read queue, error: %s " % e)
self.shutdown()
self.logger.info("Queue size is %s after" %
self.inQueue.qsize())
else:
time.sleep(.25)
def handleMessage(self, message):
self.logger.info("handleMessage not implemented!")
def putCPMessage(self, data, type):
if type == "off":
# Send off message
self.logger.info('Sending off message')
msg = CollectionPointEvent(self._collectionPointId,
self._collectionPointType, 'off', None)
self.outQueue.put(msg)
elif type == "light-edges":
# Reset collection start and now needs needs reset
collectionStart = time.time()
self.logger.info('Sending light message')
msg = CollectionPointEvent(self._collectionPointId,
self._collectionPointType,
'light-edges', data)
self.outQueue.put(msg)
elif type == "light-dominant":
# Reset collection start and now needs needs reset
collectionStart = time.time()
self.logger.info('Sending light message')
msg = CollectionPointEvent(self._collectionPointId,
self._collectionPointType,
'light-dominant', data)
self.outQueue.put(msg)
def shutdown(self):
self.alive = False
self.logger.info("Shutting down")
# self.putCPMessage(None, 'off')
cv2.destroyAllWindows()
time.sleep(1)
self.exit = True
def get_opencv_version(self):
import cv2 as lib
return lib.__version__
def check_opencv_version(self, major, lib=None):
# if the supplied library is None, import OpenCV
if lib is None:
import cv2 as lib
# return whether or not the current OpenCV version matches the
# major version number
return lib.__version__.startswith(major)
class AzureImagePrediction(AbstractImagePrediction):
def __init__(self, moduleConfig=None, loggingQueue=None):
""" Initialize new AzureImagePrediction instance.
Set parameters required by Azure Face API.
"""
logging.basicConfig(level=logging.CRITICAL)
self.logger = ThreadsafeLogger(
loggingQueue, "AzureImagePrediction") # Setup logging queue
self.config = moduleConfig
# Constants
self._subscriptionKey = self.config['Azure']['SubscriptionKey']
self._uriBase = self.config['Azure']['UriBase']
self._headers = {
'Content-Type': 'application/octet-stream',
'Ocp-Apim-Subscription-Key':
self.config['Azure']['SubscriptionKey'],
}
self._params = urllib.parse.urlencode({
"returnFaceId":
"true",
"returnFaceLandmarks":
"false",
"returnFaceAttributes":
"age,gender,glasses,facialHair"
})
def getPrediction(self, imageBytes):
""" Get prediction results from Azure Face API.
Returns object with either a predictions array property or an error property.
"""
resultData = {}
try:
tempResult = self.__getPrediction(imageBytes)
resultData['predictions'] = tempResult
except Exception as e:
self.logger.error('Error getting prediction: %s' % e)
resultData['error'] = str(e)
return resultData
def __getPrediction(self, imageBytes):
""" Execute REST API call and return result """
if len(self._subscriptionKey) < 10:
raise EnvironmentError(
'Azure subscription key - %s - is not valid' %
self._subscriptionKey)
else:
try:
api_url = "https://%s/face/v1.0/detect?%s" % (self._uriBase,
self._params)
r = requests.post(api_url,
headers=self._headers,
data=imageBytes)
if r.status_code != 200:
raise ValueError(
'Request to Azure returned an error %s, the response is:\n%s'
% (r.status_code, r.text))
jsonResult = r.json()
self.logger.debug("Got azure data %s" % jsonResult)
return jsonResult
except Exception as e:
self.logger.error(e)
# Collection point queues
cpEventOutboundChannel = mp.Queue()
cpEventInboundChannel = mp.Queue()
# For each collection module, import, initialize
for moduleName in _collectionModuleNames:
try:
sys.path.append('./collection_modules/%s' % moduleName)
_collectionModules[moduleName] = import_module(
'collection_modules.%s' % moduleName)
queues[moduleName] = {}
# queues[moduleName]['in'] = mp.Queue()
queues[moduleName]['out'] = mp.Queue()
except Exception as e:
logger.error('Error importing %s: %s' % (moduleName, e))
# For each collection module, import, initialize, and create an in/out queue
for moduleName in _communicationModuleNames:
try:
logger.debug('importing %s' % (moduleName))
sys.path.append('./communication_modules/%s' % moduleName)
_communicationModules[moduleName] = import_module(
'communication_modules.%s' % moduleName)
queues[moduleName] = {}
queues[moduleName]['in'] = mp.Queue()
queues[moduleName]['out'] = mp.Queue()
except Exception as e:
logger.error('Error importing %s: %s' % (moduleName, e))
class Tracker():
def __init__(self, bbox, frame, kind, moduleConfig, loggingQueue):
""" Create and initialize a new Tracker. Set up constants and parameters. """
if kind in ["KCF", "MIL", "MEDIANFLOW", "GOTURN", "TLD", "BOOSTING"]:
self.tracker = cv2.Tracker_create(kind)
self.tracker.init(frame,
(bbox['x'], bbox['y'], bbox['w'], bbox['h']))
self.created = time()
self.bbox = (bbox['x'], bbox['y'], bbox['w'], bbox['h'])
self.velocity = (0, 0)
self.updateTime = self.created
self.config = moduleConfig
# Constants
self._useVelocity = self.config['UseVelocity']
self._horizontalVelocityBuffer = self.config[
'HorizontalVelocityBuffer']
self._verticalVelocityBuffer = self.config[
'VerticalVelocityBuffer']
# Setup logging queue
self.logger = ThreadsafeLogger(loggingQueue, __name__)
else:
self.logger.error("Type %s not supported by mTracker" % kind)
def getCreated(self):
""" Get created time """
return self.created
def right(self):
""" Get right bound of tracker """
return self.bbox[0] + self.bbox[2]
def top(self):
""" Get top bound of tracker """
return self.bbox[1] + self.bbox[3]
def bottom(self):
""" Get bottom bound of tracker """
return self.bbox[1]
def left(self):
""" Get left bound of tracker """
return self.bbox[0]
def area(self):
""" Get area of tracker bounding box """
return abs(self.right() - self.left()) * abs(self.top() -
self.bottom())
def update(self, frame):
""" Update tracker.
If velocity hack is being used, calculate the new velocity of the midpoint.
"""
ok, bbox = self.tracker.update(frame)
if self._useVelocity:
# Set velocity (pixels/sec)
deltaT = time() - self.updateTime
centreNow = ((bbox[0] + bbox[2] / 2), (bbox[1] + bbox[3] / 2))
centreLast = ((self.bbox[0] + self.bbox[2] / 2),
(self.bbox[1] + self.bbox[3] / 2))
Vx = (centreNow[0] - centreLast[0]) / deltaT
Vy = (centreNow[1] - centreLast[1]) / deltaT
self.velocity = (Vx, Vy)
self.logger.debug('New velocity: %s' % str(self.velocity[0]) +
', ' + str(self.velocity[1]))
self.updateTime = time()
self.bbox = bbox
return ok, bbox
def getProjectedLocation(self, time):
""" Get the estimated location of the bounding box, based on previous velocity. """
deltaT = max((time - self.updateTime), 1)
centreNow = ((self.bbox[0] + self.bbox[2] / 2),
(self.bbox[1] + self.bbox[3] / 2))
projectedX = centreNow[0] + (self.velocity[0] * deltaT)
projectedY = centreNow[1] + (self.velocity[1] * deltaT)
return (projectedX, projectedY)
def getVelocityBuffer(self):
''' Another hack to improve low frame rate tracking.
"Spread" out the bounding box based on velocity.
'''
return (abs(self.velocity[0]) * self._horizontalVelocityBuffer,
abs(self.velocity[1]) * self._verticalVelocityBuffer)
class CollectionPoint(Thread):
""" Sample class to show basic structure of collecting data and passing it to communication channels """
def __init__(self,baseConfig,pOutBoundEventQueue, pInBoundEventQueue, loggingQueue):
# Standard initialization that most collection points would do
super(CollectionPoint, self).__init__()
self.alive = True
self.config = baseConfig
self.outBoundEventQueue = pOutBoundEventQueue
self.inBoundEventQueue = pInBoundEventQueue
self.logger = ThreadsafeLogger(loggingQueue,__name__)
# Initialize collection point specific variables
self.video = None
# Set constants from config
self._collectionPointId = self.config['CollectionPointId']
self._collectionPointType = self.config['CollectionPointType']
self._testMode = self.config['TestMode']
if not self.check_opencv_version("3.",cv2):
self.logger.critical("open CV is the wrong version {0}. We require version 3.x".format(self.get_opencv_version()))
def run(self):
""" Sample run function for a collection point class.
Starting point for when the thread is start()'d from main.py
Extend this to create your own, or understand how to perform specific actions.
"""
# Start a thread to monitor the inbound queue
self.threadProcessQueue = Thread(target=self.processQueue)
self.threadProcessQueue.setDaemon(True)
self.threadProcessQueue.start()
# Load the OpenCV classifier to detect faces
faceCascade = cv2.CascadeClassifier('./classifiers/haarcascades/haarcascade_frontalface_default.xml')
tracker = cv2.Tracker_create("KCF")
# Get first camera connected
video = cv2.VideoCapture(0)
if not video.isOpened():
video.open()
# Set resolution of capture
video.set(cv2.CAP_PROP_FRAME_WIDTH, 1080)
video.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
ok, frame = video.read()
if not ok:
self.logger.error('Cannot read video file')
self.shutdown()
while self.alive:
# Read a new frame
ok, frame = video.read()
if not ok:
self.logger.error('Cannot read video file')
break
# Convert to grayscale
grayFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Copy the frame to allow manipulation
outputImage = frame.copy()
# Detect faces
faces = faceCascade.detectMultiScale(
grayFrame,
scaleFactor=1.1,
minNeighbors=5,
minSize=(5, 5)
)
self.logger.info("Found " + str(len(faces)) + " faces")
# Draw a rectangle around each face
for (x, y, w, h) in faces:
cv2.rectangle(outputImage, (x, y), (x+w, y+h), (0, 255, 0), 2)
if self._testMode:
if len(faces) > 0:
msg = CollectionPointEvent(self._collectionPointId,self._collectionPointType,('Found {0} faces'.format(len(faces))))
self.outBoundEventQueue.put(msg)
# Display the image
cv2.imshow("Faces found", outputImage)
ch = 0xFF & cv2.waitKey(1)
if ch == 27: #esc key
self.shutdown()
break
video.release()
def shutdown(self):
self.logger.info("Shutting down collection point")
cv2.destroyAllWindows()
self.threadProcessQueue.join()
self.alive = False
time.sleep(1)
self.exit = True
def get_opencv_version(self):
import cv2 as lib
return lib.__version__
def check_opencv_version(self,major, lib=None):
# if the supplied library is None, import OpenCV
if lib is None:
import cv2 as lib
# return whether or not the current OpenCV version matches the
# major version number
return lib.__version__.startswith(major)
class IdsWrapper(object):
def __init__(self, loggingQueue, moduleConfig):
""" Create a new IdsWrapper instance.
IdsWrapper uses the Windows IDS DLL, wraps C calls in Python.
Tested using the IDS XS 2.0 USB camera
"""
self.config = moduleConfig
self.isOpen = False
self.width = 0
self.height = 0
self.bitspixels = 0
self.py_width = 0
self.py_height = 0
self.py_bitspixel = 0
self.cam = None
self.pcImgMem = c_char_p() #create placeholder for image memory
self.pid=c_int()
# Setup logger
self.logger = ThreadsafeLogger(loggingQueue, __name__)
# Load the correct dll
try:
if architecture()[0] == '64bit':
self.logger.info('Using IDS camera with 64-bit architecture')
self.uEyeDll = cdll.LoadLibrary("C:/Windows/System32/uEye_api_64.dll")
else:
self.logger.info('Using IDS camera with 32-bit architecture')
self.uEyeDll = cdll.LoadLibrary("C:/Windows/System32/uEye_api.dll")
except Exception as e:
self.logger.error('Failed to load IDS DLL: %s . Are you sure you are using an IDS camera?'%e)
def isOpened(self):
""" Return camera open status"""
return self.isOpen
def set(self, key, value):
""" Set the py_width, py_height or py_bitspixel properties """
if key == 'py_width':
self.py_width = value
elif key == 'py_height':
self.py_height = value
else:
self.py_bitspixel = value
def allocateImageMemory(self):
""" Wrapped call to allocate image memory """
ret=self.uEyeDll.is_AllocImageMem(self.cam, self.width, self.height, self.bitspixel, byref(self.pcImgMem), byref(self.pid))
if ret == IS_SUCCESS:
self.logger.info("Successfully allocated image memory")
else:
self.logger.error('Memory allocation failed, no camera with value ' + str(self.cam.value) + ' | Error code: ' + str(ret))
return
def setImageMemory(self):
""" Wrapped call to set image memory """
ret = self.uEyeDll.is_SetImageMem(self.cam, self.pcImgMem, self.pid)
if ret == IS_SUCCESS:
self.logger.info("Successfully set image memory")
else:
self.logger.error("Failed to set image memory; error code: " + str(ret))
return
def beginCapture(self):
""" Wrapped call to begin capture """
ret = self.uEyeDll.is_CaptureVideo (self.cam, c_long(IS_DONT_WAIT))
if ret == IS_SUCCESS:
self.logger.info("Successfully began video capture")
else:
self.logger.error("Failed to begin video capture; error code: " + str(ret))
return
def initImageData(self):
""" Initialize the ImageData numpy array """
self.ImageData = np.ones((self.py_height,self.py_width),dtype=np.uint8)
def setCTypes(self):
""" Set C Types for width, height, and bitspixel properties"""
self.width = c_int(self.py_width)
self.height = c_int(self.py_height)
self.bitspixel = c_int(self.py_bitspixel)
def start(self):
""" Start capturing frames on another thread as a daemon """
self.updateThread = Thread(target=self.update)
self.updateThread.setDaemon(True)
self.updateThread.start()
return self
def initializeCamera(self):
""" Wrapped call to initialize camera """
ret = self.uEyeDll.is_InitCamera(byref(self.cam), self.hWnd)
if ret == IS_SUCCESS:
self.logger.info("Successfully initialized camera")
else:
self.logger.error("Failed to initialize camera; error code: " + str(ret))
return
def enableAutoExit(self):
""" Wrapped call to allow allocated memory to be dropped on exit. """
ret = self.uEyeDll.is_EnableAutoExit (self.cam, c_uint(1))
if ret == IS_SUCCESS:
self.logger.info("Successfully enabled auto exit")
else:
self.logger.error("Failed to enable auto exit; error code: " + str(ret))
return
def setDisplayMode(self):
""" Wrapped call to set display mode to DIB """
ret = self.uEyeDll.is_SetDisplayMode (self.cam, c_int(IS_SET_DM_DIB))
if ret == IS_SUCCESS:
self.logger.info("Successfully set camera to DIB mode")
else:
self.logger.error("Failed to set camera mode; error code: " + str(ret))
return
def setColorMode(self):
""" Wrapped call to set camera color capture mode """
ret = self.uEyeDll.is_SetColorMode(self.cam, c_int(IS_CM_SENSOR_RAW8))
if ret == IS_SUCCESS:
self.logger.info("Successfully set color mode")
else:
self.logger.error("Failed to set color mode; error code: " + str(ret))
return
def setCompressionFactor(self):
""" Wrapped call to set image compression factor.
Required for long USB lengths when bandwidth is constrained, lowers quality.
"""
ret = self.uEyeDll.is_DeviceFeature(self.cam, IS_DEVICE_FEATURE_CMD_SET_JPEG_COMPRESSION, byref(c_int(self.config['CompressionFactor'])), c_uint(INT_BYTE_SIZE));
if ret == IS_SUCCESS:
self.logger.info("Successfully set compression factor to: " + str(self.config['CompressionFactor']))
else:
self.logger.error("Failed to set compression factor; error code: " + str(ret))
return
def setPixelClock(self):
""" Wrapped call to set pixel clock.
Required for long USB lengths when bandwidth is constrained
Lowers frame rate and increases motion blur.
"""
ret = self.uEyeDll.is_PixelClock(self.cam, IS_PIXELCLOCK_CMD_SET, byref(c_uint(self.config['PixelClock'])), c_uint(INT_BYTE_SIZE));
if ret == IS_SUCCESS:
self.logger.info("Successfully set pixel clock to: " + str(self.config['PixelClock']))
else:
self.logger.error("Failed to set pixel clock; error code: " + str(ret))
return
def setTrigger(self):
""" Wrapped call to set trigger type to software trigger. """
ret = self.uEyeDll.is_SetExternalTrigger(self.cam, c_uint(IS_SET_TRIGGER_SOFTWARE))
if ret == IS_SUCCESS:
self.logger.info("Successfully set software trigger")
else:
self.logger.error("Failed to set software trigger; error code: " + str(ret))
return
def setImageProfile(self):
""" Wrapped call to set image format.
Sets resolution of the capture to UXGA. More modes available in idsConsts.py.
"""
ret = self.uEyeDll.is_ImageFormat(self.cam, c_uint(IMGFRMT_CMD_SET_FORMAT), byref(c_int(UXGA)), c_uint(INT_BYTE_SIZE))
if ret == IS_SUCCESS:
self.logger.info("Successfully set camera image profile")
else:
self.logger.error("Failed to set camera image profile; error code: " + str(ret))
return
def open(self):
""" Open connection to IDS camera, set various modes. """
self.cam = c_uint32(0)
self.hWnd = c_voidp()
self.initializeCamera()
self.enableAutoExit()
self.setDisplayMode()
self.setColorMode()
self.setCompressionFactor()
self.setPixelClock()
self.setTrigger()
self.setImageProfile()
# Declare video open
self.isOpen = True
self.logger.info('Successfully opened camera')
def update(self):
""" Loop to update frames and copy to ImageData variable. """
while True:
if not self.isOpen:
return
self.uEyeDll.is_CopyImageMem (self.cam, self.pcImgMem, self.pid, self.ImageData.ctypes.data_as(c_char_p))
def read(self):
""" Read frame currently available in ImageData variable. """
try:
return True, self.ImageData
except Exception as e:
self.logger.error('Error getting image data: %r'% e)
return False, None
def exit(self):
""" Close camera down, release memory. """
self.uEyeDll.is_ExitCamera(self.cam)
self.isOpen = False
self.logger.info('Closing wrapper and camera')
return
class CamCollectionPoint(Thread):
def __init__(self, baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue):
""" Initialize new CamCollectionPoint instance.
Setup queues, variables, configs, predictionEngines, constants and loggers.
"""
super(CamCollectionPoint, self).__init__()
if not self.check_opencv_version("3.", cv2):
print(
"OpenCV version {0} is not supported. Use 3.x for best results."
.format(self.get_opencv_version()))
# Queues
self.outQueue = pOutBoundQueue #messages from this thread to the main process
self.inQueue = pInBoundQueue
self.loggingQueue = loggingQueue
# Variables
self.video = None
self.needsReset = False
self.needsResetMux = False
self.alive = True
# Configs
self.moduleConfig = camConfigLoader.load(
self.loggingQueue) #Get the config for this module
self.config = baseConfig
# Prediction engine
self.imagePredictionEngine = AzureImagePrediction(
moduleConfig=self.moduleConfig, loggingQueue=loggingQueue)
# Constants
self._useIdsCamera = self.moduleConfig['UseIdsCamera']
self._minFaceWidth = self.moduleConfig['MinFaceWidth']
self._minFaceHeight = self.moduleConfig['MinFaceHeight']
self._minNearestNeighbors = self.moduleConfig['MinNearestNeighbors']
self._maximumPeople = self.moduleConfig['MaximumPeople']
self._facePixelBuffer = self.moduleConfig['FacePixelBuffer']
self._collectionThreshold = self.moduleConfig['CollectionThreshold']
self._showVideoStream = self.moduleConfig['ShowVideoStream']
self._sendBlobs = self.moduleConfig['SendBlobs']
self._blobWidth = self.moduleConfig['BlobWidth']
self._blobHeight = self.moduleConfig['BlobHeight']
self._captureWidth = self.moduleConfig['CaptureWidth']
self._captureHeight = self.moduleConfig['CaptureHeight']
self._bitsPerPixel = self.moduleConfig['BitsPerPixel']
self._resetEventTimer = self.moduleConfig['ResetEventTimer']
self._collectionPointType = self.config['CollectionPointType']
self._collectionPointId = self.config['CollectionPointId']
# Logger
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def run(self):
""" Main thread method, run when the thread's start() function is called.
Controls flow of detected faces and the MultiTracker.
Determines when to send 'reset' events to clients and when to send 'found' events.
This function contains various comments along the way to help understand the flow.
You can use this flow, extend it, or build your own.
"""
# Monitor inbound queue on own thread
self.threadProcessQueue = Thread(target=self.processQueue)
self.threadProcessQueue.setDaemon(True)
self.threadProcessQueue.start()
self.initializeCamera()
# Load the OpenCV Haar classifier to detect faces
curdir = os.path.dirname(__file__)
cascadePath = os.path.join(curdir, 'classifiers', 'haarcascades',
'haarcascade_frontalface_default.xml')
faceCascade = cv2.CascadeClassifier(cascadePath)
self.mmTracker = MultiTracker("KCF", self.moduleConfig,
self.loggingQueue)
# Setup timer for FPS calculations
start = time.time()
frameCounter = 1
fps = 0
# Start timer for collection events
self.collectionStart = time.time()
ok, frame = self.video.read()
if not ok:
self.logger.error('Cannot read video file')
self.shutdown()
while self.alive:
ok, frame = self.video.read()
if not ok:
self.logger.error('Error while reading frame')
break
# Image alts
if self._useIdsCamera:
grayFrame = frame.copy()
outputImage = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
else:
outputImage = frame.copy()
grayFrame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
# Detect faces
faces = faceCascade.detectMultiScale(
grayFrame,
scaleFactor=1.1,
minNeighbors=self._minNearestNeighbors,
minSize=(self._minFaceWidth, self._minFaceHeight))
# If no faces in frame, clear tracker and start reset timer
if len(faces
) == 0 or self.mmTracker.length() > self._maximumPeople:
self.mmTracker.clear()
self.startReset()
# If there are trackers, update
if self.mmTracker.length() > 0:
ok, bboxes, failed = self.mmTracker.update(outputImage)
if failed:
self.logger.error('Update trackers failed on: %s' %
''.join(str(s) for s in failed))
for (x, y, w, h) in faces:
# If faces are detected, engagement exists, do not reset
self.needsReset = False
# Optionally add buffer to face, can improve tracking/classification accuracy
if self._facePixelBuffer > 0:
(x, y, w,
h) = self.applyFaceBuffer(x, y, w, h,
self._facePixelBuffer,
outputImage.shape)
# Get region of interest
roi_gray = grayFrame[y:y + h, x:x + w]
roi_color = outputImage[y:y + h, x:x + w]
# If the tracker is valid and doesn't already exist, add it
if self.validTracker(x, y, w, h):
self.logger.info('Adding tracker')
ok = self.mmTracker.add(bbox={
'x': x,
'y': y,
'w': w,
'h': h
},
frame=outputImage)
# Draw box around face
if self._showVideoStream:
cv2.rectangle(outputImage, (x, y), (x + w, y + h),
(0, 255, 0), 2)
# If the time since last collection is more than the set threshold
if not self.needsReset or (time.time() - self.collectionStart >
self._collectionThreshold):
# Check if the focal face has changed
check, face = self.mmTracker.checkFocus()
if check:
predictions = self.getPredictions(grayFrame, face)
if predictions:
self.putCPMessage(data={
'detectedTime':
datetime.now().isoformat('T'),
'predictions':
predictions
},
type="update")
frameCounter += 1
elapsed = time.time() - start
fps = frameCounter / max(abs(elapsed), 0.0001)
if frameCounter > sys.maxsize:
start = time.time()
frameCounter = 1
if self._showVideoStream:
cv2.putText(outputImage, "%s FPS" % fps, (20, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1,
cv2.LINE_AA)
cv2.imshow("Faces found", outputImage)
cv2.waitKey(1)
if self._sendBlobs and frameCounter % 6 == 0:
self.putCPMessage(data={
'imageArr':
cv2.resize(outputImage,
(self._blobWidth, self._blobHeight)),
'time':
datetime.now().isoformat('T')
},
type="blob")
def getPredictions(self, grayFrame, face):
""" Send face to predictionEngine as JPEG.
Return predictions array or false if no face is found.
"""
faceArr = grayFrame[int(face[1]):int(face[1] + face[3]),
int(face[0]):int(face[0] + face[2])]
img = Image.fromarray(faceArr)
buff = io.BytesIO()
img.save(buff, format="JPEG")
try:
predictions = self.imagePredictionEngine.getPrediction(
buff.getvalue())
except Exception as e:
predictions = False
return predictions
def validTracker(self, x, y, w, h):
""" Check if the coordinates are a newly detected face or already present in MultiTracker.
Only accepts new tracker candidates every _collectionThreshold seconds.
Return true if the object in those coordinates should be tracked.
"""
if not self.needsReset or (time.time() - self.collectionStart >
self._collectionThreshold):
if (self.mmTracker.length() == 0
or not self.mmTracker.contains(bbox={
'x': x,
'y': y,
'w': w,
'h': h
})):
return True
return False
def startReset(self):
"""Start a timer from reset event.
If timer completes and the reset event should still be sent, send it.
"""
if self.needsResetMux:
self.needsReset = True
self.needsResetMux = False
self.resetStart = time.time()
if self.needsReset:
if (time.time() - self.resetStart
) > 10: # 10 seconds after last face detected
self.putCPMessage(data=None, type="reset")
self.needsReset = False
def applyFaceBuffer(self, x, y, w, h, b, shape):
x = x - b if x - b >= 0 else 0
y = y - b if y - b >= 0 else 0
w = w + b if w + b <= shape[1] else shape[1]
h = h + b if h + b <= shape[0] else shape[0]
return (x, y, w, h)
def initializeCamera(self):
# Using IDS camera
if self._useIdsCamera:
self.logger.info("Using IDS Camera")
self.wrapper = IdsWrapper(self.loggingQueue, self.moduleConfig)
if not (self.wrapper.isOpened()):
self.wrapper.open()
self.wrapper.set('py_width', self._captureWidth)
self.wrapper.set('py_height', self._captureHeight)
self.wrapper.set('py_bitspixel', self._bitsPerPixel)
# Convert values to ctypes, prep memory locations
self.wrapper.setCTypes()
self.wrapper.allocateImageMemory()
self.wrapper.setImageMemory()
self.wrapper.beginCapture()
self.wrapper.initImageData()
# Start video update thread
self.video = self.wrapper.start()
# Not using IDS camera
else:
# open first webcam available
self.video = cv2.VideoCapture(0)
if not (self.video.isOpened()):
self.video.open()
#set the resolution from config
self.video.set(cv2.CAP_PROP_FRAME_WIDTH, self._captureWidth)
self.video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._captureHeight)
def processQueue(self):
self.logger.info(
"Starting to watch collection point inbound message queue")
while self.alive:
if (self.inQueue.empty() == False):
self.logger.info("Queue size is %s" % self.inQueue.qsize())
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None:
if message == "SHUTDOWN":
self.logger.info("SHUTDOWN command handled on %s" %
__name__)
self.shutdown()
else:
self.sendOutMessage(message)
except Exception as e:
self.logger.error("Unable to read queue, error: %s " % e)
self.shutdown()
self.logger.info("Queue size is %s after" %
self.inQueue.qsize())
else:
time.sleep(.25)
def putCPMessage(self, data, type):
if type == "reset":
# Send reset message
self.logger.info('Sending reset message')
msg = CollectionPointEvent(self._collectionPointId,
self._collectionPointType, 'Reset mBox',
None)
self.outQueue.put(msg)
elif type == "update":
# Reset collection start and now needs needs reset
collectionStart = time.time()
self.needsResetMux = True
self.logger.info('Sending found message')
msg = CollectionPointEvent(self._collectionPointId,
self._collectionPointType, 'Found face',
data['predictions'])
self.outQueue.put(msg)
elif type == "blob":
# Get numpy array as bytes
img = Image.fromarray(data['imageArr'])
buff = io.BytesIO()
img.save(buff, format="JPEG")
s = base64.b64encode(buff.getvalue()).decode("utf-8")
eventExtraData = {}
eventExtraData['imageData'] = s
eventExtraData['dataType'] = 'image/jpeg'
# Send found message
# self.logger.info('Sending blob message')
msg = CollectionPointEvent(self._collectionPointId,
self._collectionPointType, 'blob',
eventExtraData, True)
self.outQueue.put(msg)
def shutdown(self):
self.alive = False
self.logger.info("Shutting down")
# self.outQueue.put("SHUTDOWN")
if self._useIdsCamera and self.wrapper.isOpened():
self.wrapper.exit()
cv2.destroyAllWindows()
# self.threadProcessQueue.join()
time.sleep(1)
self.exit = True
#Custom methods for demo
def get_opencv_version(self):
import cv2 as lib
return lib.__version__
def check_opencv_version(self, major, lib=None):
# if the supplied library is None, import OpenCV
if lib is None:
import cv2 as lib
# return whether or not the current OpenCV version matches the
# major version number
return lib.__version__.startswith(major)
