class IdsWrapper(object):
def __init__(self, loggingQueue, moduleConfig):
""" Create a new IdsWrapper instance.
IdsWrapper uses the Windows IDS DLL, wraps C calls in Python.
Tested using the IDS XS 2.0 USB camera
"""
self.config = moduleConfig
self.isOpen = False
self.width = 0
self.height = 0
self.bitspixels = 0
self.py_width = 0
self.py_height = 0
self.py_bitspixel = 0
self.cam = None
self.pcImgMem = c_char_p() #create placeholder for image memory
self.pid = c_int()
# Setup logger
self.logger = ThreadsafeLogger(loggingQueue, __name__)
# Load the correct dll
try:
if architecture()[0] == '64bit':
self.logger.info('Using IDS camera with 64-bit architecture')
self.uEyeDll = cdll.LoadLibrary(
"C:/Windows/System32/uEye_api_64.dll")
else:
self.logger.info('Using IDS camera with 32-bit architecture')
self.uEyeDll = cdll.LoadLibrary(
"C:/Windows/System32/uEye_api.dll")
except Exception as e:
self.logger.error(
'Failed to load IDS DLL: %s . Are you sure you are using an IDS camera?'
% e)
def isOpened(self):
""" Return camera open status"""
return self.isOpen
def set(self, key, value):
""" Set the py_width, py_height or py_bitspixel properties """
if key == 'py_width':
self.py_width = value
elif key == 'py_height':
self.py_height = value
else:
self.py_bitspixel = value
def allocateImageMemory(self):
""" Wrapped call to allocate image memory """
ret = self.uEyeDll.is_AllocImageMem(self.cam, self.width,
self.height, self.bitspixel,
byref(self.pcImgMem),
byref(self.pid))
if ret == IS_SUCCESS:
self.logger.info("Successfully allocated image memory")
else:
self.logger.error(
'Memory allocation failed, no camera with value ' +
str(self.cam.value) + ' | Error code: ' + str(ret))
return
def setImageMemory(self):
""" Wrapped call to set image memory """
ret = self.uEyeDll.is_SetImageMem(self.cam, self.pcImgMem, self.pid)
if ret == IS_SUCCESS:
self.logger.info("Successfully set image memory")
else:
self.logger.error("Failed to set image memory; error code: " +
str(ret))
return
def beginCapture(self):
""" Wrapped call to begin capture """
ret = self.uEyeDll.is_CaptureVideo(self.cam, c_long(IS_DONT_WAIT))
if ret == IS_SUCCESS:
self.logger.info("Successfully began video capture")
else:
self.logger.error("Failed to begin video capture; error code: " +
str(ret))
return
def initImageData(self):
""" Initialize the ImageData numpy array """
self.ImageData = np.ones((self.py_height, self.py_width),
dtype=np.uint8)
def setCTypes(self):
""" Set C Types for width, height, and bitspixel properties"""
self.width = c_int(self.py_width)
self.height = c_int(self.py_height)
self.bitspixel = c_int(self.py_bitspixel)
def start(self):
""" Start capturing frames on another thread as a daemon """
self.updateThread = Thread(target=self.update)
self.updateThread.setDaemon(True)
self.updateThread.start()
return self
def initializeCamera(self):
""" Wrapped call to initialize camera """
ret = self.uEyeDll.is_InitCamera(byref(self.cam), self.hWnd)
if ret == IS_SUCCESS:
self.logger.info("Successfully initialized camera")
else:
self.logger.error("Failed to initialize camera; error code: " +
str(ret))
return
def enableAutoExit(self):
""" Wrapped call to allow allocated memory to be dropped on exit. """
ret = self.uEyeDll.is_EnableAutoExit(self.cam, c_uint(1))
if ret == IS_SUCCESS:
self.logger.info("Successfully enabled auto exit")
else:
self.logger.error("Failed to enable auto exit; error code: " +
str(ret))
return
def setDisplayMode(self):
""" Wrapped call to set display mode to DIB """
ret = self.uEyeDll.is_SetDisplayMode(self.cam, c_int(IS_SET_DM_DIB))
if ret == IS_SUCCESS:
self.logger.info("Successfully set camera to DIB mode")
else:
self.logger.error("Failed to set camera mode; error code: " +
str(ret))
return
def setColorMode(self):
""" Wrapped call to set camera color capture mode """
ret = self.uEyeDll.is_SetColorMode(self.cam, c_int(IS_CM_SENSOR_RAW8))
if ret == IS_SUCCESS:
self.logger.info("Successfully set color mode")
else:
self.logger.error("Failed to set color mode; error code: " +
str(ret))
return
def setCompressionFactor(self):
""" Wrapped call to set image compression factor.
Required for long USB lengths when bandwidth is constrained, lowers quality.
"""
ret = self.uEyeDll.is_DeviceFeature(
self.cam, IS_DEVICE_FEATURE_CMD_SET_JPEG_COMPRESSION,
byref(c_int(self.config['CompressionFactor'])),
c_uint(INT_BYTE_SIZE))
if ret == IS_SUCCESS:
self.logger.info("Successfully set compression factor to: " +
str(self.config['CompressionFactor']))
else:
self.logger.error(
"Failed to set compression factor; error code: " + str(ret))
return
def setPixelClock(self):
""" Wrapped call to set pixel clock.
Required for long USB lengths when bandwidth is constrained
Lowers frame rate and increases motion blur.
"""
ret = self.uEyeDll.is_PixelClock(
self.cam, IS_PIXELCLOCK_CMD_SET,
byref(c_uint(self.config['PixelClock'])), c_uint(INT_BYTE_SIZE))
if ret == IS_SUCCESS:
self.logger.info("Successfully set pixel clock to: " +
str(self.config['PixelClock']))
else:
self.logger.error("Failed to set pixel clock; error code: " +
str(ret))
return
def setTrigger(self):
""" Wrapped call to set trigger type to software trigger. """
ret = self.uEyeDll.is_SetExternalTrigger(
self.cam, c_uint(IS_SET_TRIGGER_SOFTWARE))
if ret == IS_SUCCESS:
self.logger.info("Successfully set software trigger")
else:
self.logger.error("Failed to set software trigger; error code: " +
str(ret))
return
def setImageProfile(self):
""" Wrapped call to set image format.
Sets resolution of the capture to UXGA. More modes available in idsConsts.py.
"""
ret = self.uEyeDll.is_ImageFormat(self.cam,
c_uint(IMGFRMT_CMD_SET_FORMAT),
byref(c_int(UXGA)),
c_uint(INT_BYTE_SIZE))
if ret == IS_SUCCESS:
self.logger.info("Successfully set camera image profile")
else:
self.logger.error(
"Failed to set camera image profile; error code: " + str(ret))
return
def open(self):
""" Open connection to IDS camera, set various modes. """
self.cam = c_uint32(0)
self.hWnd = c_voidp()
self.initializeCamera()
self.enableAutoExit()
self.setDisplayMode()
self.setColorMode()
self.setCompressionFactor()
self.setPixelClock()
self.setTrigger()
self.setImageProfile()
# Declare video open
self.isOpen = True
self.logger.info('Successfully opened camera')
def update(self):
""" Loop to update frames and copy to ImageData variable. """
while True:
if not self.isOpen:
return
self.uEyeDll.is_CopyImageMem(
self.cam, self.pcImgMem, self.pid,
self.ImageData.ctypes.data_as(c_char_p))
def read(self):
""" Read frame currently available in ImageData variable. """
try:
return True, self.ImageData
except Exception as e:
self.logger.error('Error getting image data: %r' % e)
return False, None
def exit(self):
""" Close camera down, release memory. """
self.uEyeDll.is_ExitCamera(self.cam)
self.isOpen = False
self.logger.info('Closing wrapper and camera')
return
class WebsocketServerModule(ModuleProcess):
def __init__(self, baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue):
# super(WebsocketServerModule, self).__init__()
ModuleProcess.__init__(self, baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue)
self.alive = False
self.config = baseConfig
self.inQueue = pInBoundQueue # inQueue are messages from the main process to websocket clients
self.outQueue = pOutBoundQueue # outQueue are messages from clients to main process
self.websocketServer = None
self.loggingQueue = loggingQueue
self.threadProcessQueue = None
# Configs
self.moduleConfig = configLoader.load(self.loggingQueue, __name__)
# Constants
self._port = self.moduleConfig['WebsocketPort']
self._host = self.moduleConfig['WebsocketHost']
# logging setup
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def run(self):
if not self.check_ss_version():
#cant run with wrong version so we return early
return False
""" Main thread entry point.
Sets up websocket server and event callbacks.
Starts thread to monitor inbound message queue.
"""
self.logger.info("Starting websocket server")
self.alive = True
self.listen()
self.websocketServer = WebsocketServer(self._port, host=self._host)
self.websocketServer.set_fn_new_client(self.new_websocket_client)
self.websocketServer.set_fn_message_received(
self.websocket_message_received)
self.websocketServer.run_forever()
def check_ss_version(self):
#check for min version met
self.logger.info('Module version %s' % (__version__))
if LooseVersion(self.config['ss_version']) < LooseVersion(
self.moduleConfig['MinSimpleSensorVersion']):
self.logger.error(
'This module requires a min SimpleSensor %s version. This instance is running version %s'
% (self.moduleConfig['MinSimpleSensorVersion'],
self.config['ss_version']))
return False
return True
def new_websocket_client(self, client, server):
""" Client joined callback - called whenever a new client joins. """
self.logger.debug("Client joined")
def websocket_message_received(self, client, server, message):
""" Message received callback - called whenever a new message is received. """
self.logger.debug('Message received: %s' % message)
message = json.loads(message)
self.logger.info("message jsond: %s" % message)
_msg = Message(topic=message['topic'], sender_id=message['sender_id'])
if 'sender_type' in message:
_msg.sender_type = message['sender_type']
if 'recipients' in message:
_msg.recipients = message['recipients']
if 'extended_data' in message:
_msg.extended_data = message['extended_data']
self.put_message(_msg)
def listen(self):
self.threadProcessQueue = Thread(target=self.process_queue)
self.threadProcessQueue.setDaemon(True)
self.threadProcessQueue.start()
def shutdown(self):
""" Handle shutdown message.
Close and shutdown websocket server.
Join queue processing thread.
"""
self.logger.info("Shutting down websocket server")
try:
self.logger.info("Closing websocket")
self.websocketServer.server_close()
except Exception as e:
self.logger.error("Websocket close error : %s " % e)
try:
self.logger.info("Shutdown websocket")
self.websocketServer.shutdown()
except Exception as e:
self.logger.error("Websocket shutdown error : %s " % e)
self.alive = False
self.threadProcessQueue.join()
time.sleep(1)
self.exit = True
def handle_message(self, message):
""" Send message to listening clients. """
self.websocketServer.send_message_to_all(json.dumps(message.__dict__))
def process_queue(self):
""" Monitor queue of messages from main process to this thread. """
while self.alive:
if (self.inQueue.empty() == False):
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None:
if message.topic.upper() == "SHUTDOWN":
self.logger.debug("SHUTDOWN handled")
self.shutdown()
else:
self.handle_message(message)
except Exception as e:
self.logger.error("Websocket unable to read queue : %s " %
e)
else:
time.sleep(.25)
class AzureImagePredictor(ImagePredictor):
def __init__(self, moduleConfig=None, loggingQueue=None):
"""
Initialize new AzureImagePrediction instance.
Set parameters required by Azure Face API.
"""
# logging.basicConfig(level=logging.CRITICAL)
self.logger = ThreadsafeLogger(
loggingQueue, "AzureImagePrediction") # Setup logging queue
self.config = moduleConfig
# Constants
self._subscriptionKey = self.config['Azure']['SubscriptionKey']
self._uriBase = self.config['Azure']['UriBase']
self._headers = {
'Content-Type': 'application/octet-stream',
'Ocp-Apim-Subscription-Key': self._subscriptionKey,
}
self._params = urllib.parse.urlencode({
"returnFaceId":
"true",
"returnFaceLandmarks":
"false",
"returnFaceAttributes":
"age,gender,glasses,facialHair"
})
def get_prediction(self, imageBytes):
""" Get prediction results from Azure Face API.
Returns object with either a predictions array property or an error property.
"""
resultData = {}
try:
tempResult = self._get_prediction(imageBytes)
resultData['predictions'] = tempResult
except Exception as e:
self.logger.error('Error getting prediction: %s' % e)
resultData['error'] = str(e)
return resultData
def _get_prediction(self, imageBytes):
""" Execute REST API call and return result """
if len(self._subscriptionKey) < 10:
raise EnvironmentError(
'Azure subscription key - %s - is not valid' %
self._subscriptionKey)
else:
try:
api_url = "https://%s/face/v1.0/detect?%s" % (self._uriBase,
self._params)
r = requests.post(api_url,
headers=self._headers,
data=imageBytes)
if r.status_code != 200:
raise ValueError(
'Request to Azure returned an error %s, the response is:\n%s'
% (r.status_code, r.text))
jsonResult = r.json()
self.logger.debug("Got azure data %s" % jsonResult)
return jsonResult
except Exception as e:
self.logger.error(e)
class MQTTClientModule(ModuleProcess):
""" Threaded MQTT client for processing and publishing outbound messages"""
def __init__(self, baseConfig, pInBoundEventQueue, pOutBoundEventQueue, loggingQueue):
super(MQTTClientModule, self).__init__()
self.config = baseConfig
self.alive = True
self.inQueue = pInBoundEventQueue
# Module config
self.moduleConfig = configLoader.load(self.loggingQueue, __name__)
# Constants
self._keepAlive = self.moduleConfig['MqttKeepAlive']
self._feedName = self.moduleConfig['MqttFeedName']
self._username = self.moduleConfig['MqttUsername']
self._key = self.moduleConfig['MqttKey']
self._host = self.moduleConfig['MqttHost']
self._port = self.moduleConfig['MqttPort']
self._publishJson = self.moduleConfig['MqttPublishJson']
# MQTT setup
self._client = mqtt.Client()
self._client.username_pw_set(self._username, self._key)
self._client.on_connect = self.on_connect
self._client.on_disconnect = self.on_disconnect
self._client.on_message = self.on_message
self.mqttConnected = False
# Logging setup
self.logger = ThreadsafeLogger(loggingQueue, "MQTT")
def check_ss_version(self):
#check for min version met
self.logger.info('Module version %s' %(__version__))
if LooseVersion(self.config['ss_version']) < LooseVersion(self.moduleConfig['MinSimpleSensorVersion']):
self.logger.error('This module requires a min SimpleSensor %s version. This instance is running version %s' %(self.moduleConfig['MinSimpleSensorVersion'],self.config['ss_version']))
return False
return True
def on_connect(self, client, userdata, flags, rc):
self.logger.debug('MQTT onConnect called')
# Result code 0 is success
if rc == 0:
self.mqttConnected = True
# Subscribe to feed here
else:
self.logger.error('MQTT failed to connect: %s'%rc)
raise RuntimeError('MQTT failed to connect: %s'%rc)
def on_disconnect(self, client, userdata, rc):
self.logger.debug('MQTT onDisconnect called')
self.mqttConnected = False
if rc != 0:
self.logger.debug('MQTT disconnected unexpectedly: %s'%rc)
self.handle_reconnect(rc)
def handle_reconnect(self, result_code):
pass
def on_message(self, client, userdata, msg):
self.logger.debug('MQTT onMessage called for client: %s'%client)
def connect(self):
""" Connect to MQTT broker
Skip calling connect if already connected.
"""
if self.mqttConnected:
return
self._client.connect(self._host, port=self._port, keepalive=self._keepAlive)
def disconnect(self):
""" Check if connected"""
if self.mqttConnected:
self._client.disconnect()
def subscribe(self, feed=False):
"""Subscribe to feed, defaults to feed specified in config"""
if not feed: feed = _feedName
self._client.subscribe('{0}/feeds/{1}'.format(self._username, feed))
def publish(self, value, feed=False):
"""Publish a value to a feed"""
if not feed: feed = _feedName
self._client.publish('{0}/feeds/{1}'.format(self._username, feed), payload=value)
def publish_face_values(self, message):
""" Publish face detection values to individual MQTT feeds
Parses _extendedData.predictions.faceAttributes property
"""
try:
for face in message.extended_data['predictions']:
faceAttrs = face['faceAttributes']
for key in faceAttrs:
if type(faceAttrs[key]) is dict:
val = self.flatten_dict(faceAttrs[key])
print('val: ', val)
else:
val = faceAttrs[key]
self.publish(val, key)
except Exception as e:
self.logger.error('Error publishing values: %s'%e)
def flatten_dict(self, aDict):
""" Get average of simple dictionary of numerical values """
try:
val = float(sum(aDict[key] for key in aDict)) / len(aDict)
except Exception as e:
self.logger.error('Error flattening dict, returning 0: %s'%e)
return val or 0
def publish_json_message(self, message):
self.publish(message.stringify())
def stringify_message(self, message):
""" Dump into JSON string """
return json.dumps(message.__dict__).encode('utf8')
def process_queue(self):
""" Process incoming messages. """
while self.alive:
# Pump the loop
self._client.loop(timeout=1)
if (self.inQueue.empty() == False):
try:
message = self.inQueue.get(block=False,timeout=1)
if message is not None and self.mqttConnected:
if message.topic.upper() == "SHUTDOWN":
self.logger.debug("SHUTDOWN command handled")
self.shutdown()
else:
# Send message as string or split into channels
if self._publishJson:
self.publish_json_message(message)
elif self._publishFaceData:
self.publish_face_values(message)
else:
self.publish_values(message)
except Exception as e:
self.logger.error("MQTT unable to read queue : %s " %e)
else:
time.sleep(.25)
def shutdown(self):
self.logger.info("Shutting down")
self.alive = False
time.sleep(1)
self.exit = True
def run(self):
if not self.check_ss_version():
#cant run with wrong version so we return early
return False
""" Thread start method"""
self.logger.info("Running MQTT")
self.connect()
self.alive = True
# Start queue loop
self.process_queue()
inputThread.setDaemon(True)
inputThread.start()
# For each collection module, import, initialize
for moduleName in _collectionModuleNames:
try:
logger.debug('importing %s' % (moduleName))
_collectionModules[moduleName] = import_module(
'simplesensor.collection_modules.%s' % moduleName)
queues[moduleName] = {}
# Each module has it's own incoming message queue (outbound from main proc)
# This is for cases where messages should be handled by specific modules, not all.
queues[moduleName]['out'] = mp.Queue()
except Exception as e:
logger.error('Error importing %s: %s' % (moduleName, e))
# For each collection module, import, initialize, and create an in/out queue
for moduleName in _communicationModuleNames:
try:
logger.debug('importing %s' % (moduleName))
_communicationModules[moduleName] = import_module(
'simplesensor.communication_modules.%s' % moduleName)
queues[moduleName] = {}
queues[moduleName]['out'] = mp.Queue()
except Exception as e:
logger.error('Error importing %s: %s' % (moduleName, e))
alive = True
class MultiTracker(object):
def __init__(self, kind="KCF", moduleConfig=None, loggingQueue=None):
""" Create an initialize new MultiTracker. Set up constants and parameters. """
self.config = moduleConfig
self.trackers = [] # List of trackers
self.kind = kind
self.focus = None
self.loggingQueue = loggingQueue
# Constants
self._useVelocity = self.config['UseVelocity']
self._closestThreshold = self.config["ClosestThreshold"]
self._primaryTarget = self.config['PrimaryTarget']
# Setup logging queue
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def add(self, bbox, frame, kind="KCF"):
""" Add new tracker with default type KCF. """
aTracker = Tracker(bbox, frame, kind, self.config, self.loggingQueue)
self.trackers.append(aTracker)
def removeAt(self, i):
""" Remove Tracker at index i. """
self.trackers.pop(i)
def remove(self, aTracker):
""" Remove tracker provided as parameter. """
self.trackers.remove(aTracker)
def update(self, frame):
""" Loop through each tracker updating bounding box, keep track of failures. """
bboxes = []
ind = 0
failed = []
for aTracker in self.trackers:
ok, bbox = aTracker.update(frame)
if not ok:
failed.append(ind)
else:
bboxes.append(bbox)
ind += 1
if len(failed) == 0:
return True, bboxes, None
else:
self.logger.error('Failed to update all trackers')
return False, bboxes, failed
def clear(self):
""" Remove all trackers. """
self.trackers.clear()
self.focus = None
def bbox_contains_pt(self, bbox, pt, vBuffer):
""" Check if bbox contains pt.
Optionally provide velocity buffer to spread containing space.
"""
if ((bbox['x'] - vBuffer[0] <= pt[0] <=
(bbox['x'] + bbox['w'] + vBuffer[0]))
and (bbox['y'] - vBuffer[1] <= pt[1] <=
(bbox['y'] + bbox['h'] + vBuffer[1]))):
return True
else:
return False
def projected_location_matches(self, tracker, bbox):
""" Check if the velocity of the tracker could put it in the same spot as the bbox. """
if tracker.velocity:
return self.bbox_contains_pt(
bbox, tracker.get_projected_location(time()),
tracker.get_velocity_buffer())
else:
return False
def intersects(self, tracker, bbox):
""" Check if the bbox and the trackers bounds intersect. """
if (tracker.right() < bbox['x']
or bbox['x'] + bbox['w'] < tracker.left()
or tracker.top() < bbox['y']
or bbox['y'] + bbox['h'] < tracker.bottom()):
return False # intersection is empty
else:
return True # intersection is not empty
def contains(self, bbox):
""" Check if the MultiTracker already has a tracker for the object detected.
Uses intersections and projected locations to determine if the tracker overlaps others.
This means objects that overlap when first detected will not _both_ be added to the MultiTracker.
"""
for aTracker in self.trackers:
if self._useVelocity:
if self.intersects(aTracker,
bbox) and self.projected_location_matches(
aTracker, bbox):
return True
elif self.intersects(aTracker, bbox):
return True
return False
def length(self):
""" Get number of Trackers in the MultiTracker. """
return len(self.trackers)
def get_focus(self):
""" Get focal object based on primaryTarget configuration.
Currently only closest is supported - checks whether there is a tracker
that is larger than the previous closest tracker by the configured threshold.
"""
if self._primaryTarget == "closest":
focusChanged = False
if self.focus:
# area = self.focus.area()
area = self.focus.area()
else:
area = None
for aTracker in self.trackers:
# If there's no focus or aTracker is larger than focus, and they aren't the same tracker
if not self.focus or (aTracker.area() > area *
(1 + (self._closestThreshold / 100))
and self.focus.get_created() !=
aTracker.get_created()):
focusChanged = True
self.focus = aTracker
area = aTracker.area()
if focusChanged:
return self.focus
else:
return None
elif self._primaryTarget == "closest_engaged":
#TODO
self.logger.error('Primary Target %s is not implemented.' %
self._primaryTarget)
return None
else:
self.logger.error('Primary Target %s is not implemented.' %
self._primaryTarget)
return None
def check_focus(self):
""" Check if focal Tracker has changed by updating the focus. """
focus = self.get_focus()
if focus:
return True, focus.bbox
else:
return False, None
class Tracker():
def __init__(self, bbox, frame, kind, moduleConfig, loggingQueue):
""" Create and initialize a new Tracker. Set up constants and parameters. """
if kind in ["KCF", "MIL", "MEDIANFLOW", "GOTURN", "TLD", "BOOSTING"]:
self.tracker = cv2.Tracker_create(kind)
self.tracker.init(frame, (bbox['x'], bbox['y'], bbox['w'], bbox['h']))
self.created = time()
self.bbox = (bbox['x'], bbox['y'], bbox['w'], bbox['h'])
self.velocity = (0, 0)
self.updateTime = self.created
self.config = moduleConfig
# Constants
self._useVelocity = self.config['UseVelocity']
self._horizontalVelocityBuffer = self.config['HorizontalVelocityBuffer']
self._verticalVelocityBuffer = self.config['VerticalVelocityBuffer']
# Setup logging queue
self.logger = ThreadsafeLogger(loggingQueue, __name__)
else:
self.logger.error("Type %s not supported by mTracker" % kind)
def get_created(self):
""" Get created time """
return self.created
def right(self):
""" Get right bound of tracker """
return self.bbox[0] + self.bbox[2]
def top(self):
""" Get top bound of tracker """
return self.bbox[1] + self.bbox[3]
def bottom(self):
""" Get bottom bound of tracker """
return self.bbox[1]
def left(self):
""" Get left bound of tracker """
return self.bbox[0]
def area(self):
""" Get area of tracker bounding box """
return abs(self.right() - self.left())*abs(self.top() - self.bottom())
def update(self, frame):
""" Update tracker.
If velocity hack is being used, calculate the new velocity of the midpoint.
"""
ok, bbox = self.tracker.update(frame)
if self._useVelocity:
# Set velocity (pixels/sec)
deltaT = time() - self.updateTime
centreNow = ((bbox[0]+bbox[2]/2), (bbox[1]+bbox[3]/2))
centreLast = ((self.bbox[0]+self.bbox[2]/2), (self.bbox[1]+self.bbox[3]/2))
Vx = (centreNow[0] - centreLast[0])/deltaT
Vy = (centreNow[1] - centreLast[1])/deltaT
self.velocity = (Vx, Vy)
self.logger.debug('New velocity: %s' % str(self.velocity[0])+', '+str(self.velocity[1]))
self.updateTime = time()
self.bbox = bbox
return ok, bbox
def get_projected_location(self, time):
""" Get the estimated location of the bounding box, based on previous velocity. """
deltaT = max((time - self.updateTime), 1)
centreNow = ((self.bbox[0]+self.bbox[2]/2), (self.bbox[1]+self.bbox[3]/2))
projectedX = centreNow[0]+(self.velocity[0]*deltaT)
projectedY = centreNow[1]+(self.velocity[1]*deltaT)
return (projectedX, projectedY)
def get_velocity_buffer(self):
''' Another hack to improve low frame rate tracking.
"Spread" out the bounding box based on velocity.
'''
return (abs(self.velocity[0])*self._horizontalVelocityBuffer,
abs(self.velocity[1])*self._verticalVelocityBuffer)
class CollectionPoint(ModuleProcess):
def __init__(self, baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue):
""" Initialize new CamCollectionPoint instance.
Setup queues, variables, configs, predictionEngines, constants and loggers.
"""
# ModuleProcess.__init__(self, baseConfig, pInBoundQueue, pOutBoundQueue, loggingQueue)
super().__init__(baseConfig, pInBoundQueue, pOutBoundQueue,
loggingQueue)
if not self.check_opencv_version("3.", cv2):
print(
"OpenCV version {0} is not supported. Use 3.x for best results."
.format(self.get_opencv_version()))
# Queues
self.outQueue = pOutBoundQueue #messages from this thread to the main process
self.inQueue = pInBoundQueue
self.loggingQueue = loggingQueue
# Variables
self.video = None
self.needsReset = False
self.needsResetMux = False
self.alive = False
# Configs
self.moduleConfig = configLoader.load(
self.loggingQueue, __name__) #Get the config for this module
self.config = baseConfig
# Prediction engine
self.imagePredictionEngine = AzureImagePredictor(
moduleConfig=self.moduleConfig, loggingQueue=loggingQueue)
# Constants
self._useIdsCamera = self.moduleConfig['UseIdsCamera']
self._minFaceWidth = self.moduleConfig['MinFaceWidth']
self._minFaceHeight = self.moduleConfig['MinFaceHeight']
self._minNearestNeighbors = self.moduleConfig['MinNearestNeighbors']
self._maximumPeople = self.moduleConfig['MaximumPeople']
self._facePixelBuffer = self.moduleConfig['FacePixelBuffer']
self._collectionThreshold = self.moduleConfig['CollectionThreshold']
self._showVideoStream = self.moduleConfig['ShowVideoStream']
self._sendBlobs = self.moduleConfig['SendBlobs']
self._blobWidth = self.moduleConfig['BlobWidth']
self._blobHeight = self.moduleConfig['BlobHeight']
self._captureWidth = self.moduleConfig['CaptureWidth']
self._captureHeight = self.moduleConfig['CaptureHeight']
self._bitsPerPixel = self.moduleConfig['BitsPerPixel']
self._resetEventTimer = self.moduleConfig['ResetEventTimer']
self._collectionPointType = self.moduleConfig['CollectionPointType']
self._collectionPointId = self.moduleConfig['CollectionPointId']
# Logger
self.logger = ThreadsafeLogger(loggingQueue, __name__)
def run(self):
""" Main thread method, run when the thread's start() function is called.
Controls flow of detected faces and the MultiTracker.
Determines when to send 'reset' events to clients and when to send 'found' events.
This function contains various comments along the way to help understand the flow.
You can use this flow, extend it, or build your own.
"""
if not self.check_ss_version():
#cant run with wrong version so we return early
return False
self.alive = True
# Monitor inbound queue on own thread
self.listen()
self.initialize_camera()
# Load the OpenCV Haar classifier to detect faces
curdir = os.path.dirname(__file__)
cascadePath = os.path.join(curdir, 'classifiers', 'haarcascades',
'haarcascade_frontalface_default.xml')
faceCascade = cv2.CascadeClassifier(cascadePath)
self.mmTracker = MultiTracker("KCF", self.moduleConfig,
self.loggingQueue)
# Setup timer for FPS calculations
start = time.time()
frameCounter = 1
fps = 0
# Start timer for collection events
self.collectionStart = time.time()
ok, frame = self.video.read()
if not ok:
self.logger.error('Cannot read video file')
self.shutdown()
while self.alive:
ok, frame = self.video.read()
if not ok:
self.logger.error('Error while reading frame')
break
# Image alts
if self._useIdsCamera:
grayFrame = frame.copy()
outputImage = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
else:
outputImage = frame.copy()
grayFrame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
# Detect faces
faces = faceCascade.detectMultiScale(
grayFrame,
scaleFactor=1.1,
minNeighbors=self._minNearestNeighbors,
minSize=(self._minFaceWidth, self._minFaceHeight))
# If no faces in frame, clear tracker and start reset timer
if len(faces
) == 0 or self.mmTracker.length() > self._maximumPeople:
self.mmTracker.clear()
self.start_reset()
# If there are trackers, update
if self.mmTracker.length() > 0:
ok, bboxes, failed = self.mmTracker.update(outputImage)
if failed:
self.logger.error('Update trackers failed on: %s' %
''.join(str(s) for s in failed))
for (x, y, w, h) in faces:
# If faces are detected, engagement exists, do not reset
self.needsReset = False
# Optionally add buffer to face, can improve tracking/classification accuracy
if self._facePixelBuffer > 0:
(x, y, w,
h) = self.apply_face_buffer(x, y, w, h,
self._facePixelBuffer,
outputImage.shape)
# Get region of interest
roi_gray = grayFrame[y:y + h, x:x + w]
roi_color = outputImage[y:y + h, x:x + w]
# If the tracker is valid and doesn't already exist, add it
if self.valid_tracker(x, y, w, h):
self.logger.info('Adding tracker')
ok = self.mmTracker.add(bbox={
'x': x,
'y': y,
'w': w,
'h': h
},
frame=outputImage)
# Draw box around face
if self._showVideoStream:
cv2.rectangle(outputImage, (x, y), (x + w, y + h),
(0, 255, 0), 2)
# If the time since last collection is more than the set threshold
if not self.needsReset or (time.time() - self.collectionStart >
self._collectionThreshold):
# Check if the focal face has changed
check, face = self.mmTracker.check_focus()
if check:
predictions = self.get_predictions(grayFrame, face)
if predictions:
self.send_message(topic="update",
data={
'detectedTime':
datetime.now().isoformat('T'),
'predictions':
predictions
})
frameCounter += 1
elapsed = time.time() - start
fps = frameCounter / max(abs(elapsed), 0.0001)
if frameCounter > sys.maxsize:
start = time.time()
frameCounter = 1
if self._showVideoStream:
cv2.putText(outputImage, "%s FPS" % fps, (20, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1,
cv2.LINE_AA)
cv2.imshow("Faces found", outputImage)
cv2.waitKey(1)
if self._sendBlobs and frameCounter % 6 == 0:
self.send_message(
topic="blob",
data={
'imageArr':
cv2.resize(outputImage,
(self._blobWidth, self._blobHeight)),
'time':
datetime.now().isoformat('T')
})
# self.putCPMessage(data = {
# 'imageArr': cv2.resize(outputImage, (self._blobWidth, self._blobHeight)) ,
# 'time': datetime.now().isoformat('T')
# },
# type="blob")
def check_ss_version(self):
#check for min version met
self.logger.info('Module version %s' % (__version__))
if LooseVersion(self.config['ss_version']) < LooseVersion(
self.moduleConfig['MinSimpleSensorVersion']):
self.logger.error(
'This module requires a min SimpleSensor %s version. This instance is running version %s'
% (self.moduleConfig['MinSimpleSensorVersion'],
self.config['ss_version']))
return False
return True
def get_predictions(self, grayFrame, face):
""" Send face to predictionEngine as JPEG.
Return predictions array or false if no face is found.
"""
faceArr = grayFrame[int(face[1]):int(face[1] + face[3]),
int(face[0]):int(face[0] + face[2])]
img = Image.fromarray(faceArr)
buff = io.BytesIO()
img.save(buff, format="JPEG")
predictions = self.imagePredictionEngine.get_prediction(
buff.getvalue())
if 'error' in predictions:
return False
return predictions
def valid_tracker(self, x, y, w, h):
""" Check if the coordinates are a newly detected face or already present in MultiTracker.
Only accepts new tracker candidates every _collectionThreshold seconds.
Return true if the object in those coordinates should be tracked.
"""
if not self.needsReset or (time.time() - self.collectionStart >
self._collectionThreshold):
if (self.mmTracker.length() == 0
or not self.mmTracker.contains(bbox={
'x': x,
'y': y,
'w': w,
'h': h
})):
return True
return False
def start_reset(self):
"""Start a timer from reset event.
If timer completes and the reset event should still be sent, send it.
"""
if self.needsResetMux:
self.needsReset = True
self.needsResetMux = False
self.resetStart = time.time()
if self.needsReset:
if (time.time() - self.resetStart
) > 10: # 10 seconds after last face detected
self.send_message(data=None, type="reset")
self.needsReset = False
def apply_face_buffer(self, x, y, w, h, b, shape):
x = x - b if x - b >= 0 else 0
y = y - b if y - b >= 0 else 0
w = w + b if w + b <= shape[1] else shape[1]
h = h + b if h + b <= shape[0] else shape[0]
return (x, y, w, h)
def initialize_camera(self):
# Using IDS camera
if self._useIdsCamera:
self.logger.info("Using IDS Camera")
self.wrapper = IdsWrapper(self.loggingQueue, self.moduleConfig)
if not (self.wrapper.isOpened()):
self.wrapper.open()
self.wrapper.set('py_width', self._captureWidth)
self.wrapper.set('py_height', self._captureHeight)
self.wrapper.set('py_bitspixel', self._bitsPerPixel)
# Convert values to ctypes, prep memory locations
self.wrapper.set_c_types()
self.wrapper.allocate_image_memory()
self.wrapper.set_image_memory()
self.wrapper.begin_capture()
self.wrapper.init_image_data()
# Start video update thread
self.video = self.wrapper.start()
# Not using IDS camera
else:
# open first webcam available
self.video = cv2.VideoCapture(0)
if not (self.video.isOpened()):
self.video.open()
#set the resolution from config
self.video.set(cv2.CAP_PROP_FRAME_WIDTH, self._captureWidth)
self.video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._captureHeight)
def processQueue(self):
self.logger.info(
"Starting to watch collection point inbound message queue")
while self.alive:
if (self.inQueue.empty() == False):
self.logger.info("Queue size is %s" % self.inQueue.qsize())
try:
message = self.inQueue.get(block=False, timeout=1)
if message is not None:
if message == "SHUTDOWN":
self.logger.info("SHUTDOWN command handled on %s" %
__name__)
self.shutdown()
else:
self.handleMessage(message)
except Exception as e:
self.logger.error("Unable to read queue, error: %s " % e)
self.shutdown()
self.logger.info("Queue size is %s after" %
self.inQueue.qsize())
else:
time.sleep(.25)
def handle_message(self, message):
if message._topic == 'open-stream':
self._sendBlobs = True
elif message._topic == 'close-stream':
self._sendBlobs = False
def send_message(self, topic, data=None):
message = self.build_message(topic, data)
self.put_message(message)
def build_message(self, topic, data):
if topic == "reset":
# Send reset message
self.logger.info('Sending reset message')
msg = Message(topic='reset',
sender_id=self._collectionPointId,
sender_type=self._collectionPointType,
extended_data=None,
recipients='communication_modules')
return msg
elif topic == "update":
# Reset collection start and now needs needs reset
collectionStart = time.time()
self.needsResetMux = True
self.logger.info('Sending found message')
msg = Message(
topic='face',
sender_id=self._collectionPointId,
sender_type=self._collectionPointType,
extended_data=data['predictions'],
recipients='communication_modules',
timestamp=data['detectedTime'],
)
return msg
elif topic == "blob":
# Get numpy array as bytes
img = Image.fromarray(data['imageArr'])
buff = io.BytesIO()
img.save(buff, format="JPEG")
s = base64.b64encode(buff.getvalue()).decode("utf-8")
eventExtraData = {}
eventExtraData['imageData'] = s
eventExtraData['dataType'] = 'image/jpeg'
msg = Message(topic='blob',
sender_id=self._collectionPointId,
sender_type=self._collectionPointType,
extended_data=eventExtraData,
recipients='communication_modules')
return msg
# def putCPMessage(self, data, type):
# if type == "reset":
# # Send reset message
# self.logger.info('Sending reset message')
# msg = CollectionPointEvent(
# self._collectionPointId,
# self._collectionPointType,
# 'Reset mBox',
# None)
# self.outQueue.put(msg)
# elif type == "update":
# # Reset collection start and now needs needs reset
# collectionStart = time.time()
# self.needsResetMux = True
# self.logger.info('Sending found message')
# msg = CollectionPointEvent(
# self._collectionPointId,
# self._collectionPointType,
# 'Found face',
# data['predictions']
# )
# self.outQueue.put(msg)
# elif type == "blob":
# # Get numpy array as bytes
# img = Image.fromarray(data['imageArr'])
# buff = io.BytesIO()
# img.save(buff, format="JPEG")
# s = base64.b64encode(buff.getvalue()).decode("utf-8")
# eventExtraData = {}
# eventExtraData['imageData'] = s
# eventExtraData['dataType'] = 'image/jpeg'
# # Send found message
# # self.logger.info('Sending blob message')
# msg = CollectionPointEvent(
# self._collectionPointId,
# self._collectionPointType,
# 'blob',
# eventExtraData,
# False
# )
# self.outQueue.put(msg)
def shutdown(self):
self.alive = False
self.logger.info("Shutting down")
if self._useIdsCamera and self.wrapper.is_open():
self.wrapper.exit()
cv2.destroyAllWindows()
self.threadProcessQueue.join()
time.sleep(1)
self.exit = True
#Custom methods for demo
def get_opencv_version(self):
import cv2 as lib
return lib.__version__
def check_opencv_version(self, major, lib=None):
# if the supplied library is None, import OpenCV
if lib is None:
import cv2 as lib
# return whether or not the current OpenCV version matches the
# major version number
return lib.__version__.startswith(major)