def __init__(self):
yarp.RFModule.__init__(self)
self.xtionResolution = (640, 480)
# Define port variable to recive an image from Teo's camera
self.xtionImagePort = yarp.BufferedPortImageRgb()
# Create numpy array to receive the image
self.xtionImageArray = np.zeros(
(self.xtionResolution[1], self.xtionResolution[0], 3),
dtype=np.uint8)
self.xtionObjectDetectionsPort = yarp.BufferedPortBottle()
self.glassesImagePort = yarp.BufferedPortImageRgb()
self.glassesResolution = (1080, 1080)
self.glassesImageArray = np.zeros(
(self.glassesResolution[1], self.glassesResolution[0], 3),
dtype=np.uint8)
self.minConfidenceDetection = 0.9
self.triggerProb = 0.7
self.cropsResolution = (120, 120)
self.glassesDataPort = yarp.BufferedPortBottle()
self.tripletModel = None
self.embeddingModel = None
# Output Ports
self.fixationObjectPort = yarp.Port()
self.fixationObjectImagePort = yarp.BufferedPortImageRgb()
self.glassesCropImagePort = yarp.BufferedPortImageRgb()
self.detections = yarp.Bottle()
self.bottleData = yarp.Bottle()
self.count_not_detections = 0
self.max_count_not_detections = 10
self.rightObject = False
if robot == '/teo':
self.xtionImagePortName = "/xtion/rgbImage"
elif robot == '/teoSim':
self.xtionImagePortName = "/teoSim/camera/rgbImage"
self.xtionObjectDetectionPortName = "/rgbdObjectDetection/state"
self.glassesImagePortName = "/glassesServer/images"
self.glassesDataPortName = "/glassesServer/data"
self.fixationObjectPortName = "/object"
self.fixationObjectImagePortName = "/rgbImage"
self.glassesCropImagePortName = "/Glassescrop"
# Categories
self.categories = None
self.previous_category = 0
self.glassesCropImages = []
self.allDetectionsInWindowFrames = []
self.allDetectionsLabelsInWindowFrames = []
self.period = 0.01
self.timeToGrop = 0
self.numberOfFramesJustOneObject = 0
def createPorts(self):
self.imageDataInputPort = yarp.BufferedPortImageRgb()
self.outputFacePrection = yarp.Port()
self.speakStatusPort = yarp.RpcClient()
self.speakStatusOutBottle = yarp.Bottle()
self.speakStatusInBottle = yarp.Bottle()
self.imageInputBottle = yarp.Bottle()
def __init__(self):
yarp.RFModule.__init__(self)
# handle port for the RFModule
self.handle_port = yarp.Port()
self.attach(self.handle_port)
# Define vars to receive an image
self.input_port = yarp.BufferedPortImageRgb()
# Create numpy array to receive the image and the YARP image wrapped around it
self.input_img_array = None
# Define vars for outputing image
self.output_objects_port = yarp.Port()
self.output_raw_port = yarp.Port()
self.output_img_port = yarp.Port()
self.display_buf_array = None
self.display_buf_image = yarp.ImageRgb()
self.module_name = None
self.width_img = None
self.height_img = None
self.label_path = None
self.category_index = None
self.detection_graph = None
self.model_path = None
self.cap = None
self.session = None
def __init__(self,
name,
out_port,
carrier="",
width=320,
height=240,
rgb=True):
"""
Initializing the ImagePublisher
:param name: Name of the publisher
:param out_port: The published port name preceded by "/"
:param carrier: For a list of carrier names, visit https://www.yarp.it/carrier_config.html. Default is "tcp"
:param width: Image width
:param height: Image height
:param rgb: Transmits an RGB unsigned int image when "True". Transmits a float when "False"
"""
super().__init__()
self.__name__ = name
self.width = width
self.height = height
if rgb:
self._port = yarp.BufferedPortImageRgb()
self._port.open(out_port)
self.__netconnect__ = yarp.Network.connect(out_port,
out_port + ":out",
carrier)
else:
self._port = yarp.BufferedPortImageFloat()
self._port.open(out_port)
self.__netconnect__ = yarp.Network.connect(out_port,
out_port + ":out",
carrier)
def __init__(self):
yarp.RFModule.__init__(self)
self.socket = None
self.output_dir = '.'
self.file_num = 1
self.connection = None
self.cipher_decrypt = None
self.cipher_encrypt = None
self.imageWidth = 0
self.imageHeight = 0
self.glasses_images_port = yarp.BufferedPortImageRgb() # Create a port
self.glasses_images_bbox_port = yarp.BufferedPortImageRgb()
self.glasses_data_port = yarp.Port()
self.wait_for_connection = True
self.server_ip = '2.2.2.109' #'163.117.150.88' #'2.2.2.109'
def __init__(self):
"""
param glassesImageResolution: Tuple (width, height) with the resolution of the images
"""
yarp.RFModule.__init__(self)
self.imageWidth = 0
self.imageHeight = 0
self.glasses_images_port = yarp.BufferedPortImageRgb() # Create a port
self.glasses_fixation_point_port = yarp.Port()
def __init__(self, options):
self.options = options
# Set requested GPU
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(options.gpu_id)
# Initialize YARP network
yarp.Network.init()
# Initialize RF module
yarp.RFModule.__init__(self)
# Initialize inference
self.inference = Inference(self.options.model, self.options.refine_model, self.options.width, self.options.height, self.options.cam_fx, self.options.cam_fy, self.options.cam_cx, self.options.cam_cy)
# Initialize YARP ports
self.rgb_in = yarp.BufferedPortImageRgb()
self.rgb_in.open("/dense-fusion/camera/rgb:i")
self.depth_in = yarp.BufferedPortImageFloat()
self.depth_in.open("/dense-fusion/camera/depth:i")
self.mask_in = yarp.BufferedPortImageMono()
self.mask_in.open("/dense-fusion/camera/mask:i")
self.camera_pose_in = yarp.BufferedPortVector()
self.camera_pose_in.open("/dense-fusion/camera/pose:i")
self.prediction_out = yarp.Port()
self.prediction_out.open("/dense-fusion/pose:o")
# Input buffers initialization
self.rgb_buffer = bytearray(numpy.zeros((self.options.height, self.options.width, 3), dtype = numpy.uint8))
self.rgb_image = yarp.ImageRgb()
self.rgb_image.resize(self.options.width, self.options.height)
self.rgb_image.setExternal(self.rgb_buffer, self.options.width, self.options.height)
self.depth_buffer = bytearray(numpy.zeros((self.options.height, self.options.width, 1), dtype = numpy.float32))
self.depth_image = yarp.ImageFloat()
self.depth_image.resize(self.options.width, self.options.height)
self.depth_image.setExternal(self.depth_buffer, self.options.width, self.options.height)
self.mask_buffer = bytearray(numpy.zeros((self.options.height, self.options.width, 1), dtype = numpy.uint8))
self.mask_image = yarp.ImageMono()
self.mask_image.resize(self.options.width, self.options.height)
self.mask_image.setExternal(self.mask_buffer, self.options.width, self.options.height)
# Stamp initialization for finite differences
self.last_stamp = -1
self.last_total_position = None
def configure(self, rf):
yarp.Network.init()
self.respondPort = yarp.Port()
self.inputPort = yarp.BufferedPortImageRgb()
self.outputStillPort = yarp.BufferedPortImageRgb()
self.outputImagePort = yarp.BufferedPortImageRgb()
self.respondPort.open('/imageRPC')
self.inputPort.open('/pythonRead')
self.outputImagePort.open('/imageWrite')
self.outputStillPort.open('/pictureWrite')
self.attach(self.respondPort)
yarp.Network.connect('/grabber', '/pythonRead')
yarp.Network.connect('/pictureWrite', '/pictureView')
yarp.Network.connect('/imageWrite', '/internalView')
self.imgArray = numpy.zeros((240, 320, 3), dtype=numpy.uint8)
self.imgBuffer = numpy.zeros((240, 320, 3), dtype=numpy.uint8)
self.inputImage = yarp.ImageRgb()
self.tempImage = yarp.ImageRgb()
self.tempImage.resize(320, 240)
self.tempImage.setExternal(self.imgArray, self.imgArray.shape[1],
self.imgArray.shape[0])
self.outputImage = yarp.ImageRgb()
self.outputStill = yarp.ImageRgb()
self.uniCom = 'None'
print 'configured'
return True
# BSD-3-Clause license. See the accompanying LICENSE file for details.
import numpy as np
import yarp
# Initialise YARP Network
yarp.Network.init()
if __name__ == '__main__':
# Input and Output image size
image_w = 640
image_h = 480
# Input port
in_port = yarp.BufferedPortImageRgb()
in_port_name = '/image:i'
in_port.open(in_port_name)
# Output port
out_port = yarp.Port()
out_port_name = '/image:o'
out_port.open(out_port_name)
# Preparing input image buffer (do it only once)
in_buf_array = np.ones((image_h, image_w, 3), dtype=np.uint8)
in_buf_image = yarp.ImageRgb()
in_buf_image.resize(image_w, image_h)
in_buf_image.setExternal(in_buf_array, in_buf_array.shape[1],
in_buf_array.shape[0])
#!/usr/bin/python
# Copyright (C) 2006-2020 Istituto Italiano di Tecnologia (IIT)
# Copyright (C) 2006-2010 RobotCub Consortium
# All rights reserved.
#
# This software may be modified and distributed under the terms of the
# BSD-3-Clause license. See the accompanying LICENSE file for details.
import yarp
yarp.Network.init()
p = yarp.BufferedPortImageRgb()
p.open("/python")
top = 100
for i in range(1, top):
img = p.prepare()
img.resize(320, 240)
img.zero()
img.pixel(160, 120).r = 255
p.write()
yarp.delay(0.5)
p.close()
yarp.Network.fini()
def configure(self, rf):
print sys.argv
stringCommand = 'from SAM.SAM_Drivers import ' + sys.argv[
4] + ' as Driver'
print stringCommand
exec stringCommand
self.mm = [Driver()]
self.dataPath = sys.argv[1]
self.modelPath = sys.argv[2]
self.driverName = sys.argv[4]
self.configPath = sys.argv[3]
off = 17
print '-------------------'
print 'Interaction Settings:'
print
print 'Data Path: '.ljust(off), self.dataPath
print 'Model Path: '.ljust(off), self.modelPath
print 'Config Path: '.ljust(off), self.configPath
print 'Driver:'.ljust(off), self.driverName
print '-------------------'
print 'Configuring Interaction...'
print
self.mm = SAM_utils.initialiseModels(
[self.dataPath, self.modelPath, self.driverName], 'update',
'interaction')
# parse settings from config file
parser2 = SafeConfigParser()
parser2.read(self.configPath)
self.portNameList = parser2.items(self.dataPath.split('/')[-1])
print self.portNameList
self.portsList = []
for j in range(len(self.portNameList)):
if self.portNameList[j][0] == 'rpcbase':
self.portsList.append(yarp.Port())
self.portsList[j].open(self.portNameList[j][1] + ":i")
self.svPort = j
self.attach(self.portsList[j])
elif self.portNameList[j][0] == 'callsign':
# should check for repeated call signs by getting list from samSupervisor
self.callSignList = self.portNameList[j][1].split(',')
elif self.portNameList[j][0] == 'collectionmethod':
self.collectionMethod = self.portNameList[j][1].split(' ')[0]
try:
if self.mm[0].model_mode != 'temporal':
self.bufferSize = int(
self.portNameList[j][1].split(' ')[1])
elif self.mm[0].model_mode == 'temporal':
self.bufferSize = self.mm[0].windowSize
except ValueError:
print 'collectionMethod bufferSize is not an integer'
print 'Should be e.g: collectionMethod = buffered 3'
return False
if self.collectionMethod not in [
'buffered', 'continuous', 'future_buffered'
]:
print 'collectionMethod should be set to buffered / continuous / future_buffered'
return False
else:
parts = self.portNameList[j][1].split(' ')
print parts
if parts[1].lower() == 'imagergb':
self.portsList.append(yarp.BufferedPortImageRgb())
self.portsList[j].open(parts[0])
elif parts[1].lower() == 'imagemono':
self.portsList.append(yarp.BufferedPortImageMono())
self.portsList[j].open(parts[0])
elif parts[1].lower() == 'bottle':
self.portsList.append(yarp.BufferedPortBottle())
self.portsList[j].open(parts[0])
else:
print 'Data type ', parts[1], 'for ', self.portNameList[j][
0], ' unsupported'
return False
# mrd models with label/instance training will always have:
# 1 an input data line which is used when a label is requested
# 2 an output data line which is used when a generated instance is required
if parts[0][-1] == 'i':
self.labelPort = j
elif parts[0][-1] == 'o':
self.instancePort = j
if self.svPort is None or self.labelPort is None or self.instancePort is None:
print 'Config file properties incorrect. Should look like this:'
print '[Actions]'
print 'dataIn = /sam/actions/actionData:i Bottle'
print 'dataOut = /sam/actions/actionData:o Bottle'
print 'rpcBase = /sam/actions/rpc'
print 'callSign = ask_action_label, ask_action_instance'
print 'collectionMethod = buffered 3'
# self.mm[0].configInteraction(self)
self.inputType = self.portNameList[self.labelPort][1].split(
' ')[1].lower()
self.outputType = self.portNameList[self.labelPort][1].split(
' ')[1].lower()
self.dataList = []
self.classificationList = []
yarp.Network.init()
self.test()
return True
print("")
print("**************************************************************************")
print("YARP configuration:")
print("**************************************************************************")
print("")
print("")
print("Initializing YARP network ...")
# Init YARP Network
yarp.Network.init()
print("")
print("[INFO] Opening image input port with name /facialAnalysisLiteDetection2D/img:i ...")
# Open input image port
facialAnalysisLiteDetection2D_portIn = yarp.BufferedPortImageRgb()
facialAnalysisLiteDetection2D_portNameIn = '/facialAnalysisLiteDetection2D/img:i'
facialAnalysisLiteDetection2D_portIn.open(facialAnalysisLiteDetection2D_portNameIn)
print("")
print("[INFO] Opening image output port with name /facialAnalysisLiteDetection2D/img:o ...")
# Open output image port
facialAnalysisLiteDetection2D_portOut = yarp.Port()
facialAnalysisLiteDetection2D_portNameOut = '/facialAnalysisLiteDetection2D/img:o'
facialAnalysisLiteDetection2D_portOut.open(facialAnalysisLiteDetection2D_portNameOut)
print("")
print("[INFO] Opening data output port with name /facialAnalysisLiteDetection2D/data:o ...")
# Open output data port
def configure(self, rf):
"""
Configure interactionSAMModel yarp module
Args:
rf: Yarp RF context input
argv_1 : String containing data path.
argv_2 : String containing model path.
argv_3 : String containing config file path (from `config_path` parameter of samSupervisor config file).
argv_4 : String driver name corresponding to a valid driver present in SAM_Drivers folder.
argv_5 : String `'True'` or `'False'` to switch formatting of logging depending on whether interaction is logging to a separate window or to the stdout of another process.
Returns:
Boolean indicating success or no success in initialising the yarp module
"""
stringCommand = 'from SAM.SAM_Drivers import ' + sys.argv[
4] + ' as Driver'
exec stringCommand
self.mm = [Driver()]
self.dataPath = sys.argv[1]
self.modelPath = sys.argv[2]
self.driverName = sys.argv[4]
self.configPath = sys.argv[3]
self.windowedMode = sys.argv[5] == 'True'
self.modelRoot = self.dataPath.split('/')[-1]
file_i = 0
loggerFName = join(self.dataPath,
self.baseLogFileName + '_' + str(file_i) + '.log')
# check if file exists
while os.path.isfile(loggerFName) and os.path.getsize(loggerFName) > 0:
loggerFName = join(
self.dataPath,
self.baseLogFileName + '_' + str(file_i) + '.log')
file_i += 1
if self.windowedMode:
logFormatter = logging.Formatter("[%(levelname)s] %(message)s")
else:
logFormatter = logging.Formatter(
"\033[31m%(asctime)s [%(name)-33s] [%(levelname)8s] %(message)s\033[0m"
)
rootLogger = logging.getLogger('interaction ' + self.driverName)
rootLogger.setLevel(logging.DEBUG)
fileHandler = logging.FileHandler(loggerFName)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
logging.root = rootLogger
off = 17
logging.info('Arguments: ' + str(sys.argv))
logging.info(stringCommand)
logging.info('Using log' + str(loggerFName))
logging.info('-------------------')
logging.info('Interaction Settings:')
logging.info('Data Path: '.ljust(off) + str(self.dataPath))
logging.info('Model Path: '.ljust(off) + str(self.modelPath))
logging.info('Config Path: '.ljust(off) + str(self.configPath))
logging.info('Driver: '.ljust(off) + str(self.driverName))
logging.info('-------------------')
logging.info('Configuring Interaction...')
logging.info('')
# parse settings from config file
parser2 = SafeConfigParser()
parser2.read(self.configPath)
proposedBuffer = 5
if self.modelRoot in parser2.sections():
self.portNameList = parser2.items(self.dataPath.split('/')[-1])
logging.info(str(self.portNameList))
self.portsList = []
for j in range(len(self.portNameList)):
if self.portNameList[j][0] == 'rpcbase':
self.portsList.append(yarp.Port())
self.portsList[j].open(self.portNameList[j][1] + ":i")
self.svPort = j
self.attach(self.portsList[j])
elif self.portNameList[j][0] == 'visualise':
if self.portNameList[j][1] == "True":
self.drawLatent = True
elif self.portNameList[j][0] == 'callsign':
# should check for repeated call signs by getting list from samSupervisor
self.callSignList = self.portNameList[j][1].split(',')
elif self.portNameList[j][0] == 'latentmodelport':
self.portsList.append(yarp.BufferedPortImageRgb())
self.latentPort = j
ports = self.portNameList[j][1].split(',')
self.portsList[j].open(ports[0])
yarp.Network.connect(ports[0], ports[1])
elif self.portNameList[j][0] == 'collectionmethod':
self.collectionMethod = self.portNameList[j][1].split(
' ')[0]
try:
proposedBuffer = int(
self.portNameList[j][1].split(' ')[1])
except ValueError:
logging.error(
'collectionMethod bufferSize is not an integer')
logging.error(
'Should be e.g: collectionMethod = buffered 3')
return False
if self.collectionMethod not in [
'buffered', 'continuous', 'future_buffered'
]:
logging.error(
'collectionMethod should be set to buffered / continuous / future_buffered'
)
return False
elif self.portNameList[j][0] == 'recency':
try:
self.recency = int(self.portNameList[j][1])
except ValueError:
logging.error('Recency value for ' +
str(self.driverName) +
' is not an integer')
self.recency = 5
else:
parts = self.portNameList[j][1].split(' ')
logging.info(parts)
if parts[1].lower() == 'imagergb':
self.portsList.append(yarp.BufferedPortImageRgb())
# self.portsList[j].open(parts[0])
elif parts[1].lower() == 'imagemono':
self.portsList.append(yarp.BufferedPortImageMono())
# self.portsList[j].open(parts[0])
elif parts[1].lower() == 'bottle':
self.portsList.append(yarp.BufferedPortBottle())
# self.portsList[j].open(parts[0])
else:
logging.error('Data type ' + str(parts[1]) + ' for ' +
str(self.portNameList[j][0]) +
' unsupported')
return False
# mrd models with label/instance training will always have:
# 1 an input data line which is used when a label is requested
# 2 an output data line which is used when a generated instance is required
if parts[0][-1] == 'i':
self.labelPort = j
self.labelPortName = parts[0]
elif parts[0][-1] == 'o':
self.instancePort = j
self.instancePortName = parts[0]
if self.collectionMethod == 'continuous':
self.portsList.append(yarp.BufferedPortBottle())
self.eventPort = len(self.portsList) - 1
self.eventPortName = '/'.join(
self.labelPortName.split('/')[:3]) + '/event'
self.portsList[self.eventPort].open(self.eventPortName)
self.classTimestamps = []
if self.recency is None:
logging.warning('No recency value specified for ' +
self.driverName)
logging.warning('Setting value to default of 5 seconds')
self.recency = 5
if self.svPort is None or self.labelPort is None or self.instancePort is None:
logging.warning(
'Config file properties incorrect. Should look like this:')
logging.warning('[Actions]')
logging.warning('dataIn = /sam/actions/actionData:i Bottle')
logging.warning('dataOut = /sam/actions/actionData:o Bottle')
logging.warning('rpcBase = /sam/actions/rpc')
logging.warning(
'callSign = ask_action_label, ask_action_instance')
logging.warning('collectionMethod = buffered 3')
# self.mm[0].configInteraction(self)
self.inputType = self.portNameList[self.labelPort][1].split(
' ')[1].lower()
self.outputType = self.portNameList[self.labelPort][1].split(
' ')[1].lower()
self.dataList = []
self.classificationList = []
self.probClassList = []
self.classTimestamps = []
yarp.Network.init()
self.mm = initialiseModels(
[self.dataPath, self.modelPath, self.driverName],
'update',
'interaction',
drawLatent=self.drawLatent)
self.modelLoaded = True
if self.drawLatent:
self.latentPlots = dict()
self.latentPlots['ax'], self.latentPlots['dims'] = self.mm[
0].SAMObject.visualise(plot_scales=True)
self.sendLatent(self.latentPlots['ax'])
if self.mm[0].model_mode != 'temporal':
self.bufferSize = proposedBuffer
elif self.mm[0].model_mode == 'temporal':
self.bufferSize = self.mm[0].temporalModelWindowSize
self.portsList[self.labelPort].open(self.labelPortName)
self.portsList[self.instancePort].open(self.instancePortName)
# self.test()
return True
else:
logging.error('Section ' + str(self.modelRoot) + ' not found in ' +
str(self.configPath))
return False
class Detector (yarp.RFModule):
def __init__(self,
in_port_name,
out_det_img_port_name,
out_det_port_name
out_img_port_name,
classes,
image_w,
image_h,
deeplabmodel,
cpu_mode,
gpu_id):
if tf.__version__ < '1.5.0':
raise ImportError('Please upgrade your tensorflow installation to v1.5.0 or newer!')
self.LABEL_NAMES = np.asarray([
'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle',
'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog',
'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa',
'train', 'tv'
])
self.FULL_LABEL_MAP = np.arange(len(self.LABEL_NAMES)).reshape(len(self.LABEL_NAMES), 1)
self.FULL_COLOR_MAP = get_dataset_colormap.label_to_color_image(self.FULL_LABEL_MAP)
self.deeplabmodel = deeplabmodel
#self._TARBALL_NAME = 'deeplab_model.tar.gz'
print(self.LABEL_NAMES)
self.model = DeepLabModel(self.deeplabmodel)
# Images port initialization
## Prepare ports
self._in_port = yarp.BufferedPortImageRgb()
# self._in_port = yarp.Port()
self._in_port_name = in_port_name
self._in_port.open(self._in_port_name)
self._out_det_port = yarp.BufferedPortBottle()
self._out_det_port_name = out_det_port_name
self._out_det_port.open(self._out_det_port_name)
self._out_det_img_port = yarp.Port()
self._out_det_img_port_name = out_det_img_port_name
self._out_det_img_port.open(self._out_det_img_port_name)
self._out_img_port = yarp.Port()
self._out_img_port_name = out_img_port_name
self._out_img_port.open(self._out_img_port_name)
## Prepare image buffers
### Input
print 'prepare input image'
#self._in_buf_array = np.ones((image_h, image_w, 3), dtype = np.uint8)
self._in_buf_array = Image.new(mode='RGB', size=(image_w, image_h))
self._in_buf_image = yarp.ImageRgb()
self._in_buf_image.resize(image_w, image_h)
self._in_buf_image.setExternal(self._in_buf_array, self._in_buf_array.shape[1], self._in_buf_array.shape[0])
### Output
print 'prepare output image'
self._out_buf_image = yarp.ImageRgb()
self._out_buf_image.resize(image_w, image_h)
#self._out_buf_array = np.zeros((image_h, image_w, 3), dtype = np.uint8)
self._out_buf_array = Image.new(mode='RGB', size=(image_w, image_h))
self._out_buf_image.setExternal(self._out_buf_array, self._out_buf_array.shape[1], self._out_buf_array.shape[0])
"**************************************************************************"
)
print("")
print("")
print("Initializing YARP network ...")
# Init YARP Network
yarp.Network.init()
print("")
print(
"[INFO] Opening image input port with name /humanActivityDetection2D/img:i ..."
)
# Open input image port
humanActivityDetection2D_portIn = yarp.BufferedPortImageRgb()
humanActivityDetection2D_portNameIn = '/humanActivityDetection2D/img:i'
humanActivityDetection2D_portIn.open(humanActivityDetection2D_portNameIn)
print("")
print(
"[INFO] Opening image output port with name /humanActivityDetection2D/img:o ..."
)
# Open output image port
humanActivityDetection2D_portOut = yarp.Port()
humanActivityDetection2D_portNameOut = '/humanActivityDetection2D/img:o'
humanActivityDetection2D_portOut.open(humanActivityDetection2D_portNameOut)
print("")
print(
portNameList = parser2.items(sect)
print portNameList
portsList = []
for j in range(len(portNameList)):
if (portNameList[j][0] == 'rpcbase'):
portsList.append(yarp.RpcServer())
portsList[j].open(portNameList[j][1] + ':i')
svPort = j
elif (portNameList[j][0] == 'callsign'):
callSignList = portNameList[j][1].split(',')
else:
parts = portNameList[j][1].split(' ')
print parts
if (parts[1].lower() == 'imagergb'):
portsList.append(yarp.BufferedPortImageRgb())
portsList[j].open(parts[0])
elif (parts[1].lower() == 'imagemono'):
portsList.append(yarp.BufferedPortImageMono())
portsList[j].open(parts[0])
elif (parts[1].lower() == 'bottle'):
portsList.append(yarp.BufferedPortBottle())
portsList[j].open(parts[0])
#mrd models with label/instance training will always have:
#1 an input data line which is used when a label is requested
#2 an output data line which is used when a generated instance is required
if (parts[0][-1] == 'i'):
labelPort = j
elif (parts[0][-1] == 'o'):
def configure(self, rf):
'''
Configure the module internal variables and ports according to resource finder
'''
self._rf = rf
# Input
# Image port initialization
self._port_in = yarp.BufferedPortImageRgb()
self._port_in.open('/' + self._module_name + '/RGBimage:i')
# Input buffer initialization
self._input_buf_image = yarp.ImageRgb()
self._input_buf_image.resize(self._input_img_width,
self._input_img_height)
self._input_buf_array = np.ones(
(self._input_img_height, self._input_img_width, 3), dtype=np.uint8)
self._input_buf_image.setExternal(self._input_buf_array,
self._input_buf_array.shape[1],
self._input_buf_array.shape[0])
print('Input image buffer configured')
# Output
# Output image port initialization
self._port_out = yarp.Port()
self._port_out.open('/' + self._module_name + '/RGBimage:o')
# Output blobs port initialization
self._port_out_bboxes = yarp.Port()
self._port_out_bboxes.open('/' + self._module_name + '/bboxes:o')
# Output detection info port initialization
self._port_out_info = yarp.Port()
self._port_out_info.open('/' + self._module_name + '/detectionInfo:o')
# Output buffer initialization
self._output_buf_image = yarp.ImageRgb()
self._output_buf_image.resize(self._input_img_width,
self._input_img_height)
self._output_buf_array = np.zeros(
(self._input_img_height, self._input_img_width, 3), dtype=np.uint8)
self._output_buf_image.setExternal(self._output_buf_array,
self._output_buf_array.shape[1],
self._output_buf_array.shape[0])
print('Output image buffer configured')
# Output mask port initialization
self._port_out_mask = yarp.Port()
self._port_out_mask.open('/' + self._module_name + '/maskImage:o')
# Output mask buffer initialization
self._output_mask_buf_image = yarp.ImageMono()
self._output_mask_buf_image.resize(self._input_img_width,
self._input_img_height)
print('Output mask buffer configured')
# RPC port initialization
self._port_rpc = yarp.RpcServer()
self._port_rpc.open('/' + self._module_name + '/rpc')
self.attach_rpc_server(self._port_rpc)
# Inference model setup
# Configure some parameters for inference
config = configurations.YCBVideoConfigInference()
config.POST_NMS_ROIS_INFERENCE = 300
config.PRE_NMS_LIMIT = 1000
config.DETECTION_MAX_INSTANCES = 10
config.DETECTION_MIN_CONFIDENCE = 0.75
config.display()
self._model = modellib.MaskRCNN(mode='inference',
model_dir=MODEL_DIR,
config=config)
self._detection_results = None
print('Inference model configured')
# Load class names
dataset_root = os.path.join(ROOT_DIR, "datasets",
"bottles_ycb_video_format")
# Automatically discriminate the dataset according to the config file
if isinstance(config, configurations.TabletopConfigInference):
# Load the validation dataset
self._dataset = datasets.TabletopDataset()
elif isinstance(config, configurations.YCBVideoConfigInference):
self._dataset = datasets.YCBVideoDataset()
# No need to load the whole dataset, just the class names will be ok
self._dataset.load_class_names(dataset_root)
# Create a dict for assigning colors to each class
random_class_colors = tabletop_bottles.random_colors(
len(self._dataset.class_names))
self._class_colors = {
class_id: color
for (color, class_id
) in zip(random_class_colors, self._dataset.class_names)
}
# Load model weights
try:
assert os.path.exists(self._model_weights_path)
except AssertionError as error:
print("Model weights path invalid: file does not exist")
print(error)
return False
self._model.load_weights(self._model_weights_path, by_name=True)
print("Model weights loaded")
return True
def __init__(self):
yarp.RFModule.__init__(self)
# handle port for the RFModule
self.module_name = None
self.handle_port = None
self.process = False
# Define vars to receive audio
self.audio_in_port = None
self.eventPort = None
self.is_voice = False
# Predictions parameters
self.label_outputPort = None
self.predictions = []
self.database = None
# Speaker module parameters
self.model_audio = None
self.dataset_path = None
self.db_embeddings_audio = None
self.threshold_audio = None
self.length_input = None
self.resample_trans = None
self.speaker_emb = []
# Parameters for the audio
self.sound = None
self.audio = []
self.np_audio = None
self.nb_samples_received = 0
self.sampling_rate = None
# Define port to receive an Image
self.image_in_port = yarp.BufferedPortImageRgb()
self.face_coord_port = yarp.BufferedPortBottle()
# Port to query and update the memory (OPC)
self.opc_port = yarp.RpcClient()
# Image parameters
self.width_img = None
self.height_img = None
self.input_img_array = None
self.frame = None
self.coord_face = None
self.threshold_face = None
self.face_emb = []
# Model face recognition modele
self.modele_face = None
self.db_embeddings_face = None
self.trans = None
self.faces_img = []
self.face_coord_request = None
self.face_model_path = None
# Model for cross-modale recognition
self.model_av = None
self.sm = torch.nn.Softmax(dim=1)
self.threshold_multimodal = None
self.device = None
self.save_face = False
self.name = ""
self.predict = False
