def __init__(self):
# initialize the node named image_processing
rospy.init_node('image_processing', anonymous=True)
# initialize a publisher to send images from camera1 to a topic named image_topic1
# initialize a publisher to send joints' angular position to a topic called joints_pos
self.joints_pub = rospy.Publisher("joints_pos",
Float64MultiArray,
queue_size=10)
self.target_pub = rospy.Publisher("target_pos",
Float64MultiArray,
queue_size=10)
# initialize a subscriber to recieve messages rom a topic named /robot/camera1/image_raw and use callback function to recieve data
self.image_sub1 = message_filters.Subscriber("/image_topic1", Image)
self.image_sub2 = message_filters.Subscriber("/image_topic2", Image)
sync = message_filters.TimeSynchronizer(
[self.image_sub1, self.image_sub2], 10)
sync.registerCallback(self.callback_sync)
#rospy.TimeSynchronizer
# initialize the bridge between openCV and ROS
self.bridge = CvBridge()
# iterator to capture images
self.iterator = 0
# use class for object detection
self.coord = Coordinates()
self.od = ObjectDetection()
self.svm = classifer([['obj0', 8], ['obj1', 8]])
self.svm.addTrainSamples('images_train')
self.svm.train('svm.xml')
self.pix2meter0 = 0
self.pix2meter1 = 0
def __init__(self):
self.ic1 = image_converter1()
self.ic2 = image_converter2()
self.od = ObjectDetection()
self.svm = classifer([['obj0', 8], ['obj1', 8]])
self.svm.addTrainSamples('images_train')
self.svm.train('svm.xml')
def main():
object_detection = ObjectDetection(
camera=0,
model_name='ssd_mobilenet_v2_coco_2018_03_29',
label_path="./label_maps/mscoco_label_map.pbtxt",
detect_class=90)
object_detection.begin_detection()
""" base_station = BaseStation(USER, FIREBASE_REF, object_detection)
def __init__(self):
# initialize the node named image_processing
rospy.init_node('image_processing', anonymous=True)
# initialize a publisher to send images from camera1 to a topic named image_topic1
self.image_pub1 = rospy.Publisher("image_topic1",Image, queue_size = 1)
# initialize a subscriber to recieve messages rom a topic named /robot/camera1/image_raw and use callback function to recieve data
self.image_sub1 = rospy.Subscriber("/camera1/robot/image_raw",Image,self.callback1)
# initialize the bridge between openCV and ROS
self.bridge = CvBridge()
# iterator to capture images
self.iterator = 0
#
self.od = ObjectDetection()
# targets
self.targets_pub = rospy.Publisher("targets_pos",Float64MultiArray, queue_size=10)
# initialize target array
self.targets = Float64MultiArray()
self.cv_image1 = cv2.imread("image_1_1.png", 1)
def start_from_terminal(app):
"""
Parse command line options and start the server.
"""
parser = optparse.OptionParser()
parser.add_option(
'-d', '--debug',
help="enable debug mode",
action="store_true", default=False)
parser.add_option(
'-p', '--port',
help="which port to serve content on",
type='int', default=5000)
parser.add_option(
'-g', '--gpu',
help="use gpu mode",
action='store_true', default=False)
opts, args = parser.parse_args()
app.clf = Classification()
app.face = FaceVerification()
app.od = ObjectDetection()
app.caption = ImgCaption()
# cv2.imshow("luke",img)
# cv2.waitKey(0)
# img1 = "luke1.jpg"
# img2 = "luke2.jpg"
# Same,drawImg1,drawImg2=app.face.verification(img1,img2)
# if Same:
# print "same"
# else:
# print "different"
# drawImg = np.concatenate((drawImg1,drawImg2),axis=1)
# cv2.imshow("face",drawImg)
# cv2.waitKey(3000)
# Initialize classifier + warm start by forward for allocation
# with Manager() as manager:
# ret = manager.dict()
# p = Process(target=app.clf.classify_image,args=("test.jpg","jpg",ret))
# p.start()
# p.join()
# print ret
#warm up
# for i in range(2):
# app.clf.classify_image("test.jpg",'jpg')
if opts.debug:
app.run(debug=True, host='0.0.0.0', port=opts.port)
else:
start_tornado(app, opts.port)
class controller:
def __init__(self):
self.ic1 = image_converter1()
self.ic2 = image_converter2()
self.od = ObjectDetection()
self.svm = classifer([['obj0', 8], ['obj1', 8]])
self.svm.addTrainSamples('images_train')
self.svm.train('svm.xml')
def detect_joints(self, img):
joints = []
for colour in ["red", "green", "blue", "yellow"]
mask = self.ic.filter_color(img, colour)
mask = self.od.dilate(img=mask, kernel_size=5)
cx, cy = self.od.get_center_joint(mask)
joints.append([cx, cy])
return joints
def detect_target(self, img):
img = self.od.filter_colour(img, "orange")
img = self.od.opening(img, kernel_size=3)
img = self.od.dilate(img, 3)
boundries, contours = self.od.find_boundries(img)
return img, boundries, contours
from ObjectDetection import ObjectDetection
import cv2
detector = cv2.CascadeClassifier("./haarcascade_frontalface_default.xml")
object_detection = ObjectDetection(
use_pi_camera=False,
recognize_faces=True,
face_detector=detector,
frame_callback=None,
detection_method='hog',
encodings_files='./encodings/friends_family_encodings.pkl')
object_detection.detect_objects()
recv_timeout=10,
show_frame_rate=10)
sender.run_in_background()
else:
sender = None
def new_frame_callback(frame):
sender.send_frame_async(frame)
cb = new_frame_callback
if sender is None:
cb = None
detector = cv2.CascadeClassifier("./haarcascade_frontalface_default.xml")
object_detection = ObjectDetection(
use_pi_camera=True,
recognize_faces=True,
face_detector=detector,
show_image=show_on_pi,
rotate_image=rotation,
frame_callback=cb,
detection_method='hog',
encodings_files='./encodings/pr_encodings.pkl')
print("Starting image detection")
object_detection.detect_objects()
class image2target:
# Defines publisher and subscriber
def __init__(self):
# initialize the node named image_processing
rospy.init_node('image_processing', anonymous=True)
# initialize a publisher to send images from camera1 to a topic named image_topic1
# initialize a publisher to send joints' angular position to a topic called joints_pos
self.joints_pub = rospy.Publisher("joints_pos",
Float64MultiArray,
queue_size=10)
self.target_pub = rospy.Publisher("target_pos",
Float64MultiArray,
queue_size=10)
# initialize a subscriber to recieve messages rom a topic named /robot/camera1/image_raw and use callback function to recieve data
self.image_sub1 = message_filters.Subscriber("/image_topic1", Image)
self.image_sub2 = message_filters.Subscriber("/image_topic2", Image)
sync = message_filters.TimeSynchronizer(
[self.image_sub1, self.image_sub2], 10)
sync.registerCallback(self.callback_sync)
#rospy.TimeSynchronizer
# initialize the bridge between openCV and ROS
self.bridge = CvBridge()
# iterator to capture images
self.iterator = 0
# use class for object detection
self.coord = Coordinates()
self.od = ObjectDetection()
self.svm = classifer([['obj0', 8], ['obj1', 8]])
self.svm.addTrainSamples('images_train')
self.svm.train('svm.xml')
self.pix2meter0 = 0
self.pix2meter1 = 0
def detect_joints(self, img):
joints = []
for colour in ["red", "green", "blue", "yellow"]:
mask = self.od.filter_colour(img, colour)
mask = self.od.dilate(img=mask, kernel_size=5)
cx, cy = self.od.get_center_joint(mask)
joints.append([cx, cy])
return np.array(joints)
def detect_target(self, img):
img = self.od.filter_colour(img, "orange")
img = self.od.opening(img, kernel_size=3)
img = self.od.dilate(img, 3)
boundries, contours = self.od.find_boundries(img)
return img, boundries, contours
def set_pix2meter(self, joints0, joints1):
# set conversion factor (pixels to meters) between green and blue joints (3 meters)
self.pix2meter0 = self.coord.calc_pixel_to_meter_2D(
joints0[1], joints0[2], 3)
self.pix2meter1 = self.coord.calc_pixel_to_meter_2D(
joints1[1], joints1[2], 3)
def merge_coordinates(self, joints0, joints1):
coordinates = []
for i in range(joints0.shape[0]):
coordinates.append(
self.coord.merge_coordinates_2D_to_3D(joints0[i], joints1[i]))
return np.array(coordinates)
# Recieve data from camera 1, process it, and publish
def callback_sync(self, data0, data1):
self.iterator += 1
# Recieve the image
try:
self.cv_image0 = self.bridge.imgmsg_to_cv2(data0, "bgr8")
self.cv_image1 = self.bridge.imgmsg_to_cv2(data1, "bgr8")
except CvBridgeError as e:
print(e)
# Uncomment if you want to save the image
#if self.iterator % 50 == 0:
# print(self.iterator)
# cv2.imwrite('image_1_'+str(self.iterator/50)+'.png', self.cv_image1)
# show images
#im1=cv2.imshow('window1', self.cv_image0)
#im2=cv2.imshow('window2', self.cv_image1)
#cv2.waitKey(1)
##############
### joints ###
##############
# detect joints from camera1 ans camera2
joints0 = self.detect_joints(self.cv_image0)
joints1 = self.detect_joints(self.cv_image1)
# set conversion factor between green and blue joints
self.set_pix2meter(joints0, joints1)
# merge pixel coordinates from both images to coordinates x,y,z
coordinates = self.merge_coordinates(joints0, joints1)
#joints0 = joints0 * self.pix2meter0
#joints0 = joints0.round(0)
print(coordinates[0], coordinates[1], coordinates[2], coordinates[3])
###############
### targets ###
###############
# get filterd & thresholded image, boundries, contours of target objects
img, boundries, contours = self.detect_target(self.cv_image0)
try:
# get center
cx0, cy0 = self.od.get_center_target(img, boundries[0])
obj0 = self.od.get_object(img, boundries[0])
prediction0 = self.svm.classify(obj0)
data0 = [int(prediction0[1][0]), cx0, cy0] #prediction1[1][0],
#print(prediction, cx, cy)
except:
#self.targets.data = np.array([0, 0, 0, 0])
data0 = [0, 0, 0]
try:
cx1, cy1 = self.od.get_center_target(img, boundries[1])
obj1 = self.od.get_object(img, boundries[1])
prediction1 = self.svm.classify(obj1)
data1 = [int(prediction1[1][0]), cx1, cy1] #prediction1[1][0],
except:
#self.targets.data = np.array([0, 0, 0, 0])
data1 = [0, 0, 0]
#print("target 1 ", data0, " target 2 ", data1)
# set variables for ublishing
self.target = Float64MultiArray()
self.target.data = np.array([data0, data1])
# Publish the results
try:
#self.joints_pub.publish(self.joints)
self.target_pub.publish(self.target)
except CvBridgeError as e:
print(e)
from ObjectDetection import ObjectDetection
import cv2
detector = cv2.CascadeClassifier("./haarcascade_frontalface_default.xml")
object_detection = ObjectDetection(
use_pi_camera=True,
recognize_faces=True,
face_detector=detector,
encodings_files='./encodings/friends_family_encodings.pkl')
object_detection.detect_objects()
class image_converter:
# Defines publisher and subscriber
def __init__(self):
# initialize the node named image_processing
rospy.init_node('image_processing', anonymous=True)
# initialize a publisher to send images from camera1 to a topic named image_topic1
self.image_pub1 = rospy.Publisher("image_topic1",Image, queue_size = 1)
# initialize a subscriber to recieve messages rom a topic named /robot/camera1/image_raw and use callback function to recieve data
self.image_sub1 = rospy.Subscriber("/camera1/robot/image_raw",Image,self.callback1)
# initialize the bridge between openCV and ROS
self.bridge = CvBridge()
# iterator to capture images
self.iterator = 0
#
self.od = ObjectDetection()
# targets
self.targets_pub = rospy.Publisher("targets_pos",Float64MultiArray, queue_size=10)
# initialize target array
self.targets = Float64MultiArray()
self.cv_image1 = cv2.imread("image_1_1.png", 1)
# Recieve data from camera 1, process it, and publish
def callback1(self,data):
self.iterator += 1
# Recieve the image
try:
self.cv_image1 = self.bridge.imgmsg_to_cv2(data, "bgr8")
except CvBridgeError as e:
print(e)
# Uncomment if you want to save the image
#if self.iterator % 50 == 0:
# print(self.iterator)
# cv2.imwrite('image_1_'+str(self.iterator/50)+'.png', self.cv_image1)
#im1=cv2.imshow('window1', self.cv_image1)
#cv2.waitKey(1)
img = self.od.filter_colour(self.cv_image1, "orange")
img = self.od.opening(img, kernel_size=3)
img = self.od.dilate(img, 3)
boundries, contours = self.od.find_boundries(img)
try:
cx0, cy0 = self.od.get_center(img, boundries[0])
cx1, cy1 = self.od.get_center(img, boundries[1])
#obj = od.get_object(img2, rects[j])
a = np.array([cx0, cy0, cx1, cy1])
self.targets.data = a
except:
self.targets.data = np.array([0, 0, 0, 0])
#rects, cnts = od.find_boundries(img2)
# Publish the results
try:
self.image_pub1.publish(self.bridge.cv2_to_imgmsg(self.cv_image1, "bgr8"))
self.targets_pub.publish(self.targets)
except CvBridgeError as e:
print(e)
color = np.asanyarray(color_frame.get_data())
align = rs.align(rs.stream.color)
frameset = align.process(frameset)
# Update color and depth frames:
aligned_depth_frame = frameset.get_depth_frame()
# Apply Post Processing Filters
aligned_depth_frame = rs.hole_filling_filter().process(aligned_depth_frame)
aligned_depth_frame = rs.spatial_filter().process(aligned_depth_frame)
aligned_depth_frame = rs.temporal_filter().process(aligned_depth_frame)
aligned_depth = np.asanyarray(aligned_depth_frame.get_data())
object_detect = ObjectDetection()
objects = object_detect.run(color, aligned_depth)
# print(objects)
object_detect.visualize()
colorizer = rs.colorizer()
colorized_depth = np.asanyarray(colorizer.colorize(aligned_depth_frame).get_data())
images = np.hstack((color, colorized_depth))
# Show images
cv2.namedWindow('RealSense: (RGB, DEPTH-ALL_FILTERS)', cv2.WINDOW_AUTOSIZE)
cv2.imshow('RealSense: (RGB, DEPTH-ALL_FILTERS)', images)
cv2.waitKey(1)
except Exception as e:
print(e)
finally:
from openvino.inference_engine import IECore
inference_engine = IECore()
print('[ MVPy ] OpenVINO Inference Engine created')
from MultilabelClassification import MultilabelClassification
from ObjectDetection import ObjectDetection
from MulticlassClassification import MulticlassClassification
from Part4Detection import Part4Detection
from ORingClassification import ORingClassification
from GloveClassification import GloveClassification
from PartCountDetection import PartCountDetection
multilabel = MultilabelClassification(inference_engine, device_name)
print('[ MVPy ] Multilabel Classification model loaded')
detection = ObjectDetection(inference_engine, device_name)
print('[ MVPy ] Object Detection model loaded')
multiclass = MulticlassClassification(inference_engine, device_name)
print('[ MVPy ] Multiclass Classification model loaded')
part_4_det = Part4Detection(inference_engine, device_name)
print('[ MVPy ] Part 4 Detection model loaded')
oring_class = ORingClassification(inference_engine, device_name)
print('[ MVPy ] O-Ring Classification model loaded')
glove_class = GloveClassification(inference_engine, device_name)
print('[ MVPy ] Glove Classification model loaded')
part_count = PartCountDetection(inference_engine, device_name)
print('[ MVPy ] Part Count Detection model loaded')
number_of_models = 3
number_of_steps = 8
number_of_messages = [