def __init__(self,
g_pool,
device_id=0,
frame_size=(1920, 1080),
frame_rate=30,
depth_frame_size=(640, 480),
depth_frame_rate=60,
align_streams=False,
preview_depth=False,
device_options=(),
record_depth=True,
stream_preset=None):
super().__init__(g_pool)
self._intrinsics = None
self.color_frame_index = 0
self.depth_frame_index = 0
self.device = None
self.service = pyrs.Service()
self.align_streams = align_streams
self.preview_depth = preview_depth
self.record_depth = record_depth
self.depth_video_writer = None
self.controls = None
self.pitch = 0
self.yaw = 0
self.mouse_drag = False
self.last_pos = (0, 0)
self.depth_window = None
self._needs_restart = False
self.stream_preset = stream_preset
self._initialize_device(device_id, frame_size, frame_rate,
depth_frame_size, depth_frame_rate,
device_options)
def run(self):
r = rospy.Rate(10)
with pyrs.Service() as serv:
with serv.Device() as dev:
cnt = 0
last = time.time()
smoothing = 0.9
fps_smooth = 30
while True:
cnt += 1
if (cnt % 10) == 0:
now = time.time()
dt = now - last
fps = 10 / dt
fps_smooth = (fps_smooth *
smoothing) + (fps * (1.0 - smoothing))
last = now
dev.wait_for_frames()
c = dev.color
c = cv2.cvtColor(c, cv2.COLOR_RGB2BGR)
d = dev.depth * dev.depth_scale * 1000
d = cv2.applyColorMap(d.astype(np.uint8),
cv2.COLORMAP_RAINBOW)
print(dev.depth_scale)
# self.rbg_pub.publish(self.build_multi_array(c))
# self.depth_pub.publish(self.build_multi_array(d))
r.sleep()
def frame_grab():
with pyrs.Service() as serv:
with serv.Device() as dev:
dev.wait_for_frames()
colour_frame = dev.color
colour_frame = cv2.cvtColor(colour_frame, cv2.COLOR_RGB2BGR)
return colour_frame
def test_is_wrapped(self):
service = pyrs.Service()
cam = service.Device()
self.assertTrue(cam.is_streaming())
cam.wait_for_frames()
from pyrealsense import rsutilwrapper
pc = cam.points
dm = cam.depth
nz = np.nonzero(pc)
x0, y0 = nz[0][0], nz[1][0]
pixel0 = np.ones(2, dtype=np.float32) * np.NaN
point0 = pc[x0, y0].astype(np.float32)
rsutilwrapper.project_point_to_pixel(pixel0, cam.depth_intrinsics, point0)
point1 = np.zeros(3, dtype=np.float32)
x1, y1 = np.round(pixel0[1]).astype(int), np.round(pixel0[0]).astype(int)
self.assertTrue(np.isclose([x0, y0], [x1, y1], atol=2).all())
depth = dm[x0, y0] * cam.depth_scale
rsutilwrapper.deproject_pixel_to_point(point1, cam.depth_intrinsics, pixel0, depth)
self.assertTrue(np.isclose(point0, point1, atol=10e-3).all())
cam.stop()
service.stop()
def __init__(self,
g_pool,
device_id=0,
frame_size=(1920, 1080),
frame_rate=30,
depth_frame_size=(640, 480),
depth_frame_rate=30,
align_streams=False,
preview_depth=False,
device_options=(),
record_depth=True):
super().__init__(g_pool)
self._intrinsics = None
self.color_frame_index = 0
self.depth_frame_index = 0
self.device = None
self.service = pyrs.Service()
self.align_streams = align_streams
self.preview_depth = preview_depth
self.record_depth = record_depth
self.depth_video_writer = None
self.controls = None
self._initialize_device(device_id, frame_size, frame_rate,
depth_frame_size, depth_frame_rate,
device_options)
def main():
# intrinsic paramters of Intel Realsense SR300
fx, fy, ux, uy = 463.889, 463.889, 320, 240
# paramters
dataset = 'icvl'
model = 'ren_9x6x6'
lower_ = 1
upper_ = 650
# init hand pose estimation model
hand_model = HandModel(dataset, model,
lambda img: get_center(img, lower=lower_, upper=upper_),
param=(fx, fy, ux, uy), use_gpu=False)
# realtime hand pose estimation loop
with pyrs.Service() as serv:
with serv.Device() as dev:
while True:
dev.apply_ivcam_preset(0)
dev.wait_for_frames()
depth = dev.depth * dev.depth_scale * 1000
# preprocessing depth
depth[depth == 0] = depth.max()
depth = depth[:, ::-1] # flip
# get hand pose
results = hand_model.detect_image(depth)
img_show = show_results(depth, results, dataset)
cv2.imshow('result', img_show)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def __init__(self):
self.flag_save = 0
self.rgb_img = None
self.depth_img = None
self.count = 0
self.last = time.time()
self.smoothing = 0.9
self.fps_smooth = 60
# Attempt to establish camera service
try:
self.srv = pyrs.Service()
except:
self.srv = None
print("[ERROR] Could not establish CameraR200 Service!")
# Attempt to connect to camera device
try:
self.dev = self.srv.Device(depth_control_preset=1)
except:
self.dev = None
print("[ERROR] Could not establish CameraR200 Device!")
# Attempt to configure camera settings
self.apply_configuration()
if (self.flag_save): # Configure depth and color streams
fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
self.writerD = cv2.VideoWriter("realsense_depth.avi", fourcc, 60,
(640, 480), True)
self.writerRGB = cv2.VideoWriter("realsense_rgb.avi", fourcc, 60,
(640, 480), True)
else:
self.writerD = None
self.writerRGB = None
def main():
serv = pyrs.Service()
dev = serv.Device()
for i in range(60):
dev.wait_for_frames()
color = dev.color
color = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
depth = dev.depth*dev.depth_scale
img_y, img_x = depth.shape
if len(sys.argv)<3:
print('InputError: python file.py threshold1 threshold2')
sys.exit()
else:
threshold1 = sys.argv[1]
threshold2 = sys.argv[2]
try:
objects = get_objects(color, depth, threshold1, threshold2)
for i in range(len(objects)):
x, y, w, h, d = objects[i].get_attributes()
print(x, y, w, h, d)
corr = get_correction(d, a, hfov, x)
cv2.rectangle(color, (x-corr, y), (x+w-corr, y+h), (0, 255, 0), 4)
try:
real_w, real_h = get_dimensions(d, w, h, 70, 43, 640, 480)
real_w = round(real_w, 3)
real_h = round(real_h, 3)
except:
real_x, real_y = 'ERROR'
cv2.putText(color, 'depth = ' + str(d) + 'm', (30, i*60 + 30) , cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
cv2.putText(color, 'width = ' + str(real_w)+ 'm', (30, i*60 + 45) , cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
cv2.putText(color, 'height = ' + str(real_h)+ 'm', (30, i*60 + 60) , cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
if(i < len(objects)-1):
## distance between left and right object
distance = round(distance_between_objects(objects[i], objects[i+1], 70, 640), 3)
if distance > l:
textcolor = (0, 255, 0)
else:
textcolor = (0, 0, 255)
cv2.putText(color, 'distance between objects = ' + str(distance) + 'm', (320, i*60 + 70) , cv2.FONT_HERSHEY_SIMPLEX, 0.5, textcolor, 1)
except:
cv2.putText(color, 'no objects detected between ' + str(threshold1) + 'm and '+ str(threshold2) + 'm', (30, 30) , cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
cv2.imwrite('obj_SURF.png',color)
## stop device and service
dev.stop()
serv.stop()
def initialize_(self):
self.serv = pyrs.Service()
fps = 30
self.depth_stream = pyrs.stream.DepthStream(fps=fps)
self.color_stream = pyrs.stream.ColorStream(fps=fps)
self.device = self.serv.Device(
device_id=0, streams=[self.color_stream, self.depth_stream])
self.device.apply_ivcam_preset(0)
def test_is_not_created(self):
service = pyrs.Service()
cam = service.Device()
self.assertTrue(cam.is_streaming())
cam.wait_for_frames()
self.assertTrue(cam.color.any())
cam.stop()
service.stop()
def connect(self):
"""
Establishes connection to R200 camera
"""
logging.info("Cam.py: connecting components")
self.serv = pyrs.Service()
self.dev = self.serv.Device(device_id=0,
streams=[\
pyrs.stream.DepthStream(fps=60), pyrs.stream.ColorStream(fps=60)])
def dev_selection_list():
default = (None, 'Select to activate')
try:
with pyrs.Service() as service:
dev_pairs = [default] + [pair(d) for d in service.get_devices()]
except pyrs.RealsenseError:
dev_pairs = [default]
return zip(*dev_pairs)
def start(self):
import pyrealsense as rs1
self.serv = rs1.Service().__enter__()
if self.device == "f200":
self._setup_f200()
else:
print("Invalid device")
def main():
## start the service - also available as context manager
serv = pyrs.Service()
## create a device from device id and streams of interest
dev = serv.Device()
## retrieve 60 frames of data as "warm up"
for i in range(60):
dev.wait_for_frames()
color = dev.color
depth = dev.depth*dev.depth_scale
img_y, img_x = depth.shape
if len(sys.argv)<3:
threshold1 = 0.6
threshold2 = 2
else:
threshold1 = sys.argv[1]
threshold2 = sys.argv[2]
try:
endpoint = is_endpoint(color)
if not endpoint:
objects = get_objects(color, depth, threshold1, threshold2)
for i in range(len(objects)):
x, y, w, h, d = objects[i].get_attributes()
print(x, y, w, h, d)
try:
real_x, real_y = get_dimensions(d, w, h, 70, 43, 640, 480)
real_x = round(real_x, 3)
real_y = round(real_y, 3)
except:
real_x, real_y = 'ERROR'
print('depth = ' + str(d) + 'm')
print('width = ' + str(real_x)+ 'm')
print('height = ' + str(real_y)+ 'm')
if(i < len(objects)-1):
## distance between left and right object
distance = round(distance_between_objects(objects[i], objects[i+1], 70, 640), 3)
print('distance between objects = ' + str(distance) + 'm')
else:
print('Endpoint reached!')
except:
print('no objects detected between ' + str(threshold1) + 'm and '+ str(threshold2) + 'm')
## stop device and service
dev.stop()
serv.stop()
def set_camera_presets():
with pyrs.Service() as serv:
with serv.Device() as dev:
dev.apply_ivcam_preset(0)
try:
custom_options = [(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)]
dev.set_device_options(*zip(*custom_options))
except:
pass
def test_is_started(self):
try:
service = pyrs.Service()
except RealsenseError as e:
self.assertTrue(e.function == 'rs_create_context')
else:
try:
dev = service.Device()
except RealsenseError as e:
self.assertTrue(e.function == 'rs_get_device')
else:
dev.stop()
finally:
service.stop()
def get_cam():
with pyrs.Service() as serv:
with serv.Device() as dev:
dev.apply_ivcam_preset(0)
try: # set custom gain/exposure values to obtain good depth image
custom_options = [(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)]
dev.set_device_options(*zip(*custom_options))
print(dir(rs_intrinsics.width))
print(rs_intrinsics.width.__getattribute__('size'),
rs_intrinsics.height)
print(rs_intrinsics.ppx, rs_intrinsics.ppy)
print(rs_intrinsics.fx, rs_intrinsics.fy)
print(rs_extrinsics.rotation, rs_extrinsics.translation)
except pyrs.RealsenseError:
pass # options are not available on all devices
cnt = 0
last = time.time()
smoothing = 0.9
fps_smooth = 30
while True:
cnt += 1
if (cnt % 10) == 0:
now = time.time()
dt = now - last
fps = 10 / dt
fps_smooth = (fps_smooth * smoothing) + (fps *
(1.0 - smoothing))
last = now
dev.wait_for_frames()
c = dev.color
c = cv2.cvtColor(c, cv2.COLOR_RGB2BGR)
d = dev.depth * dev.depth_scale * 1000
d = cv2.applyColorMap(d.astype(np.uint8), cv2.COLORMAP_RAINBOW)
cd = np.concatenate((c, d), axis=1)
cv2.putText(c,
str(fps_smooth)[:4], (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 0))
cv2.imshow('', c)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def run(self):
dac = pyrs.stream.DACStream(fps=60)
depth = pyrs.stream.DepthStream(fps=60)
color = pyrs.stream.ColorStream(fps=60, color_format='bgr')
with pyrs.Service() as serv:
with serv.Device(streams=(depth, color, dac)) as dev:
dev.apply_ivcam_preset(0)
while not self.quit:
if dev.poll_for_frame():
d = dev.dac
self.color_frame = dev.color
if self.position is not None:
self.coord3D = self.__find_3d_coord(
self.position, d, dev)
self.position = None
time.sleep(0.005)
def take_picture(threshold1, threshold2):
"""
Takes a picture of the scene and returns:
* if the endpoint is reached or not
* the objects between the thresholds
* the dimensions of the picture
"""
# start the service - also available as context manager
serv = pyrs.Service()
# create a device from device id and streams of interest
dev = serv.Device()
# retrieve 60 frames of data as "warm up"
for i in range(60):
dev.wait_for_frames()
print("Taking a picture")
color = dev.color
color = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
depth = dev.depth*dev.depth_scale
img_y, img_x = depth.shape
endpoint = is_endpoint(color)
objects = []
if not endpoint:
try:
objects = get_objects(color, depth, threshold1, threshold2)
draw_bounding_box(objects,color)
except:
cv2.putText(color, 'no objects detected between ' + str(threshold1) + 'm and '
+ str(threshold2) + 'm', (30, 30) , cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
else:
cv2.putText(color, 'Endpoint reached!', (30, 30) ,
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
cv2.imwrite('picture.png',color)
## stop device and service
dev.stop()
serv.stop()
return endpoint, objects
def save_camera_data_to_files(device_id=0,
rgb: bool = True,
depth: bool = True,
ir: bool = False,
frames: int = 150,
outfile_pattern: str = "output"):
## start the service - also available as context manager
with pyrs.Service() as serv:
streams = []
rgb_writer, depth_writer, ir_writer = None, None, None
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
if rgb:
rgb_stream = ColorStream()
streams.append(rgb_stream)
rgb_writer = cv2.VideoWriter(f'{outfile_pattern}_rgb.mp4', fourcc,
30,
(rgb_stream.width, rgb_stream.height))
if depth:
depth_stream = DepthStream()
dac_stream = DACStream()
streams.append(depth_stream)
streams.append(dac_stream)
depth_writer = cv2.VideoWriter(
f'{outfile_pattern}_depth.mp4', fourcc, 30,
(depth_stream.width, depth_stream.height))
if ir:
ir_stream = InfraredStream()
streams.append(ir_stream)
ir_writer = cv2.VideoWriter(f'{outfile_pattern}_ir.mp4', fourcc,
30,
(ir_stream.width, ir_stream.height),
False)
with serv.Device(device_id=device_id, streams=streams) as cam:
for _ in range(frames):
cam.wait_for_frames()
if rgb_writer:
frame = cv2.cvtColor(cam.color, cv2.COLOR_RGB2BGR)
rgb_writer.write(frame)
if depth_writer:
dac = convert_z16_to_bgr(cam.dac)
depth_writer.write(dac)
if ir_writer:
ir_writer.write(cv2.GaussianBlur(cam.infrared, (5, 5), 0))
for writer in [rgb_writer, depth_writer, ir_writer]:
if writer:
writer.release()
cv2.destroyAllWindows()
def ball_detection_driver_loop():
global calibration_mode
global target_pixel, target_lower, target_upper
global image_hsv
global gripper_start
global location
gripper_start = False
frame = np.zeros((300, 600, 3),
np.uint8) # black starting screen just in case
with pyrs.Service() as serv:
with serv.Device() as dev:
dev.apply_ivcam_preset(0)
try:
custom_options = [(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)]
dev.set_device_options(*zip(*custom_options))
except:
pass
while (True):
#print(type(dev))
#print("waiting for frames")
dev.wait_for_frames()
#print("frame arrived")
frame = dev.color
if frame is None:
return
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
frame = imutils.resize(frame, width=600)
#cv2.imshow("Raw Camera Feed",frame)
# frame = shift_frame_perspective(frame)
image_hsv, mask = process_frame(frame, target_lower,
target_upper)
cv2.imshow("Isolation", cv2.bitwise_and(frame,
frame,
mask=mask))
cv2.imshow("HSV", image_hsv)
center, x, y, radius = get_ball_pixel_location(mask)
frame = draw_circle_on_ball(frame, center, x, y, radius)
location = get_ball_actual_location(center, frame)
#print(location)
cv2.imshow("Transformed Camera Feed", frame)
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
def depthCam():
depth_fps = 90
depth_stream = pyrs.stream.DepthStream(fps=depth_fps)
with pyrs.Service() as serv:
with serv.Device(streams=(depth_stream, )) as dev:
dev.apply_ivcam_preset(0)
try: # set custom gain/exposure values to obtain good depth image
custom_options = [(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)]
dev.set_device_options(*zip(*custom_options))
except pyrs.RealsenseError:
pass # options are not available on all devices
cnt = 0
last = time.time()
smoothing = 0.9
fps_smooth = depth_fps
while True:
cnt += 1
if (cnt % 30) == 0:
now = time.time()
dt = now - last
fps = 30 / dt
fps_smooth = (fps_smooth * smoothing) + (fps *
(1.0 - smoothing))
last = now
dev.wait_for_frames()
d = dev.depth
d = convert_z16_to_bgr(d)
cv2.putText(d,
str(fps_smooth)[:4], (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255))
cv2.imshow('', d)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def keep_cam (q1, q2):
with pyrs.Service() as serv:
with serv.Device() as dev:
dev.apply_ivcam_preset(0)
try: # set custom gain/exposure values to obtain good depth image
custom_options = [(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)]
dev.set_device_options(*zip(*custom_options))
except pyrs.RealsenseError:
pass # options are not available on all devices
cnt = 0
last = time.time()
smoothing = 0.9
fps_smooth = 30
while True:
key = q1.get(True)
if key == 1:
cnt += 1
if (cnt % 10) == 0:
now = time.time()
dt = now - last
fps = 10 / dt
fps_smooth = (fps_smooth * smoothing) + (fps * (1.0-smoothing))
last = now
dev.wait_for_frames()
c = dev.color
c = cv2.cvtColor(c, cv2.COLOR_RGB2BGR)
# d = dev.depth * dev.depth_scale * 1000
# d = cv2.applyColorMap(d.astype(np.uint8), cv2.COLORMAP_RAINBOW)
q2.put(c)
# cd = np.concatenate((c, d), axis=1)
cv2.putText(c, str(fps_smooth)[:4], (0, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 0))
print('taking picture: '+str(cnt))
cv2.imwrite('/home/jeremy/1/1.jpg',c)
cv2.imwrite('/home/jeremy/DataTest/'+str(cnt)+'.jpg',c)
def save_color_intrinsics(folder):
import pyrealsense2 as rs
import json
with pyrs.Service() as serv:
with serv.Device() as dev:
serial = dev.serial
dev.wait_for_frames()
c = dev.color
H, W, _ = c.shape
dev.stop()
serv.stop()
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.color, W, H, rs.format.bgr8, 30)
# Start pipeline
profile = pipeline.start(config)
frames = pipeline.wait_for_frames()
color_frame = frames.get_color_frame()
# Color Intrinsics
intr = color_frame.profile.as_video_stream_profile().intrinsics
pipeline.stop()
camera_parameters = {
'ID': serial,
'fx': intr.fx,
'fy': intr.fy,
'ppx': intr.ppx,
'ppy': intr.ppy,
'height': intr.height,
'width': intr.width
}
with open(folder + 'intrinsics.json', 'w') as fp:
json.dump(camera_parameters, fp)
def frames():
rssv = pyrs.Service()
dev = rssv.Device()
dev.apply_ivcam_preset(0)
#camera = cv2.VideoCapture(Camera.video_source)
#cv2.VideoCapture.set(camera, 3,640)
#cv2.VideoCapture.set(camera, 4,480)
#cv2.VideoCapture.set(camera, 5,30)
#cv2.VideoCapture.set(camera, 16, True)
#if not camera.isOpened():
# raise RuntimeError('Could not start camera.')
while True:
dev.wait_for_frames()
c = dev.color
c = cv2.cvtColor(c, cv2.COLOR_RGB2BGR)
# read current frame
#_, img = camera.read()
#fRgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', c)[1].tobytes()
def __init__(self):
self.JumpWayMQTTClient = ""
self.TassTools = TassTools()
self.TassClassifier = TassClassifier(1)
self._configs = {}
with open('config.json') as configs:
self._configs = json.loads(configs.read())
framesPerSecond = 30
serv = pyrs.Service()
realsense = serv.Device(
device_id=self._configs["RealsenseCam"]["localCamID"],
streams=[pyrs.stream.ColorStream(fps=framesPerSecond)])
self.startMQTT()
self.processFrame(realsense, self._configs["RealsenseCam"]["camID"],
self._configs["RealsenseCam"]["camZone"],
self._configs["RealsenseCam"]["camSensorID"])
rgb_frame = np.empty(frame.shape[:2] + (3,), dtype=np.uint8)
zeros = frame == 0
non_zeros = frame != 0
f = hist[frame[non_zeros]] * 255 / hist[0xFFFF]
rgb_frame[non_zeros, 0] = 255 - f
rgb_frame[non_zeros, 1] = 0
rgb_frame[non_zeros, 2] = f
rgb_frame[zeros, 0] = 20
rgb_frame[zeros, 1] = 5
rgb_frame[zeros, 2] = 0
return rgb_frame
with pyrs.Service() as serv:
with serv.Device(streams=(depth_stream,)) as dev:
dev.apply_ivcam_preset(0)
try: # set custom gain/exposure values to obtain good depth image
custom_options = [(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)]
dev.set_device_options(*zip(*custom_options))
except pyrs.RealsenseError:
pass # options are not available on all devices
cnt = 0
last = time.time()
smoothing = 0.9
fps_smooth = depth_fps
from __future__ import print_function, unicode_literals
import tensorflow as tf
import numpy as np
import scipy.misc
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import pyrealsense as pyrs
from nets.ColorHandPose3DNetwork import ColorHandPose3DNetwork
from utils.general import detect_keypoints, trafo_coords, plot_hand, plot_hand_3d
if __name__ == '__main__':
# instantiate camera objects
serv = pyrs.Service()
cam = serv.Device(device_id=0, streams=[pyrs.stream.ColorStream(fps=30)])
# network input
image_tf = tf.placeholder(tf.float32, shape=(1, 240, 320, 3))
hand_side_tf = tf.constant([[1.0, 0.0]
]) # left hand (true for all samples provided)
evaluation = tf.placeholder_with_default(True, shape=())
# build network
net = ColorHandPose3DNetwork()
hand_scoremap_tf, image_crop_tf, scale_tf, center_tf,\
keypoints_scoremap_tf, keypoint_coord3d_tf = net.inference(image_tf, hand_side_tf, evaluation)
# Start TF
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
# video_capture = cv2.VideoCapture(0)
# Load a sample picture and learn how to recognize it.
obama_image = face_recognition.load_image_file("obama.jpg")
obama_face_encoding = face_recognition.face_encodings(obama_image)[0]
tyler_image = face_recognition.load_image_file("tyler.jpg")
tyler_face_encoding = face_recognition.face_encodings(tyler_image)[0]
# Initialize some variables
face_locations = []
face_encodings = []
face_names = []
process_this_frame = True
with pyrs.Service() as a, pyrs.Device() as dev:
while True:
dev.wait_for_frames()
c = dev.color
c = cv2.cvtColor(c, cv2.COLOR_RGB2BGR)
#cv2.imshow('', c)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# Grab a single frame of video
frame = c
# Resize frame of video to 1/4 size for faster face recognition processing
small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
def startStreamAndSave(self, saveRate, nCams):
if len(self.strms) != 0:
# signal.signal(signal.SIGINT, ignoreSignals)
# signal.signal(signal.SIGTSTP, ignoreSignals)
with pyrs.Service() as serv:
if nCams == 1:
with serv.Device(streams=self.strms) as dev:
dev.apply_ivcam_preset(0)
try: # set custom gain/exposure values to obtain good depth image
custom_options = [
(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)
]
dev.set_device_options(*zip(*custom_options))
except pyrs.RealsenseError:
pass # options are not available on all devices
# Implement timestamping file
timeDestFile = "./frames/one_camera/" + self.trialName + "/time/times.txt"
stopToken = "Stop Time"
timeList = multiprocessing.Queue()
timeProc = multiprocessing.Process(target=timeWriter,
args=(timeDestFile,
timeList,
stopToken))
timeProc.start()
d = datetime.utcnow()
timeList.put(str(d) + ", " + str(time.time()) + "\n")
if self.streamColor:
colList = multiprocessing.Queue()
colProc = multiprocessing.Process(
target=colWriter,
args=(colList, self.trialName))
colProc.start()
if self.streamDepth:
depList = multiprocessing.Queue()
depProc = multiprocessing.Process(
target=depWriter,
args=(dep1List, self.trialName))
depProc.start()
if self.streamPts:
ptsList = multiprocessing.Queue()
ptsProc = multiprocessing.Process(
target=ptsWriter,
args=(ptsList, self.trialName))
ptsProc.start()
# Construct viewing windows
if self.streamColor:
cv2.namedWindow('ColorStream')
if self.streamDepth:
cv2.namedWindow('DepthStream')
if self.streamPts:
cv2.namedWindow('PointStream')
# Make socket to listen to other computer
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Hardcoded values, were personally set
s.bind(('10.0.0.60', 12345))
s.listen(5)
s.setblocking(False)
stopToken = "Stop Time"
baseDir = "./frames/one_camera/" + self.trialName + "/"
makeDir(baseDir)
frame = 0
trialCounter = 0
while True:
# Set up to control streaming by socket communication
# Not fully debugged yet
streaming = 0
try:
conn, addr = s.accept()
streaming = int(conn.recv(4096))
# if turned on, increment to next trial number
if streaming:
trialCounter += 1
trialDir = baseDir + "t" + str(
trialCounter) + "/"
makeDir(trialDir)
if self.streamColor:
makeDir(trialDir + "color")
if self.streamDepth:
makeDir(trialDir + "depth")
if self.streamPts:
makeDir(trialDir + "points")
# Implement timestamping file
timeList = multiprocessing.Queue()
timeProc = multiprocessing.Process(
target=timeWriter,
args=(trialDir + "times.txt", timeList,
stopToken))
timeProc.start()
d = datetime.utcnow()
timeList.put(
str(d) + ", " + str(time.time()) +
"\n")
else:
timeList.put((stopToken, trialCounter))
except socket.error as e:
continue
print streaming
if streaming:
# Show desired streams
if self.streamDepth:
dep = (dev.dac * dev.depth_scale)
cv2.imshow('DepthStream', dep)
if self.streamPts:
pts = dev.points
# pts = cv2.cvtColor(pts, cv2.COLOR_XYZ2BGR)
cv2.imshow('PointStream', pts)
if self.streamColor:
color = cv2.cvtColor(
dev.color, cv2.COLOR_RGB2BGR)
cv2.imshow('ColorStream', color)
# wait and check if 'q' was pressed. If so, end streams
# If 'c' was pressed and not continuous saving, save current frames
# if continuous saving, save desired streams at desired rate
keyPress = cv2.waitKey(1) & 0xFF
if keyPress == ord('q'):
if self.streamColor:
colList.put(
(color, frame, 0, trialCounter))
colProc.join()
if self.streamDepth:
dep1List.put(
(dep, frame, 0, trialCounter))
dep1Proc.join()
if self.streamPts:
ptsList.put(
(pts, frame, 0, trialCounter))
ptsProc.join()
timeList.put((stopToken, trialCounter))
timeProc.join()
break
elif keyPress == ord('c') and saveRate == 0:
timeList.put((str(time.time()) + "\n",
trialCounter))
frame += 1
if self.streamColor:
colList.put(
(color, frame, 3, trialCounter))
if self.streamDepth:
dep1List.put(
(dep, frame, 3, trialCounter))
if self.streamPts:
ptsList.put(
(pts, frame, 3, trialCounter))
elif saveRate and frame % saveRate == 0:
timeList.put((str(time.time()) + "\n",
trialCounter))
if self.streamColor:
colList.put(
(color, frame, 3, trialCounter))
if self.streamDepth:
dep1List.put(
(dep, frame, 3, trialCounter))
if self.streamPts:
ptsList.put(
(pts, frame, 3, trialCounter))
elif nCams == 2:
with serv.Device(device_id=0,
streams=self.strms) as dev1, serv.Device(
device_id=1,
streams=self.strms) as dev2:
dev1.apply_ivcam_preset(0)
dev2.apply_ivcam_preset(0)
try: # set custom gain/exposure values to obtain good depth image
custom_options = [
(rs_option.RS_OPTION_R200_LR_EXPOSURE, 30.0),
(rs_option.RS_OPTION_R200_LR_GAIN, 100.0)
]
dev1.set_device_options(*zip(*custom_options))
dev2.set_device_options(*zip(*custom_options))
except pyrs.RealsenseError:
pass # options are not available on all devices
if self.streamColor:
colList = multiprocessing.Queue()
colProc = multiprocessing.Process(
target=colWriter,
args=(colList, self.trialName))
colProc.start()
if self.streamDepth:
dep1List = multiprocessing.Queue()
dep1Proc = multiprocessing.Process(
target=depWriter,
args=(dep1List, self.trialName))
dep1Proc.start()
dep2List = multiprocessing.Queue()
dep2Proc = multiprocessing.Process(
target=depWriter,
args=(dep2List, self.trialName))
dep2Proc.start()
if self.streamPts:
ptsList = multiprocessing.Queue()
ptsProc = multiprocessing.Process(
target=ptsWriter,
args=(ptsList, self.trialName))
ptsProc.start()
# Construct viewing windows
if self.streamColor:
cv2.namedWindow('ColorStream1')
cv2.namedWindow('ColorStream2')
if self.streamDepth:
cv2.namedWindow('DepthStream1')
cv2.namedWindow('DepthStream2')
if self.streamPts:
cv2.namedWindow('PointStream1')
cv2.namedWindow('PointStream2')
# Make socket to listen to other computer
# s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# s.bind(('10.0.0.60', 12345))
# s.listen(5)
stopToken = "Stop Time"
baseDir = "./frames/two_camera/" + self.trialName + "/"
makeDir(baseDir)
if self.streamColor:
makeDir(baseDir + "color1")
makeDir(baseDir + "color2")
if self.streamDepth:
makeDir(baseDir + "depth1")
makeDir(baseDir + "depth2")
if self.streamPts:
makeDir(baseDir + "points1")
makeDir(baseDir + "points2")
trialCounter = 0
frame = 0
curStreaming = False
# s.setblocking(False)
timeList = multiprocessing.Queue()
timeProc = multiprocessing.Process(
target=timeWriter,
args=(baseDir + "times.txt", timeList, stopToken))
timeProc.start()
d = datetime.utcnow()
timeList.put(str(d) + ", " + str(time.time()) + "\n")
while True:
# try:
# conn, addr = s.accept()
# conn.setblocking(False)
# streamMsg = (conn.recv(4096)) # assuming getting 1 or 0 (on/off)
# print(streamMsg)
# print(time.time())
# if turned on, increment to next trial number
# if streamMsg != curStreaming:
# curStreaming = streamMsg
# if streamMsg:
# trialCounter += 1
# trialDir = baseDir + "t" + str(trialCounter) + "/"
# makeDir(trialDir)
# if self.streamColor:
# makeDir(trialDir + "color1")
# makeDir(trialDir + "color2")
# if self.streamDepth:
# makeDir(trialDir + "depth1")
# makeDir(trialDir + "depth2")
# if self.streamPts:
# makeDir(trialDir + "points1")
# makeDir(trialDir + "points2")
# # Implement timestamping file
# timeList = multiprocessing.Queue()
# timeProc = multiprocessing.Process(target = timeWriter, args=(trialDir + "times.txt", timeList, stopToken))
# timeProc.start()
# d = datetime.utcnow()
# timeList.put(str(d) + ", " + str(time.time()) + "\n")
# elif trialCounter > 0:
# try:
# timeList.put((stopToken, trialCounter))
# except:
# print("no time records yet")
# except:
# pass
if True:
dev1.wait_for_frames()
dev2.wait_for_frames()
if saveRate != 0:
frame += 1
# Show desired streams
if self.streamColor:
color1 = cv2.cvtColor(
dev1.color, cv2.COLOR_RGB2BGR)
cv2.imshow('ColorStream1', color1)
color2 = cv2.cvtColor(
dev2.color, cv2.COLOR_RGB2BGR)
cv2.imshow('ColorStream2', color2)
if self.streamDepth:
dep1 = dev1.dac * dev1.depth_scale
cv2.imshow('DepthStream1', dep1)
dep2 = dev2.dac * dev2.depth_scale
cv2.imshow('DepthStream2', dep2)
if self.streamPts:
pts1 = dev1.points
ptStrm1 = cv2.cvtColor(
pts1, cv2.COLOR_XYZ2BGR)
cv2.imshow('PointStream1', ptStrm1)
pts2 = dev2.points
ptStrm2 = cv2.cvtColor(
pts2, cv2.COLOR_XYZ2BGR)
cv2.imshow('PointStream2', ptStrm2)
# If 'q' was pressed, save images and break
# If 'c' was pressed and not continuous saving, save current frames
# if saving frames is requested, save desired streams
keyPress = cv2.waitKey(1) & 0xFF
if keyPress == ord('q'):
if self.streamColor:
colList.put(
(color1, frame, 0, trialCounter))
colProc.join()
if self.streamDepth:
dep1List.put(
(dep1, frame, 0, trialCounter))
dep1Proc.join()
dep2List.put(
(dep1, frame, 0, trialCounter))
dep2Proc.join()
if self.streamPts:
ptsList.put(
(pts1, frame, 0, trialCounter))
ptsProc.join()
timeList.put((stopToken, trialCounter))
timeProc.join()
break
elif keyPress == ord('c') and saveRate == 0:
timeList.put((str(time.time()) + "\n",
trialCounter))
frame += 1
if self.streamColor:
colList.put(
(color1, frame, 1, trialCounter))
colList.put(
(color2, frame, 2, trialCounter))
if self.streamDepth:
dep1List.put(
(dep1, frame, 1, trialCounter))
dep2List.put(
(dep2, frame, 2, trialCounter))
if self.streamPts:
ptsList.put(
(pts1, frame, 1, trialCounter))
ptsList.put(
(pts2, frame, 2, trialCounter))
if saveRate and frame % saveRate == 0:
timeList.put((str(time.time()) + "\n",
trialCounter))
if self.streamColor:
colList.put(
(color1, frame, 1, trialCounter))
colList.put(
(color2, frame, 2, trialCounter))
if self.streamDepth:
dep1List.put(
(dep1, frame, 1, trialCounter))
dep2List.put(
(dep2, frame, 2, trialCounter))
if self.streamPts:
ptsList.put(
(pts1, frame, 1, trialCounter))
ptsList.put(
(pts2, frame, 2, trialCounter))
