def main():
# kinect_client = KinectClient(DEFAULT_HOST, DEFAULT_PORT)
kinect_client = FakeKinectClient(DEFAULT_HOST, DEFAULT_PORT)
kinect_client.navirice_capture_settings(rgb=False, ir=True, depth=True)
last_count = 0
first_img_set, last_count = _get_next_image(kinect_client)
raw_bg_depth = navirice_image_to_np(first_img_set.Depth) - 0.05
background_depth = np.where(raw_bg_depth <= 0.0, 1, raw_bg_depth)
cv2.imshow("Background Depth", background_depth) # show preview
if cv2.waitKey(1) & 0xFF == ord('q'):
return
while (1):
img_set, last_count = _get_next_image(kinect_client)
current_depth = navirice_image_to_np(img_set.Depth)
current_depth = np.where(current_depth <= 0.0, 1, current_depth)
current_ir = navirice_image_to_np(img_set.IR)
forground_depth, forground_ir = _extract_forground(
background_depth, current_depth, current_ir)
cv2.imshow("Forground Depth", forground_depth) # show preview
cv2.imshow("Forground Ir", forground_ir) # show preview
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def get_np_img_set(kinect_client):
"""Returns a img set of ir and depth iamges as numpy arrays from kinect_client.
Also uses kinect_clients last_count to keep track of what image order."""
img_set, last_count = kinect_client.navirice_get_next_image()
np_ir_image = navirice_ir_to_np(img_set.IR)
np_depth_image = navirice_image_to_np(img_set.Depth)
# Todo check if depth is actually ir rn.
(height, width, _) = np_ir_image.shape
# Quick Scaling Down
scale_value = 0.5
width = int(width * scale_value)
height = int(height * scale_value)
np_ir_image = cv2.resize(np_ir_image,
dsize=(width, height),
interpolation=cv2.INTER_CUBIC)
np_depth_image = cv2.resize(np_depth_image,
dsize=(width, height),
interpolation=cv2.INTER_CUBIC)
# Quick Cropping
# height = int((1.0/3.0) * height)
# width = int(0.5 * width)
# np_ir_image = np_ir_image[height:, :width]
# np_depth_image = np_depth_image[height:, :width]
# Todo map the values of ir max/min after scaling and cropping.
np_img_set = Np_img_set(last_count, np_ir_image, np_depth_image)
return np_img_set
def main():
scale_val = 1.0 / 8.0
x = tf.placeholder(tf.float32, [None, 424, 512, 1])
y_ = tf.placeholder(tf.float32, [None, 3])
y_conv = cnn_model_fn(x)
#cost = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)
cost = tf.square(y_ - y_conv)
train_step = tf.train.AdamOptimizer(1e-4).minimize(cost)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
print("------------------OUT SHAPES-------------------")
print(y_.get_shape())
print(y_conv.get_shape())
print("-----------------------------------------------")
cnt = 0
from navirice_get_image import KinectClient
kc = KinectClient('127.0.0.1', 29000)
kc.navirice_capture_settings(False, True, True)
saver = tf.train.Saver()
while (True):
loc = input("Enter file destination to load: ")
if (len(loc) > 0):
try:
saver.restore(sess, loc)
break
except ValueError:
print("Error: no file with that destination")
import time
while (True):
img_set, last_count = kc.navirice_get_image()
if (img_set != None and img_set.IR.width > 0
and img_set.Depth.width > 0):
depth_image = navirice_image_to_np(img_set.Depth)
ir_image = navirice_ir_to_np(img_set.IR)
inverted_depth = np.ones(depth_image.shape)
inverted_depth = inverted_depth - depth_image
tests = []
tests.append(inverted_depth)
start = time.time()
outs = sess.run(y_conv, feed_dict={x: tests})
end = time.time()
xf = outs[0][0]
yf = outs[0][1]
radiusf = outs[0][2]
print("nnoutput x:", xf, "y: ", yf, " r:", radiusf,
" time elapsed(s): ", end - start)
def load_train_set(data_list, scale_val):
real = []
expected = []
for i in range(len(data_list)):
with open(data_list[i], 'rb') as ci:
data=ci.read()
img_set = navirice_image_pb2.ProtoImageSet()
img_set.ParseFromString(data)
del data
if img_set.Depth is not None and img_set.IR is not None:
ir_image = navirice_ir_to_np(img_set.IR)
depth_image = navirice_image_to_np(img_set.Depth)
possible_bitmap = generate_bitmap_label(ir_image, depth_image)
if possible_bitmap is not None:
# Set labeled output
scaled_bitmap = cv2.resize(
possible_bitmap,None,fx=scale_val, fy=scale_val,
interpolation = cv2.INTER_CUBIC)
expected.append(scaled_bitmap)
# set input as both ir and depth into one image with 2 channels
# Both are ranges from 0 to 1
# ir_image_to_1 = navirice_ir_to_np(img_set.IR, scale=1.0, forCV=False)
# combined_image = np.concatenate((ir_image_to_1, depth_image), axis=2)
# real.append(combined_image)
real.append(depth_image)
#cv2.imshow("real-scaled-ir", depth_image)
#cv2.imshow("expected", scaled_bitmap)
#cv2.waitKey(1)
del img_set
return (real, expected)
def load_test_set(data_list):
real = []
for i in range(len(data_list)):
with open(data_list[i], 'rb') as ci:
data=ci.read()
img_set = navirice_image_pb2.ProtoImageSet()
img_set.ParseFromString(data)
del data
if img_set.IR is not None:
depth_image = navirice_image_to_np(img_set.Depth)
# ir_image = navirice_ir_to_np(img_set.IR, 1.0, forCV=False)
# combined_image = np.concatenate((ir_image, depth_image), axis=2)
real.append(depth_image)
# real.append(combined_image)
del img_set
return (real)
def thread_stream(self):
kc = KinectClient(HOST, PORT)
kc.navirice_capture_settings(False, True, True)
while (self.should_run):
img_set = None
if (self.should_pull):
img_set, self.last_count = kc.navirice_get_image()
if (img_set != None and img_set.IR.width > 0
and img_set.Depth.width > 0):
if self.should_record:
#processThread =Thread(target=navirice_img_set_write_file, args=[self.session_name, img_set, self.last_count])
#processThread.start()
navirice_img_set_write_file(self.session_name, img_set,
self.last_count)
cv2.imshow("IR", navirice_ir_to_np(img_set.IR))
cv2.imshow("DEPTH", navirice_image_to_np(img_set.Depth))
if cv2.waitKey(1) & 0xFF == ord('q'):
print("q pressed in cv window")
del img_set
def main():
count = 0
# mode = "ir_dance_"
# mode = "All_"
# mode = "head_"
# mode = "nohead_"
mode = "default_"
while True:
try:
f = open("DATA/" + mode + str(count) + ".img_set", "rb")
data = f.read()
img_set = navirice_image_pb2.ProtoImageSet()
img_set.ParseFromString(data)
depth_image = navirice_image_to_np(img_set.Depth)
IR_image = navirice_ir_to_np (img_set.IR)
# cv2.imshow("Hi Can: count: " + str(count), depth_image)
cv2.imshow("Depth", depth_image)
cv2.imshow("IR", IR_image)
cv2.waitKey(50)
except FileNotFoundError:
print("Stop asking!")
pass
count += 1
from navirice_get_image import KinectClient
from navirice_helpers import navirice_image_to_np
DEFAULT_HOST= 'navirice'
DEFAULT_PORT=29000
kin = KinectClient(DEFAULT_HOST, DEFAULT_PORT)
kin.navirice_capture_settings(rgb=False, ir=True, depth=True)
last_count=0
img_set, last_count = kin.navirice_get_next_image()
np_depth_image = navirice_image_to_np(img_set.Depth, scale=False)
hdr = OpenEXR.Header(img_set.Depth.width, img_set.Depth.height)
print(hdr)
#hdr['channels'] = {'R': FLOAT(1,1)}
#hdr['channels'] = {'R': hdr['channels']['R']}
print(hdr)
print()
print(img_set.Depth.data_type)
exr = OpenEXR.OutputFile("out.exr", hdr)
exr.writePixels({
'R': np_depth_image,
'G': np_depth_image,
'B': np_depth_image,
})
def main():
scale_val = 1.0 / 8.0
x = tf.placeholder(tf.float32, [None, 424, 512, 1])
y_ = tf.placeholder(tf.float32, [None, 3])
y_conv = cnn_model_fn(x)
#cost = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)
cost = tf.square(y_ - y_conv)
train_step = tf.train.AdamOptimizer(1e-4).minimize(cost)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
print("------------------OUT SHAPES-------------------")
print(y_.get_shape())
print(y_conv.get_shape())
print("-----------------------------------------------")
cnt = 0
from navirice_get_image import KinectClient
kc = KinectClient('127.0.0.1', 29000)
kc.navirice_capture_settings(False, True, True)
s_train = False
r_train = False
train_set_input = []
train_set_expected = []
train_set_size = 100000
saver = tf.train.Saver()
while (True):
img_set, last_count = kc.navirice_get_image()
if (s_train):
s_train = False
if (img_set != None and img_set.IR.width > 0
and img_set.Depth.width > 0):
ir_image = navirice_ir_to_np(img_set.IR)
depth_image = navirice_image_to_np(img_set.Depth)
inverted_depth = np.ones(depth_image.shape)
inverted_depth = inverted_depth - depth_image
cv_result = get_head_from_img(ir_image)
if cv_result is not None:
arr = [cv_result[0], cv_result[1], cv_result[2]]
if len(train_set_input) < train_set_size:
train_set_input.append(inverted_depth)
train_set_expected.append(arr)
else:
if (random.randint(0, 10000) > -1):
i = random.randint(0, train_set_size - 1)
train_set_input[i] = inverted_depth
train_set_expected[i] = arr
#train_step.run(session=sess, feed_dict={x: train_set_input, y_: train_set_expected})
dp = inverted_depth.copy()
cv2.circle(
dp, (int(cv_result[0] * 512), int(cv_result[1] * 424)),
int(cv_result[2] * 400), (255, 0, 0),
thickness=3,
lineType=8,
shift=0)
cv2.imshow("idl", dp)
print("db count: ", len(train_set_input))
if (img_set != None and img_set.IR.width > 0
and img_set.Depth.width > 0):
depth_image = navirice_image_to_np(img_set.Depth)
ir_image = navirice_ir_to_np(img_set.IR)
inverted_depth = np.ones(depth_image.shape)
inverted_depth = inverted_depth - depth_image
tests = []
tests.append(inverted_depth)
outs = sess.run(y_conv, feed_dict={x: tests})
xf = outs[0][0]
yf = outs[0][1]
radiusf = outs[0][2]
print("nnoutput x:", xf, "y: ", yf, " r:", radiusf)
if radiusf < 0:
radiusf = 0
cv2.circle(tests[0], (int(xf * 512), int(yf * 424)),
int(radiusf * 400), (255, 0, 0),
thickness=3,
lineType=8,
shift=0)
cv2.imshow("id", tests[0])
if (r_train):
tsi = []
tse = []
for i in range(100):
random_index = random.randint(0, len(train_set_input) - 1)
tsi.append(train_set_input[random_index])
tse.append(train_set_expected[random_index])
print("TRAINING")
train_step.run(session=sess, feed_dict={x: tsi, y_: tse})
key = cv2.waitKey(10) & 0xFF
#print("key: ", key)
# train
if (key == ord('t')):
r_train = True
# rest
if (key == ord('r')):
r_train = False
# (space) capture
if (key == 32):
s_train = True
# save model
if (key == ord('s')):
loc = input("Enter file destination to save: ")
if (len(loc) > 0):
try:
saver.save(sess, loc)
except ValueError:
print("Error: Did not enter a path..")
# load model
if (key == ord('l')):
loc = input("Enter file destination to load: ")
if (len(loc) > 0):
try:
saver.restore(sess, loc)
except ValueError:
print("Error: no file with that destination")
def get_depth_and_ir_from_kinect():
img_set, last_count = kinect_client.navirice_get_next_image()
np_depth_image = navirice_image_to_np(img_set.Depth, scale=False)
np_ir_image = navirice_image_to_np(img_set.IR, scale=False)
return (np_depth_image, np_ir_image)