def _detect_CO_balls(img, balls, cue, display_list):
PARA1 = int(float(img.shape[1]) / 640 * 15 + 0.5)
PARA2 = int(float(img.shape[1]) / 640 * 40 + 0.5)
p_cue_top, p_cue_bottom, cue_length = cue
## determine which ball is cue ball
min_dist2cue_top = img.shape[0] + 1
cue_ball_idx = -1
for idx, ball in enumerate(balls):
center, radius = ball
dist2cue = zc.calc_triangle_area(center, p_cue_top,
p_cue_bottom) * 2 / cue_length
if dist2cue < PARA1:
dist2cue_top = zc.euc_dist(center, p_cue_top)
if dist2cue_top < min_dist2cue_top:
min_dist2cue_top = dist2cue_top
cue_ball_idx = idx
if cue_ball_idx == -1:
rtn_msg = {'status': 'fail', 'message': 'Cannot find cue ball'}
return (rtn_msg, None)
cue_ball = balls[cue_ball_idx]
## determine which ball is object ball
min_dist2cue_top = img.shape[0] + 1
object_ball_idx = -1
for idx, ball in enumerate(balls):
if idx == cue_ball_idx:
continue
center, radius = ball
dist2cue = zc.calc_triangle_area(center, p_cue_top,
p_cue_bottom) * 2 / cue_length
if dist2cue < PARA2:
dist2cue_top = zc.euc_dist(center, p_cue_top)
if dist2cue_top < min_dist2cue_top:
min_dist2cue_top = dist2cue_top
object_ball_idx = idx
if object_ball_idx == -1:
rtn_msg = {'status': 'fail', 'message': 'Cannot find object ball'}
return (rtn_msg, None)
object_ball = balls[object_ball_idx]
if 'CO_balls' in display_list:
img_balls = img.copy()
cv2.circle(img_balls, (int(cue_ball[0][0]), int(cue_ball[0][1])),
int(cue_ball[1]), (255, 255, 255), -1)
cv2.circle(img_balls, (int(object_ball[0][0]), int(object_ball[0][1])),
int(object_ball[1]), (0, 0, 255), -1)
zc.display_image("CO_balls",
img_balls,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
rtn_msg = {'status': 'success'}
return (rtn_msg, (cue_ball, object_ball))
def display_state(state):
img_display = np.ones((200, 400, 3), dtype=np.uint8) * 100
if state['is_playing']:
cv2.putText(img_display, "Playing", (10, 50), cv2.FONT_HERSHEY_SIMPLEX,
1, [0, 255, 0])
cv2.putText(img_display,
"The ball was last hit to %s" % state['ball_position'],
(10, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.8, [255, 100, 100])
cv2.putText(img_display,
"Opponent is on the %s" % state['opponent_position'],
(10, 170), cv2.FONT_HERSHEY_SIMPLEX, 0.8, [255, 100, 100])
else:
cv2.putText(img_display, "Stopped", (10, 50), cv2.FONT_HERSHEY_SIMPLEX,
1, [0, 0, 255])
zc.display_image('state',
img_display,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
def get_guidance(img, cue, CO_balls, pocket, display_list):
PARA = int(float(img.shape[1]) / 640 * 8 + 0.5)
rtn_msg, objects = _detect_aim_point(cue, CO_balls, pocket)
if objects is not None:
p_aim, p_aimed = objects
print objects
if rtn_msg['status'] == 'success':
if 'all' in display_list:
img_display = img.copy()
# draw balls
cue_ball, object_ball = CO_balls
cv2.circle(img_display, (int(cue_ball[0][0]), int(cue_ball[0][1])),
int(cue_ball[1]), (255, 255, 255), -1)
cv2.circle(img_display,
(int(object_ball[0][0]), int(object_ball[0][1])),
int(object_ball[1]), (0, 0, 255), -1)
# draw cue
p_cue_top, p_cue_bottom, _ = cue
cv2.circle(img_display, p_cue_top, 3, (200, 0, 200), -1)
cv2.line(img_display, p_cue_top, p_cue_bottom, (200, 0, 200), 3)
# draw pocket
cv2.drawContours(img_display, [cnt_pocket], 0, (255, 0, 0), -1)
# draw aim point
cv2.circle(img_display, (int(p_aim[0]), int(p_aim[1])), 3,
(0, 255, 0), -1)
# draw aimed point
cv2.circle(img_display, (int(p_aimed[0]), int(p_aim[1])), 3,
(0, 0, 0), -1)
zc.display_image("all",
img_display,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
if rtn_msg['status'] == 'success':
if abs(p_aim[0] - p_aimed[0]) < PARA:
return "good"
elif p_aim[0] < p_aimed[0]:
return "left"
else:
return "right"
else:
return None
def _handle_img(self, img):
if self.is_first_frame and not config.RECOGNIZE_ONLY: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
zc.check_and_display('guidance', img_guidance, display_list, wait_time = config.DISPLAY_WAIT_TIME, resize_max = config.DISPLAY_MAX_PIXEL)
self.is_first_frame = False
result['state_index'] = 0 # first step
return json.dumps(result)
result = {'status' : "nothing"} # default
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT), interpolation = cv2.INTER_AREA)
## get bitmap for current image
zc.check_and_display('input', img, display_list, wait_time = config.DISPLAY_WAIT_TIME, resize_max = config.DISPLAY_MAX_PIXEL)
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if rtn_msg['status'] != 'success':
print rtn_msg['message']
if rtn_msg['message'] == "Not confident about reconstruction, maybe too much noise":
self.counter['not_confident'] += 1
return json.dumps(result)
self.counter['confident'] += 1
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.counter['diff_from_prev'] += 1
else:
self.counter['same_as_prev'] += 1
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap['first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap['count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
#print "\n\n\n\n\n%s\n\n\n\n\n" % self.counter
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn', img_syn, wait_time = config.DISPLAY_WAIT_TIME, resize_scale = 50)
if config.RECOGNIZE_ONLY:
return json.dumps(result)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
result, img_guidance = self.task.get_guidance()
if self.task.is_final_state():
step_idx = len(self.task.states)
else:
# get current step
step_idx = self.task.state2idx(self.task.current_state)
# make sure step index is always -1 in case of error
# also, differentiate from the default initial step (which we assign a step index 0)
# from the internal step index obtained from the task (which also begins at 0) by
# shifting the index by 1:
step_idx = -1 if step_idx < 0 else step_idx + 1
result['state_index'] = step_idx
if img_guidance is not None:
zc.check_and_display('guidance', img_guidance, display_list, wait_time = config.DISPLAY_WAIT_TIME, resize_max = config.DISPLAY_MAX_PIXEL)
return json.dumps(result)
# load test image
input_file = parse_arguments()
img = cv2.imread(input_file)
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT),
interpolation=cv2.INTER_AREA)
zc.check_and_display("input",
img,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
# process image and get the symbolic representation
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
try:
while True:
time.sleep(1)
except KeyboardInterrupt as e:
sys.stdout.write("user exits\n")
def find_opponent(img, img_prev, display_list):
def draw_flow(img, flow, step=16):
h, w = img.shape[:2]
y, x = np.mgrid[step / 2:h:step, step / 2:w:step].reshape(2, -1)
fx, fy = flow[y, x].T
lines = np.vstack([x, y, x + fx, y + fy]).T.reshape(-1, 2, 2)
lines = np.int32(lines + 0.5)
vis = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
cv2.polylines(vis, lines, 0, (0, 255, 0))
for (x1, y1), (x2, y2) in lines:
cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1)
return vis
def draw_rects(img, rects, color):
for x1, y1, x2, y2 in rects:
cv2.rectangle(img, (x1, y1), (x2, y2), color, 2)
#start_time = current_milli_time()
## General preparations
if 'opponent' in display_list:
img_opponent = img_prev.copy()
zc.check_and_display('rotated',
img,
display_list,
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
zc.check_and_display('rotated_prev',
img_prev,
display_list,
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
bw = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
bw_prev = cv2.cvtColor(img_prev, cv2.COLOR_BGR2GRAY)
# valid part of img_prev
mask_img_prev_valid = zc.get_mask(img_prev, rtn_type="mask")
bool_img_prev_valid = zc.shrink(mask_img_prev_valid, 15,
iterations=3).astype(bool)
bool_img_prev_invalid = np.bitwise_not(bool_img_prev_valid)
mask_white_prev = zc.color_inrange(img_prev, 'HSV', S_U=50, V_L=130)
bool_white_prev = zc.shrink(mask_white_prev,
13,
iterations=3,
method='circular').astype(bool)
# valid part of img
mask_img_valid = zc.get_mask(img, rtn_type="mask")
bool_img_valid = zc.shrink(mask_img_valid, 15, iterations=3).astype(bool)
bool_img_invalid = np.bitwise_not(bool_img_valid)
mask_white = zc.color_inrange(img, 'HSV', S_U=50, V_L=130)
bool_white = zc.shrink(mask_white, 13, iterations=3,
method='circular').astype(bool)
# prior score according to height
row_score, col_score = np.mgrid[0:img.shape[0], 0:img.shape[1]]
row_score = img.shape[0] * 1.2 - row_score.astype(np.float32)
#print "time0: %f" % (current_milli_time() - start_time)
## method 1: optical flow - dense
opt_flow = np.zeros((bw.shape[0], bw.shape[1], 2), dtype=np.float32)
opt_flow[::2, ::2, :] = cv2.calcOpticalFlowFarneback(bw_prev[::2, ::2],
bw[::2, ::2],
pyr_scale=0.5,
levels=1,
winsize=15,
iterations=3,
poly_n=7,
poly_sigma=1.5,
flags=0)
if 'denseflow' in display_list:
zc.display_image('denseflow',
draw_flow(bw, opt_flow, step=16),
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
# clean optical flow
mag_flow = np.sqrt(np.sum(np.square(opt_flow), axis=2))
bool_flow_valid = mag_flow > 2
bool_flow_valid = np.bitwise_and(bool_flow_valid, bool_img_prev_valid)
bool_flow_valid = np.bitwise_and(bool_flow_valid,
np.bitwise_not(bool_white_prev))
bool_flow_invalid = np.bitwise_not(bool_flow_valid)
# substract all the flow by flow average
x_ave = np.mean(opt_flow[bool_flow_valid, 0])
y_ave = np.mean(opt_flow[bool_flow_valid, 1])
opt_flow[:, :, 0] -= x_ave
opt_flow[:, :, 1] -= y_ave
opt_flow[bool_flow_invalid, :] = 0
if 'denseflow_cleaned' in display_list:
zc.display_image('denseflow_cleaned',
draw_flow(bw, opt_flow, step=16),
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
# give the flow a "score"
score_flow = np.sqrt(np.sum(np.square(opt_flow), axis=2))
score_flow = score_flow * row_score
score_horizonal = np.sum(score_flow, axis=0)
low_pass_h = np.ones(120)
low_pass_h /= low_pass_h.sum()
score_horizonal_filtered_dense = np.convolve(score_horizonal,
low_pass_h,
mode='same')
if 'dense_hist' in display_list:
plot_bar(score_horizonal_filtered_dense, name='dense_hist')
print np.argmax(score_horizonal_filtered_dense)
if 'opponent' in display_list:
cv2.circle(img_opponent,
(np.argmax(score_horizonal_filtered_dense), 220), 20,
(0, 255, 0), -1)
#print "time1: %f" % (current_milli_time() - start_time)
## method 2: optical flow - LK
feature_params = dict(maxCorners=100,
qualityLevel=0.03,
minDistance=5,
blockSize=3)
lk_params = dict(winSize=(15, 15),
maxLevel=2,
criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT,
10, 0.03))
p0 = cv2.goodFeaturesToTrack(bw_prev,
mask=mask_img_prev_valid,
useHarrisDetector=False,
**feature_params)
if p0 is None:
# TODO: this is also a possible indication that the rally is not on
rtn_msg = {
'status':
'fail',
'message':
'No good featuresToTrack at all, probably no one in the scene'
}
return (rtn_msg, None)
p1, st, err = cv2.calcOpticalFlowPyrLK(bw_prev, bw, p0, None, **lk_params)
# Select good points
good_new = p1[st == 1]
good_old = p0[st == 1]
# draw the tracks
if 'LKflow' in display_list:
img_LK = img_prev.copy()
for i, (new, old) in enumerate(zip(good_new, good_old)):
a, b = new.ravel()
c, d = old.ravel()
cv2.line(img_LK, (a, b), (c, d), (0, 255, 0), 2)
cv2.circle(img_LK, (c, d), 5, (0, 255, 0), -1)
zc.display_image('LKflow',
img_LK,
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
bool_flow_valid = np.bitwise_and(bool_img_valid,
np.bitwise_not(bool_white))
bool_flow_invalid = np.bitwise_not(bool_flow_valid)
bool_flow_valid_prev = np.bitwise_and(bool_img_prev_valid,
np.bitwise_not(bool_white_prev))
bool_flow_invalid_prev = np.bitwise_not(bool_flow_valid_prev)
is_reallygood = np.zeros((good_new.shape[0]), dtype=bool)
for i, (new, old) in enumerate(zip(good_new, good_old)):
a, b = new.ravel()
c, d = old.ravel()
if bool_flow_invalid_prev[d,
c] or max(a, b) > config.O_IMG_HEIGHT or min(
a, b) < 0 or bool_flow_invalid[b, a]:
continue
is_reallygood[i] = True
reallygood_new = good_new[is_reallygood]
reallygood_old = good_old[is_reallygood]
motion = reallygood_new - reallygood_old
motion_real = motion - np.mean(motion, axis=0)
if 'LKflow_cleaned' in display_list:
img_LK_cleaned = img_prev.copy()
img_LK_cleaned[bool_flow_invalid_prev, :] = [0, 0, 255]
for i, (new, old) in enumerate(zip(reallygood_new, reallygood_old)):
c, d = old.ravel()
cv2.line(img_LK_cleaned, (c, d),
(c + motion_real[i, 0], d + motion_real[i, 1]),
(0, 255, 0), 2)
cv2.circle(img_LK_cleaned, (c, d), 5, (0, 255, 0), -1)
zc.display_image('LKflow_cleaned',
img_LK_cleaned,
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
score_flow = np.zeros(bw.shape, dtype=np.float32)
score_flow[reallygood_old[:, 1].astype(np.int),
reallygood_old[:, 0].astype(np.int)] = np.sqrt(
np.sum(np.square(motion_real), axis=1))
score_flow = score_flow * row_score
score_horizonal = np.sum(score_flow, axis=0)
low_pass_h = np.ones(120)
low_pass_h /= low_pass_h.sum()
score_horizonal_filtered_LK = np.convolve(score_horizonal,
low_pass_h,
mode='same')
if 'LK_hist' in display_list:
plot_bar(score_horizonal_filtered_LK, name='LK_hist')
print np.argmax(score_horizonal_filtered_LK)
# if motion too small, probably no one is there...
if np.max(score_horizonal_filtered_LK) < 300:
# TODO: this is also a possible indication that the rally is not on
rtn_msg = {
'status': 'fail',
'message': 'Motion too small, probably no one in the scene'
}
return (rtn_msg, None)
if 'opponent' in display_list:
cv2.circle(img_opponent, (np.argmax(score_horizonal_filtered_LK), 220),
20, (0, 0, 255), -1)
#print "time2: %f" % (current_milli_time() - start_time)
## method 3: remove white wall
mask_white = zc.color_inrange(img_prev, 'HSV', S_U=50, V_L=130)
zc.check_and_display('mask_white_wall',
mask_white,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
score = row_score
score[bool_img_invalid] = 0
score[bool_white] = 0
score_horizonal = np.sum(score, axis=0)
low_pass_h = np.ones(120)
low_pass_h /= low_pass_h.sum()
score_horizonal_filtered_wall = np.convolve(score_horizonal,
low_pass_h,
mode='same')
if 'wall_hist' in display_list:
plot_bar(score_horizonal_filtered_wall, name='wall_hist')
print np.argmax(score_horizonal_filtered_wall)
if 'opponent' in display_list:
cv2.circle(img_opponent,
(np.argmax(score_horizonal_filtered_wall), 220), 20,
(255, 0, 0), -1)
#print "time3: %f" % (current_milli_time() - start_time)
## combining results of three methods
#score_horizonal_filtered = score_horizonal_filtered_dense * score_horizonal_filtered_LK * score_horizonal_filtered_wall
score_horizonal_filtered = score_horizonal_filtered_dense / 10 + score_horizonal_filtered_LK * 10
opponent_x = np.argmax(score_horizonal_filtered)
if 'opponent' in display_list:
cv2.circle(img_opponent, (opponent_x, 220), 20, (200, 200, 200), -1)
zc.check_and_display('opponent',
img_opponent,
display_list,
is_resize=False,
wait_time=config.DISPLAY_WAIT_TIME)
rtn_msg = {'status': 'success'}
return (rtn_msg, opponent_x)
def _generate_guidance(self, header, state_info_str, engine_id):
if config.RECOGNIZE_ONLY:
return json.dumps(result)
if self.is_first_frame: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
self.is_first_frame = False
header['status'] = result.pop('status')
result.pop('animation', None)
return json.dumps(result)
header['status'] = "success"
result = {} # default
state_info = json.loads(state_info_str)
if not state_info['trust']:
header['status'] = "fail"
return json.dumps(result)
state = state_info['state']
if state == "None":
header['status'] = "nothing"
return json.dumps(result)
bitmap = np.array(json.loads(state))
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap[
'first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap[
'count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
result, img_guidance = self.task.get_guidance()
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
header['status'] = result.pop('status')
result.pop('animation', None)
if img_guidance is not None:
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
return json.dumps(result)
def _handle_img(self, img):
if self.is_first_frame and not config.RECOGNIZE_ONLY: # do something special when the task begins
result, img_guidance = self.task.get_first_guidance()
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
result.pop('animation', None)
self.is_first_frame = False
return json.dumps(result)
result = {'status': "nothing"} # default
stretch_ratio = float(16) / 9 * img.shape[0] / img.shape[1]
if img.shape != (config.IMAGE_WIDTH, config.IMAGE_HEIGHT, 3):
img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT),
interpolation=cv2.INTER_AREA)
## get bitmap for current image
zc.check_and_display('input',
img,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
rtn_msg, bitmap = lc.process(img, stretch_ratio, display_list)
if gabriel.Debug.TIME_MEASUREMENT:
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = time.time()
if rtn_msg['status'] != 'success':
print rtn_msg['message']
if rtn_msg[
'message'] == "Not confident about reconstruction, maybe too much noise":
self.counter['not_confident'] += 1
return json.dumps(result)
self.counter['confident'] += 1
## try to commit bitmap
state_change = False
if bm.bitmap_same(self.commited_bitmap, bitmap):
pass
else:
current_time = time.time()
if not bm.bitmap_same(self.temp_bitmap['bitmap'], bitmap):
self.temp_bitmap['bitmap'] = bitmap
self.temp_bitmap['first_time'] = current_time
self.temp_bitmap['count'] = 0
self.counter['diff_from_prev'] += 1
else:
self.counter['same_as_prev'] += 1
self.temp_bitmap['count'] += 1
if current_time - self.temp_bitmap[
'first_time'] > config.BM_WINDOW_MIN_TIME or self.temp_bitmap[
'count'] >= config.BM_WINDOW_MIN_COUNT:
self.commited_bitmap = self.temp_bitmap['bitmap']
state_change = True
#print "\n\n\n\n\n%s\n\n\n\n\n" % self.counter
bitmap = self.commited_bitmap
if 'lego_syn' in display_list and bitmap is not None:
img_syn = bm.bitmap2syn_img(bitmap)
zc.display_image('lego_syn',
img_syn,
wait_time=config.DISPLAY_WAIT_TIME,
resize_scale=50)
if config.RECOGNIZE_ONLY:
return json.dumps(result)
## now user has done something, provide some feedback
img_guidance = None
if state_change:
self.task.update_state(bitmap)
sym_time = result[
gabriel.Protocol_measurement.JSON_KEY_APP_SYMBOLIC_TIME]
result, img_guidance = self.task.get_guidance()
result[gabriel.Protocol_measurement.
JSON_KEY_APP_SYMBOLIC_TIME] = sym_time
result['image'] = b64encode(zc.cv_image2raw(img_guidance))
result.pop('animation', None)
if img_guidance is not None:
zc.check_and_display('guidance',
img_guidance,
display_list,
wait_time=config.DISPLAY_WAIT_TIME,
resize_max=config.DISPLAY_MAX_PIXEL)
return json.dumps(result)
def _detect_cue(img, mask_tables, mask_balls, display_list):
CUE_MIN_LENGTH = int(float(img.shape[1]) / 640 * 40 + 0.5)
PARA1 = int(float(img.shape[1]) / 640 * 2 + 0.5)
mask_blue, mask_bluer, mask_table, mask_table_fat = mask_tables
### edges on the table
#img_table = np.zeros(img.shape, dtype=np.uint8)
#img_table = cv2.bitwise_and(img, img, dst = img_table, mask = mask_table_convex)
#bw_table = cv2.cvtColor(img_table, cv2.COLOR_BGR2GRAY)
#edge_table = cv2.Canny(bw_table, 80, 160)
#edge_table = zc.expand(edge_table, 2)
#zc.check_and_display("edge_table", edge_table, display_list, resize_max = config.DISPLAY_MAX_PIXEL, wait_time = config.DISPLAY_WAIT_TIME)
### detect cue
#lines = cv2.HoughLinesP(edge_table, 1, np.pi/180, 30, minLineLength = 70, maxLineGap = 3)
#if lines is None:
# rtn_msg = {'status': 'fail', 'message' : 'Cannot find cue'}
# return (rtn_msg, None)
#lines = lines[0]
#if 'cue_edge' in display_list:
# img_cue = img.copy()
# for line in lines:
# pt1 = (line[0], line[1])
# pt2 = (line[2], line[3])
# cv2.line(img_cue, pt1, pt2, (255, 0, 255), 2)
# zc.check_and_display("cue_edge", img_cue, display_list, resize_max = config.DISPLAY_MAX_PIXEL, wait_time = config.DISPLAY_WAIT_TIME)
## interesting parts on the table (pockets, cue, hand, etc.)
mask_table_convex, _ = zc.make_convex(mask_table.copy(),
use_approxPolyDp=False)
zc.check_and_display_mask("table_convex",
img,
mask_table_convex,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
mask_interesting = cv2.subtract(
cv2.subtract(mask_table_convex, mask_table), mask_bluer)
mask_interesting = cv2.subtract(mask_interesting, mask_balls)
mask_interesting = zc.shrink(mask_interesting, PARA1)
zc.check_and_display_mask("interesting",
img,
mask_interesting,
display_list,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
# find the blob with cue (and probably hand)
# TODO: this may be more robust with find_largest_CC function, in the case of half ball close to the bottom
mask_cue_hand = zc.get_closest_blob(
mask_interesting.copy(), (img.shape[0], img.shape[1] / 2),
min_length=CUE_MIN_LENGTH,
hierarchy_req='outer') # cue must be close to the bottom
## find cue top
p_cue_top = zc.get_edge_point(mask_cue_hand, (0, -1))
if p_cue_top is None:
rtn_msg = {'status': 'fail', 'message': 'Cannot find cue top'}
return (rtn_msg, None)
## find cue bottom
# the cue detected initially may not have reached the bottom of the image
for i in xrange(10):
mask_cue_hand = zc.expand_with_bound(mask_cue_hand,
cv2.bitwise_not(mask_bluer))
mask_cue_bottom = mask_cue_hand.copy()
mask_cue_bottom[:-2, :] = 0
mask_cue_bottom[:, :p_cue_top[0] - 40] = 0
mask_cue_bottom[:, p_cue_top[0] + 40:] = 0
nonzero = np.nonzero(mask_cue_bottom)
if len(nonzero) < 2 or len(nonzero[0]) == 0:
rtn_msg = {'status': 'fail', 'message': 'Cannot find cue bottom'}
return (rtn_msg, None)
rows, cols = nonzero
p_cue_bottom = ((np.min(cols) + np.max(cols)) / 2, img.shape[0])
## cue info
cue_length = zc.euc_dist(p_cue_top, p_cue_bottom)
if 'cue' in display_list:
img_cue = img.copy()
img_cue[mask_cue_hand > 0, :] = [0, 255, 255]
cv2.circle(img_cue, p_cue_top, 3, (255, 0, 255), -1)
cv2.line(img_cue, p_cue_top, p_cue_bottom, (255, 0, 255), 2)
zc.display_image("cue",
img_cue,
resize_max=config.DISPLAY_MAX_PIXEL,
wait_time=config.DISPLAY_WAIT_TIME)
## skeletonize
#skeleton_cue_hand = zc.skeletonize(mask_cue_hand)
rtn_msg = {'status': 'success'}
return (rtn_msg, (p_cue_top, p_cue_bottom, cue_length))