def get_curr_ball_from_prev_ball(prev_ball_center: BallCenter, ball_r,
search_margin, im_param):
im_w = im_param.size[0]
im_h = im_param.size[1]
curr_ball_horizontal_positions = None
curr_ball_vertical_positions = None
print('search_for_ball_around_prev_ball - im_h={}, im_w={}'.format(
im_h, im_w))
rgb_list, pix = get_rgb_list(im_param)
# NOTE, rgb_list is flopped, so rgb_list col is the image row to scan
start_y = max(0, prev_ball_center.y - search_margin)
end_y = min(im_h, prev_ball_center.y - search_margin)
for j in range(start_y, end_y, 1):
curr_ball_horizontal_positions = scan_line_for_objects(
rgb_list, j, 'h', (0, 255, 0), (255, 255, 255))
if len(curr_ball_horizontal_positions) > 0:
ball_center_x = int(curr_ball_horizontal_positions[0]['start'] +
(curr_ball_horizontal_positions[0]['end'] -
curr_ball_horizontal_positions[0]['start']) /
2)
curr_ball_vertical_positions = \
scan_line_for_objects(rgb_list, ball_center_x, 'v', (0, 255, 0), (255, 255, 255))
if len(curr_ball_vertical_positions) > 0:
ball_center_y = int(
curr_ball_vertical_positions[0]['start'] +
(curr_ball_vertical_positions[0]['end'] -
curr_ball_vertical_positions[0]['start']) / 2)
return BallCenter(ball_center_x, ball_center_y)
break
return None
def get_original_ball_center_and_size(im_param, bricks):
im_w = im_param.size[0]
im_h = im_param.size[1]
print('get_original_ball_center_and_size - im_h={}, im_w={}'.format(
im_h, im_w))
rgb_list, pix = get_rgb_list(im_param)
total_brick_rows = len(bricks)
brick_ref = bricks[total_brick_rows - 1][0]
# the original image shows that the ball originally is about a brick height below the last row of bricks,
# use that knowledge to set the start row to scan for the ball
ball_row_to_scan = brick_ref.y + brick_ref.h
ball_horizontal_positions = scan_line_for_objects(rgb_list,
ball_row_to_scan, 'h',
(0, 255, 0),
(255, 255, 255))
print(
'get_original_ball_center_and_size - ball_row_to_scan = {}\nball_horizontal_positions = {}'
.format(ball_row_to_scan, ball_horizontal_positions))
while (len(ball_horizontal_positions) == 0 and ball_row_to_scan < im_h):
ball_row_to_scan += 1
ball_horizontal_positions = scan_line_for_objects(
rgb_list, ball_row_to_scan, 'h', (0, 255, 0), (255, 255, 255))
if (print_debug_line_by_line):
print(
'get_original_ball_center_and_size - ball_row_to_scan = {}\nball_horizontal_positions = {}'
.format(ball_row_to_scan, ball_horizontal_positions))
if len(ball_horizontal_positions) == 0:
return False, None, None
# ball_col_to_scan = int(im_w / 2)
ball_col_to_scan = ball_horizontal_positions[0]['start'] + \
int((ball_horizontal_positions[0]['end']-ball_horizontal_positions[0]['start'])/2)
ball_vertical_positions = scan_line_for_objects(rgb_list, ball_col_to_scan,
'v', (0, 255, 0),
(255, 255, 255))
print(
'get_original_ball_center_and_size - ball_col_to_scan = {}\nball_vertical_positions = {}'
.format(ball_col_to_scan, ball_vertical_positions))
# # there is only one ball, use the middle of the ball vertical positions to determin the row to scan
# # to determine ball size
# ball_row_to_scan = ball_vertical_positions[0]['start'] + \
# int((ball_vertical_positions[0]['end'] - ball_vertical_positions[0]['start']) / 2)
# ball_horizontal_positions = scan_line_for_objects(rgb_list, ball_row_to_scan, 'h', (0, 255, 0), (255, 255, 255))
# print('get_original_ball_center_and_size - ball_row_to_scan = {}\nball_horizontal_positions = {}'.format(
# ball_row_to_scan, ball_horizontal_positions))
# note, there is only one ball, position returned is the center of the ball
if len(ball_horizontal_positions) > 0 and len(ball_vertical_positions) > 0:
ball_r = int((ball_vertical_positions[0]['end'] -
ball_vertical_positions[0]['start']) / 2)
ball_center_y = ball_vertical_positions[0]['start'] + ball_r
ball_center_x = ball_horizontal_positions[0]['start'] +\
int((ball_horizontal_positions[0]['end']-ball_horizontal_positions[0]['start'])/2)
return True, BallCenter(ball_center_x, ball_center_y), ball_r
else:
return False, None, None
def get_bat(im_param):
im_w = im_param.size[0]
im_h = im_param.size[1]
print('get_bat - im_h={}, im_w={}'.format(im_h, im_w))
rgb_list, pix = get_rgb_list(im_param)
# bat is at the bottom of the image, with a small padding, ref. game.py
row_to_scan = im_h - 10
bat_horizontal_positions = scan_line_for_objects(rgb_list, row_to_scan,
'h', (0, 0, 255),
(255, 255, 255))
print('get_bat - row_to_scan={}, \nbat_horizontal_positions={}'.format(
row_to_scan, bat_horizontal_positions))
# chose a position within the first bat width as the column to scan,
# this guarantee we won't hit a gap, note there is only one bat
col_to_scan = bat_horizontal_positions[0]['start'] + 10
bat_vertical_positions = scan_line_for_objects(rgb_list, col_to_scan, 'v',
(0, 0, 255),
(255, 255, 255))
print('get_bat - col_to_scan={}, \nbat_vertical_positions={}'.format(
col_to_scan, bat_vertical_positions))
# note, there is only one bat, so we return bat position, note, position returned is top-left corner
if len(bat_horizontal_positions) > 0 and len(bat_vertical_positions) > 0:
bat_x = bat_horizontal_positions[0]['start']
bat_y = bat_vertical_positions[0]['start']
bat_w = bat_horizontal_positions[0]['end'] - bat_horizontal_positions[
0]['start']
bat_h = bat_vertical_positions[0]['end'] - bat_vertical_positions[0][
'start']
return True, pygame.Rect(bat_x, bat_y, bat_w, bat_h)
else:
return False, None
def get_bricks_positions(im_param):
im_w = im_param.size[0]
im_h = im_param.size[1]
print('get_bricks_positions - im_h={}, im_w={}'.format(im_h, im_w))
rgb_list, pix = get_rgb_list(im_param)
# Note, the original image has 10 brick width in a row, the middle is a gap, and,
# the ball originally sits right below the bricks, just slightly off the center
# this can be explored to identify horizontal position of each brick, horizontal
# padding, as well as the ball size, use the same col_to_scan for bricks and ball
# note, rgb_list is also flopped as it's directly from pix
col_to_scan = int(im_w / 2) - 10
# print('get_bricks_positions - col_to_scan={}, \nrgb_list={}'.format(col_to_scan, rgb_list[0]))
print('get_bricks_positions - col_to_scan={}'.format(col_to_scan))
brick_vertical_positions = scan_line_for_objects(rgb_list, col_to_scan,
'v', (255, 0, 0),
(255, 255, 255))
print('get_bricks_positions - brick_vertical_positions=\n{}'.format(
brick_vertical_positions))
row_to_scan = brick_vertical_positions[0]['start'] + \
int((brick_vertical_positions[0]['end'] - brick_vertical_positions[0]['start'])/2)
# print('get_bricks_positions - row_to_scan={}, \nrgb_list={}'.format(row_to_scan, rgb_list[0]))
print('get_bricks_positions - row_to_scan={}'.format(row_to_scan))
brick_horizontal_positions = scan_line_for_objects(rgb_list, row_to_scan,
'h', (255, 0, 0),
(255, 255, 255))
print('get_bricks_positions - brick_horizontal_positions=\n{}'.format(
brick_horizontal_positions))
if len(brick_horizontal_positions) > 0 and len(
brick_vertical_positions) > 0:
return True, brick_horizontal_positions, brick_vertical_positions
else:
return False, None, None
def update_brick_states(im_param, bricks, brick_states):
rgb_list, pix = get_rgb_list(im_param)
# note, rgb_list is also flopped as it's directly from pix
states_changed = False
state_changed_brick = None
for i in range(len(bricks)):
for j in range(len(bricks[0])):
prev_state = brick_states[i][j]
# only need to check the top-left pixel to see if the color changed i.e. if the brick is knocked out
if rgb_list[bricks[i][j].y][bricks[i][j].x] != (255, 0, 0):
brick_states[i][j] = 0
if brick_states[i][j] != prev_state:
states_changed = True
state_changed_brick = bricks[i][j]
# note, only one brick gets knocked out at one time, so break
break
return states_changed, brick_states, state_changed_brick
def main():
curr_image = None
prev_image = None
curr_image_title = 'current_frame'
prev_image_title = 'previous_frame'
image_count = 0
image_w = None
image_h = None
bricks = None
brick_states = None
ball_r = None
# note, this is an artificial number, small enough to reduce search space, but big enough to include the whole ball
search_margin = 20
curr_ball_center = None
curr_ball_square = None
prev_ball_center = None
prev_ball_square = None
curr_bat = None
prev_bat = None
curr_rgb_list = None
while True:
# for sending command asking for image to return using UDP portocol,
# simply send, doesn't involve any handshake or received feedback
# note, this port is more like communication between two computers,
# not exactly server-client structure
# UDP_IP = "10.44.121.45"
# UDP_IP = "10.44.121.31"
UDP_IP = "127.0.0.1"
UDP_PORT = 46087
command_sock = socket.socket(
socket.AF_INET, # Internet
socket.SOCK_DGRAM) # UDP
# for receiving image from TCP, IP and the port to receive from is the same as UDP
# TCP_IP = '10.44.121.45'
# TCP_IP = '10.44.121.31'
TCP_IP = '127.0.0.1'
TCP_PORT = 46087
BUFFER_SIZE = 1024
image_sock = socket.socket(
socket.AF_INET, # Internet
socket.SOCK_STREAM) # TCP
try:
# construct and send game command to the server
command_message = ''
if curr_image:
curr_rgb_list = get_rgb_list(curr_image)
do_not_increment_image_count = False
# get command_message to move the bat
if image_count == 0:
command_message = 'G'
# game over, brick_stats already generated, but no image received, restart a game, reset brick_states
elif not curr_image and brick_states:
brick_states = reset_brick_states(brick_states)
command_message = 'R'
else:
command_message = '.' # to be updated by image analysis results
ball_success = False
# bricks_success = False
# bat_success = False
# got the first iamge back, need to retrieve the bricks layout, only need once for a game
bat_success, curr_bat = get_bat(curr_image)
if image_count == 1:
bricks_success, bricks, brick_states = get_bricks(
curr_image)
ball_success, org_ball_center, ball_r = get_original_ball_center_and_size(
curr_image, bricks)
if ball_success:
print(
'main - successfully getting initial bricks and ball positions...'
)
curr_ball_center = org_ball_center
curr_ball_square = BallSquare(
curr_ball_center.x - ball_r,
curr_ball_center.x - ball_r,
curr_ball_center.y - ball_r,
curr_ball_center.y + ball_r)
image_w = curr_image.size[0]
image_h = curr_image.size[1]
prev_ball_center, prev_ball_square, pre_bat, prev_image = \
set_curr_to_prev(curr_ball_center, curr_ball_square, curr_bat, curr_image)
else:
# if the game has started and the ball has dropped to the floor when we started
# running the control window, reset the game
print('main - resetting the game...')
do_not_increment_image_count = True
command_message = 'R'
###########################################################################
if command_message != 'R' and command_message != 'G':
curr_rgb_list, curr_pix = get_rgb_list(curr_image)
print(
'main - about to predict landing position...command_message = {}, '
'curr_image width = {}, curr_image_height = {}, '
'curr_rgb_list rows = {}, curr_rgb_list cols = {}'.
format(command_message, curr_image.size[0],
curr_image.size[1], len(curr_rgb_list),
len(curr_rgb_list[0])))
# predicted_ball_landing_position = curr_bat.x
# # image analysis to get the command, 'L', 'R', or '.'
# brick_states_changed, brick_states, state_changed_brick = \
# update_brick_states(curr_image, bricks, brick_states)
# print('brick_states_changed={}, brick_states={}, state_changed_brick={}'.format(
# brick_states_changed, brick_states, state_changed_brick))
# if (brick_states_changed):
# print('brick_states_changed = True, searching for ball with the following parameters:\n'
# 'search_margin={}, ball_r={}, state_changed_brick={}, \ncurr_rgb_list size=({}, {})\n'.format(
# search_margin, ball_r, state_changed_brick, len(curr_rgb_list), len(curr_rgb_list[0])
# ))
# ball_success, curr_ball_center, curr_ball_square = \
# search_for_ball('below', search_margin, ball_r, state_changed_brick,
# curr_rgb_list, image_w, image_h)
# if not ball_success:
# ball_success, curr_ball_center, curr_ball_square = \
# search_for_ball('right', search_margin, ball_r, state_changed_brick,
# curr_rgb_list, image_w, image_h)
# if not ball_success:
# ball_success, curr_ball_center, curr_ball_square = \
# search_for_ball('left', search_margin, ball_r, state_changed_brick,
# curr_rgb_list, image_w, image_h)
# if not ball_success:
# ball_success, curr_ball_center, curr_ball_square = \
# search_for_ball('above', search_margin, ball_r, state_changed_brick,
# curr_rgb_list, image_w, image_h)
# if ball_success:
# # keep under the call by default
# predicted_ball_landing_position = int(curr_ball_center.x)
# # if image_count > 1, i.e.we have an prev_image, update predicted_ball_landing_position
# if image_count > 1:
# curr_ball_relative_postion_to_brick = \
# get_relative_position_to_brick(curr_ball_square, state_changed_brick)
# prev_ball_relative_postion_to_brick = \
# get_relative_position_to_brick(prev_ball_square, state_changed_brick)
# predicted_ball_landing_position = \
# get_predicted_ball_landing_position(
# curr_ball_relative_postion_to_brick, prev_ball_relative_postion_to_brick,
# curr_ball_center, prev_ball_center, curr_ball_square, curr_bat, image_w, image_h)
# else: # if not brick knocked off, just follow the ball
# curr_ball_center = get_curr_ball_from_prev_ball(prev_ball_center, ball_r, search_margin, curr_image)
# predicted_ball_landing_position = curr_ball_center.x
curr_ball_center = get_curr_ball_from_prev_ball(
prev_ball_center, ball_r, search_margin, curr_image)
if curr_ball_center:
predicted_ball_landing_position = curr_ball_center.x
command_message = get_command_message(
curr_bat, predicted_ball_landing_position)
print(
'main - completed predicting landing position, curr_bat = {}. curr_ball_center = {}. '
'predicted_ball_landing_position = {}, command_message = {}'
.format(curr_bat, curr_ball_center,
predicted_ball_landing_position,
command_message))
###########################################################################
# done our calculations, set the curr's to prev's
prev_ball_center, prev_ball_square, pre_bat, prev_image = \
set_curr_to_prev(curr_ball_center, curr_ball_square, curr_bat, curr_image)
# curr_image = None
if command_message != '':
# send the command to the server
message_bytes_obj = bytearray(command_message,
'utf-8') # needed by sock.sendto
command_sock.sendto(message_bytes_obj, (UDP_IP, UDP_PORT))
# connect to the server to receive image
image_sock.connect((TCP_IP, TCP_PORT))
buffer = io.BytesIO()
while True:
data = image_sock.recv(BUFFER_SIZE)
if not data:
break
buffer.write(data)
curr_image = Image.open(buffer)
rgb_list, pix = get_rgb_list(curr_image)
print('current image size={}'.format(curr_image.size))
if not do_not_increment_image_count:
image_count += 1
print('do_not_increment_image_count = {}, image_count = {}'.
format(do_not_increment_image_count, image_count))
pairs = dict()
if curr_image:
pairs[curr_image_title] = curr_image
if not prev_image:
prev_image = copy.deepcopy(curr_image)
pairs[prev_image_title] = prev_image
display_all_images_in_plot(1, pairs, fig, plot_canvas)
# time.sleep(0.1)
except ConnectionRefusedError:
print("Server not running. Waiting.")
