def get_ult_colors_from_image(self):
"""Get ultimate charge number colors from this frame.
Author:
Rigel
Args:
None
Returns:
@ult_color: array of int, -1: white number, 1: black number
"""
left_pre_pos = OW.get_ult_charge_color_pre_pos(True)[self.game.game_type]
left_pre_image = ImageUtils.rgb_to_gray(ImageUtils.crop(self.image, left_pre_pos))
left_shear = ImageUtils.shear(left_pre_image, OW.get_tf_shear(True)[self.game.game_type])
left_pos = OW.get_ult_charge_color_pos(True)[self.game.game_type]
left_image = ImageUtils.crop(left_shear, left_pos)
left_image_g = ImageUtils.contrast_adjust_log(left_image, OW.ULT_ADJUST_LOG_INDEX)
left_bin = ImageUtils.binary_otsu(left_image_g)
right_pre_pos = OW.get_ult_charge_color_pre_pos(False)[self.game.game_type]
right_pre_image = ImageUtils.rgb_to_gray(ImageUtils.crop(self.image, right_pre_pos))
right_shear = ImageUtils.shear(right_pre_image, OW.get_tf_shear(False)[self.game.game_type])
right_pos = OW.get_ult_charge_color_pos(False)[self.game.game_type]
right_image = ImageUtils.crop(right_shear, right_pos)
right_image_g = ImageUtils.contrast_adjust_log(right_image, OW.ULT_ADJUST_LOG_INDEX)
right_bin = ImageUtils.binary_otsu(right_image_g)
return {
"left": np.sign(2 * np.sum(left_bin) - np.size(left_bin)),
"right": np.sign(2 * np.sum(right_bin) - np.size(right_bin))
}
def get_living_status(self, avatar_ref):
"""Retrieves chara living status for current player.
If the chara is dead, general variation of avatar brightness gets lower
than reference.
Author:
Appcell
Args:
avatar_ref: reference avatar image
Returns:
None
"""
avatar = []
if self.is_observed:
avatar = ImageUtils.crop(
self.image,
OW.get_avatar_pos(self.index)[self.frame.game.game_type])
else:
avatar = ImageUtils.crop(
self.image,
OW.get_avatar_pos_small(self.index)[self.frame.game.game_type])
brightness = np.mean(avatar, 2)
brightness_ref = np.mean(avatar_ref, 2)
variation = brightness.max() - brightness.min()
variation_ref = brightness_ref.max() - brightness_ref.min()
# TODO: write consts here into ow.py
if abs(variation_ref - variation) > 45:
self.is_dead = True
def validate(self):
"""Validate this frame, set up Game obj if it's not set.
Validation by:
1) Test if there's any players detectable. If none, frame is invalid
2) Test if top-right corner is white. If not, frame is invalid
If frame is valid and Game info (i.e. team colors, avatars) are not
set, set them up.
Author:
Appcell
Args:
None
Returns:
None
"""
flag = False
for player in self.players:
if player.is_dead is False:
flag = True
validation_roi = ImageUtils.crop(
self.image, OW.FRAME_VALIDATION_POS[self.game.game_type])
std = np.max([
np.std(validation_roi[:, :, 0]),
np.std(validation_roi[:, :, 1]),
np.std(validation_roi[:, :, 2])
])
mean = [
np.mean(validation_roi[:, :, 0]),
np.mean(validation_roi[:, :, 1]),
np.mean(validation_roi[:, :, 2])
]
if std < OW.FRAME_VALIDATION_COLOR_STD[self.game.game_type] \
and np.mean(mean) > OW.FRAME_VALIDATION_COLOR_MEAN[self.game.game_type] \
and flag is True:
self.is_valid = True
replay_icon = ImageUtils.crop(
self.image,
OW.get_replay_icon_pos()[self.game.game_type])
# cv2.imshow('t', self.image)
# cv2.waitKey(0)
replay_match_result = cv2.matchTemplate(replay_icon,
self.game.replay_icon_ref,
cv2.TM_CCOEFF_NORMED)
_, max_val, _, _ = cv2.minMaxLoc(replay_match_result)
if max_val < OW.FRAME_VALIDATION_REPLAY_PROB[self.game.game_type]:
self.is_valid = True
if self.is_valid is True and self.game.team_colors is None:
self.game.set_team_colors(self)
self.game.avatars_ref = self._get_avatars_before_validation()
def crop_image(self, img, mask, bbox, kp, vis, sfm_pose):
# crop image and mask and translate kps
img = image_utils.crop(img, bbox, bgval=1)
mask = image_utils.crop(mask, bbox, bgval=0)
kp[vis, 0] -= bbox[0]
kp[vis, 1] -= bbox[1]
sfm_pose[1][0] -= bbox[0]
sfm_pose[1][1] -= bbox[1]
return img, mask, kp, sfm_pose
def crop_image(self, img, mask, bbox, kp, vis, sfm_pose):
# crop image and mask and translate kps
img = image_utils.crop(img, bbox, bgval=1)
mask = image_utils.crop(mask, bbox, bgval=0)
# subtract x1, y1 to translate the coordinate system to (0, 0)
kp[vis, 0] -= bbox[0]
kp[vis, 1] -= bbox[1]
sfm_pose[1][0] -= bbox[0]
sfm_pose[1][1] -= bbox[1]
return img, mask, kp, sfm_pose
def get_chara(self):
"""Retrieves chara name for current player in current frame.
Compare cropped avatar with reference avatars, pick the best match as
the chara current player plays with. In OWL, currently observed player
has a larger avatar. To differentiate between the two, comparison has
to run twice and the better match gets chosen.
Author:
Appcell
Args:
None
Returns:
None
"""
all_avatars = self.frame.get_avatars(self.index)
avatars_ref = all_avatars["normal"]
avatars_small_ref = all_avatars["small"]
# Crop avatar from frame
avatar = ImageUtils.crop(
self.image,
OW.get_avatar_pos(self.index)[self.frame.game.game_type])
avatar_small = ImageUtils.crop(
avatar, [1, avatar.shape[0] - 4, 0, avatar.shape[1]])
score = 0
# if self.index == 0:
# cv2.imshow('t',avatar)
# cv2.imshow('t',avatar_small)
# cv2.waitKey(0)
for (name, avatar_ref) in avatars_ref.iteritems():
s = cv2.matchTemplate(avatar, avatar_ref, cv2.TM_CCOEFF_NORMED)
_, s, _, _ = cv2.minMaxLoc(s)
s_small = cv2.matchTemplate(avatar_small, avatars_small_ref[name],
cv2.TM_CCOEFF_NORMED)
_, s_small, _, _ = cv2.minMaxLoc(s_small)
s_final = s if s > s_small else s_small
# if self.index == 0 and (name == 'roadhog' or name == 'symmetra' or name == 'orisa'):
# print [name, s_small, s]
if s_final > score:
score = s_final
self.chara = name
self.is_observed = True if s > s_small else False
if self.chara == None:
self.chara = "empty"
self.is_dead = True
return
self.get_living_status(avatars_ref[self.chara])
def get_ult_status(self):
"""Retrieves ultimate statues info for current player in current frame.
Author:
Appcell
Args:
None
Returns:
None
"""
# Crop icon from current frame
ult_icon_pos = OW.get_ult_icon_pos(
self.index)[self.frame.game.game_type]
ult_icon = ImageUtils.crop(self.image, ult_icon_pos)
# Get reference icon image
ult_icon_ref = OW.get_ult_icon_ref(
self.index)[self.frame.game.game_type]
# Tranfer both to grayscale for comparison
ult_icon_ref, ult_icon = ImageUtils.rgb_to_gray(
ult_icon_ref), ImageUtils.rgb_to_gray(ult_icon)
# Compare cropped icon with reference, get the probability of them
# being similar
prob_map = cv2.matchTemplate(ult_icon, ult_icon_ref,
cv2.TM_CCOEFF_NORMED)
prob_map_cropped = prob_map[0:(ult_icon.shape[0] -
ult_icon_ref.shape[0]), :]
_, prob, _, loc = cv2.minMaxLoc(prob_map_cropped)
# To avoid possible explosion effect.
# When ult gets ready, brightness of icon goes above limit.
brightness = np.mean(ult_icon)
deviation = np.std(ult_icon)
if brightness > OW.ULT_ICON_MAX_BRIGHTNESS[self.frame.game.game_type] \
and deviation < OW.ULT_ICON_MAX_DEVIATION[self.frame.game.game_type]:
prob = 1
self.is_ult_ready = True
return
temp_ult_icon = ImageUtils.crop(
ult_icon,
[loc[1], ult_icon_ref.shape[0], loc[0], ult_icon_ref.shape[1]])
prob_ssim = measure.compare_ssim(temp_ult_icon,
ult_icon_ref,
multichannel=False)
if prob > OW.ULT_ICON_MAX_PROB[self.frame.game.game_type]:
if prob_ssim > OW.ULT_ICON_MAX_PROB_SSIM[
self.frame.game.game_type]:
self.is_ult_ready = True
def __getitem__(self, index: int) -> Sample:
# image
image_filepath = os.path.join(self._root, self._image_list[index])
image = cv2.imread(image_filepath)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
h, w = image.shape[:2]
# roi
bbox = self._bbox_list[index].copy()
# expanding
bbox = self._expand(bbox, (w, h))
# cropping roi
image = crop(image, bbox)
# landmarks
landmarks = self._landmark_list[index].copy()
landmarks_wrapper = LandmarksWrapper(landmarks)
# 0-1 normalization
landmarks_wrapper.norm_01(bbox)
sample = {"image": image, "landmarks": landmarks_wrapper.landmarks}
if self._transform:
sample = self._transform(sample)
return sample
def preprocess_image(img_path, img_size=256):
img = io.imread(img_path) / 255.
# Scale the max image size to be img_size
#-這邊將圖片的大小scale到257
scale_factor = float(img_size) / np.max(img.shape[:2])
img, _ = img_util.resize_img(img, scale_factor) #256x256x3
# Crop img_size x img_size from the center
#---------------其實看不太懂它為什麼要切割，因為它切割的大小是257x257，而它縮放的大小是256x256
#--------------他是不是在耍人阿！？？
#--------------切割是由中心點往外切出一個bounding box
center = np.round(np.array(img.shape[:2]) / 2).astype(int)
#p
# print("center1:"+str(center))
# img center in (x, y)
center = center[::-1]
#p
# print("center2:"+str(center))
bbox = np.hstack([center - img_size / 2., center + img_size / 2.])
#p
# print("bbox:"+str(bbox))
img = img_util.crop(img, bbox, bgval=1.) #257x257x3
# Transpose the image to 3xHxW
img = np.transpose(img, (2, 0, 1)) #3x257x257
return img
def preprocess_image(img_path, img_size=256):
img = io.imread(img_path) / 255.
# if grayscale, convert to RGB
if len(img.shape) == 2:
img = np.repeat(np.expand_dims(img, 2), 3, axis=2)
# Scale the max image size to be img_size
scale_factor = float(img_size) / np.max(img.shape[:2])
img, _ = img_util.resize_img(img, scale_factor)
# Crop img_size x img_size from the center
center = np.round(np.array(img.shape[:2]) / 2).astype(int)
# img center in (x, y)
center = center[::-1]
bbox = np.hstack([center - img_size / 2., center + img_size / 2.])
img = img_util.crop(img, bbox, bgval=1.)
# Transpose the image to 3xHxW
img = np.transpose(img, (2, 0, 1))
# necessary preprocessing for resnet
img = torch.tensor(img, dtype=torch.float)
img = resnet_transform(img)
# random flip
if np.random.rand(1) > 0.5:
img = torch.flip(img, (2,))
return img
def get_images(self, orig_image: OrthographicImage):
image = clone(orig_image)
draw_around_box(image, box=self.box)
background_color = image.value_from_depth(
get_distance_to_box(image, self.box))
mat_image_resized = cv2.resize(image.mat, self.size_resized)
mat_images = []
for a in self.a_space:
rot_mat = cv2.getRotationMatrix2D(
(self.size_resized[0] / 2, self.size_resized[1] / 2),
a * 180.0 / np.pi, 1.0)
rot_mat[:,
2] += [(self.size_rotated[0] - self.size_resized[0]) / 2,
(self.size_rotated[1] - self.size_resized[1]) / 2]
dst_depth = cv2.warpAffine(mat_image_resized,
rot_mat,
self.size_rotated,
borderValue=background_color)
mat_images.append(crop(dst_depth, self.size_cropped))
mat_images = np.array(mat_images) / np.iinfo(image.mat.dtype).max
if len(mat_images.shape) == 3:
mat_images = np.expand_dims(mat_images, axis=-1)
# mat_images = 2 * mat_images - 1.0
return mat_images
def __init__(self, frame, index):
"""Initialize a Killfeed object.
Author:
Appcell
Args:
frame: the Frame obj current killfeed is in
index: row number of current killfeed, ranges from 0 to 5.
Returns:
None
"""
self.player1 = {
"chara": "empty",
"player": "empty",
"team": "empty",
"pos": -1
}
self.player2 = {
"chara": "empty",
"player": "empty",
"team": "empty",
"pos": -1
}
self.ability = 0
self.assists = []
self.index = index
self.is_valid = True
self.is_headshot = False
self.frame = frame
self.game_type = frame.game.game_type
killfeed_pos = OW.get_killfeed_pos(index)[frame.game.game_type]
killfeed_with_gap_pos = OW.get_killfeed_with_gap_pos(index)[
frame.game.game_type]
self.image = ImageUtils.crop(frame.image, killfeed_pos)
self.image_with_gap = ImageUtils.crop(
frame.image, killfeed_with_gap_pos)
self.get_players()
self.get_ability_and_assists()
self.get_headshot()
self.free()
def get_image_roi(wc):
config = get_config()
imgd = get_absolute_path(config.get('KWS', 'images'))
imgs = glob(os.path.join(imgd, '*.jpg'))
dids = [os.path.basename(x).replace('.jpg', '') for x in imgs]
imgp = imgs[dids.index(wc.doc_id)]
img = imread(imgp)
path = get_svg_path(wc)
poly = path2polygon(path)
return crop(img, poly)
def get_image_roi(wc):
config = get_config()
imgd = get_absolute_path(config.get('KWS', 'images'))
imgs = glob(os.path.join(imgd, '*.jpg'))
dids = [os.path.basename(x).replace('.jpg', '') for x in imgs]
imgp = imgs[dids.index(wc.doc_id)]
img = imread(imgp)
path = get_svg_path(wc)
poly = path2polygon(path)
return crop(img, poly)
def grasp_convert(self, action: Action,
images: List[OrthographicImage]) -> None:
image = images[0]
mat_area_image = get_area_of_interest_new(image,
action.pose,
border_color=np.nan,
project_3d=False,
flags=cv2.INTER_NEAREST).mat
mat_area_image = mat_area_image.astype(np.float32)
mat_area_image[mat_area_image < 10 *
255] = np.nan # Make every not found pixel NaN
# Get distance at gripper for possible collisions
gripper_one_side_size = 0.5 * image.pixel_size * (action.pose.d + 0.002
) # [px]
area_center = crop(
mat_area_image,
(image.pixel_size * 0.012, image.pixel_size * 0.012))
side_gripper_image_size = (image.pixel_size * 0.025,
image.pixel_size * 0.025)
area_left = crop(mat_area_image, side_gripper_image_size,
(-gripper_one_side_size, 0))
area_right = crop(mat_area_image, side_gripper_image_size,
(gripper_one_side_size, 0))
# Z is positive!
z_raw = image.depth_from_value(np.nanmedian(area_center))
if z_raw is np.NaN:
area_center = crop(
mat_area_image,
(image.pixel_size * 0.03, image.pixel_size * 0.03))
z_raw = image.depth_from_value(np.nanmedian(area_center))
z_raw_left = image.depth_from_value(np.nanmin(area_left))
z_raw_right = image.depth_from_value(np.nanmin(area_right))
z_raw += self.grasp_z_offset
z_raw_collision = min(z_raw_left, z_raw_right) - 0.008 # [m]
z_raw_max = min(
z_raw, z_raw_collision) # Get the maximum [m] for impedance mode
action.pose.z = -z_raw_max
def place_convert(self, action: Action,
images: List[OrthographicImage]) -> None:
image = images[0]
mat_area_image = get_area_of_interest_new(image,
action.pose,
border_color=np.nan).mat
mat_area_image = mat_area_image.astype(np.float32)
mat_area_image[mat_area_image ==
0] = np.nan # Make every not found pixel NaN
# Get distance at gripper for possible collisions
gripper_one_side_size = 0.5 * image.pixel_size * (action.pose.d + 0.002
) # [px]
area_center = crop(
mat_area_image,
(image.pixel_size * 0.025, image.pixel_size * 0.025))
side_gripper_image_size = (image.pixel_size * 0.025,
image.pixel_size * 0.025)
area_left = crop(mat_area_image, side_gripper_image_size,
(-gripper_one_side_size, 0))
area_right = crop(mat_area_image, side_gripper_image_size,
(gripper_one_side_size, 0))
z_raw = image.depth_from_value(np.nanmedian(area_center))
z_raw_left = image.depth_from_value(np.nanmin(area_left))
z_raw_right = image.depth_from_value(np.nanmin(area_right))
if z_raw is np.NaN:
area_center = crop(
mat_area_image,
(image.pixel_size * 0.022, image.pixel_size * 0.03))
z_raw = image.depth_from_value(np.nanmedian(area_center))
z_raw += self.place_z_offset
z_raw_collision = min(z_raw_left, z_raw_right) - 0.008 # [m]
# z_raw_max = min(z_raw, z_raw_collision) # Get the maximum [m] for impedance mode
z_raw_max = z_raw
action.pose.z = -z_raw_max # [m] Move slightly up for gripper center point
def shift_convert(self, action: Action,
images: List[OrthographicImage]) -> None:
image = images[0]
mat_area_image = get_area_of_interest_new(image,
action.pose,
border_color=np.nan).mat
mat_area_image = mat_area_image.astype(np.float32)
mat_area_image[mat_area_image ==
0] = np.nan # Make every not found pixel NaN
# Get distance at gripper for possible collisions
area_center = crop(mat_area_image,
(image.pixel_size * 0.01, image.pixel_size * 0.03))
z_raw = image.depth_from_value(np.nanmax(area_center))
z_raw += self.shift_z_offset
action.pose.z = -z_raw # [m] Move slightly up for gripper center point
def preprocess_image(img_path, img_size=256):
img = io.imread(img_path) / 255.
# Scale the max image size to be img_size
scale_factor = float(img_size) / np.max(img.shape[:2])
img, _ = img_util.resize_img(img, scale_factor)
# Crop img_size x img_size from the center
center = np.round(np.array(img.shape[:2]) / 2).astype(int)
# img center in (x, y)
center = center[::-1]
bbox = np.hstack([center - img_size / 2., center + img_size / 2.])
img = img_util.crop(img, bbox, bgval=1.)
# Transpose the image to 3xHxW
img = np.transpose(img, (2, 0, 1))
return img
def detect_gesture(pd, tracker, img, action='activate'):
# Inference
_, t, status, box = pd.predict(img)
print('Gesture detection estimated time {:.4f}'.format(t / 1000))
# Calculate result
ok = False
# 0: None, 1: lefthand, 2: righthand, 3: both hands
if action == 'activate' and (status == 1 or status == 2):
height, width, _ = img.shape
xmin, ymin = int(box[0] * width), int(box[1] * height)
xmax, ymax = int(box[2] * width), int(box[3] * height)
box = np.array([xmin, ymin, xmax, ymax])
obj_img = image.crop(img, box)
obj_img = image.resize(obj_img, tracker.input_shape)
obj_img = np.array(obj_img / 127.5 - 1, dtype=np.float32)
ok = tracker.set_anchor(obj_img, box)
if action == 'deactivate' and status == 3:
ok = True
# Return
return ok
def get_ult_status(self):
"""Retrieves ultimate statues info for current player in current frame.
Author:
Appcell
Args:
None
Returns:
None
"""
# Crop icon from current frame
ult_icon_pos = OW.get_ult_icon_pos(
self.index)[self.frame.game.game_type]
ult_icon = ImageUtils.crop(self.image, ult_icon_pos)
# Get reference icon image
ult_icon_ref = OW.get_ult_icon_ref(
self.index)[self.frame.game.game_type]
# Tranfer both to grayscale for comparison
ult_icon_ref, ult_icon = ImageUtils.rgb_to_gray(
ult_icon_ref), ImageUtils.rgb_to_gray(ult_icon)
# Compare cropped icon with reference, get the probability of them
# being similar
prob = cv2.matchTemplate(ult_icon, ult_icon_ref,
cv2.TM_CCOEFF_NORMED).max()
# To avoid possible explosion effect.
# When ult gets ready, brightness of icon goes above limit.
brightness = np.mean(ult_icon)
if brightness > OW.ULT_ICON_MAX_BRIGHTNESS[self.frame.game.game_type]:
prob = 1
if prob > OW.ULT_ICON_MAX_PROB[self.frame.game.game_type]:
self.is_ult_ready = True
def test_crop(self):
uncropped = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12],
[13, 14, 15, 16]]])
#Test without offset
croppedTest1 = np.array([[[6, 7], [10, 11]]])
croppedTest1Crop = (2, 2)
croppedTest1Offset = (0, 0)
self.assertTrue(
np.allclose(
pre.crop(uncropped, croppedTest1Crop, croppedTest1Offset),
croppedTest1))
#Test offset
croppedTest2 = np.array([[[11, 12], [15, 16]]])
croppedTest2Crop = (2, 2)
croppedTest2Offset = (1, 1)
self.assertTrue(
np.allclose(
pre.crop(uncropped, croppedTest2Crop, croppedTest2Offset),
croppedTest2))
#Test non-square factor regions and preference for top and left with odd remainders.
croppedTest3 = np.array([[[2], [6], [10]]])
croppedTest3Crop = (3, 1)
croppedTest3Offset = (0, 0)
self.assertTrue(
np.allclose(
pre.crop(uncropped, croppedTest3Crop, croppedTest3Offset),
croppedTest3))
with self.assertRaises(ValueError):
#Fails when crop region is too big.
croppedTest4Crop = (6, 3)
croppedTest4Offset = (0, 0)
pre.crop(uncropped, croppedTest4Crop, croppedTest4Offset)
#Fails when offset is too large.
croppedTest5Crop = (2, 2)
croppedTest5Offset = (2, 1)
pre.crop(uncropped, croppedTest5Crop, croppedTest5Offset)
def _get_icons_weights(self, edge_validation):
"""Get icon comparison scores of row image and icon references.
Use match template in cv2 to get possible icon and their weights in the
killfeed image.
Author:
Leavebody
Args:
edge_validation: A list of booleans. Should be the result of
_validate_edge()
Returns:
A list of dict of matching results, including all possible icons
in this killfeed image. A matching result is in the form of:
{
"chara": "empty", # name of chara, or "empty"
"prob": "empty", # score from comparison
"pos": "empty", # x-axis position of icon in killfeed
row image
}
"""
result = []
for (chara, icon) in self.frame.game.killfeed_icons_ref.iteritems():
match_result = cv2.matchTemplate(self.image, icon,
cv2.TM_CCOEFF_NORMED)
# Find two most possible location of this character's icon in the killfeed image.
# Mask the pixels around the first location to find the second one.
_, max_val, _, max_loc = cv2.minMaxLoc(match_result)
# Here we have to allow some error, thus comes +/- 2
if sum(edge_validation[max_loc[0] - 2: max_loc[0] + 2]) > 0 \
and max_val > OW.KILLFEED_MAX_PROB[self.game_type]:
temp_icon = ImageUtils.crop(self.image, [
3, icon.shape[0], max_loc[0],
OW.KILLFEED_ICON_WIDTH[self.game_type]
])
score_ssim = measure.compare_ssim(temp_icon,
icon,
multichannel=True)
if score_ssim >= OW.KILLFEED_SSIM_THRESHOLD[self.game_type]:
result.append({
'chara': chara,
'prob': max_val,
'pos': max_loc[0]
})
half_mask_width = 5
mask_index_left = max((max_loc[0] + half_mask_width \
- OW.ABILITY_GAP_NORMAL[self.game_type] \
- OW.KILLFEED_ICON_WIDTH[self.game_type], 0))
mask_index_right = min((
max_loc[0] - half_mask_width + 1 \
+ OW.ABILITY_GAP_NORMAL[self.game_type] \
+ OW.KILLFEED_ICON_WIDTH[self.game_type],
OW.KILLFEED_WIDTH[self.game_type] \
- OW.KILLFEED_ICON_WIDTH[self.game_type]))
match_result_masked = np.matrix(match_result)
match_result_masked[0:match_result_masked.shape[0],
mask_index_left:mask_index_right] = -1
_, max_val2, _, max_loc2 = cv2.minMaxLoc(match_result_masked)
if sum(edge_validation[max_loc2[0]-2: max_loc2[0]+2]) > 0 \
and max_val2 > OW.KILLFEED_MAX_PROB[self.game_type]:
temp_icon2 = ImageUtils.crop(self.image, [
3, icon.shape[0], max_loc2[0],
OW.KILLFEED_ICON_WIDTH[self.game_type]
])
score_ssim2 = measure.compare_ssim(temp_icon2,
icon,
multichannel=True)
if score_ssim2 >= OW.KILLFEED_SSIM_THRESHOLD[self.game_type]:
result.append({
'chara': chara,
'prob': max_val2,
'pos': max_loc2[0]
})
# Since there's often a 'fake' avatar appearing in the middle, here we
# have to do some filtering. The idea is, if positions of recoged
# avatars range largely, this kf usually comes with 2 avatars (killer
# and killed player). A pair of 2 avatars have a minimum distance
# requirement: distance >= ABILITY_GAP_NORMAL + OW.KILLFEED_ICON_WIDTH
# Another possible error: tm takes assist icon as a full avatar. This
# is a bit tricky and my solution is not really optimal. I directly
# remove those far from left/right edges.
min_pos = 1000
max_pos = 0
for chara in result:
if chara['pos'] < min_pos:
min_pos = chara['pos']
if chara['pos'] > max_pos:
max_pos = chara['pos']
if len(result) < 2:
return result
# calculate distance between each pair of potential recogs, then
# remove those which are never visited
result_validation = [False] * len(result)
if max_pos - min_pos \
> OW.ABILITY_GAP_NORMAL[self.game_type] + OW.KILLFEED_ICON_WIDTH[self.game_type] - 10:
for ind1, chara1 in enumerate(result):
for ind2, chara2 in enumerate(result):
if abs(chara1['pos'] - chara2['pos']) \
> OW.ABILITY_GAP_NORMAL[self.game_type] \
+ OW.KILLFEED_ICON_WIDTH[self.game_type] - 10 \
or abs(chara1['pos'] - chara2['pos']) < 10:
result_validation[ind1] = True
result_validation[ind2] = True
# If 2 charas, remove those far from left/right edges
result_filtered = []
for ind, val in enumerate(result_validation):
if val is True and abs(result[ind]['pos'] - min_pos) < 15 \
or abs(result[ind]['pos'] - max_pos) < 15:
result_filtered.append(result[ind])
return result_filtered
def main(imgpath=None, svgpath=None, outputfile=None, retake=True, saveimgs=True):
print('Word pre-processing')
config = get_config()
# create an output file
if outputfile is None:
txtp = get_absolute_path(config.get('KWS.features', 'file'))
else:
txtp = get_absolute_path(os.path.join(outputfile))
processed = []
if retake and os.path.exists(txtp):
takenext = False
for line in open(txtp, 'r'):
line = line.strip()
if takenext and (len(line) >= 9):
processed.append(line.strip())
takenext = False
elif line == "###":
takenext = True
else:
handle = open(txtp, 'w+')
for param, value in config.items('KWS.prepro'):
handle.write('%s: %s%s' % (param, value, os.linesep))
for param, value in config.items('KWS.features'):
handle.write('%s: %s%s' % (param, value, os.linesep))
handle.write('###' + os.linesep)
handle.close()
# get the data
if svgpath is None:
svgd = get_absolute_path(config.get('KWS', 'locations'))
else:
svgd = get_absolute_path(svgpath)
svgs = glob(os.path.join(svgd, '*.svg'))
if imgpath is None:
imgd = get_absolute_path(config.get('KWS', 'images'))
else:
imgd = get_absolute_path(imgpath)
imgs = glob(os.path.join(imgd, '*.jpg'))
# parse some parameter
threshold = float(config.get('KWS.prepro', 'segmentation_threshold'))
relative_height = float(config.get('KWS.prepro', 'relative_height'))
skew_resolution = float(config.get('KWS.prepro', 'angular_resolution'))
primary_peak_height = float(config.get('KWS.prepro', 'primary_peak_height'))
secondary_peak_height = float(config.get('KWS.prepro', 'secondary_peak_height'))
window_width = int(config.get('KWS.features', 'window_width'))
step_size = int(config.get('KWS.features', 'step_size'))
blocks = int(config.get('KWS.features', 'number_of_blocks'))
svgs.sort()
imgs.sort()
for svgp, imgp in zip(svgs, imgs):
svgid = os.path.basename(svgp).replace('.svg', '')
imgid = os.path.basename(imgp).replace('.jpg', '')
print('\t%s\n\t%s' % (svgp, imgp))
if svgid != imgid:
raise IOError('the id\'s of the image file (%s) and the svg file (%s) are not the same' % (svgid, imgid))
trans = get_transcription(svgid)
print('\tdoc id: %s' % svgid)
wids, paths = parse_svg(svgp)
img = imread(imgp)
for wid, path in zip(wids, paths):
print('\tword id: %s' % wid)
if retake and (processed.count(wid) == 1):
print('\talready processed')
continue
# look up the corresponding word
if saveimgs:
imgfile = wid
word = get_word(wid, data=trans)
if word is not None:
imgfile = word.code2string() + '_' + imgfile
else:
imgfile = None
# get the word image
poly = path2polygon(path)
roi = crop(img, poly)
pre, sym = word_preprocessor(roi,
threshold=threshold,
rel_height=relative_height,
skew_res=skew_resolution,
ppw=primary_peak_height,
spw=secondary_peak_height,
save=imgfile)
if type(pre) is str:
print('\tpre-processing failed\n\t\t%s' % pre)
continue
fea = compute_features(pre,
window_width=window_width,
step_size=step_size,
blocks=blocks)
write_word_features(txtp, wid, fea, [pre.shape[0], pre.shape[1], sym])
print('...')
def validate(self):
"""Validate this frame, set up Game obj if it's not set.
Validation by:
1) Test if there's any players detectable. If none, frame is invalid
2) Test if top-right corner is white. If not, frame is invalid
If frame is valid and Game info (i.e. team colors, avatars) are not
set, set them up.
Author:
Appcell
Args:
None
Returns:
None
"""
flag = False
for player in self.players:
if player.is_dead is False:
flag = True
if flag == False:
self.is_valid = False
return
else:
self.is_valid = True
validation_roi = ImageUtils.crop(
self.image, OW.FRAME_VALIDATION_POS[self.game.game_type])
std = np.max([
np.std(validation_roi[:, :, 0]),
np.std(validation_roi[:, :, 1]),
np.std(validation_roi[:, :, 2])
])
mean = [
np.mean(validation_roi[:, :, 0]),
np.mean(validation_roi[:, :, 1]),
np.mean(validation_roi[:, :, 2])
]
if std < OW.FRAME_VALIDATION_COLOR_STD[self.game.game_type] \
and np.mean(mean) > OW.FRAME_VALIDATION_COLOR_MEAN[self.game.game_type] \
and flag is True:
self.is_valid = True
else:
self.is_valid = False
return
replay_icon = ImageUtils.crop(
self.image,
OW.get_replay_icon_pos()[self.game.game_type])
replay_icon_preseason = ImageUtils.crop(
self.image,
OW.get_replay_icon_preseason_pos()[self.game.game_type])
max_val = measure.compare_ssim(replay_icon,
self.game.replay_icon_ref,
multichannel=True)
max_val_preseason = measure.compare_ssim(replay_icon_preseason,
self.game.replay_icon_ref,
multichannel=True)
# TODO: another situation: after replay effect there might be a blue
# rectangle remaining on screen.
max_val = max_val if max_val > max_val_preseason else max_val_preseason
if max_val < OW.FRAME_VALIDATION_REPLAY_PROB[self.game.game_type]:
self.is_valid = True
else:
self.is_valid = False
return
if self.is_valid is True and self.game.team_colors is None:
self.game.set_team_colors(self)
self.game.avatars_ref = self._get_avatars_before_validation()
def main(imgpath=None,
svgpath=None,
outputfile=None,
retake=True,
saveimgs=True):
print('Word pre-processing')
config = get_config()
# create an output file
if outputfile is None:
txtp = get_absolute_path(config.get('KWS.features', 'file'))
else:
txtp = get_absolute_path(os.path.join(outputfile))
processed = []
if retake and os.path.exists(txtp):
takenext = False
for line in open(txtp, 'r'):
line = line.strip()
if takenext and (len(line) >= 9):
processed.append(line.strip())
takenext = False
elif line == "###":
takenext = True
else:
handle = open(txtp, 'w+')
for param, value in config.items('KWS.prepro'):
handle.write('%s: %s%s' % (param, value, os.linesep))
for param, value in config.items('KWS.features'):
handle.write('%s: %s%s' % (param, value, os.linesep))
handle.write('###' + os.linesep)
handle.close()
# get the data
if svgpath is None:
svgd = get_absolute_path(config.get('KWS', 'locations'))
else:
svgd = get_absolute_path(svgpath)
svgs = glob(os.path.join(svgd, '*.svg'))
if imgpath is None:
imgd = get_absolute_path(config.get('KWS', 'images'))
else:
imgd = get_absolute_path(imgpath)
imgs = glob(os.path.join(imgd, '*.jpg'))
# parse some parameter
threshold = float(config.get('KWS.prepro', 'segmentation_threshold'))
relative_height = float(config.get('KWS.prepro', 'relative_height'))
skew_resolution = float(config.get('KWS.prepro', 'angular_resolution'))
primary_peak_height = float(config.get('KWS.prepro',
'primary_peak_height'))
secondary_peak_height = float(
config.get('KWS.prepro', 'secondary_peak_height'))
window_width = int(config.get('KWS.features', 'window_width'))
step_size = int(config.get('KWS.features', 'step_size'))
blocks = int(config.get('KWS.features', 'number_of_blocks'))
svgs.sort()
imgs.sort()
for svgp, imgp in zip(svgs, imgs):
svgid = os.path.basename(svgp).replace('.svg', '')
imgid = os.path.basename(imgp).replace('.jpg', '')
print('\t%s\n\t%s' % (svgp, imgp))
if svgid != imgid:
raise IOError(
'the id\'s of the image file (%s) and the svg file (%s) are not the same'
% (svgid, imgid))
trans = get_transcription(svgid)
print('\tdoc id: %s' % svgid)
wids, paths = parse_svg(svgp)
img = imread(imgp)
for wid, path in zip(wids, paths):
print('\tword id: %s' % wid)
if retake and (processed.count(wid) == 1):
print('\talready processed')
continue
# look up the corresponding word
if saveimgs:
imgfile = wid
word = get_word(wid, data=trans)
if word is not None:
imgfile = word.code2string() + '_' + imgfile
else:
imgfile = None
# get the word image
poly = path2polygon(path)
roi = crop(img, poly)
pre, sym = word_preprocessor(roi,
threshold=threshold,
rel_height=relative_height,
skew_res=skew_resolution,
ppw=primary_peak_height,
spw=secondary_peak_height,
save=imgfile)
if type(pre) is str:
print('\tpre-processing failed\n\t\t%s' % pre)
continue
fea = compute_features(pre,
window_width=window_width,
step_size=step_size,
blocks=blocks)
write_word_features(txtp, wid, fea,
[pre.shape[0], pre.shape[1], sym])
print('...')
def get_chara(self):
"""Retrieves chara name for current player in current frame.
Compare cropped avatar with reference avatars, pick the best match as
the chara current player plays with. In OWL, currently observed player
has a larger avatar. To differentiate between the two, comparison has
to run twice and the better match gets chosen.
Author:
Appcell
Args:
None
Returns:
None
"""
all_avatars = self.frame.get_avatars(self.index)
avatars_ref = all_avatars["normal"]
avatars_small_ref = all_avatars["small"]
team_color = avatars_ref['ana'][0, 0]
# Crop avatar from frame
avatar = ImageUtils.crop(
self.image,
OW.get_avatar_pos(self.index)[self.frame.game.game_type])
avatar_small = ImageUtils.crop(
avatar, [4, avatar.shape[0] - 4, 0, avatar.shape[1]])
# If player is observed, not sure about this tho
avatar_diff = ImageUtils.crop(
self.image,
OW.get_avatar_diff_pos(self.index)[self.frame.game.game_type])
max_diff = 0
for i in range(avatar_diff.shape[0]):
for j in range(avatar_diff.shape[1]):
if ImageUtils.color_distance(avatar_diff[i, j],
team_color) > max_diff:
max_diff = ImageUtils.color_distance(
avatar_diff[i, j], team_color)
if max_diff < 40 and self.is_ult_ready is False:
self.is_observed = True
score = 0
for (name, avatar_ref) in avatars_ref.iteritems():
s = cv2.matchTemplate(avatar, avatar_ref, cv2.TM_CCOEFF_NORMED)
_, s, _, loc1 = cv2.minMaxLoc(s)
temp_avatar = ImageUtils.crop(
avatar,
[loc1[1], avatar_ref.shape[0], loc1[0], avatar_ref.shape[1]])
s_ssim1 = measure.compare_ssim(temp_avatar,
avatar_ref,
multichannel=True)
s_small = cv2.matchTemplate(avatar_small, avatars_small_ref[name],
cv2.TM_CCOEFF_NORMED)
_, s_small, _, loc2 = cv2.minMaxLoc(s_small)
temp_avatar2 = ImageUtils.crop(avatar_small, [
loc2[1], avatars_small_ref[name].shape[0], loc2[0],
avatars_small_ref[name].shape[1]
])
s_ssim2 = measure.compare_ssim(temp_avatar2,
avatars_small_ref[name],
multichannel=True)
s_ssim = s_ssim1 if s > s_small else s_ssim2
s_final = s if s > s_small else s_small
loc = loc1 if s > s_small else loc2
if s_final * 0.4 + s_ssim * 0.6 > score:
score = s_final * 0.4 + s_ssim * 0.6
self.chara = name
if self.chara is None:
self.chara = "empty"
self.is_dead = True
return
self.get_living_status(avatars_ref[self.chara])
def get_ability_and_assists(self):
"""Retrieve info of ability and assisting players in a row
If distance between 2 avatar icons, width of arrow icon removed, is
not divisible by width of an assist icon, then an ability icon must
exist somewhere. Cut it off and compare with all possible ability
icons to find the best match.
After removing ability icon & arrow icon, what's left between 2
avatars must be n assist icons. Cut each from killfeed image, and then
compare with references. Pick the one with maximum score as result.
All results are written into self.ability and self.assists.
Author:
Appcell
Args:
None
Returns:
None
"""
if self.player1['pos'] == -1 or self.player2['pos'] == -1:
return
distance = self.player2[
'pos'] - self.player1['pos'] - OW.KILLFEED_ICON_WIDTH[self.game_type]
gap = ImageUtils.crop(
self.image_with_gap,
[0, self.image_with_gap.shape[0], self.player1['pos'], distance])
ability_icon = ImageUtils.crop(
self.image_with_gap,
OW.get_ability_icon_pos(self.player2['pos'])[self.game_type])
# Error gets too much with lowQ videos. Use edge detection instead.
# Honestly it's not the best choice, since for non-OWL videos it
# doesn't work anymore. But again, for non-OWL videos we expect a
# better resolution.
edge_image = cv2.Canny(self.image, 100, 200)
# Get the "spanned" edge image.
roi_x_min = self.player1['pos'] + OW.KILLFEED_ICON_WIDTH[self.game_type] + 4
roi_x_max = self.player2['pos'] - OW.ABILITY_GAP_NORMAL[self.game_type]
if roi_x_max - roi_x_min < OW.ASSIST_GAP[self.game_type]:
return
edge_span = (np.sum(edge_image, 0) / 255)[roi_x_min:roi_x_max]
edges = list(filter(
lambda i: edge_span[i] >= self.image.shape[0] * 0.7,
range(0, roi_x_max - roi_x_min)))
if not edges:
# Assist avatar doesn't exist
return
edge = edges[-1] # The end of assist avatars list
assist_num = int(round(float(edge) / OW.ASSIST_GAP[self.game_type]))
ability_list = OW.ABILITY_LIST[self.player1['chara']]
ability_icons_ref = self.frame.game.ability_icons_ref[
self.player1['chara']]
if (distance - OW.ABILITY_GAP_NORMAL[self.game_type]) % OW.ASSIST_GAP[self.game_type] > 5:
# Has ability icon
max_prob = -10
for (ind, ability_index) in enumerate(ability_list):
filtered_icon = self._preprocess_ability_icon(ability_icon)
score = measure.compare_ssim(
filtered_icon,
ability_icons_ref[ind],
multichannel=True)
if score > max_prob:
max_prob = score
self.ability = ability_index
if max_prob < 0.1 and self.player1['chara'] == OW.GENJI:
self.ability = OW.ABILITY_E
for i in range(assist_num):
# TODO: write this into ow.py!
assist_icon = ImageUtils.crop(
self.image,
[
OW.ABILITY_ICON_Y_MIN[self.game_type],
OW.ASSIST_ICON_HEIGHT[self.game_type],
8 + self.player1['pos'] + i * OW.ASSIST_GAP[self.game_type] \
+ OW.KILLFEED_ICON_WIDTH[self.game_type],
OW.ASSIST_ICON_WIDTH[self.game_type]])
assist = {
"chara": "empty",
"player": "empty",
"team": self.player1['team']
}
max_score = -10
for (chara, icon) in self.frame.game.assist_icons_ref.iteritems():
score = measure.compare_ssim(assist_icon,
icon, multichannel=True)
if score > max_score:
max_score = score
assist['chara'] = chara
self.assists.append(self._set_assist_info(assist))
def get_ult_charge(self):
"""Retrieves ultimate charge for current player.
Author:
Args:
None
Returns:
None
"""
if self.is_ult_ready:
self.ult_charge = 100
return
if self.is_dead:
return
ult_charge_pre_pos = OW.get_ult_charge_pre_pos(
self.index)[self.frame.game.game_type]
ult_charge_pre_image = ImageUtils.rgb_to_gray(
ImageUtils.crop(self.image, ult_charge_pre_pos))
ult_charge_shear = ImageUtils.shear(
ult_charge_pre_image,
OW.get_tf_shear(self.index)[self.frame.game.game_type])
ult_charges = [0, 0]
# Here's another thought: we need to find the gap more intellectually,
# not relying only on fixed position.
# In detail, after shearing, find the gap by telling if there are more
# than 2 colors in same column.
ult_charge_image = ImageUtils.crop(
ult_charge_shear,
OW.get_ult_charge_pos(self.index)[self.frame.game.game_type])
# TODO: I see there's no difference at all of brightness deviation!!
# Our contrast adjusting must be seriously problematic. For grayscale
# img, a simple normalization based on std would do.
# ult_charge_image_g = ImageUtils.contrast_adjust_log(
# ult_charge_image, OW.ULT_ADJUST_LOG_INDEX)
ult_charge_image_g = ImageUtils.normalize_gray(ult_charge_image)
# tell if player is observed (more accurate than previous)
# Here I use another local variable flag_observed, since the global one
# might be inaccurate
flag_observed = False
deviation_row = ult_charge_image_g.max(
axis=1) - ult_charge_image_g.min(axis=1)
if deviation_row[2] - deviation_row[0] > \
OW.ULT_GAP_DEVIATION_LIMIT[self.frame.game.game_type]:
self.is_observed = True
flag_observed = True
# If current player is observed, there's a white dot on right side
# needs to be removed.
# TODO: write this into ow.py as well
if flag_observed is True:
ult_charge_image_g = ImageUtils.crop(ult_charge_image_g, [
0, ult_charge_image_g.shape[0], 0,
ult_charge_image_g.shape[1] - 5
])
width = ult_charge_image_g.shape[1]
height = ult_charge_image_g.shape[0]
# Find the gap
deviation = ult_charge_image_g.max(axis=0) - ult_charge_image_g.min(
axis=0)
gap = -1
for i in range(width - 4, 3, -1):
if deviation[i-3] - deviation[i] \
> OW.ULT_GAP_DEVIATION_LIMIT[self.frame.game.game_type] \
and deviation[i+3] - deviation[i] \
> OW.ULT_GAP_DEVIATION_LIMIT[self.frame.game.game_type]:
gap = i
break
bg_color = ult_charge_image_g[:, 0].mean()
if bg_color < 0.6:
# Dark background
ult_charge_image_g = ImageUtils.inverse_gray(ult_charge_image_g)
# No need to switch to BW here.
if gap == -1:
# Only one digit
num = ImageUtils.remove_digit_vertical_edge(
ult_charge_image_g,
OW.ULT_GAP_DEVIATION_LIMIT[self.frame.game.game_type],
ImageUtils.REMOVE_NUMBER_VERTICAL_EDGE_BOTH)
else:
# 2 digits
num_left = ImageUtils.crop(
ult_charge_image_g,
[0, ult_charge_image_g.shape[0], 0, gap + 1])
num_right = ImageUtils.crop(ult_charge_image_g, [
0, ult_charge_image_g.shape[0], gap,
ult_charge_image_g.shape[1] - gap
])
if flag_observed is True:
num_left = ImageUtils.crop(
num_left,
[0, num_left.shape[0], num_left.shape[1] \
- OW.ULT_CHARGE_NUMBER_WIDTH_OBSERVED[self.frame.game.game_type] - 1,
OW.ULT_CHARGE_NUMBER_WIDTH_OBSERVED[self.frame.game.game_type]])
num_right = ImageUtils.crop(num_right, [
0, num_left.shape[0], 0, OW.
ULT_CHARGE_NUMBER_WIDTH_OBSERVED[self.frame.game.game_type]
])
else:
num_left = ImageUtils.crop(
num_left,
[0, num_left.shape[0], num_left.shape[1] \
- OW.ULT_CHARGE_NUMBER_WIDTH_OBSERVED[self.frame.game.game_type] - 1,
OW.ULT_CHARGE_NUMBER_WIDTH_OBSERVED[self.frame.game.game_type]])
num_right = ImageUtils.crop(num_right, [
0, num_left.shape[0], 0, OW.
ULT_CHARGE_NUMBER_WIDTH_OBSERVED[self.frame.game.game_type]
])
# if self.index == 5:
# cv2.imshow('t1', num_left)
# cv2.waitKey(0)
# cv2.imshow('t2', num_right)
# cv2.waitKey(0)
# Since when cropping img we also included the slope on left side,
# num_left could actually be empty
# Also we need another recognition method. Simple MSE wouldn't work due to error.
# ult_charge_image_g = ImageUtils.contrast_adjust_log(
# ult_charge_image, OW.ULT_ADJUST_LOG_INDEX)
# for i in (0,1):
# cv2.imshow('t', ult_charge_image)
# cv2.waitKey(0)
# try:
# ult_charge_image_binary = ImageUtils.binary_otsu(ult_charge_image_g)
# except ValueError:
# self.ult_charge = None
# return
# ult_charge_similarities = np.zeros(11)
# for j in range(1 - i, 11-i):
# # 1st number can't be 0, 2nd number can't be empty
# ult_charge_ref = self.frame.game.ult_charge_numbers_ref[j - i]
# ult_charge_similarities[j] = ImageUtils.similarity(ult_charge_ref, ult_charge_image_binary)
# ult_charges[i] = np.argmax(ult_charge_similarities)
# print ult_charges[i]
# if ult_charges[i] == 10:
# ult_charges[i] = 0
self.ult_charge = ult_charges[0] * 10 + ult_charges[1]
return
