def im_detect(sess, net, image_id, Test_RCNN, object_thres, human_thres, detection):
# save image information
This_image = []
im_orig, im_shape = get_blob(image_id)
blobs = {}
blobs['H_num'] = 1
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (Human_out[1] == 'Human'): # This is a valid human
blobs['H_boxes'] = np.array([0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]).reshape(1,5)
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (np.all(Object[2] == Human_out[2])): # This is a valid object
blobs['O_boxes'] = np.array([0, Object[2][0], Object[2][1], Object[2][2], Object[2][3]]).reshape(1,5)
blobs['sp'] = Get_next_sp(Human_out[2], Object[2]).reshape(1, 64, 64, 2)
mask = np.zeros(shape=(1, im_shape[0], im_shape[1], 1), dtype=np.float32)
obj_box = blobs['O_boxes'][0][1:].astype(np.int32)
# print(obj_box)
# print(obj_box, blobs['O_boxes'])
mask[:, obj_box[0]:obj_box[2], obj_box[1]:obj_box[3]] = 1
blobs['O_mask'] = mask
print(image_id, blobs); exit()
# prediction_HO = net.test_image_HO(sess, im_orig, blobs)
prediction_HO, pH, pO, pSp, pVerbs = net.obtain_all_preds(sess, im_orig, blobs)
# print("DEBUG:", type(prediction_HO), len(prediction_HO), prediction_HO[0].shape, prediction_HO[0][0].shape)
temp = []
temp.append(Human_out[2]) # Human box
temp.append(Object[2]) # Object box
temp.append(Object[4]) # Object class
temp.append(prediction_HO[0]) # Score
temp.append(Human_out[5]) # Human score
temp.append(Object[5]) # Object score
temp.append(pH[0]) # 6
temp.append(pO[0])
temp.append(pSp[0])
temp.append(pVerbs[0])
This_image.append(temp)
detection[image_id] = This_image
def im_detect(sess, net, image_id, Test_RCNN, object_thres, human_thres,
detection):
# save image information
This_image = []
im_orig, im_shape = get_blob(image_id)
blobs = {}
blobs['H_num'] = 1
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (
Human_out[1] == 'Human'): # This is a valid human
blobs['H_boxes'] = np.array([
0, Human_out[2][0], Human_out[2][1], Human_out[2][2],
Human_out[2][3]
]).reshape(1, 5)
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (np.all(
Object[2] == Human_out[2])): # This is a valid object
blobs['O_boxes'] = np.array([
0, Object[2][0], Object[2][1], Object[2][2],
Object[2][3]
]).reshape(1, 5)
blobs['sp'] = Get_next_sp(Human_out[2],
Object[2]).reshape(1, 64, 64, 2)
prediction_HO = net.test_image_HO(sess, im_orig, blobs)
temp = []
temp.append(Human_out[2]) # Human box
temp.append(Object[2]) # Object box
temp.append(Object[4]) # Object class
temp.append(prediction_HO[0][0]) # Score
temp.append(Human_out[5]) # Human score
temp.append(Object[5]) # Object score
This_image.append(temp)
detection[image_id] = This_image
def im_detect(sess, net, image_id, Test_RCNN, prior_mask, Action_dic_inv,
object_thres, human_thres, prior_flag, detection):
im_orig, im_shape = get_blob(image_id)
blobs = {}
blobs['H_num'] = 1
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (
Human_out[1] == 'Human'): # This is a valid human
# Predict actrion using human appearance only
blobs['H_boxes'] = np.array([
0, Human_out[2][0], Human_out[2][1], Human_out[2][2],
Human_out[2][3]
]).reshape(1, 5)
prediction_H = net.test_image_H(sess, im_orig, blobs)
# save image information
dic = {}
dic['image_id'] = image_id
dic['person_box'] = Human_out[2]
# Predict actrion using human and object appearance
Score_obj = np.empty((0, 4 + 29), dtype=np.float32)
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (np.all(
Object[2] == Human_out[2])): # This is a valid object
blobs['O_boxes'] = np.array([
0, Object[2][0], Object[2][1], Object[2][2],
Object[2][3]
]).reshape(1, 5)
blobs['sp'] = Get_next_sp(Human_out[2],
Object[2]).reshape(1, 64, 64, 2)
prediction_HO = net.test_image_HO(sess, im_orig, blobs)
if prior_flag == 1:
prediction_HO = apply_prior(Object, prediction_HO)
if prior_flag == 2:
prediction_HO = prediction_HO * prior_mask[:, Object[
4]].reshape(1, 29)
if prior_flag == 3:
prediction_HO = apply_prior(Object, prediction_HO)
prediction_HO = prediction_HO * prior_mask[:, Object[
4]].reshape(1, 29)
This_Score_obj = np.concatenate((Object[2].reshape(
1, 4), prediction_HO[0] * np.max(Object[5])),
axis=1)
Score_obj = np.concatenate((Score_obj, This_Score_obj),
axis=0)
# There is only a single human detected in this image. I just ignore it. Might be better to add Nan as object box.
if Score_obj.shape[0] == 0:
continue
# Find out the object box associated with highest action score
max_idx = np.argmax(Score_obj, 0)[4:]
# agent mAP
for i in range(29):
#'''
# walk, smile, run, stand
if (i == 3) or (i == 17) or (i == 22) or (i == 27):
agent_name = Action_dic_inv[i] + '_agent'
dic[agent_name] = np.max(
Human_out[5]) * prediction_H[0][0][i]
continue
# cut
if i == 2:
agent_name = 'cut_agent'
dic[agent_name] = np.max(Human_out[5]) * max(
Score_obj[max_idx[2]][4 + 2],
Score_obj[max_idx[4]][4 + 4])
continue
if i == 4:
continue
# eat
if i == 9:
agent_name = 'eat_agent'
dic[agent_name] = np.max(Human_out[5]) * max(
Score_obj[max_idx[9]][4 + 9],
Score_obj[max_idx[16]][4 + 16])
continue
if i == 16:
continue
# hit
if i == 19:
agent_name = 'hit_agent'
dic[agent_name] = np.max(Human_out[5]) * max(
Score_obj[max_idx[19]][4 + 19],
Score_obj[max_idx[20]][4 + 20])
continue
if i == 20:
continue
# These 2 classes need to save manually because there is '_' in action name
if i == 6:
agent_name = 'talk_on_phone_agent'
dic[agent_name] = np.max(
Human_out[5]) * Score_obj[max_idx[i]][4 + i]
continue
if i == 8:
agent_name = 'work_on_computer_agent'
dic[agent_name] = np.max(
Human_out[5]) * Score_obj[max_idx[i]][4 + i]
continue
# all the rest
agent_name = Action_dic_inv[i].split("_")[0] + '_agent'
dic[agent_name] = np.max(
Human_out[5]) * Score_obj[max_idx[i]][4 + i]
#'''
'''
if i == 6:
agent_name = 'talk_on_phone_agent'
dic[agent_name] = np.max(Human_out[5]) * prediction_H[0][0][i]
continue
if i == 8:
agent_name = 'work_on_computer_agent'
dic[agent_name] = np.max(Human_out[5]) * prediction_H[0][0][i]
continue
agent_name = Action_dic_inv[i].split("_")[0] + '_agent'
dic[agent_name] = np.max(Human_out[5]) * prediction_H[0][0][i]
'''
# role mAP
for i in range(29):
# walk, smile, run, stand. Won't contribute to role mAP
if (i == 3) or (i == 17) or (i == 22) or (i == 27):
dic[Action_dic_inv[i]] = np.append(
np.full(4, np.nan).reshape(1, 4),
np.max(Human_out[5]) * prediction_H[0][0][i])
continue
# Impossible to perform this action
if np.max(Human_out[5]) * Score_obj[max_idx[i]][4 + i] == 0:
dic[Action_dic_inv[i]] = np.append(
np.full(4, np.nan).reshape(1, 4),
np.max(Human_out[5]) * Score_obj[max_idx[i]][4 + i])
# Action with >0 score
else:
dic[Action_dic_inv[i]] = np.append(
Score_obj[max_idx[i]][:4],
np.max(Human_out[5]) * Score_obj[max_idx[i]][4 + i])
detection.append(dic)
def im_detect(sess, net, path, image_id, Test_RCNN, prior_mask, Action_dic_inv,
object_thres, human_thres, prior_flag, detection):
im_orig, im_shape = get_blob(path, image_id)
blobs = {}
blobs['H_num'] = 1
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (
Human_out[1] == 'Human'): # This is a valid human
# Predict actrion using human appearance only
blobs['H_boxes'] = np.array([
0, Human_out[2][0], Human_out[2][1], Human_out[2][2],
Human_out[2][3]
]).reshape(1, 5)
prediction_H = net.test_image_H(sess, im_orig, blobs)
# save image information
dic = {}
dic['image_id'] = image_id
dic['person_box'] = Human_out[2]
# Predict action using human and object appearance
Score_obj = np.empty((0, 5 + 29), dtype=np.float32)
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (np.all(
Object[2] == Human_out[2])): # This is a valid object
#if (np.max(Object[5]) > object_thres) and not (np.all(Object[2] == Human_out[2])) and (Object[2][3] - Object[2][1]) * (Object[2][2] - Object[2][0]) > 10000: # This is a valid object
blobs['O_boxes'] = np.array([
0, Object[2][0], Object[2][1], Object[2][2],
Object[2][3]
]).reshape(1, 5)
blobs['sp'] = Get_next_sp(Human_out[2],
Object[2]).reshape(1, 64, 64, 2)
prediction_HO = net.test_image_HO(sess, im_orig, blobs)
if prior_flag == 1:
prediction_HO = apply_prior(Object, prediction_HO)
if prior_flag == 2:
prediction_HO = prediction_HO * prior_mask[:, Object[
4]].reshape(1, 29)
if prior_flag == 3:
prediction_HO = apply_prior(Object, prediction_HO)
prediction_HO = prediction_HO * prior_mask[:, Object[
4]].reshape(1, 29)
This_Score_obj = np.concatenate(
(Object[2].reshape(1, 4), np.array(Object[4]).reshape(
1, 1), prediction_HO[0] * np.max(Object[5])),
axis=1)
Score_obj = np.concatenate((Score_obj, This_Score_obj),
axis=0)
# There is only a single human detected in this image. I just ignore it. Might be better to add Nan as object box.
if Score_obj.shape[0] == 0:
continue
# Find out the object box associated with highest action score
max_idx = np.argmax(Score_obj, 0)[5:]
# agent mAP
for i in range(29):
#'''
# walk, smile, run, stand
agent_name = Action_dic_inv[i] + '_agent'
dic[agent_name] = np.max(Human_out[5]) * prediction_H[0][0][i]
# role mAP
for i in range(29):
# walk, smile, run, stand. Won't contribute to role mAP
if (i == 3) or (i == 17) or (i == 22) or (i == 27):
dic[Action_dic_inv[i]] = np.append(
np.full(4, np.nan).reshape(1, 4),
np.max(Human_out[5]) * prediction_H[0][0][i])
continue
# Impossible to perform this action
if np.max(Human_out[5]) * Score_obj[max_idx[i]][5 + i] == 0:
dic[Action_dic_inv[i]] = np.append(
np.full(4, np.nan).reshape(1, 4),
np.max(Human_out[5]) * Score_obj[max_idx[i]][5 + i])
# Action with >0 score
else:
dic[Action_dic_inv[i]] = np.append(
Score_obj[max_idx[i]][:5],
np.max(Human_out[5]) * Score_obj[max_idx[i]][5 + i])
detection.append(dic)
def generator1():
np.random.seed(cfg.RNG_SEED)
i = 0
for line in glob.iglob(cfg.DATA_DIR + '/' + 'hico_20160224_det/images/'+dataset_name+'/*.jpg'):
i += 1
# if i > 30: # TODO remove
# break
image_id = int(line[-9:-4])
# save image information
im_orig, im_shape = get_blob(image_id)
mask_all = np.zeros(shape=(1, im_shape[0], im_shape[1], 1), dtype=np.float32)
blobs = {}
blobs['H_num'] = 0
blobs['H_boxes'] = []
blobs['O_boxes'] = []
blobs['sp'] = []
blobs['O_cls'] = []
blobs['H_score'] = []
blobs['O_score'] = []
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (Human_out[1] == 'Human'): # This is a valid human
# blobs['H_boxes'] = np.array(
# [0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]).reshape(1, 5)
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (
np.all(Object[2] == Human_out[2])): # This is a valid object
blobs['H_num'] += 1
blobs['H_boxes'].append(np.array([0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]))
obj_box = np.array(
[0, Object[2][0], Object[2][1], Object[2][2], Object[2][3]])
blobs['O_boxes'].append(obj_box)
if len(Human_out) <= 6:
blobs['sp'].append(Get_next_sp(Human_out[2], Object[2]))
else:
blobs['sp'].append(Get_next_sp_with_pose(Human_out[2], Object[2], Human_out[6]))
blobs['O_cls'].append(Object[4])
blobs['H_score'].append(Human_out[5])
blobs['O_score'].append(Object[5])
if blobs['H_num'] == 0:
print('\rDealing with zero-sample test Image ' + str(image_id), end='', flush=True)
list_human_included = []
list_object_included = []
Human_out_list = []
Object_list = []
test_pair_all = Test_RCNN[image_id]
length = len(test_pair_all)
flag_continue_searching = 1
while (len(list_human_included) < human_num_thres) or (
len(list_object_included) < object_num_thres):
h_max = [-1, -1.0]
o_max = [-1, -1.0]
flag_continue_searching = 0
for i in range(length):
if test_pair_all[i][1] == 'Human':
if (np.max(test_pair_all[i][5]) > h_max[1]) and not (i in list_human_included) and len(
list_human_included) < human_num_thres:
h_max = [i, np.max(test_pair_all[i][5])]
flag_continue_searching = 1
else:
if np.max(test_pair_all[i][5]) > o_max[1] and not (i in list_object_included) and len(
list_object_included) < object_num_thres:
o_max = [i, np.max(test_pair_all[i][5])]
flag_continue_searching = 1
if flag_continue_searching == 0:
break
list_human_included.append(h_max[0])
list_object_included.append(o_max[0])
Human_out_list.append(test_pair_all[h_max[0]])
Object_list.append(test_pair_all[o_max[0]])
for Human_out in Human_out_list:
for Object in Object_list:
blobs['H_num'] += 1
blobs['H_boxes'].append(
np.array([0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]))
obj_box = np.array(
[0, Object[2][0], Object[2][1], Object[2][2], Object[2][3]])
blobs['O_boxes'].append(obj_box)
if len(Human_out) <= 6:
blobs['sp'].append(Get_next_sp(Human_out[2], Object[2]))
else:
blobs['sp'].append(Get_next_sp_with_pose(Human_out[2], Object[2], Human_out[6]))
blobs['O_cls'].append(Object[4])
blobs['H_score'].append(Human_out[5])
blobs['O_score'].append(Object[5])
if blobs['H_num'] == 0:
# print('None ', image_id)
continue
# print(im_orig.shape, image_id, end=' ')
# for k, v in blobs.items():
# if not k == 'H_num':
# blobs[k] = np.asarray(v)
# print(k, blobs[k].shape, end=' ')
# print('\n')
im_mask = np.multiply(im_orig, mask_all)
im_orig = np.concatenate([im_orig, im_mask], axis=0)
start = 0
stride = 200
while start < blobs['H_num']:
b_temp = {}
for k ,v in blobs.items():
if not k == 'H_num':
b_temp[k] = blobs[k][start:start+stride]
b_temp['H_num'] = min(start + stride, blobs['H_num']) - start
start += stride
yield im_orig, b_temp, image_id
def generator1():
np.random.seed(cfg.RNG_SEED)
i = 0
for line in glob.iglob(cfg.DATA_DIR + '/' + 'hico_20160224_det/images/'+dataset_name+'/*.jpg'):
i += 1
# if i > 30: # TODO remove
# break
image_id = int(line[-9:-4])
# save image information
im_orig, im_shape = get_blob(image_id)
mask_all = np.zeros(shape=(1, im_shape[0], im_shape[1], 1), dtype=np.float32)
blobs = {}
blobs['H_num'] = 0
blobs['H_boxes'] = []
blobs['O_boxes'] = []
blobs['sp'] = []
blobs['O_cls'] = []
blobs['H_score'] = []
blobs['O_score'] = []
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (Human_out[1] == 'Human'): # This is a valid human
# blobs['H_boxes'] = np.array(
# [0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]).reshape(1, 5)
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (
np.all(Object[2] == Human_out[2])): # This is a valid object
blobs['H_num'] += 1
blobs['H_boxes'].append(np.array([0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]))
obj_box = np.array(
[0, Object[2][0], Object[2][1], Object[2][2], Object[2][3]])
blobs['O_boxes'].append(obj_box)
blobs['sp'].append(Get_next_sp(Human_out[2], Object[2]))
mask = np.zeros(shape=(im_shape[0], im_shape[1], 1), dtype=np.float32)
obj_box = obj_box[1:].astype(np.int32)
# print(obj_box)
# print(obj_box, blobs['O_boxes'])
mask[obj_box[0]:obj_box[2], obj_box[1]:obj_box[3]] = 1
mask_all[:, obj_box[0]:obj_box[2], obj_box[1]:obj_box[3]] = 1
# from skimage import transform
# mask = transform.resize(mask, [im_shape[0] // 16, im_shape[1] // 16, 1], order=0,
# preserve_range=True)
blobs['O_cls'].append(Object[4])
blobs['H_score'].append(Human_out[5])
blobs['O_score'].append(Object[5])
if blobs['H_num'] == 0:
# print('None ', image_id)
continue
# print(im_orig.shape, image_id, end=' ')
# for k, v in blobs.items():
# if not k == 'H_num':
# blobs[k] = np.asarray(v)
# print(k, blobs[k].shape, end=' ')
# print('\n')
im_mask = np.multiply(im_orig, mask_all)
im_orig = np.concatenate([im_orig, im_mask], axis=0)
start = 0
stride = 200
while start < blobs['H_num']: # save GPU memory
b_temp = {}
for k ,v in blobs.items():
if not k == 'H_num':
b_temp[k] = blobs[k][start:start+stride]
b_temp['H_num'] = min(start + stride, blobs['H_num']) - start
start += stride
# print('b_temp' , im_orig.shape, image_id, end=' ')
# for k, v in blobs.items():
# if not k == 'H_num':
# blobs[k] = np.asarray(v)
# print(k, blobs[k].shape, end=' ')
# print('\n')
yield im_orig, b_temp, image_id
def generator1():
np.random.seed(cfg.RNG_SEED)
i = 0
# for line in glob.iglob(cfg.DATA_DIR + '/' + 'hico_20160224_det/images/'+dataset_name+'/*.jpg'):
for image_id in Test_RCNN:
i += 1
# if i > 30: # TODO remove
# break
im_orig, im_shape = get_blob(image_id)
blobs = {}
blobs['H_num'] = 0
blobs['H_boxes'] = []
blobs['O_boxes'] = []
blobs['sp'] = []
blobs['O_cls'] = []
blobs['H_score'] = []
blobs['O_score'] = []
for Human_out in Test_RCNN[image_id]:
if (np.max(Human_out[5]) > human_thres) and (Human_out[1] == 'Human'): # This is a valid human
for Object in Test_RCNN[image_id]:
if (np.max(Object[5]) > object_thres) and not (
np.all(Object[2] == Human_out[2])): # This is a valid object
blobs['H_num'] += 1
blobs['H_boxes'].append(np.array([0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]))
obj_box = np.array(
[0, Object[2][0], Object[2][1], Object[2][2], Object[2][3]])
blobs['O_boxes'].append(obj_box)
blobs['sp'].append(Get_next_sp(Human_out[2], Object[2], pattern_type))
assert Object[4] > 0, (Object[4])
blobs['O_cls'].append(Object[4])
blobs['H_score'].append(Human_out[5])
blobs['O_score'].append(Object[5])
if blobs['H_num'] == 0 and has_human_threhold:
# copy from previous work (TIN). This is useless for better object detector.
# This also illustrates the importance of fine-tuned object detector!
print('\rDealing with zero-sample test Image ' + str(image_id), end='', flush=True)
list_human_included = []
list_object_included = []
Human_out_list = []
Object_list = []
test_pair_all = Test_RCNN[image_id]
length = len(test_pair_all)
while (len(list_human_included) < human_num_thres) or (
len(list_object_included) < object_num_thres):
h_max = [-1, -1.0]
o_max = [-1, -1.0]
flag_continue_searching = 0
for i in range(length):
if test_pair_all[i][1] == 'Human':
if (np.max(test_pair_all[i][5]) > h_max[1]) and not (i in list_human_included) and len(
list_human_included) < human_num_thres:
h_max = [i, np.max(test_pair_all[i][5])]
flag_continue_searching = 1
else:
if np.max(test_pair_all[i][5]) > o_max[1] and not (i in list_object_included) and len(
list_object_included) < object_num_thres:
o_max = [i, np.max(test_pair_all[i][5])]
flag_continue_searching = 1
if flag_continue_searching == 0:
break
list_human_included.append(h_max[0])
list_object_included.append(o_max[0])
Human_out_list.append(test_pair_all[h_max[0]])
Object_list.append(test_pair_all[o_max[0]])
for Human_out in Human_out_list:
for Object in Object_list:
blobs['H_num'] += 1
blobs['H_boxes'].append(
np.array([0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]))
obj_box = np.array(
[0, Object[2][0], Object[2][1], Object[2][2], Object[2][3]])
blobs['O_boxes'].append(obj_box)
blobs['sp'].append(Get_next_sp(Human_out[2], Object[2], pattern_type))
blobs['O_cls'].append(Object[4])
blobs['H_score'].append(Human_out[5])
blobs['O_score'].append(Object[5])
if blobs['H_num'] == 0:
# print('None ', image_id)
continue
start = 0
# stride = 200
while start < blobs['H_num']:
b_temp = {}
for k ,v in blobs.items():
if not k == 'H_num':
b_temp[k] = blobs[k][start:start+stride]
b_temp['H_num'] = min(start + stride, blobs['H_num']) - start
start += stride
yield im_orig, b_temp, image_id