def worker(path):
print('worker start ' + path)
batch_rois = []
batch_featuremap_inds = []
batch_categories = []
batch_bboxes = []
fc_index = 0
dataset = {}
#'/ImageSets/DET/train_*'
for fname in glob.glob(ILSVRC_dataset_path + path):
print(fname)
with open(fname, 'r') as f:
basename = os.path.basename(fname)
category = int(basename.split('_')[1].split('.')[0])
content = []
for line in f:
if 'extra' not in line:
content.append(line)
dataset[category] = content
print(len(dataset))
from random import randint
while 1:
try:
category = randint(1, 200)
content = dataset[category]
n = randint(0, len(content))
line = content[n]
_, gt_boxes, h_w = parse_label(anno_path + line.split()[0] +
'.xml')
if len(gt_boxes) == 0:
continue
rois, bboxes, categories = produce_batch(
img_path + line.split()[0] + '.JPEG', gt_boxes, h_w, category)
except Exception:
# print('parse label or produce batch failed: for: '+line.split()[0])
# traceback.print_exc()
continue
if len(rois) <= 0:
continue
for i in range(len(rois)):
batch_rois.append(rois[i])
batch_featuremap_inds.append(fc_index)
batch_categories.append(categories[i])
batch_bboxes.append(bboxes[i])
a = feature_map
b = np.asarray(batch_rois)
c = np.asarray(batch_featuremap_inds)
d = np.asarray(batch_categories)
e = np.asarray(batch_bboxes)
f = np.zeros((len(rois), a.shape[1], a.shape[2], a.shape[3]))
f[0] = feature_map[0]
yield [f, b, c], [d, e]
batch_rois = []
batch_featuremap_inds = []
batch_categories = []
batch_bboxes = []
fc_index = 0
def worker(path, q):
print('worker start ' + path)
batch_rois = []
batch_featuremap_inds = []
batch_categories = []
batch_bboxes = []
fc_index = 0
#'/ImageSets/DET/train_*'
for fname in glob.glob(ILSVRC_dataset_path + path):
print(fname)
with open(fname, 'r') as f:
basename = os.path.basename(fname)
category = int(basename.split('_')[1].split('.')[0])
for line in f:
if 'extra' not in line:
feature_map_file = feature_map_path + line.split()[0]
if not os.path.exists(feature_map_file):
continue
try:
feature_map = np.load(feature_map_file)['fc']
_, gt_boxes, h_w = parse_label(anno_path +
line.split()[0] +
'.xml')
if len(gt_boxes) == 0:
continue
rois, bboxes, categories = produce_batch(
feature_map, gt_boxes, h_w, category)
except Exception:
print('parse label or produce batch failed: for: ' +
line.split()[0])
traceback.print_exc()
continue
if len(rois) <= 0:
continue
for i in range(len(rois)):
batch_rois.append(rois[i])
batch_featuremap_inds.append(fc_index)
batch_categories.append(categories[i])
batch_bboxes.append(bboxes[i])
a = feature_map
b = np.asarray(batch_rois)
c = np.asarray(batch_featuremap_inds)
d = np.asarray(batch_categories)
e = np.asarray(batch_bboxes)
f = np.zeros(
(len(rois), a.shape[1], a.shape[2], a.shape[3]))
f[0] = feature_map[0]
q.put([a, b, c, d, e])
# yield [f,b,c], [d,e]
batch_rois = []
batch_featuremap_inds = []
batch_categories = []
batch_bboxes = []
fc_index = 0
def input_generator():
batch_tiles = []
batch_labels = []
batch_bboxes = []
count = 0
while 1:
for fname in glob.glob(ILSVRC_dataset_path + '/ImageSets/DET/train_*'):
with open(fname, 'r') as f:
for line in f:
if 'extra' not in line:
try:
category, gt_boxes, scale = parse_label(
anno_path + line.split()[0] + '.xml')
if len(gt_boxes) == 0:
continue
tiles, labels, bboxes = produce_batch(
img_path + line.split()[0] + '.JPEG', gt_boxes,
scale)
except Exception:
print(
'parse label or produce batch failed: for: ' +
line.split()[0])
traceback.print_exc()
continue
for i in range(len(tiles)):
batch_tiles.append(tiles[i])
batch_labels.append(labels[i])
batch_bboxes.append(bboxes[i])
if (len(batch_tiles) == BATCH_SIZE):
a = np.asarray(batch_tiles)
b = np.asarray(batch_labels)
c = np.asarray(batch_bboxes)
if not a.any() or not b.any() or not c.any():
print("empty array found.")
yield a, [b, c]
batch_tiles = []
batch_labels = []
batch_bboxes = []
def run_in_movie(data_dir, subset, algorithm, temporal_link, gpu_id):
affinity_dir = osp.join(data_dir, 'affinity', subset, 'in')
list_file = osp.join(data_dir, 'meta', subset + '.json')
mid_list, meta_info = read_meta(list_file)
average_mAP = 0
search_count = 0
average_top1 = 0
average_top3 = 0
average_top5 = 0
for i, mid in enumerate(mid_list):
# read data
tnum = meta_info[mid]['num_tracklet']
pids = meta_info[mid]['pids']
gt_list, gt_dict = parse_label(meta_info, mid)
# read affinity matrix
if temporal_link:
link_type = 'max'
else:
link_type = 'mean'
ct_affmat = read_affmat_of_one_movie(affinity_dir,
mid,
region='face',
data_type='ct',
link_type=link_type)
tt_affmat = read_affmat_of_one_movie(affinity_dir,
mid,
region='body',
data_type='tt',
link_type=link_type)
# run algorithm
if algorithm == 'ppcc':
result = run_ccpp(ct_affmat, tt_affmat, gpu_id)
elif algorithm == 'lp':
result = run_lp(ct_affmat, tt_affmat, gpu_id)
else:
raise ValueError('No such algrothm: {}'.format(algorithm))
# parse results and get performance
ret_dict = affmat2retdict(result, pids)
ret_list = affmat2retlist(result, pids)
mAP = get_mAP(gt_dict, ret_dict)
topk = get_topk(gt_list, ret_list)
average_mAP += mAP * len(pids)
search_count += len(pids)
max_k = len(topk)
if max_k < 3:
top3 = 1
else:
top3 = topk[2]
if max_k < 5:
top5 = 1
else:
top5 = topk[4]
average_top1 += topk[0]
average_top3 += top3
average_top5 += top5
# get average performance
average_mAP = average_mAP / search_count
average_top1 = average_top1 / len(mid_list)
average_top3 = average_top3 / len(mid_list)
average_top5 = average_top5 / len(mid_list)
print(
'Average mAP: {:.4f}\tAverage top1: {:.4f}\tAverage top3: {:.4f}\tAverage top5: {:.4f}'
.format(average_mAP, average_top1, average_top3, average_top5))
def run_in_movie(data_dir, subset, data_type, face_ratio):
affinity_dir = osp.join(data_dir, 'affinity', subset, 'in')
list_file = osp.join(data_dir, 'meta', subset + '.json')
mid_list, meta_info = read_meta(list_file)
average_mAP = 0
search_count = 0
average_top1 = 0
average_top3 = 0
average_top5 = 0
for i, mid in enumerate(mid_list):
# read data
tnum = meta_info[mid]['num_tracklet']
pids = meta_info[mid]['pids']
gt_list, gt_dict = parse_label(meta_info, mid)
# read affinity matrix
if data_type == 'face':
affmat = read_affmat_of_one_movie(affinity_dir,
mid,
region='face',
data_type='ct')
elif data_type == 'body':
affmat = read_affmat_of_one_movie(affinity_dir,
mid,
region='body',
data_type='ct')
else:
face_affmat = read_affmat_of_one_movie(affinity_dir,
mid,
region='face',
data_type='ct')
body_affmat = read_affmat_of_one_movie(affinity_dir,
mid,
region='body',
data_type='ct')
if data_type == 'ave_fusion':
affmat = face_ratio * face_affmat + (1 -
face_ratio) * body_affmat
else:
affmat = np.maximum(face_affmat, body_affmat)
# parse results and get performance
ret_dict = affmat2retdict(affmat, pids)
ret_list = affmat2retlist(affmat, pids)
mAP = get_mAP(gt_dict, ret_dict)
topk = get_topk(gt_list, ret_list)
average_mAP += mAP * len(pids)
search_count += len(pids)
max_k = len(topk)
if max_k < 3:
top3 = 1
else:
top3 = topk[2]
if max_k < 5:
top5 = 1
else:
top5 = topk[4]
average_top1 += topk[0]
average_top3 += top3
average_top5 += top5
# get average performance
average_mAP = average_mAP / search_count
average_top1 = average_top1 / len(mid_list)
average_top3 = average_top3 / len(mid_list)
average_top5 = average_top5 / len(mid_list)
print(
'Average mAP: {:.4f}\tAverage top1: {:.4f}\tAverage top3: {:.4f}\tAverage top5: {:.4f}'
.format(average_mAP, average_top1, average_top3, average_top5))
train_mr_img_path = "data/example-data/tubulin/train/mr/" using_batch_norm = False train_test_data_path = None train_valid_lr_path = None # valid on_the_fly valid_lr_img_path = "data/example-data/tubulin/inference/lr/" valid_block_size = [64, 64, 64, 1] # [depth, height, width, channels] valid_block_overlap = 0.2 config = edict() config.TRAIN = edict() config.VALID = edict() params = parse_label(label) archi2 = params['archi2'] #['rdn', 'unet', 'dbpn'] archi1 = params['archi1'] # [None, 'dbpn' 'denoise'] # None if 1stage loss = params['loss'] #['mse', 'mae'] factor = params['factor'] normalization = params['norm'] config.archi1 = archi1 config.archi2 = archi2 config.loss = loss config.TRAIN.ckpt_saving_interval = 10 config.TRAIN.batch_size = 1 config.TRAIN.beta1 = 0.9 config.TRAIN.n_epoch = 500 config.TRAIN.decay_every = 50