def experiment_c(args, max_steps=30000):
print(
'[*] starting experiment C: keypoint estimation test loss/inference speed: EfficientNet vs VGG16'
)
config = model.get_config(args.project_id)
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'C'
config['kp_train_loss'] = 'focal'
config['kp_test_losses'] = ['focal'] #['cce','focal']
config['kp_max_steps'] = max_steps
#config['kp_max_steps'] = 15000
config['early_stopping'] = False
config['kp_lr'] = 1e-4
config['kp_num_hourglass'] = 1
config['batch_size'] = 8
#for backbone in ['hourglass2']:
for backbone in [
'vgg16', 'efficientnetLarge', 'psp', 'hourglass2', 'hourglass4',
'hourglass8'
]:
print('[*] starting sub experiment backbone %s' % backbone)
config['kp_backbone'] = backbone
print(config, '\n')
checkpoint_path = roi_segm.train(config, log_images=False)
clear_session()
def experiment_e(args, train_video_ids=None):
print(
'[*] starting experiment E: object detection test loss under different number of training samples'
)
config = model.get_config(args.project_id)
if train_video_ids is not None:
config['train_video_ids'] = train_video_ids
else:
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'E'
config['early_stopping'] = False
config['object_finetune_warmup'] = 1000
config['maxsteps_objectdetection'] = 10000
config['lr_objectdetection'] = 0.005
config['object_augm_gaussian'] = bool(0)
config['object_augm_image'] = bool(0)
config['object_augm_mixup'] = bool(0)
for data_ratio in [0.01, 0.1, 0.5, 1.0][::-1]: #,
now = str(datetime.now()).replace(' ', '_').replace(':',
'-').split('.')[0]
print('[*] starting sub experiment with %i/100 of data used' %
int(100. * data_ratio))
config['data_ratio'] = data_ratio
checkpoint_dir = os.path.expanduser(
'~/checkpoints/experiments/%s/E/%i-%s' %
(config['project_name'], int(100. * data_ratio), now))
#config['maxsteps_objectdetection'] = {0.01: 12000, 0.1: 15000, 0.5: 20000, 1.0: 25000}[data_ratio]
finetune(config, checkpoint_dir)
clear_session()
print(10 * '\n')
def experiment_a(args, max_steps=30000):
print(
'[*] starting experiment A: keypoint estimation test loss under different number of training samples (100%, 50%, 10%, 1%)'
)
config = model.get_config(args.project_id)
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'A'
config['kp_train_loss'] = 'focal'
config['kp_test_losses'] = ['focal'] #['cce','focal']
config['kp_max_steps'] = max_steps
config['early_stopping'] = False
config['kp_lr'] = 1e-4
config['batch_size'] = 8
#config['kp_lr'] = 2e-5
#for data_ratio in [0.01,0.1,0.5,1.0][::-1]:
for data_ratio in [0.01, 0.1, 0.5, 1.0]:
print('[*] starting sub experiment with %i/100 of data used' %
int(100. * data_ratio))
config['data_ratio'] = data_ratio
print(config, '\n')
checkpoint_path = roi_segm.train(config, log_images=False)
clear_session()
print(10 * '\n')
def get_next_bbox_frame(project_id, video_id):
project_id = int(project_id)
config = model.get_config(project_id)
#video_id = db.get_random_project_video(project_id)
video_id = int(video_id)
config['video_id'] = video_id
# load labeled frame idxs
labeled_frames_keypoints = db.get_labeled_frames(video_id)
labeled_frame_idxs = db.get_labeled_bbox_frames(video_id)
num_db_frames = len(labeled_frame_idxs)
# load frame files from disk
frames_dir = os.path.join(video.get_frames_dir(video.get_project_dir(video.base_dir_default, project_id), video_id),'train')
frames = sorted(glob(os.path.join(frames_dir, '*.png')))
frames_keypoints = [os.path.join(frames_dir, '%s.png' % ff) for ff in labeled_frames_keypoints]
shuffle(frames_keypoints)
unlabeled_frame_found = False
if len(frames_keypoints) > 0:
# first try to label bounding boxes where keypoints are labeled but bboxes not
tries = 0
while not unlabeled_frame_found and tries < 50:
ridx = int(np.random.uniform(len(frames_keypoints)))
frame_idx = frames_keypoints[ridx]
frame_idx = '.'.join(frame_idx.split('/')[-1].split('.')[:-1])
unlabeled_frame_found = not (frame_idx in labeled_frame_idxs)
tries += 1
# then take random unlabeled frame
tries, nearest_labeled_frame_diff = 0, 0
while not unlabeled_frame_found and tries < 50:
ridx = int(np.random.uniform(len(frames)))
frame_idx = frames[ridx]
frame_idx = '.'.join(frame_idx.split('/')[-1].split('.')[:-1])
if len(labeled_frame_idxs) == 0:
unlabeled_frame_found = True
else:
nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx)))
if nearest_labeled_frame_diff > 20:
unlabeled_frame_found = not (frame_idx in labeled_frame_idxs)
tries += 1
if len(labeled_frame_idxs) > 0:
nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx)))
else:
nearest_labeled_frame_diff = -1
if unlabeled_frame_found:
print('[*] serving bounding box label job for frame %s %s. nearest frame already labeled %i frames away'%(frame_idx,get_frame_time(frame_idx),nearest_labeled_frame_diff))
return render_template('labeling.html',project_id = int(project_id), video_id = int(video_id), frame_idx = frame_idx, num_db_frames = num_db_frames, keypoint_names = db.list_sep.join(config['keypoint_names']), sep = db.list_sep, labeling_mode = 'bbox')
else:
print('[*] redirecting to keypoint labeling')
return render_labeling(project_id, video_id)
def experiment_g(args, train_video_ids=None):
print(
'[*] starting experiment G: object detection pretrained vs random init network, w/wo augmentation'
)
config = model.get_config(args.project_id)
config['data_dir'] = '/home/alex/data/multitracker'
if train_video_ids is not None:
config['train_video_ids'] = train_video_ids
else:
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'G'
config['early_stopping'] = False
config['object_finetune_warmup'] = 1000
config['maxsteps_objectdetection'] = 20000
config['lr_objectdetection'] = 0.005
config['object_augm_gaussian'] = bool(0)
config['object_augm_image'] = bool(0)
config['object_augm_mixup'] = bool(0)
#for should_init_pretrained in [True]:
# for augm in [True]:
for should_init_pretrained in [False, True]:
for augm in [False, True]:
now = str(datetime.now()).replace(' ',
'_').replace(':',
'-').split('.')[0]
if not augm:
config['object_augm_flip'] = bool(0)
config['object_augm_rot90'] = bool(0)
else:
config['object_augm_flip'] = bool(1)
config['object_augm_rot90'] = bool(1)
config['object_pretrained'] = should_init_pretrained
str_augm = 'augm' if augm else 'noaugm'
str_pretrained = 'pretrained' if should_init_pretrained else 'scratch'
print('[*] starting sub experiment', str_augm, str_pretrained)
checkpoint_dir = os.path.expanduser(
'~/checkpoints/experiments/%s/G/%s-%s-%s' %
(config['project_name'], str_augm, str_pretrained, now))
#config['maxsteps_objectdetection'] = {0.01: 12000, 0.1: 15000, 0.5: 20000, 1.0: 25000}[data_ratio]
print(finetune)
finetune(config, checkpoint_dir)
clear_session()
print(10 * '\n')
def get_annotation(project_id, video_id, frame_id):
project_id = int(project_id)
video_id = int(video_id)
frame_id = int(frame_id)
config = model.get_config(int(project_id))
#if args.video_id is None:
# raise Exception("\n please restart the app with argument --video_id and --upper_bound")
global loaded_video_id
#if loaded_video_id is None or not loaded_video_id == video_id:
# get_next_annotation_frame(project_id, video_id, int(np.random.uniform(1e6)))
# load labeled frame idxs
labeled_frame_idxs = db.get_labeled_frames(video_id)
labeled_frame_idxs_boundingboxes = db.get_labeled_bbox_frames(video_id)
num_db_frames = len(labeled_frame_idxs)
# load annotation from database
animals = db.get_frame_annotations(video_id, frame_id )
if animals is None:
animals = []
# if no annotation around, init upper_bound many animals with keypoints
for i in range(int(args.upper_bound)):
x1,y1,x2,y2 = [200*i,200*i,200*(i+1),200*(i+1)]
keypoints = [ ]
for j,kp_name in enumerate(config['keypoint_names']):
keypoints.append({
'name': kp_name,
'x': x1 + 20*j,
'y': y1 + 23*j,
'db_id': None,
'is_visible': True
})
animals.append({'id': str(i+1), 'box': [x1,y1,x2,y2], 'keypoints': keypoints, 'db_id': None, 'is_visible': True})
else:
# parse db animals
#for i, animal in enumerate(animals):
# print(i, animal)
pass
animals_json = json.dumps(animals)
return render_template('annotate.html', animals = animals, animals_json = animals_json, project_id = int(project_id), video_id = int(video_id), frame_idx = frame_id, keypoint_names = db.list_sep.join(config['keypoint_names']), sep = db.list_sep, num_db_frames = num_db_frames, labeling_mode = 'annotate')
def experiment_b(args, max_steps=12000):
print(
'[*] starting experiment B: keypoint estimation test loss: pretrained vs random init network'
)
config = model.get_config(args.project_id)
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'B'
config['kp_backbone'] = 'hourglass2'
config['kp_train_loss'] = 'focal'
config['kp_test_losses'] = ['focal'] #['cce','focal']
config['kp_max_steps'] = max_steps
#config['kp_max_steps'] = 15000
config['early_stopping'] = False
config['batch_size'] = 8
config['kp_lr'] = 1e-4
config_b = deepcopy(config)
run_configs = [
[False, False],
[False, True],
[True, False],
[True, True],
]
for should_init_pretrained, augm in run_configs:
print('[*] starting sub experiment %s weight initialization' %
['without', 'with'][int(should_init_pretrained)])
if not augm:
config_b['kp_mixup'] = False
config_b['kp_cutmix'] = False
config_b['kp_hflips'] = False
config_b['kp_vflips'] = False
config_b['kp_rotation_augmentation'] = False
config_b['kp_rot90s'] = False
config_b['kp_im_noise'] = False
config_b['kp_im_transf'] = False
config_b['should_init_pretrained'] = should_init_pretrained
print(config_b, '\n')
checkpoint_path = roi_segm.train(config_b, log_images=False)
clear_session()
def experiment_f(args, train_video_ids=None):
print(
'[*] starting experiment F: object detection test loss SSD vs FasterRCNN'
)
config = model.get_config(args.project_id)
if train_video_ids is not None:
config['train_video_ids'] = train_video_ids
else:
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'F'
config['maxsteps_objectdetection'] = 15000
config['early_stopping'] = False
for od_backbone in ['fasterrcnn', 'ssd']:
print('[*] starting subexperiment for backbone type', od_backbone)
config['object_detection_backbone'] = od_backbone
config['object_detection_backbonepath'] = {
'ssd': 'ssd_resnet50_v1_fpn_640x640_coco17_tpu-8',
'fasterrcnn':
'faster_rcnn_inception_resnet_v2_640x640_coco17_tpu-8'
}[config['object_detection_backbone']]
config['object_detection_batch_size'] = {
'ssd': 16,
'fasterrcnn': 4
}[config['object_detection_backbone']]
print(config, '\n')
now = str(datetime.now()).replace(' ', '_').replace(':',
'-').split('.')[0]
checkpoint_directory = os.path.expanduser(
"~/checkpoints/experiments/%s/F/%s-%s" %
(config['project_name'], od_backbone, now))
finetune(config, checkpoint_directory)
clear_session()
print(10 * '\n')
def experiment_d(args, max_steps=25000, train_video_ids=None):
print(
'[*] starting experiment D: keypoint estimation categorical cross entropy / focal loss test trained with categorical cross entropy and focal loss'
)
config = model.get_config(args.project_id)
if train_video_ids is not None:
config['train_video_ids'] = train_video_ids
else:
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['kp_test_losses'] = ['focal', 'cce', 'l2']
config['experiment'] = 'D'
'''config['kp_mixup'] = False
config['kp_cutmix'] = False
config['kp_rot90s'] = False
config['kp_hflips'] = False
config['kp_vflips'] = False
config['kp_blurpool'] = False
config['kp_rotation_augmentation'] = bool(0)
config['kp_backbone'] = 'hourglass2'''
#config['kp_max_steps'] = 15000
config['kp_max_steps'] = max_steps
config['early_stopping'] = False
config['batch_size'] = 8
config['kp_lr'] = 1e-4
for loss_name in ['focal', 'cce', 'l2']:
print('[*] starting sub experiment loss function %s' % loss_name)
config['kp_train_loss'] = loss_name
print(config, '\n')
checkpoint_path = roi_segm.train(config, log_images=False)
clear_session()
def augment_dataset(project_id):
config = model.get_config(project_id)
config['input_image_shape'] = cv.imread(glob(os.path.join(config['data_dir'],'train/*.png'))[0]).shape[:2]
h = config['input_image_shape'][0]
w = config['input_image_shape'][1] // (2+ len(config['keypoint_names'])//3)
print(config)
print(h,w, 4 * w)
file_list, dataset_train = model.load_raw_dataset(config,'train')
inputs = tf.keras.layers.Input(shape=[config['img_height'], config['img_width'], 3])
inputss = preprocess_input(inputs)
net = tf.keras.applications.ResNet152V2(input_tensor=inputss,
include_top=False,
weights='imagenet',
pooling='avg')
net.summary()
if 0:
for layer in net.layers:
try:
print(layer.name,layer.outputs[0].shape)
except:
pass
layer_name = 'conv1_conv'
feature_activation = net.get_layer(layer_name)
feature_extractor = tf.keras.models.Model(name="ImageNet Encoder",inputs=net.input,outputs=[feature_activation.output])
output_dir = '/tmp/feature_augment'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
files = [f for f in sorted(glob(os.path.join(config['data_dir'],'train/*.png'))) if not 'augment-' in f]
for i,f in enumerate(files):
im = cv.imread(f)
rgb = im[:,:w,:]
rgb = rgb.reshape((1,) + rgb.shape)
rgb = preprocess_input(rgb)
features = feature_extractor(rgb,training=False).numpy()[0,:,:,:]
features = cv.resize(features,None,None,fx=2.,fy=2.)
print('[*] wrote %i/%i:'%(i,len(files)),im.shape,features.shape,features.min(),features.max())
for j in range(2):
fo = os.path.join(config['data_dir'],'train/augment-%i-%i.png' % (i,j) )
#print(fo)
a = np.zeros_like(features[:,:,0])
b = np.zeros_like(features[:,:,0])
c = np.zeros_like(features[:,:,0])
for k in range(features.shape[-1] // 3):
a += features[:,:,int(np.random.uniform(features.shape[-1]))]
b += features[:,:,int(np.random.uniform(features.shape[-1]))]
c += features[:,:,int(np.random.uniform(features.shape[-1]))]
abc = cv.merge((a,b,c))
thresh = 1.
abc[thresh > np.abs(abc)] = thresh
abc8 = 255. * (abc-abc.min())/(1e-5+abc.max()-abc.min())
abc8 = np.uint8(abc8)
abc8 = np.hstack((abc8,im[:,w:,:]))
cv.imwrite(fo,abc8)
'''for x,y in dataset_train:
def main(args):
config = model.get_config(args.project_id)
config['video_id'] = int(args.video_id)
print(config,'\n')
checkpoint_path = train(config)
def check_labeling_handler(project_id, video_id):
"""
serve a little page to see the drawn labeling and two buttons for good and bad. bads should be deleted and added as a labeling job
"""
project_id, video_id = int(project_id), int(video_id)
config = model.get_config(project_id=project_id)
if video_id not in idx_checked:
idx_checked[video_id] = 0
# first get all frames
q = "select frame_idx from keypoint_positions where video_id=%i order by id;" % video_id
db.execute(q)
frame_idxs = [x[0] for x in db.cur.fetchall()]
frame_idxs = list(set(frame_idxs))
frame_idx = frame_idxs[idx_checked[video_id]]
q = "select keypoint_name, individual_id, keypoint_x, keypoint_y from keypoint_positions where video_id=%i and frame_idx='%s' order by individual_id, keypoint_name desc;" % (video_id,frame_idx)
db.execute(q)
keypoints = [x for x in db.cur.fetchall()]
filepath = os.path.join(dbconnection.base_data_dir, "projects/%i/%i/frames/train/%s.png" % (int(project_id), int(video_id), frame_idx))
im = cv.imread(filepath)
hm = heatmap_drawing.generate_hm(im.shape[0], im.shape[1] , [ [int(kp[2]),int(kp[3]),kp[0]] for kp in keypoints ], config['keypoint_names'])
colors = util.get_colors()
vis = np.float32(im)
hm_color = np.zeros_like(vis)
for i in range(hm.shape[2]):
hm8 = hm[:,:,i]
hm8[hm8<0.5]=0
c = np.array(colors[i]).reshape((1,1,3))
c = np.tile(c,[hm.shape[0],hm.shape[1],1])
hm8 = np.expand_dims(hm8,axis=2)
hmc = c * np.tile(hm8,[1,1,3])
hm_color += hmc
vis = np.uint8(vis/2. + hm_color/2.)
vis_path = '/tmp/check_%i-%s.png' % (video_id, str(frame_idx))
cv.imwrite(vis_path, vis)
#print('[*] wrote',vis_path,vis.shape,vis.dtype,vis.min(),vis.max())
idx_checked[video_id] += 1
print('idx_checked',idx_checked[video_id])
dom = """
<html>
<head>
<link rel="stylesheet" type="text/css" href="https://semantic-ui.com/dist/semantic.min.css">
<script
src="https://code.jquery.com/jquery-3.1.1.min.js"
integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8="
crossorigin="anonymous"></script>
<script src="https://semantic-ui.com/dist/semantic.min.js"></script>
<script src="/static/js/network.js"></script>
<style>
* {
margin: 0;
padding: 0;
}
.imgbox {
height: 100%
}
</style>
</head>
<body>
<div class="imgbox">
<img height="80%" class="center-fit" src="/get_labeled_frame/[project_id]/[video_id]/[frame_idx]">
<script type="text/javascript">
function onclick_ok(){ window.location.reload() }
function onclick_bad(){
get("/delete_labeling/[project_id]/[video_id]/[frame_idx]");
window.location.reload();
}
</script>
<button id="bu_ok" class="ui positive button" onclick="onclick_ok();">OK</button>
<button id="bu_bad" class="ui negative button" onclick='onclick_bad();'>BAD</button>
</div>
</body>
</html>
"""
for _ in range(3):
dom = dom.replace('[project_id]',str(project_id)).replace('[video_id]',str(video_id)).replace('[frame_idx]',str(frame_idx))
return dom
def render_labeling(project_id,video_id):
config = model.get_config(int(project_id))
video_id = int(video_id)
config['video_id'] = video_id
# load labeled frame idxs
labeled_frame_idxs = db.get_labeled_frames(video_id)
labeled_frame_idxs_boundingboxes = db.get_labeled_bbox_frames(video_id)
num_db_frames = len(labeled_frame_idxs)
# load frame files from disk
frames = []
while len(frames) == 0:
#video_id = db.get_random_project_video(project_id)
frames_dir = os.path.join(video.get_frames_dir(video.get_project_dir(video.base_dir_default, project_id), video_id),'train')
frames = sorted(glob(os.path.join(frames_dir, '*.png')))
#print('[E] no frames found in directory %s.'%frames_dir)
# look for unfinished labeling jobs
db.execute('select id, frame_name from frame_jobs where project_id=%i and video_id=%i;' % (int(project_id),int(video_id)))
labeling_jobs = [ {'id':x[0],'frame_name':x[1]} for x in db.cur.fetchall()]
if len(labeling_jobs) > 0:
if not labeling_jobs[0]['frame_name'][:-4] == '.png':
labeling_jobs[0]['frame_name'] += '.png'
frame_idx = '.'.join(labeling_jobs[0]['frame_name'].split('.')[:-1])
#print('A',labeling_jobs[0]['frame_name'],'->',frame_idx)
print('[*] found %i labeling jobs, giving %i' % (len(labeling_jobs),labeling_jobs[0]['id']))
# delete job
db.execute('delete from frame_jobs where id=%i;' % labeling_jobs[0]['id'])
db.commit()
else:
## first check if labeled bounding box data with unlabeled keypoint data is in db
unlabeled_labeled_boxes_frame_idx = []
for frame_idx_bbox in labeled_frame_idxs_boundingboxes:
if frame_idx_bbox not in labeled_frame_idxs:
unlabeled_labeled_boxes_frame_idx.append(frame_idx_bbox)
if len(unlabeled_labeled_boxes_frame_idx) > 0:
print('[*] found %i labeled boxes frames, where no keypoints are annotated. here is one of them' % len(unlabeled_labeled_boxes_frame_idx))
shuffle(unlabeled_labeled_boxes_frame_idx)
frame_idx = unlabeled_labeled_boxes_frame_idx[0]
else:
if num_db_frames < args.num_labeling_base:
'' # randomly sample frame
# choose random frame that is not already labeled
unlabeled_frame_found = False
while not unlabeled_frame_found:
ridx = int( len(frames) * num_db_frames / float(args.num_labeling_base) ) + int(np.random.uniform(-5,5))
if ridx > 0 and ridx < len(frames):
frame_idx = frames[ridx]
#frame_idx = frames[int(len(frames)*np.random.random())] # random sampling
frame_idx = '.'.join(frame_idx.split('/')[-1].split('.')[:-1])
unlabeled_frame_found = not (frame_idx in labeled_frame_idxs)
print('[*] serving keypoint label job for frame %s %s.'%(frame_idx,get_frame_time(frame_idx)))
else:
shuffle(frames)
nn = 1#32
unlabeled = []
while len(unlabeled) < nn:
frame_f = frames[int(len(frames)*np.random.random())]
frame_idx = '.'.join(frame_f.split('/')[-1].split('.')[:-1])
nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx)))
if frame_f not in unlabeled and frame_idx not in labeled_frame_idxs and nearest_labeled_frame_diff > 20:
unlabeled.append(frame_f)
if training_model is not None:
# active learning: load some unlabeled frames, inference multiple time, take one with biggest std variation
# predict whole image, height like trained height and variable width
# to keep aspect ratio and relative size
w = 1+int(2*config['img_height']/(float(cv.imread(unlabeled[0]).shape[0]) / cv.imread(unlabeled[0]).shape[1]))
batch = np.array( [cv.resize(cv.imread(f), (w,2*config['img_height'])) for f in unlabeled] ).astype(np.float32)
# inference multiple times
ni = 5
pred = np.array([ training_model(batch,training=True)[-1].numpy() for _ in range(ni)])
pred = pred[:,:,:,:,:-1] # cut background channel
# calculate max std item
stds = np.std(pred,axis=(0,2,3,4))
maxidx = np.argmax(stds)
frame_idx = '.'.join(unlabeled[maxidx].split('/')[-1].split('.')[:-1])
print('[*] serving keypoint label job for frame %s with std %f %s.'%(frame_idx,stds[maxidx],get_frame_time(frame_idx)))
else:
if len(unlabeled) ==0:
return "<h1>You have labeled all frames for this video! :)</h1>"
shuffle(unlabeled)
frame_idx = '.'.join(unlabeled[0].split('/')[-1].split('.')[:-1])
frame_candidates = []
if len(labeled_frame_idxs) > 0:
nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx)))
else:
nearest_labeled_frame_diff = -1
print('[*] serving keypoint label job for frame %s %s. nearest frame already labeled %i frames away'%(frame_idx,get_frame_time(frame_idx),nearest_labeled_frame_diff))
if args.open_gallery:
p = subprocess.Popen(['eog',unlabeled[maxidx]])
return render_template('labeling.html',project_id = int(project_id), video_id = int(video_id), frame_idx = frame_idx, keypoint_names = db.list_sep.join(config['keypoint_names']), sep = db.list_sep, num_db_frames = num_db_frames, frame_candidates = frame_candidates, labeling_mode = 'keypoint')
parser.add_argument('--data_dir', required=False, default = os.path.expanduser('~/data/multitracker'))
args = parser.parse_args()
os.environ['MULTITRACKER_DATA_DIR'] = args.data_dir
from multitracker.be import dbconnection
db = dbconnection.DatabaseConnection(file_db=os.path.join(args.data_dir, 'data.db'))
# load neural network from disk (or init new one)
if args.model is not None and os.path.isfile(args.model):
assert args.project_id is not None
import h5py
import tensorflow as tf
tf.get_logger().setLevel(logging.ERROR)
training_model = tf.keras.models.load_model(h5py.File(args.model, 'r'))
print('[*] loaded model %s from disk' % args.model)
config = model.get_config(project_id=args.project_id)
optimizer = tf.keras.optimizers.Adam(config['kp_lr'])
config = model.get_config(int(args.project_id))
else:
training_model = None
count_active_steps = 0
def get_frame_time(frame_idx):
frame_idx = int(frame_idx)
tmin = int(frame_idx/(30.*60.))
tsec = int(frame_idx/30.-tmin*60.)
return '%i:%imin'%(tmin,tsec)
def main(args):
os.environ['MULTITRACKER_DATA_DIR'] = args.data_dir
from multitracker.be import dbconnection
if args.minutes > 0 or args.video_resolution is not None:
tpreprocessvideostart = time.time()
sscale = args.video_resolution if args.video_resolution is not None else ''
smins = str(args.minutes) if args.minutes > 0 else ''
fvo = args.video[:-4] + sscale + smins + args.video[-4:]
if not os.path.isfile(fvo):
commands = ['ffmpeg']
if args.minutes > 0:
commands.extend(['-t', str(int(60. * args.minutes))])
commands.extend(['-i', args.video])
if args.video_resolution is not None:
commands.extend([
'-vf',
'scale=%s' % args.video_resolution.replace('x', ':')
])
commands.extend([fvo])
print('[*] preprocess video', ' '.join(commands))
subprocess.call(commands)
tpreprocessvideoend = time.time()
print('[*] preprocessing of video to %s took %f seconds' %
(fvo, tpreprocessvideoend - tpreprocessvideostart))
args.video = fvo
tstart = time.time()
config = model.get_config(project_id=args.project_id)
config['project_id'] = args.project_id
config['video'] = args.video
config['keypoint_model'] = args.keypoint_model
config['autoencoder_model'] = args.autoencoder_model
config['objectdetection_model'] = args.objectdetection_model
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['minutes'] = args.minutes
#config['upper_bound'] = db.get_video_fixednumber(args.video_id)
#config['upper_bound'] = None
config['upper_bound'] = args.upper_bound
config['n_blocks'] = 4
config['tracking_method'] = args.tracking_method
config['track_tail'] = args.track_tail
config['sketch_file'] = args.sketch_file
config['file_tracking_results'] = args.output_tracking_results
config['use_all_data4train'] = args.use_all_data4train
config['object_detection_backbone'] = args.objectdetection_method
config = model.update_config_object_detection(config)
config['object_detection_resolution'] = [
int(r) for r in args.objectdetection_resolution.split('x')
]
config['keypoint_resolution'] = [
int(r) for r in args.keypoint_resolution.split('x')
]
config['img_height'], config['img_width'] = config[
'keypoint_resolution'][::-1]
config['kp_backbone'] = args.keypoint_method
if 'hourglass' in args.keypoint_method:
config['kp_num_hourglass'] = int(args.keypoint_method[9:])
config['kp_backbone'] = 'efficientnetLarge'
if args.inference_objectdetection_batchsize > 0:
config[
'inference_objectdetection_batchsize'] = args.inference_objectdetection_batchsize
if args.inference_keypoint_batchsize > 0:
config[
'inference_keypoint_batchsize'] = args.inference_keypoint_batchsize
if args.delete_all_checkpoints:
if os.path.isdir(os.path.expanduser('~/checkpoints/multitracker')):
shutil.rmtree(os.path.expanduser('~/checkpoints/multitracker'))
if args.data_dir:
db = dbconnection.DatabaseConnection(
file_db=os.path.join(args.data_dir, 'data.db'))
config['data_dir'] = args.data_dir
config['kp_data_dir'] = os.path.join(
args.data_dir, 'projects/%i/data' % config['project_id'])
config['kp_roi_dir'] = os.path.join(
args.data_dir, 'projects/%i/data_roi' % config['project_id'])
config['keypoint_names'] = db.get_keypoint_names(config['project_id'])
config['project_name'] = db.get_project_name(config['project_id'])
# <train models>
# 1) animal bounding box finetuning -> trains and inferences
config['objectdetection_max_steps'] = 30000
# train object detector
now = str(datetime.now()).replace(' ', '_').replace(':', '-').split('.')[0]
checkpoint_directory_object_detection = os.path.expanduser(
'~/checkpoints/multitracker/bbox/vids%s-%s' %
(config['train_video_ids'], now))
object_detect_restore = None
if 'objectdetection_model' in config and config[
'objectdetection_model'] is not None:
object_detect_restore = config['objectdetection_model']
detection_model = None
if object_detect_restore is None:
detection_model = finetune.finetune(
config,
checkpoint_directory_object_detection,
checkpoint_restore=object_detect_restore)
print('[*] trained object detection model',
checkpoint_directory_object_detection)
config[
'object_detection_model'] = checkpoint_directory_object_detection
## crop bbox detections and train keypoint estimation on extracted regions
#point_classification.calculate_keypoints(config, detection_file_bboxes)
# 2) train autoencoder for tracking appearence vector
if config['autoencoder_model'] is None and config[
'tracking_method'] == 'DeepSORT':
config_autoencoder = autoencoder.get_autoencoder_config()
config_autoencoder['project_id'] = config['project_id']
config_autoencoder['video_ids'] = natsorted(
list(
set([int(iid)
for iid in config['train_video_ids'].split(',')] +
[int(iid)
for iid in config['test_video_ids'].split(',')])))
config_autoencoder['project_name'] = config['project_name']
config_autoencoder['data_dir'] = config['data_dir']
config['autoencoder_model'] = autoencoder.train(config_autoencoder)
print('[*] trained autoencoder model', config['autoencoder_model'])
# 4) train keypoint estimator model
if config['keypoint_model'] is None and not config['kp_backbone'] == 'none':
config['kp_max_steps'] = 25000
config['keypoint_model'] = roi_segm.train(config)
print('[*] trained keypoint_model', config['keypoint_model'])
# </train models>
# <load models>
# load trained object detection model
if detection_model is None:
# load config json to know which backbone was used
with open(os.path.join(config['objectdetection_model'],
'config.json')) as json_file:
objconfig = json.load(json_file)
objconfig['objectdetection_model'] = config['objectdetection_model']
detection_model = finetune.load_trained_model(objconfig)
# load trained autoencoder model for Deep Sort Tracking
encoder_model = None
if config['tracking_method'] == 'DeepSORT':
encoder_model = inference.load_autoencoder_feature_extractor(config)
# load trained keypoint model
if config['kp_backbone'] == 'none':
keypoint_model = None
else:
keypoint_model = inference.load_keypoint_model(
config['keypoint_model'])
# </load models>
# 3) run bbox tracking deep sort with fixed tracks
nms_max_overlap = 1.0 # Non-maxima suppression threshold: Maximum detection overlap
max_cosine_distance = 0.2 # Gating threshold for cosine distance metric (object appearance).
nn_budget = None # Maximum size of the appearance descriptors gallery. If None, no budget is enforced.
print(4 * '\n', config)
ttrack_start = time.time()
if config['tracking_method'] == 'DeepSORT':
deep_sort_app.run(config, detection_model, encoder_model,
keypoint_model, args.min_confidence_boxes,
args.min_confidence_keypoints, nms_max_overlap,
max_cosine_distance, nn_budget)
elif config['tracking_method'] == 'UpperBound':
upperbound_tracker.run(config, detection_model, encoder_model,
keypoint_model, args.min_confidence_boxes,
args.min_confidence_keypoints)
elif config['tracking_method'] == 'VIoU':
viou_tracker.run(config, detection_model, encoder_model,
keypoint_model, args.min_confidence_boxes,
args.min_confidence_keypoints)
ttrack_end = time.time()
ugly_big_video_file_out = inference.get_video_output_filepath(config)
video_file_out = ugly_big_video_file_out.replace('.avi', '.mp4')
convert_video_h265(ugly_big_video_file_out, video_file_out)
print(
'[*] done tracking after %f minutes. outputting file' %
float(int((ttrack_end - ttrack_start) * 10.) / 10.), video_file_out)
"""
plot results of experiments
"""
import os
from glob import glob
import numpy as np
import matplotlib.pyplot as plt
import json
from multitracker.keypoint_detection import model
from multitracker.experiments import bg_accuracy
from multitracker.experiments import roi_curve
from multitracker.be import dbconnection
db = dbconnection.DatabaseConnection()
config = model.get_config(project_id=7)
video_id = 9
config['max_steps'] = 25000
dpi=300
figsize = (12,8)
output_dir = os.path.expanduser('~/Documents/Multitracker_experiments')
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
colors = {1: 'tab:brown', 10: 'tab:blue',20: 'tab:orange', 50: 'tab:green', 100: 'tab:red',200: 'tab:black',300: 'tab:yellow', 400: 'tab:gray'}
def plot_experiment_a_roi(args):
num_train_samples = len(db.get_labeled_bbox_frames(args.video_id))
title = 'Experiment A - Keypoint Estimation: using fractions of training data ({0} samples total)'.format(num_train_samples)
experiment_dirs = [
