def experiment_c(args, max_steps=30000): print( '[*] starting experiment C: keypoint estimation test loss/inference speed: EfficientNet vs VGG16' ) config = model.get_config(args.project_id) config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['experiment'] = 'C' config['kp_train_loss'] = 'focal' config['kp_test_losses'] = ['focal'] #['cce','focal'] config['kp_max_steps'] = max_steps #config['kp_max_steps'] = 15000 config['early_stopping'] = False config['kp_lr'] = 1e-4 config['kp_num_hourglass'] = 1 config['batch_size'] = 8 #for backbone in ['hourglass2']: for backbone in [ 'vgg16', 'efficientnetLarge', 'psp', 'hourglass2', 'hourglass4', 'hourglass8' ]: print('[*] starting sub experiment backbone %s' % backbone) config['kp_backbone'] = backbone print(config, '\n') checkpoint_path = roi_segm.train(config, log_images=False) clear_session()
def experiment_e(args, train_video_ids=None): print( '[*] starting experiment E: object detection test loss under different number of training samples' ) config = model.get_config(args.project_id) if train_video_ids is not None: config['train_video_ids'] = train_video_ids else: config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['experiment'] = 'E' config['early_stopping'] = False config['object_finetune_warmup'] = 1000 config['maxsteps_objectdetection'] = 10000 config['lr_objectdetection'] = 0.005 config['object_augm_gaussian'] = bool(0) config['object_augm_image'] = bool(0) config['object_augm_mixup'] = bool(0) for data_ratio in [0.01, 0.1, 0.5, 1.0][::-1]: #, now = str(datetime.now()).replace(' ', '_').replace(':', '-').split('.')[0] print('[*] starting sub experiment with %i/100 of data used' % int(100. * data_ratio)) config['data_ratio'] = data_ratio checkpoint_dir = os.path.expanduser( '~/checkpoints/experiments/%s/E/%i-%s' % (config['project_name'], int(100. * data_ratio), now)) #config['maxsteps_objectdetection'] = {0.01: 12000, 0.1: 15000, 0.5: 20000, 1.0: 25000}[data_ratio] finetune(config, checkpoint_dir) clear_session() print(10 * '\n')
def experiment_a(args, max_steps=30000): print( '[*] starting experiment A: keypoint estimation test loss under different number of training samples (100%, 50%, 10%, 1%)' ) config = model.get_config(args.project_id) config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['experiment'] = 'A' config['kp_train_loss'] = 'focal' config['kp_test_losses'] = ['focal'] #['cce','focal'] config['kp_max_steps'] = max_steps config['early_stopping'] = False config['kp_lr'] = 1e-4 config['batch_size'] = 8 #config['kp_lr'] = 2e-5 #for data_ratio in [0.01,0.1,0.5,1.0][::-1]: for data_ratio in [0.01, 0.1, 0.5, 1.0]: print('[*] starting sub experiment with %i/100 of data used' % int(100. * data_ratio)) config['data_ratio'] = data_ratio print(config, '\n') checkpoint_path = roi_segm.train(config, log_images=False) clear_session() print(10 * '\n')
def get_next_bbox_frame(project_id, video_id): project_id = int(project_id) config = model.get_config(project_id) #video_id = db.get_random_project_video(project_id) video_id = int(video_id) config['video_id'] = video_id # load labeled frame idxs labeled_frames_keypoints = db.get_labeled_frames(video_id) labeled_frame_idxs = db.get_labeled_bbox_frames(video_id) num_db_frames = len(labeled_frame_idxs) # load frame files from disk frames_dir = os.path.join(video.get_frames_dir(video.get_project_dir(video.base_dir_default, project_id), video_id),'train') frames = sorted(glob(os.path.join(frames_dir, '*.png'))) frames_keypoints = [os.path.join(frames_dir, '%s.png' % ff) for ff in labeled_frames_keypoints] shuffle(frames_keypoints) unlabeled_frame_found = False if len(frames_keypoints) > 0: # first try to label bounding boxes where keypoints are labeled but bboxes not tries = 0 while not unlabeled_frame_found and tries < 50: ridx = int(np.random.uniform(len(frames_keypoints))) frame_idx = frames_keypoints[ridx] frame_idx = '.'.join(frame_idx.split('/')[-1].split('.')[:-1]) unlabeled_frame_found = not (frame_idx in labeled_frame_idxs) tries += 1 # then take random unlabeled frame tries, nearest_labeled_frame_diff = 0, 0 while not unlabeled_frame_found and tries < 50: ridx = int(np.random.uniform(len(frames))) frame_idx = frames[ridx] frame_idx = '.'.join(frame_idx.split('/')[-1].split('.')[:-1]) if len(labeled_frame_idxs) == 0: unlabeled_frame_found = True else: nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx))) if nearest_labeled_frame_diff > 20: unlabeled_frame_found = not (frame_idx in labeled_frame_idxs) tries += 1 if len(labeled_frame_idxs) > 0: nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx))) else: nearest_labeled_frame_diff = -1 if unlabeled_frame_found: print('[*] serving bounding box label job for frame %s %s. nearest frame already labeled %i frames away'%(frame_idx,get_frame_time(frame_idx),nearest_labeled_frame_diff)) return render_template('labeling.html',project_id = int(project_id), video_id = int(video_id), frame_idx = frame_idx, num_db_frames = num_db_frames, keypoint_names = db.list_sep.join(config['keypoint_names']), sep = db.list_sep, labeling_mode = 'bbox') else: print('[*] redirecting to keypoint labeling') return render_labeling(project_id, video_id)
def experiment_g(args, train_video_ids=None): print( '[*] starting experiment G: object detection pretrained vs random init network, w/wo augmentation' ) config = model.get_config(args.project_id) config['data_dir'] = '/home/alex/data/multitracker' if train_video_ids is not None: config['train_video_ids'] = train_video_ids else: config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['experiment'] = 'G' config['early_stopping'] = False config['object_finetune_warmup'] = 1000 config['maxsteps_objectdetection'] = 20000 config['lr_objectdetection'] = 0.005 config['object_augm_gaussian'] = bool(0) config['object_augm_image'] = bool(0) config['object_augm_mixup'] = bool(0) #for should_init_pretrained in [True]: # for augm in [True]: for should_init_pretrained in [False, True]: for augm in [False, True]: now = str(datetime.now()).replace(' ', '_').replace(':', '-').split('.')[0] if not augm: config['object_augm_flip'] = bool(0) config['object_augm_rot90'] = bool(0) else: config['object_augm_flip'] = bool(1) config['object_augm_rot90'] = bool(1) config['object_pretrained'] = should_init_pretrained str_augm = 'augm' if augm else 'noaugm' str_pretrained = 'pretrained' if should_init_pretrained else 'scratch' print('[*] starting sub experiment', str_augm, str_pretrained) checkpoint_dir = os.path.expanduser( '~/checkpoints/experiments/%s/G/%s-%s-%s' % (config['project_name'], str_augm, str_pretrained, now)) #config['maxsteps_objectdetection'] = {0.01: 12000, 0.1: 15000, 0.5: 20000, 1.0: 25000}[data_ratio] print(finetune) finetune(config, checkpoint_dir) clear_session() print(10 * '\n')
def get_annotation(project_id, video_id, frame_id): project_id = int(project_id) video_id = int(video_id) frame_id = int(frame_id) config = model.get_config(int(project_id)) #if args.video_id is None: # raise Exception("\n please restart the app with argument --video_id and --upper_bound") global loaded_video_id #if loaded_video_id is None or not loaded_video_id == video_id: # get_next_annotation_frame(project_id, video_id, int(np.random.uniform(1e6))) # load labeled frame idxs labeled_frame_idxs = db.get_labeled_frames(video_id) labeled_frame_idxs_boundingboxes = db.get_labeled_bbox_frames(video_id) num_db_frames = len(labeled_frame_idxs) # load annotation from database animals = db.get_frame_annotations(video_id, frame_id ) if animals is None: animals = [] # if no annotation around, init upper_bound many animals with keypoints for i in range(int(args.upper_bound)): x1,y1,x2,y2 = [200*i,200*i,200*(i+1),200*(i+1)] keypoints = [ ] for j,kp_name in enumerate(config['keypoint_names']): keypoints.append({ 'name': kp_name, 'x': x1 + 20*j, 'y': y1 + 23*j, 'db_id': None, 'is_visible': True }) animals.append({'id': str(i+1), 'box': [x1,y1,x2,y2], 'keypoints': keypoints, 'db_id': None, 'is_visible': True}) else: # parse db animals #for i, animal in enumerate(animals): # print(i, animal) pass animals_json = json.dumps(animals) return render_template('annotate.html', animals = animals, animals_json = animals_json, project_id = int(project_id), video_id = int(video_id), frame_idx = frame_id, keypoint_names = db.list_sep.join(config['keypoint_names']), sep = db.list_sep, num_db_frames = num_db_frames, labeling_mode = 'annotate')
def experiment_b(args, max_steps=12000): print( '[*] starting experiment B: keypoint estimation test loss: pretrained vs random init network' ) config = model.get_config(args.project_id) config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['experiment'] = 'B' config['kp_backbone'] = 'hourglass2' config['kp_train_loss'] = 'focal' config['kp_test_losses'] = ['focal'] #['cce','focal'] config['kp_max_steps'] = max_steps #config['kp_max_steps'] = 15000 config['early_stopping'] = False config['batch_size'] = 8 config['kp_lr'] = 1e-4 config_b = deepcopy(config) run_configs = [ [False, False], [False, True], [True, False], [True, True], ] for should_init_pretrained, augm in run_configs: print('[*] starting sub experiment %s weight initialization' % ['without', 'with'][int(should_init_pretrained)]) if not augm: config_b['kp_mixup'] = False config_b['kp_cutmix'] = False config_b['kp_hflips'] = False config_b['kp_vflips'] = False config_b['kp_rotation_augmentation'] = False config_b['kp_rot90s'] = False config_b['kp_im_noise'] = False config_b['kp_im_transf'] = False config_b['should_init_pretrained'] = should_init_pretrained print(config_b, '\n') checkpoint_path = roi_segm.train(config_b, log_images=False) clear_session()
def experiment_f(args, train_video_ids=None): print( '[*] starting experiment F: object detection test loss SSD vs FasterRCNN' ) config = model.get_config(args.project_id) if train_video_ids is not None: config['train_video_ids'] = train_video_ids else: config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['experiment'] = 'F' config['maxsteps_objectdetection'] = 15000 config['early_stopping'] = False for od_backbone in ['fasterrcnn', 'ssd']: print('[*] starting subexperiment for backbone type', od_backbone) config['object_detection_backbone'] = od_backbone config['object_detection_backbonepath'] = { 'ssd': 'ssd_resnet50_v1_fpn_640x640_coco17_tpu-8', 'fasterrcnn': 'faster_rcnn_inception_resnet_v2_640x640_coco17_tpu-8' }[config['object_detection_backbone']] config['object_detection_batch_size'] = { 'ssd': 16, 'fasterrcnn': 4 }[config['object_detection_backbone']] print(config, '\n') now = str(datetime.now()).replace(' ', '_').replace(':', '-').split('.')[0] checkpoint_directory = os.path.expanduser( "~/checkpoints/experiments/%s/F/%s-%s" % (config['project_name'], od_backbone, now)) finetune(config, checkpoint_directory) clear_session() print(10 * '\n')
def experiment_d(args, max_steps=25000, train_video_ids=None): print( '[*] starting experiment D: keypoint estimation categorical cross entropy / focal loss test trained with categorical cross entropy and focal loss' ) config = model.get_config(args.project_id) if train_video_ids is not None: config['train_video_ids'] = train_video_ids else: config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['kp_test_losses'] = ['focal', 'cce', 'l2'] config['experiment'] = 'D' '''config['kp_mixup'] = False config['kp_cutmix'] = False config['kp_rot90s'] = False config['kp_hflips'] = False config['kp_vflips'] = False config['kp_blurpool'] = False config['kp_rotation_augmentation'] = bool(0) config['kp_backbone'] = 'hourglass2''' #config['kp_max_steps'] = 15000 config['kp_max_steps'] = max_steps config['early_stopping'] = False config['batch_size'] = 8 config['kp_lr'] = 1e-4 for loss_name in ['focal', 'cce', 'l2']: print('[*] starting sub experiment loss function %s' % loss_name) config['kp_train_loss'] = loss_name print(config, '\n') checkpoint_path = roi_segm.train(config, log_images=False) clear_session()
def augment_dataset(project_id): config = model.get_config(project_id) config['input_image_shape'] = cv.imread(glob(os.path.join(config['data_dir'],'train/*.png'))[0]).shape[:2] h = config['input_image_shape'][0] w = config['input_image_shape'][1] // (2+ len(config['keypoint_names'])//3) print(config) print(h,w, 4 * w) file_list, dataset_train = model.load_raw_dataset(config,'train') inputs = tf.keras.layers.Input(shape=[config['img_height'], config['img_width'], 3]) inputss = preprocess_input(inputs) net = tf.keras.applications.ResNet152V2(input_tensor=inputss, include_top=False, weights='imagenet', pooling='avg') net.summary() if 0: for layer in net.layers: try: print(layer.name,layer.outputs[0].shape) except: pass layer_name = 'conv1_conv' feature_activation = net.get_layer(layer_name) feature_extractor = tf.keras.models.Model(name="ImageNet Encoder",inputs=net.input,outputs=[feature_activation.output]) output_dir = '/tmp/feature_augment' if not os.path.isdir(output_dir): os.makedirs(output_dir) files = [f for f in sorted(glob(os.path.join(config['data_dir'],'train/*.png'))) if not 'augment-' in f] for i,f in enumerate(files): im = cv.imread(f) rgb = im[:,:w,:] rgb = rgb.reshape((1,) + rgb.shape) rgb = preprocess_input(rgb) features = feature_extractor(rgb,training=False).numpy()[0,:,:,:] features = cv.resize(features,None,None,fx=2.,fy=2.) print('[*] wrote %i/%i:'%(i,len(files)),im.shape,features.shape,features.min(),features.max()) for j in range(2): fo = os.path.join(config['data_dir'],'train/augment-%i-%i.png' % (i,j) ) #print(fo) a = np.zeros_like(features[:,:,0]) b = np.zeros_like(features[:,:,0]) c = np.zeros_like(features[:,:,0]) for k in range(features.shape[-1] // 3): a += features[:,:,int(np.random.uniform(features.shape[-1]))] b += features[:,:,int(np.random.uniform(features.shape[-1]))] c += features[:,:,int(np.random.uniform(features.shape[-1]))] abc = cv.merge((a,b,c)) thresh = 1. abc[thresh > np.abs(abc)] = thresh abc8 = 255. * (abc-abc.min())/(1e-5+abc.max()-abc.min()) abc8 = np.uint8(abc8) abc8 = np.hstack((abc8,im[:,w:,:])) cv.imwrite(fo,abc8) '''for x,y in dataset_train:
def main(args): config = model.get_config(args.project_id) config['video_id'] = int(args.video_id) print(config,'\n') checkpoint_path = train(config)
def check_labeling_handler(project_id, video_id): """ serve a little page to see the drawn labeling and two buttons for good and bad. bads should be deleted and added as a labeling job """ project_id, video_id = int(project_id), int(video_id) config = model.get_config(project_id=project_id) if video_id not in idx_checked: idx_checked[video_id] = 0 # first get all frames q = "select frame_idx from keypoint_positions where video_id=%i order by id;" % video_id db.execute(q) frame_idxs = [x[0] for x in db.cur.fetchall()] frame_idxs = list(set(frame_idxs)) frame_idx = frame_idxs[idx_checked[video_id]] q = "select keypoint_name, individual_id, keypoint_x, keypoint_y from keypoint_positions where video_id=%i and frame_idx='%s' order by individual_id, keypoint_name desc;" % (video_id,frame_idx) db.execute(q) keypoints = [x for x in db.cur.fetchall()] filepath = os.path.join(dbconnection.base_data_dir, "projects/%i/%i/frames/train/%s.png" % (int(project_id), int(video_id), frame_idx)) im = cv.imread(filepath) hm = heatmap_drawing.generate_hm(im.shape[0], im.shape[1] , [ [int(kp[2]),int(kp[3]),kp[0]] for kp in keypoints ], config['keypoint_names']) colors = util.get_colors() vis = np.float32(im) hm_color = np.zeros_like(vis) for i in range(hm.shape[2]): hm8 = hm[:,:,i] hm8[hm8<0.5]=0 c = np.array(colors[i]).reshape((1,1,3)) c = np.tile(c,[hm.shape[0],hm.shape[1],1]) hm8 = np.expand_dims(hm8,axis=2) hmc = c * np.tile(hm8,[1,1,3]) hm_color += hmc vis = np.uint8(vis/2. + hm_color/2.) vis_path = '/tmp/check_%i-%s.png' % (video_id, str(frame_idx)) cv.imwrite(vis_path, vis) #print('[*] wrote',vis_path,vis.shape,vis.dtype,vis.min(),vis.max()) idx_checked[video_id] += 1 print('idx_checked',idx_checked[video_id]) dom = """ <html> <head> <link rel="stylesheet" type="text/css" href="https://semantic-ui.com/dist/semantic.min.css"> <script src="https://code.jquery.com/jquery-3.1.1.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script> <script src="https://semantic-ui.com/dist/semantic.min.js"></script> <script src="/static/js/network.js"></script> <style> * { margin: 0; padding: 0; } .imgbox { height: 100% } </style> </head> <body> <div class="imgbox"> <img height="80%" class="center-fit" src="/get_labeled_frame/[project_id]/[video_id]/[frame_idx]"> <script type="text/javascript"> function onclick_ok(){ window.location.reload() } function onclick_bad(){ get("/delete_labeling/[project_id]/[video_id]/[frame_idx]"); window.location.reload(); } </script> <button id="bu_ok" class="ui positive button" onclick="onclick_ok();">OK</button> <button id="bu_bad" class="ui negative button" onclick='onclick_bad();'>BAD</button> </div> </body> </html> """ for _ in range(3): dom = dom.replace('[project_id]',str(project_id)).replace('[video_id]',str(video_id)).replace('[frame_idx]',str(frame_idx)) return dom
def render_labeling(project_id,video_id): config = model.get_config(int(project_id)) video_id = int(video_id) config['video_id'] = video_id # load labeled frame idxs labeled_frame_idxs = db.get_labeled_frames(video_id) labeled_frame_idxs_boundingboxes = db.get_labeled_bbox_frames(video_id) num_db_frames = len(labeled_frame_idxs) # load frame files from disk frames = [] while len(frames) == 0: #video_id = db.get_random_project_video(project_id) frames_dir = os.path.join(video.get_frames_dir(video.get_project_dir(video.base_dir_default, project_id), video_id),'train') frames = sorted(glob(os.path.join(frames_dir, '*.png'))) #print('[E] no frames found in directory %s.'%frames_dir) # look for unfinished labeling jobs db.execute('select id, frame_name from frame_jobs where project_id=%i and video_id=%i;' % (int(project_id),int(video_id))) labeling_jobs = [ {'id':x[0],'frame_name':x[1]} for x in db.cur.fetchall()] if len(labeling_jobs) > 0: if not labeling_jobs[0]['frame_name'][:-4] == '.png': labeling_jobs[0]['frame_name'] += '.png' frame_idx = '.'.join(labeling_jobs[0]['frame_name'].split('.')[:-1]) #print('A',labeling_jobs[0]['frame_name'],'->',frame_idx) print('[*] found %i labeling jobs, giving %i' % (len(labeling_jobs),labeling_jobs[0]['id'])) # delete job db.execute('delete from frame_jobs where id=%i;' % labeling_jobs[0]['id']) db.commit() else: ## first check if labeled bounding box data with unlabeled keypoint data is in db unlabeled_labeled_boxes_frame_idx = [] for frame_idx_bbox in labeled_frame_idxs_boundingboxes: if frame_idx_bbox not in labeled_frame_idxs: unlabeled_labeled_boxes_frame_idx.append(frame_idx_bbox) if len(unlabeled_labeled_boxes_frame_idx) > 0: print('[*] found %i labeled boxes frames, where no keypoints are annotated. here is one of them' % len(unlabeled_labeled_boxes_frame_idx)) shuffle(unlabeled_labeled_boxes_frame_idx) frame_idx = unlabeled_labeled_boxes_frame_idx[0] else: if num_db_frames < args.num_labeling_base: '' # randomly sample frame # choose random frame that is not already labeled unlabeled_frame_found = False while not unlabeled_frame_found: ridx = int( len(frames) * num_db_frames / float(args.num_labeling_base) ) + int(np.random.uniform(-5,5)) if ridx > 0 and ridx < len(frames): frame_idx = frames[ridx] #frame_idx = frames[int(len(frames)*np.random.random())] # random sampling frame_idx = '.'.join(frame_idx.split('/')[-1].split('.')[:-1]) unlabeled_frame_found = not (frame_idx in labeled_frame_idxs) print('[*] serving keypoint label job for frame %s %s.'%(frame_idx,get_frame_time(frame_idx))) else: shuffle(frames) nn = 1#32 unlabeled = [] while len(unlabeled) < nn: frame_f = frames[int(len(frames)*np.random.random())] frame_idx = '.'.join(frame_f.split('/')[-1].split('.')[:-1]) nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx))) if frame_f not in unlabeled and frame_idx not in labeled_frame_idxs and nearest_labeled_frame_diff > 20: unlabeled.append(frame_f) if training_model is not None: # active learning: load some unlabeled frames, inference multiple time, take one with biggest std variation # predict whole image, height like trained height and variable width # to keep aspect ratio and relative size w = 1+int(2*config['img_height']/(float(cv.imread(unlabeled[0]).shape[0]) / cv.imread(unlabeled[0]).shape[1])) batch = np.array( [cv.resize(cv.imread(f), (w,2*config['img_height'])) for f in unlabeled] ).astype(np.float32) # inference multiple times ni = 5 pred = np.array([ training_model(batch,training=True)[-1].numpy() for _ in range(ni)]) pred = pred[:,:,:,:,:-1] # cut background channel # calculate max std item stds = np.std(pred,axis=(0,2,3,4)) maxidx = np.argmax(stds) frame_idx = '.'.join(unlabeled[maxidx].split('/')[-1].split('.')[:-1]) print('[*] serving keypoint label job for frame %s with std %f %s.'%(frame_idx,stds[maxidx],get_frame_time(frame_idx))) else: if len(unlabeled) ==0: return "<h1>You have labeled all frames for this video! :)</h1>" shuffle(unlabeled) frame_idx = '.'.join(unlabeled[0].split('/')[-1].split('.')[:-1]) frame_candidates = [] if len(labeled_frame_idxs) > 0: nearest_labeled_frame_diff = np.min(np.abs(np.array([int(idx) for idx in labeled_frame_idxs]) - int(frame_idx))) else: nearest_labeled_frame_diff = -1 print('[*] serving keypoint label job for frame %s %s. nearest frame already labeled %i frames away'%(frame_idx,get_frame_time(frame_idx),nearest_labeled_frame_diff)) if args.open_gallery: p = subprocess.Popen(['eog',unlabeled[maxidx]]) return render_template('labeling.html',project_id = int(project_id), video_id = int(video_id), frame_idx = frame_idx, keypoint_names = db.list_sep.join(config['keypoint_names']), sep = db.list_sep, num_db_frames = num_db_frames, frame_candidates = frame_candidates, labeling_mode = 'keypoint')
parser.add_argument('--data_dir', required=False, default = os.path.expanduser('~/data/multitracker')) args = parser.parse_args() os.environ['MULTITRACKER_DATA_DIR'] = args.data_dir from multitracker.be import dbconnection db = dbconnection.DatabaseConnection(file_db=os.path.join(args.data_dir, 'data.db')) # load neural network from disk (or init new one) if args.model is not None and os.path.isfile(args.model): assert args.project_id is not None import h5py import tensorflow as tf tf.get_logger().setLevel(logging.ERROR) training_model = tf.keras.models.load_model(h5py.File(args.model, 'r')) print('[*] loaded model %s from disk' % args.model) config = model.get_config(project_id=args.project_id) optimizer = tf.keras.optimizers.Adam(config['kp_lr']) config = model.get_config(int(args.project_id)) else: training_model = None count_active_steps = 0 def get_frame_time(frame_idx): frame_idx = int(frame_idx) tmin = int(frame_idx/(30.*60.)) tsec = int(frame_idx/30.-tmin*60.) return '%i:%imin'%(tmin,tsec)
def main(args): os.environ['MULTITRACKER_DATA_DIR'] = args.data_dir from multitracker.be import dbconnection if args.minutes > 0 or args.video_resolution is not None: tpreprocessvideostart = time.time() sscale = args.video_resolution if args.video_resolution is not None else '' smins = str(args.minutes) if args.minutes > 0 else '' fvo = args.video[:-4] + sscale + smins + args.video[-4:] if not os.path.isfile(fvo): commands = ['ffmpeg'] if args.minutes > 0: commands.extend(['-t', str(int(60. * args.minutes))]) commands.extend(['-i', args.video]) if args.video_resolution is not None: commands.extend([ '-vf', 'scale=%s' % args.video_resolution.replace('x', ':') ]) commands.extend([fvo]) print('[*] preprocess video', ' '.join(commands)) subprocess.call(commands) tpreprocessvideoend = time.time() print('[*] preprocessing of video to %s took %f seconds' % (fvo, tpreprocessvideoend - tpreprocessvideostart)) args.video = fvo tstart = time.time() config = model.get_config(project_id=args.project_id) config['project_id'] = args.project_id config['video'] = args.video config['keypoint_model'] = args.keypoint_model config['autoencoder_model'] = args.autoencoder_model config['objectdetection_model'] = args.objectdetection_model config['train_video_ids'] = args.train_video_ids config['test_video_ids'] = args.test_video_ids config['minutes'] = args.minutes #config['upper_bound'] = db.get_video_fixednumber(args.video_id) #config['upper_bound'] = None config['upper_bound'] = args.upper_bound config['n_blocks'] = 4 config['tracking_method'] = args.tracking_method config['track_tail'] = args.track_tail config['sketch_file'] = args.sketch_file config['file_tracking_results'] = args.output_tracking_results config['use_all_data4train'] = args.use_all_data4train config['object_detection_backbone'] = args.objectdetection_method config = model.update_config_object_detection(config) config['object_detection_resolution'] = [ int(r) for r in args.objectdetection_resolution.split('x') ] config['keypoint_resolution'] = [ int(r) for r in args.keypoint_resolution.split('x') ] config['img_height'], config['img_width'] = config[ 'keypoint_resolution'][::-1] config['kp_backbone'] = args.keypoint_method if 'hourglass' in args.keypoint_method: config['kp_num_hourglass'] = int(args.keypoint_method[9:]) config['kp_backbone'] = 'efficientnetLarge' if args.inference_objectdetection_batchsize > 0: config[ 'inference_objectdetection_batchsize'] = args.inference_objectdetection_batchsize if args.inference_keypoint_batchsize > 0: config[ 'inference_keypoint_batchsize'] = args.inference_keypoint_batchsize if args.delete_all_checkpoints: if os.path.isdir(os.path.expanduser('~/checkpoints/multitracker')): shutil.rmtree(os.path.expanduser('~/checkpoints/multitracker')) if args.data_dir: db = dbconnection.DatabaseConnection( file_db=os.path.join(args.data_dir, 'data.db')) config['data_dir'] = args.data_dir config['kp_data_dir'] = os.path.join( args.data_dir, 'projects/%i/data' % config['project_id']) config['kp_roi_dir'] = os.path.join( args.data_dir, 'projects/%i/data_roi' % config['project_id']) config['keypoint_names'] = db.get_keypoint_names(config['project_id']) config['project_name'] = db.get_project_name(config['project_id']) # <train models> # 1) animal bounding box finetuning -> trains and inferences config['objectdetection_max_steps'] = 30000 # train object detector now = str(datetime.now()).replace(' ', '_').replace(':', '-').split('.')[0] checkpoint_directory_object_detection = os.path.expanduser( '~/checkpoints/multitracker/bbox/vids%s-%s' % (config['train_video_ids'], now)) object_detect_restore = None if 'objectdetection_model' in config and config[ 'objectdetection_model'] is not None: object_detect_restore = config['objectdetection_model'] detection_model = None if object_detect_restore is None: detection_model = finetune.finetune( config, checkpoint_directory_object_detection, checkpoint_restore=object_detect_restore) print('[*] trained object detection model', checkpoint_directory_object_detection) config[ 'object_detection_model'] = checkpoint_directory_object_detection ## crop bbox detections and train keypoint estimation on extracted regions #point_classification.calculate_keypoints(config, detection_file_bboxes) # 2) train autoencoder for tracking appearence vector if config['autoencoder_model'] is None and config[ 'tracking_method'] == 'DeepSORT': config_autoencoder = autoencoder.get_autoencoder_config() config_autoencoder['project_id'] = config['project_id'] config_autoencoder['video_ids'] = natsorted( list( set([int(iid) for iid in config['train_video_ids'].split(',')] + [int(iid) for iid in config['test_video_ids'].split(',')]))) config_autoencoder['project_name'] = config['project_name'] config_autoencoder['data_dir'] = config['data_dir'] config['autoencoder_model'] = autoencoder.train(config_autoencoder) print('[*] trained autoencoder model', config['autoencoder_model']) # 4) train keypoint estimator model if config['keypoint_model'] is None and not config['kp_backbone'] == 'none': config['kp_max_steps'] = 25000 config['keypoint_model'] = roi_segm.train(config) print('[*] trained keypoint_model', config['keypoint_model']) # </train models> # <load models> # load trained object detection model if detection_model is None: # load config json to know which backbone was used with open(os.path.join(config['objectdetection_model'], 'config.json')) as json_file: objconfig = json.load(json_file) objconfig['objectdetection_model'] = config['objectdetection_model'] detection_model = finetune.load_trained_model(objconfig) # load trained autoencoder model for Deep Sort Tracking encoder_model = None if config['tracking_method'] == 'DeepSORT': encoder_model = inference.load_autoencoder_feature_extractor(config) # load trained keypoint model if config['kp_backbone'] == 'none': keypoint_model = None else: keypoint_model = inference.load_keypoint_model( config['keypoint_model']) # </load models> # 3) run bbox tracking deep sort with fixed tracks nms_max_overlap = 1.0 # Non-maxima suppression threshold: Maximum detection overlap max_cosine_distance = 0.2 # Gating threshold for cosine distance metric (object appearance). nn_budget = None # Maximum size of the appearance descriptors gallery. If None, no budget is enforced. print(4 * '\n', config) ttrack_start = time.time() if config['tracking_method'] == 'DeepSORT': deep_sort_app.run(config, detection_model, encoder_model, keypoint_model, args.min_confidence_boxes, args.min_confidence_keypoints, nms_max_overlap, max_cosine_distance, nn_budget) elif config['tracking_method'] == 'UpperBound': upperbound_tracker.run(config, detection_model, encoder_model, keypoint_model, args.min_confidence_boxes, args.min_confidence_keypoints) elif config['tracking_method'] == 'VIoU': viou_tracker.run(config, detection_model, encoder_model, keypoint_model, args.min_confidence_boxes, args.min_confidence_keypoints) ttrack_end = time.time() ugly_big_video_file_out = inference.get_video_output_filepath(config) video_file_out = ugly_big_video_file_out.replace('.avi', '.mp4') convert_video_h265(ugly_big_video_file_out, video_file_out) print( '[*] done tracking after %f minutes. outputting file' % float(int((ttrack_end - ttrack_start) * 10.) / 10.), video_file_out)
""" plot results of experiments """ import os from glob import glob import numpy as np import matplotlib.pyplot as plt import json from multitracker.keypoint_detection import model from multitracker.experiments import bg_accuracy from multitracker.experiments import roi_curve from multitracker.be import dbconnection db = dbconnection.DatabaseConnection() config = model.get_config(project_id=7) video_id = 9 config['max_steps'] = 25000 dpi=300 figsize = (12,8) output_dir = os.path.expanduser('~/Documents/Multitracker_experiments') if not os.path.isdir(output_dir): os.makedirs(output_dir) colors = {1: 'tab:brown', 10: 'tab:blue',20: 'tab:orange', 50: 'tab:green', 100: 'tab:red',200: 'tab:black',300: 'tab:yellow', 400: 'tab:gray'} def plot_experiment_a_roi(args): num_train_samples = len(db.get_labeled_bbox_frames(args.video_id)) title = 'Experiment A - Keypoint Estimation: using fractions of training data ({0} samples total)'.format(num_train_samples) experiment_dirs = [