def experiment_a(args, max_steps=30000):
print(
'[*] starting experiment A: keypoint estimation test loss under different number of training samples (100%, 50%, 10%, 1%)'
)
config = model.get_config(args.project_id)
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'A'
config['kp_train_loss'] = 'focal'
config['kp_test_losses'] = ['focal'] #['cce','focal']
config['kp_max_steps'] = max_steps
config['early_stopping'] = False
config['kp_lr'] = 1e-4
config['batch_size'] = 8
#config['kp_lr'] = 2e-5
#for data_ratio in [0.01,0.1,0.5,1.0][::-1]:
for data_ratio in [0.01, 0.1, 0.5, 1.0]:
print('[*] starting sub experiment with %i/100 of data used' %
int(100. * data_ratio))
config['data_ratio'] = data_ratio
print(config, '\n')
checkpoint_path = roi_segm.train(config, log_images=False)
clear_session()
print(10 * '\n')
def experiment_c(args, max_steps=30000):
print(
'[*] starting experiment C: keypoint estimation test loss/inference speed: EfficientNet vs VGG16'
)
config = model.get_config(args.project_id)
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'C'
config['kp_train_loss'] = 'focal'
config['kp_test_losses'] = ['focal'] #['cce','focal']
config['kp_max_steps'] = max_steps
#config['kp_max_steps'] = 15000
config['early_stopping'] = False
config['kp_lr'] = 1e-4
config['kp_num_hourglass'] = 1
config['batch_size'] = 8
#for backbone in ['hourglass2']:
for backbone in [
'vgg16', 'efficientnetLarge', 'psp', 'hourglass2', 'hourglass4',
'hourglass8'
]:
print('[*] starting sub experiment backbone %s' % backbone)
config['kp_backbone'] = backbone
print(config, '\n')
checkpoint_path = roi_segm.train(config, log_images=False)
clear_session()
def experiment_b(args, max_steps=12000):
print(
'[*] starting experiment B: keypoint estimation test loss: pretrained vs random init network'
)
config = model.get_config(args.project_id)
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['experiment'] = 'B'
config['kp_backbone'] = 'hourglass2'
config['kp_train_loss'] = 'focal'
config['kp_test_losses'] = ['focal'] #['cce','focal']
config['kp_max_steps'] = max_steps
#config['kp_max_steps'] = 15000
config['early_stopping'] = False
config['batch_size'] = 8
config['kp_lr'] = 1e-4
config_b = deepcopy(config)
run_configs = [
[False, False],
[False, True],
[True, False],
[True, True],
]
for should_init_pretrained, augm in run_configs:
print('[*] starting sub experiment %s weight initialization' %
['without', 'with'][int(should_init_pretrained)])
if not augm:
config_b['kp_mixup'] = False
config_b['kp_cutmix'] = False
config_b['kp_hflips'] = False
config_b['kp_vflips'] = False
config_b['kp_rotation_augmentation'] = False
config_b['kp_rot90s'] = False
config_b['kp_im_noise'] = False
config_b['kp_im_transf'] = False
config_b['should_init_pretrained'] = should_init_pretrained
print(config_b, '\n')
checkpoint_path = roi_segm.train(config_b, log_images=False)
clear_session()
def experiment_d(args, max_steps=25000, train_video_ids=None):
print(
'[*] starting experiment D: keypoint estimation categorical cross entropy / focal loss test trained with categorical cross entropy and focal loss'
)
config = model.get_config(args.project_id)
if train_video_ids is not None:
config['train_video_ids'] = train_video_ids
else:
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['kp_test_losses'] = ['focal', 'cce', 'l2']
config['experiment'] = 'D'
'''config['kp_mixup'] = False
config['kp_cutmix'] = False
config['kp_rot90s'] = False
config['kp_hflips'] = False
config['kp_vflips'] = False
config['kp_blurpool'] = False
config['kp_rotation_augmentation'] = bool(0)
config['kp_backbone'] = 'hourglass2'''
#config['kp_max_steps'] = 15000
config['kp_max_steps'] = max_steps
config['early_stopping'] = False
config['batch_size'] = 8
config['kp_lr'] = 1e-4
for loss_name in ['focal', 'cce', 'l2']:
print('[*] starting sub experiment loss function %s' % loss_name)
config['kp_train_loss'] = loss_name
print(config, '\n')
checkpoint_path = roi_segm.train(config, log_images=False)
clear_session()
def main(args):
os.environ['MULTITRACKER_DATA_DIR'] = args.data_dir
from multitracker.be import dbconnection
if args.minutes > 0 or args.video_resolution is not None:
tpreprocessvideostart = time.time()
sscale = args.video_resolution if args.video_resolution is not None else ''
smins = str(args.minutes) if args.minutes > 0 else ''
fvo = args.video[:-4] + sscale + smins + args.video[-4:]
if not os.path.isfile(fvo):
commands = ['ffmpeg']
if args.minutes > 0:
commands.extend(['-t', str(int(60. * args.minutes))])
commands.extend(['-i', args.video])
if args.video_resolution is not None:
commands.extend([
'-vf',
'scale=%s' % args.video_resolution.replace('x', ':')
])
commands.extend([fvo])
print('[*] preprocess video', ' '.join(commands))
subprocess.call(commands)
tpreprocessvideoend = time.time()
print('[*] preprocessing of video to %s took %f seconds' %
(fvo, tpreprocessvideoend - tpreprocessvideostart))
args.video = fvo
tstart = time.time()
config = model.get_config(project_id=args.project_id)
config['project_id'] = args.project_id
config['video'] = args.video
config['keypoint_model'] = args.keypoint_model
config['autoencoder_model'] = args.autoencoder_model
config['objectdetection_model'] = args.objectdetection_model
config['train_video_ids'] = args.train_video_ids
config['test_video_ids'] = args.test_video_ids
config['minutes'] = args.minutes
#config['upper_bound'] = db.get_video_fixednumber(args.video_id)
#config['upper_bound'] = None
config['upper_bound'] = args.upper_bound
config['n_blocks'] = 4
config['tracking_method'] = args.tracking_method
config['track_tail'] = args.track_tail
config['sketch_file'] = args.sketch_file
config['file_tracking_results'] = args.output_tracking_results
config['use_all_data4train'] = args.use_all_data4train
config['object_detection_backbone'] = args.objectdetection_method
config = model.update_config_object_detection(config)
config['object_detection_resolution'] = [
int(r) for r in args.objectdetection_resolution.split('x')
]
config['keypoint_resolution'] = [
int(r) for r in args.keypoint_resolution.split('x')
]
config['img_height'], config['img_width'] = config[
'keypoint_resolution'][::-1]
config['kp_backbone'] = args.keypoint_method
if 'hourglass' in args.keypoint_method:
config['kp_num_hourglass'] = int(args.keypoint_method[9:])
config['kp_backbone'] = 'efficientnetLarge'
if args.inference_objectdetection_batchsize > 0:
config[
'inference_objectdetection_batchsize'] = args.inference_objectdetection_batchsize
if args.inference_keypoint_batchsize > 0:
config[
'inference_keypoint_batchsize'] = args.inference_keypoint_batchsize
if args.delete_all_checkpoints:
if os.path.isdir(os.path.expanduser('~/checkpoints/multitracker')):
shutil.rmtree(os.path.expanduser('~/checkpoints/multitracker'))
if args.data_dir:
db = dbconnection.DatabaseConnection(
file_db=os.path.join(args.data_dir, 'data.db'))
config['data_dir'] = args.data_dir
config['kp_data_dir'] = os.path.join(
args.data_dir, 'projects/%i/data' % config['project_id'])
config['kp_roi_dir'] = os.path.join(
args.data_dir, 'projects/%i/data_roi' % config['project_id'])
config['keypoint_names'] = db.get_keypoint_names(config['project_id'])
config['project_name'] = db.get_project_name(config['project_id'])
# <train models>
# 1) animal bounding box finetuning -> trains and inferences
config['objectdetection_max_steps'] = 30000
# train object detector
now = str(datetime.now()).replace(' ', '_').replace(':', '-').split('.')[0]
checkpoint_directory_object_detection = os.path.expanduser(
'~/checkpoints/multitracker/bbox/vids%s-%s' %
(config['train_video_ids'], now))
object_detect_restore = None
if 'objectdetection_model' in config and config[
'objectdetection_model'] is not None:
object_detect_restore = config['objectdetection_model']
detection_model = None
if object_detect_restore is None:
detection_model = finetune.finetune(
config,
checkpoint_directory_object_detection,
checkpoint_restore=object_detect_restore)
print('[*] trained object detection model',
checkpoint_directory_object_detection)
config[
'object_detection_model'] = checkpoint_directory_object_detection
## crop bbox detections and train keypoint estimation on extracted regions
#point_classification.calculate_keypoints(config, detection_file_bboxes)
# 2) train autoencoder for tracking appearence vector
if config['autoencoder_model'] is None and config[
'tracking_method'] == 'DeepSORT':
config_autoencoder = autoencoder.get_autoencoder_config()
config_autoencoder['project_id'] = config['project_id']
config_autoencoder['video_ids'] = natsorted(
list(
set([int(iid)
for iid in config['train_video_ids'].split(',')] +
[int(iid)
for iid in config['test_video_ids'].split(',')])))
config_autoencoder['project_name'] = config['project_name']
config_autoencoder['data_dir'] = config['data_dir']
config['autoencoder_model'] = autoencoder.train(config_autoencoder)
print('[*] trained autoencoder model', config['autoencoder_model'])
# 4) train keypoint estimator model
if config['keypoint_model'] is None and not config['kp_backbone'] == 'none':
config['kp_max_steps'] = 25000
config['keypoint_model'] = roi_segm.train(config)
print('[*] trained keypoint_model', config['keypoint_model'])
# </train models>
# <load models>
# load trained object detection model
if detection_model is None:
# load config json to know which backbone was used
with open(os.path.join(config['objectdetection_model'],
'config.json')) as json_file:
objconfig = json.load(json_file)
objconfig['objectdetection_model'] = config['objectdetection_model']
detection_model = finetune.load_trained_model(objconfig)
# load trained autoencoder model for Deep Sort Tracking
encoder_model = None
if config['tracking_method'] == 'DeepSORT':
encoder_model = inference.load_autoencoder_feature_extractor(config)
# load trained keypoint model
if config['kp_backbone'] == 'none':
keypoint_model = None
else:
keypoint_model = inference.load_keypoint_model(
config['keypoint_model'])
# </load models>
# 3) run bbox tracking deep sort with fixed tracks
nms_max_overlap = 1.0 # Non-maxima suppression threshold: Maximum detection overlap
max_cosine_distance = 0.2 # Gating threshold for cosine distance metric (object appearance).
nn_budget = None # Maximum size of the appearance descriptors gallery. If None, no budget is enforced.
print(4 * '\n', config)
ttrack_start = time.time()
if config['tracking_method'] == 'DeepSORT':
deep_sort_app.run(config, detection_model, encoder_model,
keypoint_model, args.min_confidence_boxes,
args.min_confidence_keypoints, nms_max_overlap,
max_cosine_distance, nn_budget)
elif config['tracking_method'] == 'UpperBound':
upperbound_tracker.run(config, detection_model, encoder_model,
keypoint_model, args.min_confidence_boxes,
args.min_confidence_keypoints)
elif config['tracking_method'] == 'VIoU':
viou_tracker.run(config, detection_model, encoder_model,
keypoint_model, args.min_confidence_boxes,
args.min_confidence_keypoints)
ttrack_end = time.time()
ugly_big_video_file_out = inference.get_video_output_filepath(config)
video_file_out = ugly_big_video_file_out.replace('.avi', '.mp4')
convert_video_h265(ugly_big_video_file_out, video_file_out)
print(
'[*] done tracking after %f minutes. outputting file' %
float(int((ttrack_end - ttrack_start) * 10.) / 10.), video_file_out)