def __init__(self, config_path, model_path): args = TrackArgs() args.config = config_path args.resume = model_path cfg = load_config(args) if args.arch == 'Custom': from custom import Custom self.model = Custom(anchors=cfg['anchors']) else: parser.error('invalid architecture: {}'.format(args.arch)) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format(args.resume) self.model = load_pretrain(self.model, args.resume) self.model.eval() self.device = torch.device('cuda' if (torch.cuda.is_available() and not args.cpu) else 'cpu') self.model = self.model.to(self.device) ################# Dangerous self.p = TrackerConfig() self.p.update(cfg['hp'] if 'hp' in cfg.keys() else None, self.model.anchors) self.p.renew() self.p.scales = self.model.anchors['scales'] self.p.ratios = self.model.anchors['ratios'] self.p.anchor_num = self.model.anchor_num self.p.anchor = generate_anchor(self.model.anchors, self.p.score_size) if self.p.windowing == 'cosine': self.window = np.outer(np.hanning(self.p.score_size), np.hanning(self.p.score_size)) elif self.p.windowing == 'uniform': self.window = np.ones((self.p.score_size, self.p.score_size)) self.window = np.tile(self.window.flatten(), self.p.anchor_num)
def processed_feature(mode, read_local=True, do_transform=True, woe=True): """ 模型训练主程序入口 """ assert os.path.exists('result/feature_engineering.pickle') if do_transform else True data = load_data(mode=mode, read_local=read_local) numeric_var, category_var, datetime_var, y_var, identifier_var, text_var = get_variable_type() feature = data[numeric_var + category_var + datetime_var + text_var] y = data[y_var] if mode == "train" else pd.Series([np.nan]*len(feature), name=y_var) # 自定义数据清洗 cs = Custom() feature, y = cs.clean_data(feature, y) # 数据类型转换 feature[numeric_var] = feature[numeric_var].astype('float') feature[category_var] = feature[category_var].astype('category') feature[datetime_var] = feature[datetime_var].astype('datetime64[ns]') # 自定义特征组合,全部为数值变量 feature = cs.feature_combination(feature, y) # 数据处理 if do_transform: feature_engineering = load_pickle('result/feature_engineering.pickle') feature = feature[feature_engineering['feature_selected']] feature = transform(feature, feature_engineering, woe=woe) return feature, y
def siammask_init(self, im, init_gt): im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) parser = argparse.ArgumentParser(description='PyTorch Tracking Demo') parser.add_argument('--resume', default=os.path.join(base_path, 'DiMP_LTMU/SiamMask/experiments/siammask/SiamMask_VOT_LD.pth'), type=str, metavar='PATH', help='path to latest checkpoint (default: none)') parser.add_argument('--config', dest='config', default=os.path.join(base_path, 'DiMP_LTMU/SiamMask/experiments/siammask/config_vot19lt.json'), help='hyper-parameter of SiamMask in json format') args = parser.parse_args() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model cfg = load_config(args) self.siammask = Custom(anchors=cfg['anchors']) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format(args.resume) self.siammask = load_pretrain(self.siammask, args.resume) self.siammask.eval().to(device) x = init_gt[0] y = init_gt[1] w = init_gt[2] h = init_gt[3] target_pos = np.array([x + w / 2, y + h / 2]) target_sz = np.array([w, h]) self.siamstate = siamese_init(im, target_pos, target_sz, self.siammask, cfg['hp'])
def __init__(self, cfg, min_iou=0.3, scale_factor=2, model="SiamMask_DAVIS.pth"): device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu' ) # to do - check if this should be instantiated for multiple SiamMasks objects self.internal_id = uuid.uuid4() # object identity can also work self.cfg = cfg self.siammask = Custom(anchors=cfg['anchors']) self.siammask = load_pretrain(self.siammask, model) self.siammask.eval().to(device) self.state = None self.prev_bbox = None self.is_recruited = False self.class_id = None self.min_iou = min_iou self.iou = None self.frames_elapsed_from_set_state = 0 self.last_tracking_result = None self.counter = Counter() self.scale_factor = scale_factor
def main(): global args, logger, v_id args = parser.parse_args() cfg = load_config(args) init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') logger.info(args) # setup model if args.arch == 'Custom': from custom import Custom model = Custom(anchors=cfg['anchors']) else: parser.error('invalid architecture: {}'.format(args.arch)) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format(args.resume) model = load_pretrain(model, args.resume) model.eval() device = torch.device('cuda' if (torch.cuda.is_available()) else 'cpu') model = model.to(device) # setup dataset dataset = load_dataset(args.dataset) # VOS or VOT? if args.dataset in ['DAVIS2016', 'DAVIS2017', 'ytb_vos'] and args.mask: vos_enable = True # enable Mask output else: vos_enable = False total_lost = 0 # VOT iou_lists = [] # VOS speed_list = [] for v_id, video in enumerate(dataset.keys(), start=1): if args.video != '' and video != args.video: continue if vos_enable: iou_list, speed = track_vos(model, dataset[video], cfg['hp'] if 'hp' in cfg.keys() else None, args.mask, args.refine, args.dataset in ['DAVIS2017', 'ytb_vos'], device=device) iou_lists.append(iou_list) else: lost, speed = track_vot(model, dataset[video], cfg['hp'] if 'hp' in cfg.keys() else None, args.mask, args.refine, device=device) total_lost += lost speed_list.append(speed) # report final result if vos_enable: for thr, iou in zip(thrs, np.mean(np.concatenate(iou_lists), axis=0)): logger.info('Segmentation Threshold {:.2f} mIoU: {:.3f}'.format(thr, iou)) else: logger.info('Total Lost: {:d}'.format(total_lost)) logger.info('Mean Speed: {:.2f} FPS'.format(np.mean(speed_list)))
def __init__(self,sample_im, base_dir='', x=0 ,y=0,w=10,h=10, use_tensorrt=False,fp16_mode=True,features_trt=True,rpn_trt=False,mask_trt=False,refine_trt=False): self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model args = argparse.Namespace() args.config = base_dir + 'SiamMask/experiments/siammask_sharp/config_vot.json' args.resume = base_dir + 'SiamMask/experiments/siammask_sharp/SiamMask_VOT.pth' self.cfg = load_config(args) from custom import Custom siammask = Custom(anchors=self.cfg['anchors']) siammask = load_pretrain(siammask, args.resume) if args.resume: assert isfile(args.resume), 'Please download {} first.'.format(args.resume) siammask = load_pretrain(siammask, args.resume) siammask.eval().to(self.device) target_pos = np.array([x + w / 2, y + h / 2]) target_sz = np.array([w, h]) self.state = siamese_init(sample_im, target_pos, target_sz, siammask, self.cfg['hp'], device=self.device) # init tracker if use_tensorrt: self.state['net'].init_trt(fp16_mode,features_trt,rpn_trt,mask_trt,refine_trt, trt_weights_path='/root/msl_raptor_ws/src/msl_raptor/src/front_end/SiamMask/weights_trt') self.keys_to_share = ['target_pos','target_sz','score','mask','ploygon'] self.states_each_object = [] self.current_classes = []
class SiamMaskWrapper(): def __init__(self, base_path=DEFAULT_BASE_PATH, config=DEFAULT_CONFIG, resume=DEFAULT_RESUME, cpu=False): args = Namespace(base_path=base_path, config=config, resume=resume, cpu=cpu) self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True self.state = None self.cfg = load_config(args) # TODO figure out the important parts of this self.siammask = Custom(anchors=self.cfg['anchors']) if args.resume: assert isfile(args.resume), 'Please download {} first.'.format(args.resume) self.siammask = load_pretrain(self.siammask, args.resume) self.siammask.eval().to(self.device) def select_region(self, image, xywh=None): """ image : 3 channel image The initial image with the object xywh : ArrayLike the position of the initial bounding rectangle as [x, y, w, h] If unspecified, a pop up selection will be used """ if xywh is None: xywh = cv2.selectROI('SiamMask', image, False, False) x, y, w, h = xywh # simply expand for convenience target_pos = np.array([x + w / 2, y + h / 2]) target_sz = np.array([w, h]) # init tracker self.state = siamese_init(image, target_pos, target_sz, self.siammask, self.cfg['hp'], device=self.device) def predict(self, image, visualize=False, verbose=False): self.state = siamese_track(self.state, image, mask_enable=True, refine_enable=True, device=self.device) target_pos = self.state["target_pos"] target_sz = self.state["target_sz"] score = self.state["score"] location = self.state['ploygon'].flatten() # compute as ltwh ltwh = np.concatenate((location[0:2], location[4:6] - location[0:2])) transformed_loc = [np.int0(location).reshape((-1, 1, 2))] if verbose: print("transformed loc : {}".format(transformed_loc)) mask = self.state['mask'] > self.state['p'].seg_thr image[:, :, 2] = (mask > 0) * 255 + (mask == 0) * image[:, :, 2] cv2.polylines(image, transformed_loc, True, (0, 255, 0), 3) #cv2.line(image, tuple(loc[0:2]), tuple(loc[0:2] + loc[2:]), (0, 255, 0)) if visualize: # return mask cv2.imshow('SiamMask', image) cv2.waitKey(10000) return ltwh, score, image # TODO the image should be a crop
def main(): global args, best_acc, tb_writer, logger args = parser.parse_args() # args通过解析获得的 init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') # 实例化一个记录器 logger.info("\n" + collect_env_info()) logger.info(args) cfg = load_config(args) logger.info("config \n{}".format(json.dumps( cfg, indent=4))) # 转变成json格式的文件,缩进4格 if args.log_dir: tb_writer = SummaryWriter(args.log_dir) else: tb_writer = Dummy() # build dataset train_loader, val_loader = build_data_loader(cfg) if args.arch == 'Custom': from custom import Custom model = Custom(pretrain=True, anchors=cfg['anchors']) else: exit() logger.info(model) if args.pretrained: model = load_pretrain(model, args.pretrained) model = model.cuda() # 模型转移到GPU上 dist_model = torch.nn.DataParallel( model, list(range(torch.cuda.device_count()))).cuda() # 多GPU训练 if args.resume and args.start_epoch != 0: # 这是在干啥?蒙蔽了!!!!! model.features.unfix((args.start_epoch - 1) / args.epochs) optimizer, lr_scheduler = build_opt_lr(model, cfg, args, args.start_epoch) # 如何构建优化器和学习策略??? # optionally resume from a checkpoint if args.resume: assert os.path.isfile(args.resume), '{} is not a valid file'.format( args.resume) model, optimizer, args.start_epoch, best_acc, arch = restore_from( model, optimizer, args.resume) dist_model = torch.nn.DataParallel( model, list(range(torch.cuda.device_count()))).cuda() logger.info(lr_scheduler) logger.info('model prepare done') train(train_loader, dist_model, optimizer, lr_scheduler, args.start_epoch, cfg)
def get_siammask(): siammask = Custom(anchors=cfg['anchors']) if args.resume: assert isfile(args.resume), 'Please download {} first.'.format( args.resume) siammask = load_pretrain(siammask, args.resume) siammask.eval().to(device) return siammask
def track_init(img, x, y, w, h, device): cfg = load_config('config_davis.json') # Setup Model from custom import Custom siammask = Custom(anchors=cfg['anchors']) siammask = load_pretrain(siammask, 'SiamMask_VOT.pth') siammask.eval().to(device) target_pos = np.array([x + w / 2, y + h / 2]) target_sz = np.array([w, h]) state = siamese_init(img, target_pos, target_sz, siammask, cfg['hp'], device=device) return state
def main(): global args, best_acc, tb_writer, logger args = parser.parse_args() init_log('global', logging.INFO) # 返回一个logger对象,logging_INFO是日志的等级 if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') # 获取上面初始化的logger对象 logger.info("\n" + collect_env_info()) logger.info(args) cfg = load_config(args) # 返回修改后的配置文件对象 logger.info("config \n{}".format(json.dumps(cfg, indent=4))) #json.loads()是将str转化成dict格式,json.dumps()是将dict转化成str格式。 if args.log_dir: tb_writer = SummaryWriter(args.log_dir) else: tb_writer = Dummy() # build dataset train_loader, val_loader = build_data_loader(cfg) if args.arch == 'Custom': from custom import Custom model = Custom(pretrain=True, anchors=cfg['anchors']) else: exit() logger.info(model) if args.pretrained: model = load_pretrain(model, args.pretrained) model = model.cuda() dist_model = torch.nn.DataParallel(model, list(range(torch.cuda.device_count()))).cuda() if args.resume and args.start_epoch != 0: model.features.unfix((args.start_epoch - 1) / args.epochs) optimizer, lr_scheduler = build_opt_lr(model, cfg, args, args.start_epoch) # optionally resume from a checkpoint if args.resume: assert os.path.isfile(args.resume), '{} is not a valid file'.format(args.resume) model, optimizer, args.start_epoch, best_acc, arch = restore_from(model, optimizer, args.resume) dist_model = torch.nn.DataParallel(model, list(range(torch.cuda.device_count()))).cuda() logger.info(lr_scheduler) logger.info('model prepare done') train(train_loader, dist_model, optimizer, lr_scheduler, args.start_epoch, cfg)
def __init__(self, resume=os.path.join(os.path.dirname(__file__), 'experiments/siammask_sharp/SiamMask_DAVIS.pth'), \ config=os.path.join(os.path.dirname(__file__), 'experiments/siammask_sharp/config_davis.json')): self.args = EasyDict(resume=resume, config=config) self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model self.cfg = load_config(self.args) self.siammask = Custom(anchors=self.cfg['anchors']) assert isfile(self.args.resume), 'Please download {} first.'.format( self.args.resume) self.siammask = load_pretrain(self.siammask, self.args.resume) self.siammask = self.siammask.eval().half().to(self.device)
def __init__(self): resume = '/home/saad/Root/vision/Computer_Vision/Tracking-systems/SiamMask_DAVIS.pth' config = '/home/saad/Root/vision/Computer_Vision/Tracking-systems/config_davis.json' self.cfg = load_config(config=config) self.siammask = Custom(anchors=self.cfg['anchors']) self.siammask = load_pretrain(self.siammask, resume) self.active_boxes = [] self.frames_generator = get_detection_output_as_frames_generator() self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.siammask.eval().to(self.device) self.tracker_initialized = False self.state = None self.times = np_.asarray([])
def __init__(self, base_path=DEFAULT_BASE_PATH, config=DEFAULT_CONFIG, resume=DEFAULT_RESUME, cpu=False): args = Namespace(base_path=base_path, config=config, resume=resume, cpu=cpu) self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True self.state = None self.cfg = load_config(args) # TODO figure out the important parts of this self.siammask = Custom(anchors=self.cfg['anchors']) if args.resume: assert isfile(args.resume), 'Please download {} first.'.format(args.resume) self.siammask = load_pretrain(self.siammask, args.resume) self.siammask.eval().to(self.device)
def main(): global args, best_acc, tb_writer, logger args = parser.parse_args() args = args_process(args) init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') logger.info("\n" + collect_env_info()) logger.info(args) cfg = load_config(args) logger.info("config \n{}".format(json.dumps(cfg, indent=4))) # build dataset train_loader, val_loader = build_data_loader(cfg) args.img_size = int(cfg['train_datasets']['search_size']) args.nms_threshold = float(cfg['train_datasets']['RPN_NMS']) if args.arch == 'Custom': from custom import Custom model = Custom(pretrain=True, opts=args, anchors=train_loader.dataset.anchors) else: exit() logger.info(model) if args.pretrained: model = load_pretrain(model, args.pretrained) else: raise Exception("Pretrained weights must be loaded!") model = model.cuda() dist_model = torch.nn.DataParallel(model, list(range( torch.cuda.device_count()))).cuda() logger.info('model prepare done') logger = logging.getLogger('global') val_avg = AverageMeter() validation(val_loader, dist_model, cfg, val_avg)
def __init__(self): # Setup device device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model args = FakeArgParser() cfg = libsiam.load_config(args) siammask = Custom(anchors=cfg['anchors']) if args.resume: assert libsiam.isfile( args.resume), '{} is not a valid file'.format(args.resume) siammask = libsiam.load_pretrain(siammask, args.resume) siammask.eval().to(device) # -- Output self.siammask = siammask self.args = args self.cfg = cfg self.state = None
def main(mode, model_path, do_transform, read_local, woe, score=False, save_remote=False): # 数据、模型加载 model, threshold = load_pickle(model_path) data = load_data(mode=mode, read_local=read_local) feature, y = processed_feature(do_transform=do_transform, mode=mode, read_local=read_local, woe=woe) cs = Custom() # 应用预测 print(">>> 应用预测") res_label = pd.DataFrame(model.predict(feature), columns=['label_predict']) res_prob = pd.DataFrame(model.predict_proba(feature), columns=['probability_0', "probability_1"]) res_prob[ 'res_odds'] = res_prob['probability_0'] / res_prob["probability_1"] res_prob['label_threshold'] = res_prob['probability_1'].apply( lambda x: 0 if x < threshold else 1) res = pd.concat([data, res_label, res_prob], axis=1) if score: print(">>> 概率转换评分") odds = config.get('SCORECARD', 'odds') score = config.get('SCORECARD', 'score') pdo = config.get('SCORECARD', 'pdo') a, b = make_score(odds, score, pdo) res['score'] = res_prob['res_odds'].apply( lambda x: a + b * log(float(x))) bins = tree_binning(res[y.name], res['score'].to_frame( ))[0]["result"]["score"] if mode == "train" else cs.adjust_bins if bins: print(">>> 数据集分组") res['level'] = pd.cut(res['score'], bins) temp = res.groupby("level", as_index=False).count() temp['rate'] = temp['label_threshold'] / feature.shape[0] temp = temp[['level', 'rate']] print(temp) print(res.head()) # 结果保存 print(f">>> 结果保存中,保存模式:{save_remote}") res['load_date'] = str(date.today()) save_result(res, filename=f"{mode}_result.csv", remote=save_remote)
def __init__(self, is_cpu=True, epsilon=0.01): is_cpu = not torch.cuda.is_available() and is_cpu configarg = Namespace(resume="backends/SiamMask/experiments/siammask_sharp/SiamMask_DAVIS.pth", config="backends/SiamMask/experiments/siammask_sharp/config_davis.json", base_path="", cpu=is_cpu, ) device = torch.device("cuda" if not is_cpu else "cpu") self.cfg = load_config(configarg) siammask = Custom(anchors=self.cfg['anchors']) self.siammask = load_pretrain(siammask, configarg.resume) self.siammask.eval().to(device) self.device = device self.epsilon=epsilon
def __init__(self): # Setup device self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True base_dir = os.path.abspath(os.environ.get("MODEL_PATH", "/opt/nuclio/SiamMask/experiments/siammask_sharp")) class configPath: config = os.path.join(base_dir, "config_davis.json") self.config = load_config(configPath) from custom import Custom siammask = Custom(anchors=self.config['anchors']) self.siammask = load_pretrain(siammask, os.path.join(base_dir, "SiamMask_DAVIS.pth")) self.siammask.eval().to(self.device)
def feature_engineering(read_local): """ 项目特征工程程序入口 """ # 加载数据、变量类型划分、特征集与标签列划分 print(" >>> 数据加载") feature, y = processed_feature(mode="train", do_transform=False, read_local=read_local) print(f" >>> 当前数据规模: {feature.shape}\n" f" 特征工程预处理数:{feature.shape[1]}\n" ) cs = Custom() # 标准化特征工程 print(">>> 特征工程标准进程") fe = FeatureEngineering(feature, y, cs.target) fe.run()
class SiamTracker(object): def __init__(self, resume=os.path.join(os.path.dirname(__file__), 'experiments/siammask_sharp/SiamMask_DAVIS.pth'), \ config=os.path.join(os.path.dirname(__file__), 'experiments/siammask_sharp/config_davis.json')): self.args = EasyDict(resume=resume, config=config) self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model self.cfg = load_config(self.args) self.siammask = Custom(anchors=self.cfg['anchors']) assert isfile(self.args.resume), 'Please download {} first.'.format( self.args.resume) self.siammask = load_pretrain(self.siammask, self.args.resume) self.siammask = self.siammask.eval().half().to(self.device) def get_state(self, im, bbox): x, y = bbox[0], bbox[1] w, h = bbox[2] - x, bbox[3] - y target_pos = np.array([x + w / 2, y + h / 2]) target_sz = np.array([w, h]) state = siamese_init(im, target_pos, target_sz, self.siammask, self.cfg['hp'], device=self.device) state['track_id'] = np.random.randint(1000000) state['box'] = bbox state['score'] = 1. return state def track(self, state, im): new_state = siamese_track(state, im, self.siammask, mask_enable=False, refine_enable=False, device=self.device) new_state['track_id'] = state['track_id'] p = new_state['target_pos'] sz = new_state['target_sz'] new_state['box'] = np.concatenate((p - sz / 2, p + sz / 2)) return new_state
help='hyper-parameter of SiamMask in json format') parser.add_argument('--base_path', default='../../data/input', help='datasets') #parser.add_argument('--base_path', default='../../data/tennis', help='datasets') parser.add_argument('--cpu', action='store_true', help='cpu mode') args = parser.parse_args() if __name__ == '__main__': # Setup device device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model cfg = load_config(args) from custom import Custom siammask = Custom(anchors=cfg['anchors']) if args.resume: assert isfile(args.resume), 'Please download {} first.'.format(args.resume) siammask = load_pretrain(siammask, args.resume) siammask.eval().to(device) # Parse Image file img_files = sorted(glob.glob(join(args.base_path, '*.PN*'))) print(img_files) ims = [cv2.imread(imf) for imf in img_files[130:150]] #img_files = sorted(glob.glob(join(args.base_path, '*.jp*'))) #ims = [cv2.imread(imf) for imf in img_files] # Select ROI
def main(): init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) params = {'penalty_k': args.penalty_k, 'window_influence': args.window_influence, 'lr': args.lr, 'instance_size': args.search_region} num_search = len(params['penalty_k']) * len(params['window_influence']) * \ len(params['lr']) * len(params['instance_size']) print(params) print(num_search) cfg = load_config(args) if args.arch == 'Custom': from custom import Custom model = Custom(anchors=cfg['anchors']) else: model = models.__dict__[args.arch](anchors=cfg['anchors']) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format(args.resume) model = load_pretrain(model, args.resume) model.eval() model = model.to(device) default_hp = cfg.get('hp', {}) p = dict() p['network'] = model p['network_name'] = args.arch+'_'+args.resume.split('/')[-1].split('.')[0] p['dataset'] = args.dataset global ims, gt, image_files dataset_info = load_dataset(args.dataset) videos = list(dataset_info.keys()) np.random.shuffle(videos) for video in videos: print(video) if isfile('finish.flag'): return p['video'] = video ims = None image_files = dataset_info[video]['image_files'] gt = dataset_info[video]['gt'] np.random.shuffle(params['penalty_k']) np.random.shuffle(params['window_influence']) np.random.shuffle(params['lr']) for penalty_k in params['penalty_k']: for window_influence in params['window_influence']: for lr in params['lr']: for instance_size in params['instance_size']: p['hp'] = default_hp.copy() p['hp'].update({'penalty_k':penalty_k, 'window_influence':window_influence, 'lr':lr, 'instance_size': instance_size, }) tune(p)
def main(): global args, best_acc, tb_writer, logger args = parser.parse_args() init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') logger.info(args) cfg = load_config(args) logger.info("config \n{}".format(json.dumps(cfg, indent=4))) logger.info("\n" + collect_env_info()) if args.log_dir: tb_writer = SummaryWriter(args.log_dir) else: tb_writer = Dummy() # build dataset train_loader, val_loader = build_data_loader(cfg) if args.arch == 'Custom': from custom import Custom model = Custom(pretrain=True, anchors=cfg['anchors']) else: model = models.__dict__[args.arch](anchors=cfg['anchors']) logger.info(model) if args.pretrained: model = load_pretrain(model, args.pretrained) model = model.cuda() dist_model = torch.nn.DataParallel(model, list(range(torch.cuda.device_count()))).cuda() if args.resume and args.start_epoch != 0: model.features.unfix((args.start_epoch - 1) / args.epochs) optimizer, lr_scheduler = build_opt_lr(model, cfg, args, args.start_epoch) logger.info(lr_scheduler) # optionally resume from a checkpoint if args.resume: assert os.path.isfile(args.resume), '{} is not a valid file'.format(args.resume) model, optimizer, args.start_epoch, best_acc, arch = restore_from(model, optimizer, args.resume) dist_model = torch.nn.DataParallel(model, list(range(torch.cuda.device_count()))).cuda() epoch = args.start_epoch if dist_model.module.features.unfix(epoch/args.epochs): logger.info('unfix part model.') optimizer, lr_scheduler = build_opt_lr(dist_model.module, cfg, args, epoch) lr_scheduler.step(epoch) cur_lr = lr_scheduler.get_cur_lr() logger.info('epoch:{} resume lr {}'.format(epoch, cur_lr)) logger.info('model prepare done') train(train_loader, dist_model, optimizer, lr_scheduler, args.start_epoch, cfg)
class Dimp_LTMU_Tracker(object): def __init__(self, image, region, p=None, groundtruth=None): self.p = p self.i = 0 self.t_id = 0 if groundtruth is not None: self.groundtruth = groundtruth tfconfig = tf.ConfigProto() tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.3 self.sess = tf.Session(config=tfconfig) init_gt1 = [region.x, region.y, region.width, region.height] init_gt = [ init_gt1[1], init_gt1[0], init_gt1[1] + init_gt1[3], init_gt1[0] + init_gt1[2] ] # ymin xmin ymax xmax self.last_gt = init_gt self.init_pymdnet(image, init_gt1) self.local_init(image, init_gt1) self.Golbal_Track_init(image, init_gt1) if self.p.use_mask: self.siammask_init(image, init_gt1) self.tc_init(self.p.model_dir) self.metric_init(image, np.array(init_gt1)) self.dis_record = [] self.state_record = [] self.rv_record = [] self.all_map = [] self.count = 0 local_state1, self.score_map, update, self.score_max, dis, flag, update_score = self.local_track( image) self.local_Tracker.pos = torch.FloatTensor([ (self.last_gt[0] + self.last_gt[2] - 1) / 2, (self.last_gt[1] + self.last_gt[3] - 1) / 2 ]) self.local_Tracker.target_sz = torch.FloatTensor([ (self.last_gt[2] - self.last_gt[0]), (self.last_gt[3] - self.last_gt[1]) ]) def get_first_state(self): return self.score_map, self.score_max def siammask_init(self, im, init_gt): im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) parser = argparse.ArgumentParser(description='PyTorch Tracking Demo') parser.add_argument( '--resume', default='SiamMask/experiments/siammask/SiamMask_VOT_LD.pth', type=str, metavar='PATH', help='path to latest checkpoint (default: none)') parser.add_argument( '--config', dest='config', default='SiamMask/experiments/siammask/config_vot19lt.json', help='hyper-parameter of SiamMask in json format') args = parser.parse_args() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True # Setup Model cfg = load_config(args) self.siammask = Custom(anchors=cfg['anchors']) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format( args.resume) self.siammask = load_pretrain(self.siammask, args.resume) self.siammask.eval().to(device) x = init_gt[0] y = init_gt[1] w = init_gt[2] h = init_gt[3] target_pos = np.array([x + w / 2, y + h / 2]) target_sz = np.array([w, h]) self.siamstate = siamese_init(im, target_pos, target_sz, self.siammask, cfg['hp']) def siammask_track(self, im): im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) self.siamstate = siamese_track(self.siamstate, im, mask_enable=True, refine_enable=True) # track # pdb.set_trace() score = np.max(self.siamstate['score']) location = self.siamstate['ploygon'].flatten() mask = self.siamstate['mask'] > self.siamstate['p'].seg_thr # im[:, :, 2] = (mask > 0) * 255 + (mask == 0) * im[:, :, 2] # # cv2.namedWindow("SiamMask", cv2.WINDOW_NORMAL) # cv2.rectangle(im, (int(self.siamstate['target_pos'][0] - self.siamstate['target_sz'][0] / 2.0), # int(self.siamstate['target_pos'][1] - self.siamstate['target_sz'][1] / 2.0)), # (int(self.siamstate['target_pos'][0] + self.siamstate['target_sz'][0] / 2.0), # int(self.siamstate['target_pos'][1] + self.siamstate['target_sz'][1] / 2.0)), [0, 255, 0], 2) # # cv2.imwrite("/home/xiaobai/Desktop/MBMD_vot_code/figure/%05d.jpg"%frame_id, im[:, :, -1::-1]) # cv2.imshow("SiamMask", im) # cv2.waitKey(1) return score, mask def Golbal_Track_init(self, image, init_box): init_box = [ init_box[0], init_box[1], init_box[0] + init_box[2], init_box[1] + init_box[3] ] cfg_file = 'Global_Track/configs/qg_rcnn_r50_fpn.py' ckp_file = 'Global_Track/checkpoints/qg_rcnn_r50_fpn_coco_got10k_lasot.pth' transforms = data.BasicPairTransforms(train=False) self.Global_Tracker = GlobalTrack(cfg_file, ckp_file, transforms, name_suffix='qg_rcnn_r50_fpn') self.Global_Tracker.init(image, init_box) def Global_Track_eval(self, image, num): # xywh results = self.Global_Tracker.update(image) index = np.argsort(results[:, -1])[::-1] max_index = index[:num] can_boxes = results[max_index][:, :4] can_boxes = np.array([ can_boxes[:, 0], can_boxes[:, 1], can_boxes[:, 2] - can_boxes[:, 0], can_boxes[:, 3] - can_boxes[:, 1] ]).transpose() return can_boxes def init_pymdnet(self, image, init_bbox): target_bbox = np.array(init_bbox) self.last_result = target_bbox self.pymodel = MDNet('./pyMDNet/models/mdnet_imagenet_vid.pth') if opts['use_gpu']: self.pymodel = self.pymodel.cuda() self.pymodel.set_learnable_params(opts['ft_layers']) # Init criterion and optimizer self.criterion = BCELoss() init_optimizer = set_optimizer(self.pymodel, opts['lr_init'], opts['lr_mult']) self.update_optimizer = set_optimizer(self.pymodel, opts['lr_update'], opts['lr_mult']) tic = time.time() # Draw pos/neg samples pos_examples = SampleGenerator('gaussian', image.size, opts['trans_pos'], opts['scale_pos'])( target_bbox, opts['n_pos_init'], opts['overlap_pos_init']) neg_examples = np.concatenate([ SampleGenerator('uniform', image.size, opts['trans_neg_init'], opts['scale_neg_init'])(target_bbox, int(opts['n_neg_init'] * 0.5), opts['overlap_neg_init']), SampleGenerator('whole', image.size)(target_bbox, int(opts['n_neg_init'] * 0.5), opts['overlap_neg_init']) ]) neg_examples = np.random.permutation(neg_examples) # Extract pos/neg features pos_feats = forward_samples(self.pymodel, image, pos_examples, opts) neg_feats = forward_samples(self.pymodel, image, neg_examples, opts) self.feat_dim = pos_feats.size(-1) # Initial training train(self.pymodel, self.criterion, init_optimizer, pos_feats, neg_feats, opts['maxiter_init'], opts=opts) del init_optimizer, neg_feats torch.cuda.empty_cache() # Train bbox regressor bbreg_examples = SampleGenerator( 'uniform', image.size, opts['trans_bbreg'], opts['scale_bbreg'], opts['aspect_bbreg'])(target_bbox, opts['n_bbreg'], opts['overlap_bbreg']) bbreg_feats = forward_samples(self.pymodel, image, bbreg_examples, opts) self.bbreg = BBRegressor(image.size) self.bbreg.train(bbreg_feats, bbreg_examples, target_bbox) del bbreg_feats torch.cuda.empty_cache() # Init sample generators self.sample_generator = SampleGenerator('gaussian', image.size, opts['trans'], opts['scale']) self.pos_generator = SampleGenerator('gaussian', image.size, opts['trans_pos'], opts['scale_pos']) self.neg_generator = SampleGenerator('uniform', image.size, opts['trans_neg'], opts['scale_neg']) # Init pos/neg features for update neg_examples = self.neg_generator(target_bbox, opts['n_neg_update'], opts['overlap_neg_init']) neg_feats = forward_samples(self.pymodel, image, neg_examples, opts) self.pos_feats_all = [pos_feats] self.neg_feats_all = [neg_feats] spf_total = time.time() - tic def pymdnet_eval(self, image, samples): sample_scores = forward_samples(self.pymodel, image, samples, out_layer='fc6', opts=opts) return sample_scores[:, 1][:].cpu().numpy() # def pymdnet_track(self, image): # self.image = image # target_bbox = self.last_result # samples = self.sample_generator(target_bbox, opts['n_samples']) # sample_scores = forward_samples(self.pymodel, image, samples, out_layer='fc6', opts=opts) # # top_scores, top_idx = sample_scores[:, 1].topk(5) # top_idx = top_idx.cpu().numpy() # target_score = top_scores.mean() # target_bbox = samples[top_idx].mean(axis=0) # # success = target_score > 0 # # # Expand search area at failure # if success: # self.sample_generator.set_trans(opts['trans']) # else: # self.sample_generator.expand_trans(opts['trans_limit']) # # self.last_result = target_bbox # # Bbox regression # bbreg_bbox = self.pymdnet_bbox_reg(success, samples, top_idx) # # # Save result # region = bbreg_bbox # # # Data collect # if success: # self.collect_samples_pymdnet() # # # Short term update # if not success: # self.pymdnet_short_term_update() # # # Long term update # elif self.i % opts['long_interval'] == 0: # self.pymdnet_long_term_update() # # return region, target_score def collect_samples_pymdnet(self, image): self.t_id += 1 target_bbox = np.array([ self.last_gt[1], self.last_gt[0], self.last_gt[3] - self.last_gt[1], self.last_gt[2] - self.last_gt[0] ]) pos_examples = self.pos_generator(target_bbox, opts['n_pos_update'], opts['overlap_pos_update']) if len(pos_examples) > 0: pos_feats = forward_samples(self.pymodel, image, pos_examples, opts) self.pos_feats_all.append(pos_feats) if len(self.pos_feats_all) > opts['n_frames_long']: del self.pos_feats_all[0] neg_examples = self.neg_generator(target_bbox, opts['n_neg_update'], opts['overlap_neg_update']) if len(neg_examples) > 0: neg_feats = forward_samples(self.pymodel, image, neg_examples, opts) self.neg_feats_all.append(neg_feats) if len(self.neg_feats_all) > opts['n_frames_short']: del self.neg_feats_all[0] def pymdnet_short_term_update(self): # Short term update nframes = min(opts['n_frames_short'], len(self.pos_feats_all)) pos_data = torch.cat(self.pos_feats_all[-nframes:], 0) neg_data = torch.cat(self.neg_feats_all, 0) train(self.pymodel, self.criterion, self.update_optimizer, pos_data, neg_data, opts['maxiter_update'], opts=opts) def pymdnet_long_term_update(self): if self.t_id % opts['long_interval'] == 0: # Long term update pos_data = torch.cat(self.pos_feats_all, 0) neg_data = torch.cat(self.neg_feats_all, 0) train(self.pymodel, self.criterion, self.update_optimizer, pos_data, neg_data, opts['maxiter_update'], opts=opts) # # def pymdnet_bbox_reg(self, success, samples, top_idx): # target_bbox = self.last_result # if success: # bbreg_samples = samples[top_idx] # if top_idx.shape[0] == 1: # bbreg_samples = bbreg_samples[None, :] # bbreg_feats = forward_samples(self.pymodel, self.image, bbreg_samples, opts) # bbreg_samples = self.bbreg.predict(bbreg_feats, bbreg_samples) # bbreg_bbox = bbreg_samples.mean(axis=0) # else: # bbreg_bbox = target_bbox # return bbreg_bbox def metric_init(self, im, init_box): self.metric_model = ft_net(class_num=1120) path = '../utils/metric_net/metric_model/metric_model.pt' self.metric_model.eval() self.metric_model = self.metric_model.cuda() self.metric_model.load_state_dict(torch.load(path)) tmp = np.random.rand(1, 3, 107, 107) tmp = (Variable(torch.Tensor(tmp))).type(torch.FloatTensor).cuda() # get target feature self.metric_model(tmp) init_box = init_box.reshape((1, 4)) anchor_region = me_extract_regions(im, init_box) anchor_region = process_regions(anchor_region) anchor_region = torch.Tensor(anchor_region) anchor_region = (Variable(anchor_region)).type( torch.FloatTensor).cuda() self.anchor_feature, _ = self.metric_model(anchor_region) def metric_eval(self, im, boxes, anchor_feature): box_regions = me_extract_regions(np.array(im), boxes) box_regions = process_regions(box_regions) box_regions = torch.Tensor(box_regions) box_regions = (Variable(box_regions)).type(torch.FloatTensor).cuda() box_features, class_result = self.metric_model(box_regions) class_result = torch.softmax(class_result, dim=1) ap_dist = torch.norm(anchor_feature - box_features, 2, dim=1).view(-1) return ap_dist def tc_init(self, model_dir): self.tc_model = tclstm() self.X_input = tf.placeholder( "float", [None, tcopts['time_steps'], tcopts['lstm_num_input']]) self.maps = tf.placeholder("float", [None, 19, 19, 1]) self.map_logits = self.tc_model.map_net(self.maps) self.Inputs = tf.concat((self.X_input, self.map_logits), axis=2) self.logits, _ = self.tc_model.net(self.Inputs) variables_to_restore = [ var for var in tf.global_variables() if (var.name.startswith('tclstm') or var.name.startswith('mapnet')) ] saver = tf.train.Saver(var_list=variables_to_restore) if self.p.checkpoint is None: checkpoint = tf.train.latest_checkpoint( os.path.join('./meta_updater', model_dir)) else: checkpoint = './meta_updater/' + self.p.model_dir + '/lstm_model.ckpt-' + str( self.p.checkpoint) saver.restore(self.sess, checkpoint) def local_init(self, image, init_bbox): local_tracker = Tracker('dimp', 'dimp50') params = local_tracker.get_parameters() debug_ = getattr(params, 'debug', 0) params.debug = debug_ params.tracker_name = local_tracker.name params.param_name = local_tracker.parameter_name self.local_Tracker = local_tracker.tracker_class(params) init_box = dict() init_box['init_bbox'] = init_bbox self.local_Tracker.initialize(image, init_box) def local_track(self, image): state, score_map, test_x, scale_ind, sample_pos, sample_scales, flag, s = self.local_Tracker.track_updater( image) update_score = 0 update_flag = flag not in ['not_found', 'uncertain'] update = update_flag max_score = max(score_map.flatten()) self.all_map.append(score_map) local_state = np.array(state).reshape((1, 4)) ap_dis = self.metric_eval(image, local_state, self.anchor_feature) self.dis_record.append(ap_dis.data.cpu().numpy()[0]) h = image.shape[0] w = image.shape[1] self.state_record.append([ local_state[0][0] / w, local_state[0][1] / h, (local_state[0][0] + local_state[0][2]) / w, (local_state[0][1] + local_state[0][3]) / h ]) self.rv_record.append(max_score) if len(self.state_record) >= self.p.start_frame: dis = np.array(self.dis_record[-tcopts["time_steps"]:]).reshape( (tcopts["time_steps"], 1)) rv = np.array(self.rv_record[-tcopts["time_steps"]:]).reshape( (tcopts["time_steps"], 1)) state_tc = np.array(self.state_record[-tcopts["time_steps"]:]) map_input = np.array(self.all_map[-tcopts["time_steps"]:]) map_input = np.reshape(map_input, [tcopts['time_steps'], 1, 19, 19]) map_input = map_input.transpose((0, 2, 3, 1)) X_input = np.concatenate((state_tc, rv, dis), axis=1) logits = self.sess.run(self.logits, feed_dict={ self.X_input: np.expand_dims(X_input, axis=0), self.maps: map_input }) update = logits[0][0] < logits[0][1] update_score = logits[0][1] hard_negative = (flag == 'hard_negative') learning_rate = getattr(self.local_Tracker.params, 'hard_negative_learning_rate', None) if hard_negative else None if update: # Get train sample train_x = test_x[scale_ind:scale_ind + 1, ...] # Create target_box and label for spatial sample target_box = self.local_Tracker.get_iounet_box( self.local_Tracker.pos, self.local_Tracker.target_sz, sample_pos[scale_ind, :], sample_scales[scale_ind]) # Update the classifier model self.local_Tracker.update_classifier(train_x, target_box, learning_rate, s[scale_ind, ...]) self.last_gt = [ state[1], state[0], state[1] + state[3], state[0] + state[2] ] return state, score_map, update, max_score, ap_dis.data.cpu().numpy( )[0], flag, update_score def locate(self, image): # Convert image im = numpy_to_torch(image) self.local_Tracker.im = im # For debugging only # ------- LOCALIZATION ------- # # Get sample sample_pos = self.local_Tracker.pos.round() sample_scales = self.local_Tracker.target_scale * self.local_Tracker.params.scale_factors test_x = self.local_Tracker.extract_processed_sample( im, self.local_Tracker.pos, sample_scales, self.local_Tracker.img_sample_sz) # Compute scores scores_raw = self.local_Tracker.apply_filter(test_x) translation_vec, scale_ind, s, flag = self.local_Tracker.localize_target( scores_raw) return translation_vec, scale_ind, s, flag, sample_pos, sample_scales, test_x def local_update(self, sample_pos, translation_vec, scale_ind, sample_scales, s, test_x, update_flag=None): # Check flags and set learning rate if hard negative if update_flag is None: update_flag = self.flag not in ['not_found', 'uncertain'] hard_negative = (self.flag == 'hard_negative') learning_rate = self.local_Tracker.params.hard_negative_learning_rate if hard_negative else None if update_flag: # Get train sample train_x = TensorList( [x[scale_ind:scale_ind + 1, ...] for x in test_x]) # Create label for sample train_y = self.local_Tracker.get_label_function( sample_pos, sample_scales[scale_ind]) # Update memory self.local_Tracker.update_memory(train_x, train_y, learning_rate) # Train filter if hard_negative: self.local_Tracker.filter_optimizer.run( self.local_Tracker.params.hard_negative_CG_iter) elif (self.local_Tracker.frame_num - 1) % self.local_Tracker.params.train_skipping == 0: self.local_Tracker.filter_optimizer.run( self.local_Tracker.params.CG_iter) def tracking(self, image): self.i += 1 mask = None candidate_bboxes = None # state, pyscore = self.pymdnet_track(image) # self.last_gt = [state[1], state[0], state[1] + state[3], state[0] + state[2]] self.local_Tracker.pos = torch.FloatTensor([ (self.last_gt[0] + self.last_gt[2] - 1) / 2, (self.last_gt[1] + self.last_gt[3] - 1) / 2 ]) self.local_Tracker.target_sz = torch.FloatTensor([ (self.last_gt[2] - self.last_gt[0]), (self.last_gt[3] - self.last_gt[1]) ]) tic = time.time() local_state, self.score_map, update, local_score, dis, flag, update_score = self.local_track( image) md_score = self.pymdnet_eval(image, np.array(local_state).reshape([-1, 4]))[0] self.score_max = md_score if md_score > 0 and flag == 'normal': self.flag = 'found' if self.p.use_mask: self.siamstate['target_pos'] = self.local_Tracker.pos.numpy( )[::-1] self.siamstate[ 'target_sz'] = self.local_Tracker.target_sz.numpy()[::-1] siamscore, mask = self.siammask_track( cv2.cvtColor(image, cv2.COLOR_RGB2BGR)) self.local_Tracker.pos = torch.FloatTensor( self.siamstate['target_pos'][::-1].copy()) self.local_Tracker.target_sz = torch.FloatTensor( self.siamstate['target_sz'][::-1].copy()) local_state = torch.cat( (self.local_Tracker.pos[[1, 0]] - (self.local_Tracker.target_sz[[1, 0]] - 1) / 2, self.local_Tracker.target_sz[[1, 0]])).data.cpu().numpy() self.last_gt = np.array([ local_state[1], local_state[0], local_state[1] + local_state[3], local_state[0] + local_state[2] ]) elif md_score < 0 or flag == 'not_found': self.count += 1 self.flag = 'not_found' candidate_bboxes = self.Global_Track_eval(image, 10) candidate_scores = self.pymdnet_eval(image, candidate_bboxes) max_id = np.argmax(candidate_scores) if candidate_scores[max_id] > 0: redet_bboxes = candidate_bboxes[max_id] if self.count >= 5: self.last_gt = np.array([ redet_bboxes[1], redet_bboxes[0], redet_bboxes[1] + redet_bboxes[3], redet_bboxes[2] + redet_bboxes[0] ]) self.local_Tracker.pos = torch.FloatTensor([ (self.last_gt[0] + self.last_gt[2] - 1) / 2, (self.last_gt[1] + self.last_gt[3] - 1) / 2 ]) self.local_Tracker.target_sz = torch.FloatTensor([ (self.last_gt[2] - self.last_gt[0]), (self.last_gt[3] - self.last_gt[1]) ]) self.score_max = candidate_scores[max_id] self.count = 0 if update: self.collect_samples_pymdnet(image) self.pymdnet_long_term_update() width = self.last_gt[3] - self.last_gt[1] height = self.last_gt[2] - self.last_gt[0] toc = time.time() - tic print(toc) # if self.flag == 'found' and self.score_max > 0: # confidence_score = 0.99 # elif self.flag == 'not_found': # confidence_score = 0.0 # else: # confidence_score = np.clip((local_score+np.arctan(0.2*self.score_max)/math.pi+0.5)/2, 0, 1) confidence_score = np.clip( (local_score + np.arctan(0.2 * self.score_max) / math.pi + 0.5) / 2, 0, 1) if self.p.visualization: show_res(cv2.cvtColor(image, cv2.COLOR_RGB2BGR), np.array(self.last_gt, dtype=np.int32), '2', groundtruth=self.groundtruth, update=update_score, can_bboxes=candidate_bboxes, frame_id=self.i, tracker_score=md_score, mask=mask) return [ float(self.last_gt[1]), float(self.last_gt[0]), float(width), float(height) ], self.score_map, 0, confidence_score, 0
def main(): global args, logger, v_id args = parser.parse_args() cfg = load_config(args) init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') logger.info(args) # setup model if args.arch == 'Custom': from custom import Custom model = Custom(anchors=cfg['anchors']) else: parser.error('invalid architecture: {}'.format(args.arch)) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format( args.resume) model = load_pretrain(model, args.resume) model.eval() device = torch.device('cuda' if ( torch.cuda.is_available() and not args.cpu) else 'cpu') model = model.to(device) # setup dataset dataset = load_dataset(args.dataset) # VOS or VOT? if args.dataset in ['DAVIS2016', 'DAVIS2017', 'ytb_vos'] and args.mask: vos_enable = True # enable Mask output else: vos_enable = False total_lost = 0 # VOT # iou_lists = [] # VOS # speed_list = [] for v_id, video in enumerate(dataset.keys(), start=1): if args.video != '' and video != args.video: continue if vos_enable: iou_list, speed = track_vos( model, dataset[video], cfg['hp'] if 'hp' in cfg.keys() else None, args.mask, args.refine, args.dataset in ['DAVIS2017', 'ytb_vos'], device=device) # iou_lists.append(iou_list) else: lost, speed = track_vot(model, dataset[video], cfg['hp'] if 'hp' in cfg.keys() else None, args.mask, args.refine, device=device) total_lost += lost
def main(): global args, best_acc, tb_writer, logger args = parser.parse_args() args = args_process(args) init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') logger.info("\n" + collect_env_info()) logger.info(args) cfg = load_config(args) logger.info("config \n{}".format(json.dumps(cfg, indent=4))) if args.log_dir: tb_writer = SummaryWriter(args.log_dir) else: tb_writer = Dummy() # build dataset train_loader, val_loader = build_data_loader(cfg) args.img_size = int(cfg['train_datasets']['search_size']) args.nms_threshold = float(cfg['train_datasets']['RPN_NMS']) if args.arch == 'Custom': from custom import Custom model = Custom(pretrain=True, opts=args, anchors=cfg['anchors']) else: exit() logger.info(model) if args.pretrained: model = load_pretrain(model, args.pretrained) model = model.cuda() dist_model = torch.nn.DataParallel(model, list(range( torch.cuda.device_count()))).cuda() if args.resume and args.start_epoch != 0: model.features.unfix((args.start_epoch - 1) / args.epochs) optimizer, lr_scheduler = build_opt_lr(model, cfg, args, args.start_epoch) # optionally resume from a checkpoint if args.resume: assert os.path.isfile(args.resume), '{} is not a valid file'.format( args.resume) model, optimizer, args.start_epoch, best_acc, arch = restore_from( model, optimizer, args.resume) dist_model = torch.nn.DataParallel( model, list(range(torch.cuda.device_count()))).cuda() logger.info(lr_scheduler) logger.info('model prepare done') train(train_loader, dist_model, optimizer, lr_scheduler, args.start_epoch, cfg)
class TrackingManager: def __init__(self): resume = '/home/saad/Root/vision/Computer_Vision/Tracking-systems/SiamMask_DAVIS.pth' config = '/home/saad/Root/vision/Computer_Vision/Tracking-systems/config_davis.json' self.cfg = load_config(config=config) self.siammask = Custom(anchors=self.cfg['anchors']) self.siammask = load_pretrain(self.siammask, resume) self.active_boxes = [] self.frames_generator = get_detection_output_as_frames_generator() self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.siammask.eval().to(self.device) self.tracker_initialized = False self.state = None self.times = np_.asarray([]) def track(self): current_frame, final_one = next(self.frames_generator) if final_one: yield 1 s_ = time.time() init_boxes = select_active_boxes(current_frame, history_queue, 0.1) targets = [] for box_ in init_boxes: target_pos = np.array([box_.x1 + box_.w / 2, box_.y1 + box_.h / 2]) target_sz = np.array([box_.w, box_.h]) # print("x1 = {}, y1 = {}, w = {}, h = {}".format(box.x1, box.y1, box.w, box.h)) s = {"target_pos": target_pos, "target_sz": target_sz, "x": box_.x1, "y": box_.y1, "w": box_.w, "h": box_.h} targets.append(s) self.active_boxes.extend(init_boxes) if len(init_boxes) > 0: if self.state is not None: targets.extend(self.state['targets']) self.state = siamese_init(current_frame.img, self.siammask, self.cfg['hp'], device=self.device, targets=targets) # init tracker self.tracker_initialized = True if self.tracker_initialized and self.state is not None and len(self.state['targets']) > 0: self.state = siamese_track(self.state, current_frame.img) t = 0 while t < len(self.state['targets']): # check that the tracked object still exist score = self.state['targets'][t]['score'] if score <= .001: print("remove box because its score is {}".format(self.state['targets'][t]['score'])) self.remove_gone_boxes(self.state['targets'][t]) del self.state['targets'][t] continue target = self.state['targets'][t] boxx = select_matching_box(target['ploygon'], current_frame) self.state['targets'][t]['ploygon'] = [[boxx.x1, boxx.y1], [boxx.x1, boxx.y2], [boxx.x2, boxx.y2], [boxx.x2, boxx.y1]] # assign ID to the tracked object x, y, w, h = target['x'], target['y'], target['w'], target['h'] for o, active_box in enumerate(self.active_boxes): if active_box.x1 == x and active_box.y1 == y and active_box.w == w and active_box.h == h: boxx.ID = active_box.ID boxx.type = active_box.type # frame.get_coord_depend_seg(mask,boxx.x1,boxx.y1,boxx.x2,boxx.y2, id) cv2.rectangle(current_frame.img, (int(boxx.x1), int(boxx.y1)), (int(boxx.x2), int(boxx.y2)), (255, 0, 0), 2) center = [int(x) for x in target['target_pos']] cv2.putText(current_frame.img, str(boxx.ID), tuple(center), cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0)) t += 1 # frame.add_polygon(target['ploygon'], id) current_frame.add_box(boxx) history_queue.append(current_frame) print(f"tracked in {time.time() - s_}") self.times = np_.append(self.times, [time.time() - s_], axis=0) cv2.putText(current_frame.img, "current frame", (20, 20), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0)) current_frame.img = cv2.cvtColor(current_frame.img, cv2.COLOR_BGR2RGB) cv2.imwrite(f"/home/saad/Root/datasets/tracking/tracking_facepass_case/{current_frame.frame_indx}.jpg", current_frame.img) yield current_frame def remove_gone_boxes(self, target): x, y, w, h = target['x'], target['y'], target['w'], target['h'] i = 0 for active_box in self.active_boxes: if active_box.x1 == x and active_box.y1 == y and active_box.w == w and active_box.h == h: del self.active_boxes[i] break i += 1 def get_tracker_fps(self): avg = np_.average(self.times) return 1 / avg, avg
class SingleTracker(object): def __init__(self, config_path, model_path): args = TrackArgs() args.config = config_path args.resume = model_path cfg = load_config(args) if args.arch == 'Custom': from custom import Custom self.model = Custom(anchors=cfg['anchors']) else: parser.error('invalid architecture: {}'.format(args.arch)) if args.resume: assert isfile(args.resume), '{} is not a valid file'.format(args.resume) self.model = load_pretrain(self.model, args.resume) self.model.eval() self.device = torch.device('cuda' if (torch.cuda.is_available() and not args.cpu) else 'cpu') self.model = self.model.to(self.device) ################# Dangerous self.p = TrackerConfig() self.p.update(cfg['hp'] if 'hp' in cfg.keys() else None, self.model.anchors) self.p.renew() self.p.scales = self.model.anchors['scales'] self.p.ratios = self.model.anchors['ratios'] self.p.anchor_num = self.model.anchor_num self.p.anchor = generate_anchor(self.model.anchors, self.p.score_size) if self.p.windowing == 'cosine': self.window = np.outer(np.hanning(self.p.score_size), np.hanning(self.p.score_size)) elif self.p.windowing == 'uniform': self.window = np.ones((self.p.score_size, self.p.score_size)) self.window = np.tile(self.window.flatten(), self.p.anchor_num) ################ def get_examplar_feature(self, img, target_pos, target_sz): avg_chans = np.mean(img, axis=(0, 1)) wc_z = target_sz[0] + self.p.context_amount * sum(target_sz) hc_z = target_sz[1] + self.p.context_amount * sum(target_sz) s_z = round(np.sqrt(wc_z * hc_z)) # initialize the exemplar examplar = get_subwindow_tracking(img, target_pos, self.p.exemplar_size, s_z, avg_chans) z = Variable(examplar.unsqueeze(0)) return self.model.template(z.to(self.device)) def siamese_track(self, img, target_pos, target_sz, examplar_feature, debug=False, mask_enable=True, refine_enable=True): avg_chans = np.mean(img, axis=(0, 1)) im_h = img.shape[0] im_w = img.shape[1] wc_x = target_sz[0] + self.p.context_amount * sum(target_sz) hc_x = target_sz[1] + self.p.context_amount * sum(target_sz) s_x = np.sqrt(wc_x * hc_x) ''' scale_x = self.p.exemplar_size / s_x d_search = (self.p.instance_size - self.p.exemplar_size) / 2 pad = d_search / scale_x s_x = s_x + 2 * pad crop_box = [target_pos[0] - round(s_x) / 2, target_pos[1] - round(s_x) / 2, round(s_x), round(s_x)] ''' # myy # 上面注释的部分, 原作者写的代码可以简化为下面三句 scale_x = self.p.exemplar_size / s_x s_x = self.p.instance_size / self.p.exemplar_size * s_x crop_box = [target_pos[0] - round(s_x) / 2, target_pos[1] - round(s_x) / 2, round(s_x), round(s_x)] # extract scaled crops for search region x at previous target position x_crop = Variable(get_subwindow_tracking(img, target_pos, self.p.instance_size, round(s_x), avg_chans).unsqueeze(0)) if mask_enable: score, delta, mask = self.model.track_mask(examplar_feature, x_crop.to(self.device)) else: score, delta = self.model.track(examplar_feature, x_crop.to(self.device)) delta = delta.permute(1, 2, 3, 0).contiguous().view(4, -1).data.cpu().numpy() score = F.softmax(score.permute(1, 2, 3, 0).contiguous().view(2, -1).permute(1, 0), dim=1).data[:, 1].cpu().numpy() delta[0, :] = delta[0, :] * self.p.anchor[:, 2] + self.p.anchor[:, 0] delta[1, :] = delta[1, :] * self.p.anchor[:, 3] + self.p.anchor[:, 1] delta[2, :] = np.exp(delta[2, :]) * self.p.anchor[:, 2] delta[3, :] = np.exp(delta[3, :]) * self.p.anchor[:, 3] def change(r): return np.maximum(r, 1. / r) def sz(w, h): pad = (w + h) * 0.5 sz2 = (w + pad) * (h + pad) return np.sqrt(sz2) def sz_wh(wh): pad = (wh[0] + wh[1]) * 0.5 sz2 = (wh[0] + pad) * (wh[1] + pad) return np.sqrt(sz2) # size penalty target_sz_in_crop = target_sz*scale_x s_c = change(sz(delta[2, :], delta[3, :]) / (sz_wh(target_sz_in_crop))) # scale penalty r_c = change((target_sz_in_crop[0] / target_sz_in_crop[1]) / (delta[2, :] / delta[3, :])) # ratio penalty penalty = np.exp(-(r_c * s_c - 1) * self.p.penalty_k) pscore = penalty * score # cos window (motion model) pscore = pscore * (1 - self.p.window_influence) + self.window * self.p.window_influence best_pscore_id = np.argmax(pscore) pred_in_crop = delta[:, best_pscore_id] / scale_x lr = penalty[best_pscore_id] * score[best_pscore_id] * self.p.lr # lr for OTB res_x = pred_in_crop[0] + target_pos[0] res_y = pred_in_crop[1] + target_pos[1] res_w = target_sz[0] * (1 - lr) + pred_in_crop[2] * lr res_h = target_sz[1] * (1 - lr) + pred_in_crop[3] * lr target_pos = np.array([res_x, res_y]) target_sz = np.array([res_w, res_h]) # for Mask Branch if mask_enable: best_pscore_id_mask = np.unravel_index(best_pscore_id, (5, self.p.score_size, self.p.score_size)) delta_x, delta_y = best_pscore_id_mask[2], best_pscore_id_mask[1] if refine_enable: mask = self.model.track_refine((delta_y, delta_x)).to(self.device).sigmoid().squeeze().view( self.p.out_size, self.p.out_size).cpu().data.numpy() else: mask = mask[0, :, delta_y, delta_x].sigmoid(). \ squeeze().view(self.p.out_size, self.p.out_size).cpu().data.numpy() def crop_back(image, bbox, out_sz, padding=-1): a = (out_sz[0] - 1) / bbox[2] b = (out_sz[1] - 1) / bbox[3] c = -a * bbox[0] d = -b * bbox[1] mapping = np.array([[a, 0, c], [0, b, d]]).astype(np.float) crop = cv2.warpAffine(image, mapping, (out_sz[0], out_sz[1]), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=padding) return crop s = crop_box[2] / self.p.instance_size sub_box = [crop_box[0] + (delta_x - self.p.base_size / 2) * self.p.total_stride * s, crop_box[1] + (delta_y - self.p.base_size / 2) * self.p.total_stride * s, s * self.p.exemplar_size, s * self.p.exemplar_size] s = self.p.out_size / sub_box[2] back_box = [-sub_box[0] * s, -sub_box[1] * s, im_w * s, im_h * s] mask_in_img = crop_back(mask, back_box, (im_w, im_h)) target_mask = (mask_in_img > self.p.seg_thr).astype(np.uint8) if cv2.__version__[-5] == '4': contours, _ = cv2.findContours(target_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) else: _, contours, _ = cv2.findContours(target_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) cnt_area = [cv2.contourArea(cnt) for cnt in contours] if len(contours) != 0 and np.max(cnt_area) > 100: contour = contours[np.argmax(cnt_area)] # use max area polygon polygon = contour.reshape(-1, 2) # pbox = cv2.boundingRect(polygon) # Min Max Rectangle prbox = cv2.boxPoints(cv2.minAreaRect(polygon)) # Rotated Rectangle # box_in_img = pbox rbox_in_img = prbox else: # empty mask location = cxy_wh_2_rect(target_pos, target_sz) rbox_in_img = np.array([[location[0], location[1]], [location[0] + location[2], location[1]], [location[0] + location[2], location[1] + location[3]], [location[0], location[1] + location[3]]]) target_pos[0] = max(0, min(im_w, target_pos[0])) target_pos[1] = max(0, min(im_h, target_pos[1])) target_sz[0] = max(10, min(im_w, target_sz[0])) target_sz[1] = max(10, min(im_h, target_sz[1])) score = score[best_pscore_id] mask = mask_in_img if mask_enable else [] return target_pos, target_sz, score, mask
def main(): global args, best_acc, tb_writer, logger args = parser.parse_args() args = args_process(args) init_log('global', logging.INFO) if args.log != "": add_file_handler('global', args.log, logging.INFO) logger = logging.getLogger('global') logger.info("\n" + collect_env_info()) logger.info(args) cfg = load_config(args) logger.info("config \n{}".format(json.dumps(cfg, indent=4))) if args.log_dir: tb_writer = SummaryWriter(args.log_dir) else: tb_writer = Dummy() # build dataset train_loader, val_loader = build_data_loader(cfg) args.img_size = int(cfg['train_datasets']['search_size']) args.nms_threshold = float(cfg['train_datasets']['RPN_NMS']) if args.arch == 'Custom': from custom import Custom model = Custom(pretrain=True, opts=args, anchors=train_loader.dataset.anchors) else: exit() logger.info(model) if args.pretrained: model = load_pretrain(model, args.pretrained) model = model.cuda() dist_model = torch.nn.DataParallel(model, list(range( torch.cuda.device_count()))).cuda() if args.resume and args.start_epoch != 0: model.features.unfix((args.start_epoch - 1) / args.epochs) optimizer, lr_scheduler = build_opt_lr(model, cfg, args, args.start_epoch) # optionally resume from a checkpoint if args.resume: assert os.path.isfile(args.resume), '{} is not a valid file'.format( args.resume) model, optimizer, args.start_epoch, best_acc, arch = restore_from( model, optimizer, args.resume) dist_model = torch.nn.DataParallel( model, list(range(torch.cuda.device_count()))).cuda() logger.info(lr_scheduler) logger.info('model prepare done') global cur_lr if not os.path.exists(args.save_dir): # makedir/save model os.makedirs(args.save_dir) num_per_epoch = len(train_loader.dataset) // args.batch num_per_epoch_val = len(val_loader.dataset) // args.batch for epoch in range(args.start_epoch, args.epochs): lr_scheduler.step(epoch) cur_lr = lr_scheduler.get_cur_lr() logger = logging.getLogger('global') train_avg = AverageMeter() val_avg = AverageMeter() if dist_model.module.features.unfix(epoch / args.epochs): logger.info('unfix part model.') optimizer, lr_scheduler = build_opt_lr(dist_model.module, cfg, args, epoch) train(train_loader, dist_model, optimizer, lr_scheduler, epoch, cfg, train_avg, num_per_epoch) if dist_model.module.features.unfix(epoch / args.epochs): logger.info('unfix part model.') optimizer, lr_scheduler = build_opt_lr(dist_model.module, cfg, args, epoch) if (epoch + 1) % args.save_freq == 0: save_checkpoint( { 'epoch': epoch, 'arch': args.arch, 'state_dict': dist_model.module.state_dict(), 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), 'anchor_cfg': cfg['anchors'] }, False, os.path.join(args.save_dir, 'checkpoint_e%d.pth' % (epoch)), os.path.join(args.save_dir, 'best.pth')) validation(val_loader, dist_model, epoch, cfg, val_avg, num_per_epoch_val)