def track(opt): result_root = opt.output_root if opt.output_root != '' else '.' mkdir_if_missing(result_root) cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking timer = Timer() accs = [] n_frame = 0 logger.info('Starting tracking...') dataloader = datasets.LoadVideo(opt.input_video, opt.img_size) result_filename = os.path.join(result_root, 'results.txt') frame_rate = dataloader.frame_rate frame_dir = None if opt.output_format == 'text' else osp.join( result_root, 'frame') try: ob = eval_seq(opt, dataloader, 'mot', result_filename, save_dir=frame_dir, show_image=False, frame_rate=frame_rate) o = ob[3][1:] #bd{ i = ob[4] l_count = 0 l_count = ob[5] ai = sum(i) / len(i) ao = sum(o) / len(o) print("\nAverage Inflow=", ai) print("\nAverage Outflow=", ao) if (ai > ao): print( "\n Congratulations!! The ROI is proving to be a hotspot !!!\n" ) elif (ao > ai): print( "\n The ROI is losing people attention !!!\n" ) else: print( "\n The ROI has a moderate people attention !!!\n" ) #bd} print( "\n The ROI has a current crowd of %d people !!!\n" % l_count) except Exception as e: logger.info(e) if opt.output_format == 'video': output_video_path = osp.join(result_root, 'result.mp4') cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format( osp.join(result_root, 'frame'), output_video_path) os.system(cmd_str) return o, i, l_count
def __init__(self, cfg, img_size=(416, 416)): super(Darknet, self).__init__() # 1、读取 cfg 文件,得到 module_defs( 是一个包含多个 dict 的list,比如索引 082 对应的 dict 为 # <class 'dict'>: {'type': 'yolo', 'mask': '6,7,8', 'anchors': array([[ 10., 13.], # [ 16., 30.], # [ 33., 23.], # [ 30., 61.], # [ 62., 45.], # [ 59., 119.], # [116., 90.], # [156., 198.], # [373., 326.]]), 'classes': '80', 'num': '9', 'jitter': '.3', 'ignore_thresh': '.7', 'truth_thresh': '1', 'random': '1'} # 内容上和 .cfg 文件本身完全一样 self.module_defs = parse_model_cfg( cfg) # 注意这里除了包含索引 0-106 的所有层之外,还包含了[net]的信息,比如输入通道数channels=3, # 这个在下一步 create_modules() 搭建conv层时会用到,所以还不能删;等进入 create_modules() 后, # 才可以 module_defs.pop(0);另外 get_yolo_layers() 也因此放在再下一步,也就是第三步,注意顺序不能颠倒 # 2、将 module_defs 转化为 module_list,并且找到所有 路由层 的索引,包括 residual模块的 shortcut、 # 有一个OrderDict类型的变量_module ,比如 key '82' 对应的 value 为 YOLOLayer()对象 # 这里根据 1 中每个 dict, 创建相应层的对象,放入 nn.ModuleList()类型的 module_list, self.module_list, self.routs = create_modules(self.module_defs) # 3、这里存的只是三个 yolo_layer 的索引, # 通过遍历 module_defs 很容易得到,即 [82, 94, 106] self.yolo_layers = get_yolo_layers(self)
def track(opt): result_root = opt.output_root if opt.output_root != '' else '.' mkdir_if_missing(result_root) cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking timer = Timer() accs = [] n_frame = 0 logger.info('Starting tracking...') dataloader = datasets.LoadVideo(opt.input_video, opt.img_size) result_filename = os.path.join(result_root, 'results.txt') frame_rate = dataloader.frame_rate frame_dir = None if opt.output_format == 'text' else osp.join( result_root, 'frame') try: eval_seq(opt, dataloader, 'mot', result_filename, save_dir=frame_dir, show_image=False, frame_rate=frame_rate) except Exception as e: logger.info(e) if opt.output_format == 'video': output_video_path = osp.join(result_root, 'result.mp4') cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format( osp.join(result_root, 'frame'), output_video_path) os.system(cmd_str)
def __init__(self, cfg_path, img_size=416): super(DarkRFBNet, self).__init__() self.module_defs = parse_model_cfg(cfg_path) self.module_defs[0]['cfg'] = cfg_path self.module_defs[0]['height'] = img_size self.hyperparams, self.module_list = create_modules(self.module_defs) self.img_size = img_size
def track(opt): if opt.AVA: if opt.ov: if opt.skip: outPath = '../../output/trmot_ov/{}'.format(opt.skip) else: outPath = '../../output/trmot_ov' elif opt.cpu: if opt.skip: outPath = '../../output/trmot_cpu/{}'.format(opt.skip) else: outPath = '../../output/trmot_cpu' else: if opt.skip: outPath = '../../output/trmot_gpu/{}'.format(opt.skip) else: outPath = '../../output/trmot_gpu' if opt.skip==0: opt.skip=1 if not os.path.exists(outPath): os.makedirs(outPath) if opt.video_path!=0: countFileName = '{}/{}.csv'.format(outPath, opt.video_path.split('/')[-1].split('.mp4')[0]) else: countFileName = '{}/results_live_cam.csv'.format(outPath) countFile = open(countFileName, 'w') countFile.close() result_root = opt.output_root if opt.output_root!='' else '.' mkdir_if_missing(result_root) cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking timer = Timer() accs = [] n_frame = 0 logger.info('Starting tracking...') dataloader = datasets.LoadVideo(opt.video_path, opt.img_size) result_filename = os.path.join(result_root, 'results.txt') frame_rate = dataloader.frame_rate frame_dir = None if opt.output_format=='text' else osp.join(result_root, 'frame') try: eval_seq(opt, dataloader, 'mot', result_filename, countFileName, save_dir=frame_dir, show_image=False, frame_rate=frame_rate) except Exception as e: logger.info(e) if opt.output_format == 'video': output_video_path = osp.join(result_root, 'result.mp4') cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(osp.join(result_root, 'frame'), output_video_path) os.system(cmd_str)
def __init__(self, batch_size=19): cfg_path = "./cfg/network.cfg" net_block = parse_model_cfg(cfg_path)[0] # [net] self.n_classes = net_block["n_classes"] self.train_list = read_txt(net_block["train_set"]) self.test_list = read_txt(net_block["test_set"]) self.total_list = read_txt(net_block["total_set"]) self.BATCH_SIZE = batch_size # 一次读入多少个样本 self.num_workers = 2 # 加载batch的线程数
def __init__(self, cfg_path, img_size=416): super(YOLOv3, self).__init__() self.module_defs = parse_model_cfg(cfg_path) self.module_defs[0]['cfg'] = cfg_path self.module_defs[0]['height'] = img_size self.hyperparams, self.module_list = create_modules(self.module_defs) self.img_size = img_size self.loss_names = ['loss', 'xy', 'wh', 'conf', 'cls', 'nT'] self.losses = []
def main(opt, data_root='/data/MOT16/train', det_root=None, seqs=('MOT16-05',), exp_name='demo', save_images=False, save_videos=False, show_image=True): logger.setLevel(logging.INFO) result_root = os.path.join(data_root, '..', 'results', exp_name) mkdir_if_missing(result_root) data_type = 'mot' # Read config cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking accs = [] n_frame = 0 timer_avgs, timer_calls = [], [] for seq in seqs: output_dir = os.path.join(data_root, '..','outputs', exp_name, seq) if save_images or save_videos else None logger.info('start seq: {}'.format(seq)) dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size) result_filename = os.path.join(result_root, '{}.txt'.format(seq)) meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read() frame_rate = int(meta_info[meta_info.find('frameRate')+10:meta_info.find('\nseqLength')]) nf, ta, tc = eval_seq(opt, dataloader, data_type, result_filename, save_dir=output_dir, show_image=show_image, frame_rate=frame_rate) n_frame += nf timer_avgs.append(ta) timer_calls.append(tc) # eval logger.info('Evaluate seq: {}'.format(seq)) evaluator = Evaluator(data_root, seq, data_type) accs.append(evaluator.eval_file(result_filename)) if save_videos: output_video_path = osp.join(output_dir, '{}.mp4'.format(seq)) cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(output_dir, output_video_path) os.system(cmd_str) timer_avgs = np.asarray(timer_avgs) timer_calls = np.asarray(timer_calls) all_time = np.dot(timer_avgs, timer_calls) avg_time = all_time / np.sum(timer_calls) logger.info('Time elapsed: {:.2f} seconds, FPS: {:.2f}'.format(all_time, 1.0 / avg_time)) # get summary metrics = mm.metrics.motchallenge_metrics mh = mm.metrics.create() summary = Evaluator.get_summary(accs, seqs, metrics) strsummary = mm.io.render_summary( summary, formatters=mh.formatters, namemap=mm.io.motchallenge_metric_names ) print(strsummary) sys.stdout.flush() Evaluator.save_summary(summary, os.path.join(result_root, 'summary_{}.xlsx'.format(exp_name)))
def __init__(self, cfg, img_size=(416, 416), verbose=False): super(YOLOV3_SPP, self).__init__() self.input_size = [img_size] * 2 if isinstance(img_size, int) else img_size self.module_defs = parse_model_cfg(cfg) self.module_list, self.routs = self.create_modules(self.module_defs, img_size, cfg) self.yolo_layers = self.get_index('YOLOLayer') # YOLOV3_SPP Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346 self.version = np.array([0, 2, 5], dtype=np.int32) # (int32) version info: major, minor, revision self.seen = np.array([0], dtype=np.int64) # (int64) number of images seen during training self.info(verbose) if not ONNX_EXPORT else None # print model description
def __init__(self, cfg, img_size=(416, 416)): super(Darknet, self).__init__() self.module_defs = parse_model_cfg(cfg) self.module_defs[0]['cfg'] = cfg self.module_defs[0]['height'] = img_size self.hyperparams, self.module_list = create_modules(self.module_defs) self.yolo_layers = get_yolo_layers(self) # Needed to write header when saving weights self.header_info = np.zeros(5, dtype=np.int32) # First five are header values self.seen = self.header_info[3] # number of images seen during training
def mask_cfg_and_converted(mask_replace_layer, cfg='cfg/yolov3-voc.cfg', weight_path='../weights/converted-voc.pt', target='../weights/maskconverted-voc.pt'): mask_cfg = '/'.join(cfg.split('/')[:-1]) + '/mask' + cfg.split('/')[-1] origin_mdfs = parse_model_cfg(cfg) mask_mdfs = [] mask_mdfs.append(origin_mdfs.pop(0)) for i, mdf in enumerate(origin_mdfs): if str(i) in mask_replace_layer: mdf['type'] = 'maskconvolutional' mask_mdfs.append(mdf) write_cfg(mask_cfg, mask_mdfs) mask_weight = OrderedDict() origin_weight = torch.load(weight_path)['model'] for k, v in origin_weight.items(): key_list = k.split('.') idx = key_list[1] if idx in mask_replace_layer and key_list[2] == 'Conv2d': key_list[2] = 'Mask' + key_list[2] key = '.'.join(key_list) mask_weight[key] = v mask_weight[key.replace('weight', 'selected_channels_mask')] = torch.ones( v.size(1), dtype=torch.float32) else: key = '.'.join(key_list) mask_weight[key] = v model = Darknet(mask_cfg) model.load_state_dict(mask_weight, strict=True) if target is not None: chkpt = { 'epoch': -1, 'best_fitness': None, 'training_results': None, 'model': model.state_dict(), 'optimizer': None } torch.save(chkpt, target) return mask_cfg, model.state_dict()
def load_model(model_path, device, cfg): # select device print("device: %s" % device) # load network all_props = parse_model_cfg(cfg) yolo_props = [] for item in all_props: if item['type'] == 'yolo': yolo_props.append(item) # print(yolo_props[0]) model = YMPNet(yolo_props[0]).to(device) model.inference = True if model_path.endswith('.pt'): # pytorch format # possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc. midas_chkpt = torch.load(model_path, map_location=device) # load model try: # yolo_chkpt['model'] = {k: v for k, v in yolo_chkpt['model'].items() if model.state_dict()[k].numel() == v.numel()} # model.load_state_dict(yolo_chkpt['model'], strict=False) # own_state = load_my_state_dict(chkpt, model) # model.load_state_dict(own_state, strict=False) model_dict = model.state_dict() # 1. filter out unnecessary keys # midas_weighs_dict = {k: v for k, v in midas_chkpt.items() if k in model_dict} for k, v in midas_chkpt['model'].items(): if k in model_dict: model_dict[k] = v # 2. overwrite entries in the existing state dict # model_dict.update(midas_weighs_dict) # 3. load the new state dict model.load_state_dict(model_dict) print("Loaded Model weights successfully") except KeyError as e: s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \ "See https://github.com/ultralytics/yolov3/issues/657" % (opt.midas_weights, opt.cfg, opt.midas_weights) raise KeyError(s) from e return model
def __init__(self): super(runModel, self).__init__() self.keep_going = False self.load_path = "weight/91acc_9category_1024_512_256.pt" # 继续训练时,要加载的参数文件 self.batch_size = 30 # 分批训练数据、每批数据量 self.learning_rate = 0.01 # 1e-2 # 学习率 self.num_epoches = 700 # 训练次数 cfg_path = "./cfg/network.cfg" net_block = parse_model_cfg(cfg_path)[0] # [net] self.n_classes = net_block["n_classes"] # 类别数 # 保存用于可视化观察参数变化的列表 self.loss_list, self.lr_list, self.acc_list = [], [], [ ] # 记录损失值\学习率\准确率变化 self.layer_list = [1024, 512, 256] # self.model = MyFC(self.layer_list, 42, self.n_classes) self.model = classifier("./cfg/network.cfg") self.train_loader = MyDataLoader(batch_size=self.batch_size).train() self.test_loader = MyDataLoader(batch_size=self.batch_size).test() self.best_acc = 0 # 最高准确类 self.writer = SummaryWriter(logdir='./log') # 记录训练日志
def track(opt): result_root = opt.output_root if opt.output_root!='' else '.' mkdir_if_missing(result_root) cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking logger.info('Starting tracking...') dataloader = datasets.LoadVideo(opt.input_video, opt.img_size) result_filename = os.path.join(result_root, 'results.txt') frame_rate = dataloader.frame_rate frame_dir = None if opt.output_format=='text' else osp.join(result_root, 'frame') try: eval_seq(opt, dataloader, 'mot', result_filename, save_dir=frame_dir, show_image=True, frame_rate=frame_rate) except Exception as e: logger.info(e)
def __init__(self, args, video_path): self.args = args self.video_path = video_path self.logger = get_logger("root") use_cuda = args.use_cuda and torch.cuda.is_available() if not use_cuda: warnings.warn("Running in cpu mode which maybe very slow!", UserWarning) if args.display: cv2.namedWindow("test", cv2.WINDOW_NORMAL) cv2.resizeWindow("test", args.display_width, args.display_height) if args.rtsp != "": print("Using rtsp " + str(args.rtsp)) self.vdo = cv2.VideoCapture(args.rtsp) elif args.cam != -1: print("Using webcam " + str(args.cam)) self.vdo = cv2.VideoCapture(args.cam) else: self.vdo = cv2.VideoCapture() frame_rate = 30 cfg_dict = parse_model_cfg(args.cfg) self.width, self.height = int(cfg_dict[0]['width']), int( cfg_dict[0]['height']) args.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] self.tracker = JDETracker(args, frame_rate=frame_rate) self.known_faces = {} self.faceUtils = FaceUtils() self.camUtils = CamUtils(args) self.tmp_imgs = [] self.tmp_moves = [] self.result_imgs = queue.Queue() self.is_running = True print("Loading Control Done.")
def track(opt): # set saving dir result_root = opt.output_root if opt.output_root != '' else '.' mkdir_if_missing(result_root) # set configuration cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] logger.info('Starting tracking...') # use camera to track dataloader = datasets.LoadCamera(img_size=opt.img_size) result_filename = os.path.join(result_root, 'results.txt') try: # start tracking eval_seq(opt, dataloader, 'mot', result_filename, save_dir=result_root, show_image=True) except Exception as e: logger.info(e)
def prune(mask_cfg, progress_weights, mask_replace_layer, new_cfg_file, new_weights): only_in = mask_replace_layer[-3:] mask_replace_layer = mask_replace_layer[:-2] device_in = torch.device('cpu') model = Darknet(mask_cfg) chkpt = torch.load(progress_weights, map_location=device_in) model.load_state_dict(chkpt['model']) new_cfg = parse_model_cfg(mask_cfg) for layer in mask_replace_layer[:-1]: assert isinstance(model.module_list[int(layer)][0], MaskConv2d), "Not a pruned model!" tail_layer = mask_replace_layer[mask_replace_layer.index(layer) + 1] assert isinstance(model.module_list[int(tail_layer)][0], MaskConv2d), "Not a pruned model!" in_channels_mask = model.module_list[int( layer)][0].selected_channels_mask out_channels_mask = model.module_list[int( tail_layer)][0].selected_channels_mask in_channels = int(torch.sum(in_channels_mask)) out_channels = int(torch.sum(out_channels_mask)) new_cfg[int(layer) + 1]['type'] = 'convolutional' new_cfg[int(layer) + 1]['filters'] = str(out_channels) new_conv = nn.Conv2d( in_channels, out_channels, kernel_size=model.module_list[int(layer)][0].kernel_size, stride=model.module_list[int(layer)][0].stride, padding=model.module_list[int(layer)][0].padding, bias=False) thin_weight = model.module_list[int(layer)][0].weight[ out_channels_mask.bool()] thin_weight = thin_weight[:, in_channels_mask.bool()] assert new_conv.weight.numel() == thin_weight.numel( ), 'Do not match in shape!' new_conv.weight.data.copy_(thin_weight.data) new_batch = nn.BatchNorm2d(out_channels, momentum=0.1) new_batch.weight.data.copy_(model.module_list[int(layer)][1].weight[ out_channels_mask.bool()].data) new_batch.bias.data.copy_(model.module_list[int(layer)][1].bias[ out_channels_mask.bool()].data) new_batch.running_mean.copy_(model.module_list[int( layer)][1].running_mean[out_channels_mask.bool()].data) new_batch.running_var.copy_(model.module_list[int( layer)][1].running_var[out_channels_mask.bool()].data) new_module = nn.Sequential() new_module.add_module('Conv2d', new_conv) new_module.add_module('BatchNorm2d', new_batch) new_module.add_module('activation', model.module_list[int(layer)][2]) model.module_list[int(layer)] = new_module for layer in only_in: new_cfg[int(layer) + 1]['type'] = 'convolutional' assert isinstance(model.module_list[int(layer)][0], MaskConv2d), "Not a pruned model!" in_channels_mask = model.module_list[int( layer)][0].selected_channels_mask > 0.1 in_channels = int(torch.sum(in_channels_mask)) new_conv = nn.Conv2d( in_channels, out_channels=model.module_list[int(layer)][0].out_channels, kernel_size=model.module_list[int(layer)][0].kernel_size, stride=model.module_list[int(layer)][0].stride, padding=model.module_list[int(layer)][0].padding, bias=False) new_conv.weight.data.copy_(model.module_list[int(layer)] [0].weight[:, in_channels_mask.bool()].data) new_module = nn.Sequential() new_module.add_module('Conv2d', new_conv) new_module.add_module('BatchNorm2d', model.module_list[int(layer)][1]) new_module.add_module('activation', model.module_list[int(layer)][2]) model.module_list[int(layer)] = new_module write_cfg(new_cfg_file, new_cfg) chkpt = { 'epoch': -1, 'best_fitness': None, 'training_results': None, 'model': model.state_dict(), 'optimizer': None } torch.save(chkpt, new_weights)
def __init__(self, cfg: str): super(classifier, self).__init__() # 解析网络对应的.cfg文件,得到模型定义 self.module_definitions = parse_model_cfg(cfg) # 根据解析的网络结构一层一层去搭建 self.module_list = create_modules(self.module_definitions)
if i != len(args[1]) - 1: next_item = args[1][i + 1] else: next_item = "1" if "--" in curr_item and "--" in next_item: key = curr_item.replace('-', '') if key not in params: params[key] = "1" elif "--" in curr_item and "--" not in next_item: key = curr_item.replace('-', '') if key not in params: params[key] = next_item opWrapper = op.WrapperPython() opWrapper.configure(params) opWrapper.start() cfg_dict = parse_model_cfg(path=command_args.cfg) command_args.img_size = [int(cfg_dict[0]["width"]), int(cfg_dict[0]["height"])] timer = Timer() logger.info('Starting tracking...') dataloader = datasets.LoadVideo(command_args.input_video, command_args.img_size) frame_rate = dataloader.frame_rate orig_width = dataloader.vw orig_height = dataloader.vh width = dataloader.w height = dataloader.h video_writer = cv2.VideoWriter(os.path.join(command_args.output_dir, os.path.splitext(os.path.basename(command_args.input_video))[0] + ".mp4"), cv2.VideoWriter_fourcc(*"mp4v"), float(frame_rate), (orig_width, orig_height)) tracker = JDETracker(opt=command_args, frame_rate=frame_rate)
def run(input_path, output_path, model_path): """Run MonoDepthNN to compute depth maps. Args: input_path (str): path to input folder output_path (str): path to output folder model_path (str): path to saved model """ print("initialize") # select device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("device: %s" % device) # load network all_props = parse_model_cfg(cfg) yolo_props = [] for item in all_props: if item['type'] == 'yolo': yolo_props.append(item) # print(yolo_props[0]) model = PPENet(yolo_props[0]).to(device) model.inference = True if model_path.endswith('.pt'): # pytorch format # possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc. midas_chkpt = torch.load(model_path, map_location=device) # load model try: # yolo_chkpt['model'] = {k: v for k, v in yolo_chkpt['model'].items() if model.state_dict()[k].numel() == v.numel()} # model.load_state_dict(yolo_chkpt['model'], strict=False) # own_state = load_my_state_dict(chkpt, model) # model.load_state_dict(own_state, strict=False) model_dict = model.state_dict() # 1. filter out unnecessary keys # midas_weighs_dict = {k: v for k, v in midas_chkpt.items() if k in model_dict} for k, v in midas_chkpt['model'].items(): if k in model_dict: model_dict[k] = v # 2. overwrite entries in the existing state dict # model_dict.update(midas_weighs_dict) # 3. load the new state dict model.load_state_dict(model_dict) print("Loaded Model weights successfully") except KeyError as e: s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \ "See https://github.com/ultralytics/yolov3/issues/657" % (opt.midas_weights, opt.cfg, opt.midas_weights) raise KeyError(s) from e transform = Compose([ Resize( 384, 384, resize_target=None, keep_aspect_ratio=True, ensure_multiple_of=32, resize_method="upper_bound", image_interpolation_method=cv2.INTER_CUBIC, ), NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), PrepareForNet(), ]) model.to(device) model.eval() # get input img_names = glob.glob(os.path.join(input_path, "*")) num_images = len(img_names) # create output folder os.makedirs(output_path, exist_ok=True) print("start processing") for ind, img_name in enumerate(img_names): print(" processing {} ({}/{})".format(img_name, ind + 1, num_images)) # input img = midas_utils.read_image(img_name) img_input = transform({"image": img})["image"] # print(f"DBG run img.shape - {img.shape}") # compute with torch.no_grad(): sample = torch.from_numpy(img_input).to(device).unsqueeze(0) prediction, _ = model(sample) # print(f"DBG prediction.shape - {prediction.shape}") # print(f"prediction.min() - {prediction.min()}") # print(f"prediction.max() - {prediction.max()}") prediction = (torch.nn.functional.interpolate( prediction.unsqueeze(1), size=img.shape[:2], mode="bicubic", align_corners=False, ).squeeze().cpu().numpy()) # output filename = os.path.join( output_path, os.path.splitext(os.path.basename(img_name))[0]) midas_utils.write_depth(filename, prediction, bits=2) print("finished")
def sort_confidence(preds): """分析错分样本的置信度""" values, indexes = preds.topk(k=n_classes, dim=-1) # 将置信度前三的从大到小排列得到v,每个置信度相应的标签为index # values, indexes = values.tolist(), indexes.tolist() # values = [round(v, 3) for v in values] return values, indexes if __name__ == '__main__': cfg_path = "./cfg/network.cfg" pt_file = "weight/91acc_9category_1024_512_256.pt" # pt_file = "runs/2021-04-16/93acc_699epoch_9category.pt" net_block = parse_model_cfg(cfg_path)[0] # [net] n_classes = net_block["n_classes"] batch_size = net_block["batch"] model = classifier(cfg_path).cuda() param_dict = torch.load(pt_file) model.load_state_dict(param_dict["model_state"]) data_loader = MyDataLoader(batch_size=batch_size).test() confidence_threshold = 0.98 # 置信度阈值,大于该值才预测为真,否则pass # torch.save(model, './classification_model.pth') model = torch.load('./classification_model.pth') classes = [ '0-other', '1-OK', '2-palm', '3-up', '4-down', '5-right', '6-left', '7-heart', '8-hush' ] if n_classes == 8:
def test( cfg, data, weights=None, batch_size=16, img_size=512, conf_thres=0.001, iou_thres=0.6, # for nms save_json=False, single_cls=False, augment=False, model=None, dataloader=None, bbox_lambda=None, depth_lambda=None): # Initialize/load model and set device if model: model.inference = True if model is None: device = torch_utils.select_device(opt.device, batch_size=batch_size) verbose = opt.task == 'test' # Remove previous for f in glob.glob('test_batch*.png'): os.remove(f) # Initialize model # model = Darknet(cfg, img_size) all_props = parse_model_cfg(cfg) yolo_props = [] for item in all_props: if item['type'] == 'yolo': yolo_props.append(item) # print(yolo_props[0]) model = YMPNet(yolo_props[0], inference=True).to(device) # model.module_defs = parse_model_cfg(cfg) # model.module_list, model.routs = create_modules(model.module_defs, img_size) # model.yolo_layers = get_yolo_layers(model) # Load weights attempt_download(weights) if weights.endswith('.pt'): # pytorch format model.load_state_dict( torch.load(weights, map_location=device)['model']) # else: # darknet format # load_darknet_weights(model, weights) # Fuse model.fuse() model.to(device) if device.type != 'cpu' and torch.cuda.device_count() > 1: model = nn.DataParallel(model) else: # called by train.py device = next(model.parameters()).device # get model device verbose = False # Configure run data = parse_data_cfg(data) nc = 1 if single_cls else int(data['classes']) # number of classes path = data['valid'] # path to test images names = load_classes(data['names']) # class names iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 iouv = iouv[0].view(1) # comment for [email protected]:0.95 niou = iouv.numel() # Dataloader if dataloader is None: dataset = LoadImagesAndLabels(path, img_size, batch_size, rect=True, single_cls=opt.single_cls) batch_size = min(batch_size, len(dataset)) dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=min([ os.cpu_count(), batch_size if batch_size > 1 else 0, 8 ]), pin_memory=True, collate_fn=dataset.collate_fn) seen = 0 model.eval() # _ = model(torch.zeros((1, 3, img_size, img_size), device=device)) if device.type != 'cpu' else None # run once coco91class = coco80_to_coco91_class() s = ('%20s' + '%17s' * 10) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1', 'RmseGradientLoss', 'SSIMLoss', 'DLoss', 'TotalLoss') p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. bbox_loss = torch.zeros(3, device=device) rmse_gradient_loss = 0 ssim_loss = 0 depth_loss = 0 total_loss = 0 jdict, stats, ap, ap_class = [], [], [], [] for batch_i, (imgs, targets, paths, shapes, depth_targets, _, _) in enumerate(tqdm(dataloader, desc=s)): imgs = imgs.to( device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0 targets = targets.to(device) # print(f"DBG Imgs.shape {imgs.shape}") nb, _, height, width = imgs.shape # batch size, channels, height, width whwh = torch.Tensor([width, height, width, height]).to(device) # Plot images with bounding boxes f = 'test_batch%g.png' % batch_i # filename if batch_i < 1 and not os.path.exists(f): plot_images(imgs=imgs, targets=targets, paths=paths, fname=f) # Disable gradients with torch.no_grad(): # Run model t = torch_utils.time_synchronized() # inf_out, train_out = model(imgs, augment=augment) # inference and training outputs depth_out, (inf_out, train_out) = model( imgs) # inference and training outputs # print("DBG1", type(inf_out), type(train_out)) t0 += torch_utils.time_synchronized() - t # Compute loss # if hasattr(model, 'hyp'): # if model has loss hyperparameters # loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls # Compute loss if hasattr(model, 'hyp'): # if model has loss hyperparameters bbox_loss += bbox_lambda * compute_loss( train_out, targets, model)[1][:3] # GIoU, obj, cls rmse_gradient_mean_loss, ssim_mean_loss = [ depth_lambda * item for item in compute_depth_loss( depth_out, depth_targets, batch_size, device) ] rmse_gradient_loss += rmse_gradient_mean_loss ssim_loss += ssim_mean_loss # depth_loss += depth_lambda * compute_depth_loss(depth_out, depth_targets, batch_size, device) depth_loss = rmse_gradient_loss + ssim_loss # print(f"DBG bbox_loss.shape - {bbox_loss.shape}") # print(f"DBG bbox_loss.shape - {bbox_loss.shape}") # Run NMS t = torch_utils.time_synchronized() output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres) # nms t1 += torch_utils.time_synchronized() - t # Statistics per image for si, pred in enumerate(output): labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class seen += 1 if pred is None: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) continue # Append to text file # with open('test.txt', 'a') as file: # [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred] # Clip boxes to image bounds clip_coords(pred, (height, width)) # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(Path(paths[si]).stem.split('_')[-1]) box = pred[:, :4].clone() # xyxy scale_coords(imgs[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape box = xyxy2xywh(box) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(pred.tolist(), box.tolist()): jdict.append({ 'image_id': image_id, 'category_id': coco91class[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5) }) # Assign all predictions as incorrect correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero().view( -1) # prediction indices pi = (cls == pred[:, 5]).nonzero().view(-1) # target indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(pred[pi, :4], tbox[ti]).max( 1) # best ious, indices # Append detections for j in (ious > iouv[0]).nonzero(): d = ti[i[j]] # detected target if d not in detected: detected.append(d) correct[ pi[j]] = ious[j] > iouv # iou_thres is 1xn if len( detected ) == nl: # all targets already located in image break # Append statistics (correct, conf, pcls, tcls) stats.append( (correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) total_loss += sum(bbox_loss.tolist()) + depth_loss # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if len(stats): p, r, ap, f1, ap_class = ap_per_class(*stats) if niou > 1: p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean( 1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]] mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean() nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) # print(f"DBG bbox_loss: {bbox_loss}") # print(f"DBG depth_loss: {depth_loss}") # print(f"DBG total_loss: {total_loss}") # print(f"DBG seen, nt.sum(), mp, mr, map, mf1 - {seen, nt.sum(), mp, mr, map, mf1}") # Print results pf = '%20s' + '%17.3g' * 10 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1, rmse_gradient_loss, ssim_loss, depth_loss, total_loss)) # Print results per class if verbose and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i])) # Print speeds if verbose or save_json: t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + ( img_size, img_size, batch_size) # tuple print( 'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] return (mp, mr, map, mf1, *(bbox_loss.cpu() / len(dataloader)).tolist(), depth_loss / len(dataloader), total_loss / len(dataloader)), maps
def main(opt, data_root='/media/dh/data/MOT16/train', det_root=None, seqs=('MOT16-05', ), exp_name='demo', save_images=False, save_videos=False, show_image=True): logger.setLevel(logging.INFO) result_root = os.path.join(data_root, '..', 'results', exp_name) mkdir_if_missing(result_root) data_type = 'mot' # Read config cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking accs = [] n_frame = 0 timer_avgs, timer_calls = [], [] for seq in seqs: output_dir = os.path.join(data_root, '..', 'outputs', exp_name, seq) if save_images or save_videos else None logger.info('start seq: {}'.format(seq)) #dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size) print(osp.join(data_root, seq)) dataloader = datasets.LoadVideo(osp.join(data_root, seq)) # print ("DATALOADER", dataloader.vw) result_filename = os.path.join(result_root, '{}.csv'.format(seq)) #meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read() #frame_rate = int(meta_info[meta_info.find('frameRate')+10:meta_info.find('\nseqLength')]) frame_rate = 30 nf, ta, tc = eval_seq(opt, dataloader, data_type, result_filename, save_dir=output_dir, show_image=show_image, frame_rate=frame_rate, vw=dataloader.vw) n_frame += nf timer_avgs.append(ta) timer_calls.append(tc) # eval logger.info('Evaluate seq: {}'.format(seq)) evaluator = Evaluator(data_root, seq, data_type) accs.append(evaluator.eval_file(result_filename)) if save_videos: output_video_path = osp.join(output_dir, '{}.mp4'.format(seq)) cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format( output_dir, output_video_path) os.system(cmd_str) timer_avgs = np.asarray(timer_avgs) timer_calls = np.asarray(timer_calls) all_time = np.dot(timer_avgs, timer_calls) avg_time = all_time / np.sum(timer_calls) logger.info('Time elapsed: {:.2f} seconds, FPS: {:.2f}'.format( all_time, 1.0 / avg_time))
in_flw.append(count_up)} # save results write_results(result_filename, results, data_type) return frame_id, timer.average_time, timer.calls,out_flw,in_flw,l #bd def main(opt, data_root='/data/MOT16/train', det_root=None, seqs=('MOT16-05',), exp_name='demo', save_images=False, save_videos=False, show_image=True): logger.setLevel(logging.INFO) result_root = os.path.join(data_root, '..', 'results', exp_name) mkdir_if_missing(result_root) data_type = 'mot' # Read config cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] # run tracking accs = [] n_frame = 0 timer_avgs, timer_calls = [], [] for seq in seqs: output_dir = os.path.join(data_root, '..','outputs', exp_name, seq) if save_images or save_videos else None logger.info('start seq: {}'.format(seq)) dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size) result_filename = os.path.join(result_root, '{}.txt'.format(seq)) meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read() frame_rate = int(meta_info[meta_info.find('frameRate')+10:meta_info.find('\nseqLength')]) nf, ta, tc = eval_seq(opt, dataloader, data_type, result_filename,
def track(opt): result_root = opt.output_root if opt.output_root != '' else '.' mkdir_if_missing(result_root) cfg_dict = parse_model_cfg(opt.cfg) opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])] batch_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] batch_list gpu_used_list = [] fps_list = [] for batch in batch_list: # run tracking opt.batch_size = batch timer = Timer() accs = [] n_frame = 0 logger.info('Starting processing for batch_size: {}'.format(batch)) dataloader = datasets.LoadVideoBatches(opt.input_video, opt.batch_size, opt.img_size) result_filename = os.path.join(result_root, 'results.txt') frame_rate = dataloader.frame_rate frame_dir = None if opt.output_format == 'text' else osp.join( result_root, 'frame') # try: fps, gpu_used = batch_size_effect_measure(opt, dataloader, 'mot', result_filename, save_dir=None, show_image=False, frame_rate=frame_rate) gpu_used_list.append(gpu_used) fps_list.append(fps) filename_for_analsis = "Batch_analysis_results/results_" + str( opt.video_number) + ".csv" with open(filename_for_analsis, 'w', newline='') as file: writer = csv.writer(file) writer.writerow(["Batch_size", "GPU_used", "fps"]) print("\n\n\n") print("Batch_size GPU_used fps") for i in range(len(batch_list)): print(batch_list[i], " ", gpu_used_list[i], " ", fps_list[i]) writer.writerow([batch_list[i], gpu_used_list[i], fps_list[i]]) import matplotlib.pyplot as plt # x axis values x = batch_list # corresponding y axis values y = gpu_used_list # plotting the points plt.plot(x, y, label="GPU usage") # naming the x axis plt.xlabel('Batch size') # naming the y axis plt.ylabel('GPU Utilization (in MB)') # giving a title to my graph plt.title('GPU usage for varying batch sizes') plt.legend(loc="upper right") plt.grid() # function to show the plot graphname = "GPU_graphs/GPU_" + str(opt.video_number) + ".svg" plt.savefig(graphname) plt.clf() plt.cla() plt.close() # x axis values x = batch_list # corresponding y axis values y = fps_list # plotting the points plt.plot(x, y, label="Speed") # naming the x axis plt.xlabel('Batch size') # naming the y axis plt.ylabel('Speed (in FPS)') # giving a title to my graph plt.title('Speed for different batch sizes') plt.legend(loc="upper right") plt.grid() # function to show the plot graphname = "FPS_graphs/FPS_" + str(opt.video_number) + ".svg" plt.savefig(graphname)
def __init__(self, cfg, img_size): super(SSD, self).__init__() self.module_defs = parse_model_cfg(cfg) self.module_list, self.routs, self.num_cls = create_modules( self.module_defs, img_size)
def train(): cfg = opt.cfg data = opt.data epochs = opt.epochs # 500200 batches at bs 64, 117263 images = 273 epochs batch_size = opt.batch_size accumulate = opt.accumulate # effective bs = batch_size * accumulate = 16 * 4 = 64 midas_weights = opt.midas_weights # initial training weights yolo_weights = opt.yolo_weights weights = opt.weights imgsz_min, imgsz_max, imgsz_test = opt.img_size # img sizes (min, max, test) init_train = opt.init_train midasnet_freeze = opt.midasnet_freeze yolo_freeze = opt.yolo_freeze bbox_lambda = float(opt.bbox_lambda) depth_lambda = float(opt.depth_lambda) # Image Sizes gs = 64 # (pixels) grid size assert math.fmod( imgsz_min, gs) == 0, '--img-size %g must be a %g-multiple' % (imgsz_min, gs) opt.multi_scale |= imgsz_min != imgsz_max # multi if different (min, max) if opt.multi_scale: if imgsz_min == imgsz_max: imgsz_min //= 1.5 imgsz_max //= 0.667 grid_min, grid_max = imgsz_min // gs, imgsz_max // gs imgsz_min, imgsz_max = grid_min * gs, grid_max * gs img_size = imgsz_max # initialize with max size # Configure run init_seeds() data_dict = parse_data_cfg(data) train_path = data_dict['train'] test_path = data_dict['valid'] nc = 1 if opt.single_cls else int( data_dict['classes']) # number of classes hyp['cls'] *= nc / 80 # update coco-tuned hyp['cls'] to current dataset # Remove previous results for f in glob.glob('*_batch*.png') + glob.glob(results_file): os.remove(f) print(f"cfg - {cfg}") print(f"data - {data}") print(f"epochs - {epochs}") print(f"batch_size - {batch_size}") print(f"accumulate - {accumulate}") print(f"yolo weights - {yolo_weights}") print(f"midas weights - {midas_weights}") print( f"imgsz_min- {imgsz_min}, imgsz_max- {imgsz_max}, imgsz_test- {imgsz_test}" ) print(f"opt.rect - {opt.rect}") print(f"train_path - {train_path}") print(f"test_path - {test_path}") print(f"init_train - {init_train}") print(f"weights - {weights}") print(f"midasnet_freeze - {midasnet_freeze}") print(f"yolo_freeze - {yolo_freeze}") print(f"mixed_precision enabled - {mixed_precision}") print(f"Init LR - {hyp['lr0']}") # Initialize model # model = MidasNetEncoder(cfg).to(device) # print_summary(model,'MidasNet Encoder Model') all_props = parse_model_cfg(cfg) yolo_props = [] for item in all_props: if item['type'] == 'yolo': yolo_props.append(item) # print(yolo_props[0]) model = PPENet(yolo_props[0]).to(device) # print("\n\n****** Model Information *******\n\n") # print(model) # print_summary(model,'PPENet Model') # for name, param in model.named_parameters(): # print(name) # print(type(param)) # print(param.shape) # print('param.requires_grad: ', param.requires_grad) # print('=====') # for name, child in model.named_children(): # print(name, child) # model.info() # Optimizer pg0, pg1, pg2 = [], [], [] # optimizer parameter groups for k, v in dict(model.named_parameters()).items(): if '.bias' in k: pg2 += [v] # biases elif 'Conv2d.weight' in k: pg1 += [v] # apply weight_decay else: pg0 += [v] # all else if opt.adam: # hyp['lr0'] *= 0.1 # reduce lr (i.e. SGD=5E-3, Adam=5E-4) optimizer = optim.Adam(pg0, lr=hyp['lr0']) # optimizer = AdaBound(pg0, lr=hyp['lr0'], final_lr=0.1) else: optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True) optimizer.add_param_group({ 'params': pg1, 'weight_decay': hyp['weight_decay'] }) # add pg1 with weight_decay optimizer.add_param_group({'params': pg2}) # add pg2 (biases) del pg0, pg1, pg2 start_epoch = 0 best_fitness = 0.0 # attempt_download(midas_weights) attempt_download(yolo_weights) if midas_weights.endswith('.pt'): # pytorch format # possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc. midas_chkpt = torch.load(midas_weights, map_location=device) # load model try: # yolo_chkpt['model'] = {k: v for k, v in yolo_chkpt['model'].items() if model.state_dict()[k].numel() == v.numel()} # model.load_state_dict(yolo_chkpt['model'], strict=False) # own_state = load_my_state_dict(chkpt, model) # model.load_state_dict(own_state, strict=False) model_dict = model.state_dict() # 1. filter out unnecessary keys midas_weighs_dict = { k: v for k, v in midas_chkpt.items() if k in model_dict } # 2. overwrite entries in the existing state dict model_dict.update(midas_weighs_dict) # 3. load the new state dict model.load_state_dict(model_dict) print("Loaded Midas Encoder weights successfully") except KeyError as e: s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \ "See https://github.com/ultralytics/yolov3/issues/657" % (opt.midas_weights, opt.cfg, opt.midas_weights) raise KeyError(s) from e # print("dbg Model dict") # for key in model_dict.keys(): # if key.startswith('yolo'): # if (not (key.endswith('num_batches_tracked') or ('custom_learner' in key) or key.endswith('running_mean') or key.endswith('running_var'))): # print(f"{key}") # print(f"{model_dict[key].size()}") if yolo_weights.endswith('.pt'): # pytorch format # possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc. yolo_chkpt = torch.load(yolo_weights, map_location=device) if init_train == 'True': # print("Loading Yolo weights") model_dict = model.state_dict() # 1. filter out mapping keys of yolo_chkpt['model'] <- orig weights and yolo_weights_mapping <- model for k, v in model_dict.items(): if k in yolo_weights_mapping.keys(): model_dict[k] = yolo_chkpt['model'][ yolo_weights_mapping[k]] # 3. load the new state dict model.load_state_dict(model_dict) print("Loaded Yolo Decoder weights successfully") if weights.endswith('.pt'): # pytorch format # possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc. chkpt = torch.load(weights, map_location=device) # load model try: # print(f"model state dict {model.state_dict().keys()}") # print(f"best weights {chkpt['model'].keys()}") # print("model state dict") # for k, v in model.state_dict().items(): # print(k) # print(v.shape) # print("best weights dict") # for k,v in chkpt['model'].items(): # print(k) # print(v.shape) chkpt['model'] = { k: v for k, v in chkpt['model'].items() if model.state_dict()[k].numel() == v.numel() } model.load_state_dict(chkpt['model'], strict=False) except KeyError as e: s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \ "See https://github.com/ultralytics/yolov3/issues/657" % (opt.weights, opt.cfg, opt.weights) raise KeyError(s) from e # load optimizer if chkpt['optimizer'] is not None: optimizer.load_state_dict(chkpt['optimizer']) best_fitness = chkpt['best_fitness'] # load results if chkpt.get('training_results') is not None: with open(results_file, 'w') as file: file.write(chkpt['training_results']) # write results.txt start_epoch = chkpt['epoch'] + 1 del chkpt # elif len(weights) > 0: # darknet format # possible weights are '*.weights', 'yolov3-tiny.conv.15', 'darknet53.conv.74' etc. # load_darknet_weights(model, weights) # Mixed precision training https://github.com/NVIDIA/apex if mixed_precision: model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0) # Scheduler https://github.com/ultralytics/yolov3/issues/238 lf = lambda x: ( ((1 + math.cos(x * math.pi / epochs)) / 2 )**1.0) * 0.95 + 0.05 # cosine https://arxiv.org/pdf/1812.01187.pdf scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf, last_epoch=start_epoch - 1) # scheduler = lr_scheduler.MultiStepLR(optimizer, [round(epochs * x) for x in [0.8, 0.9]], 0.1, start_epoch - 1) # Plot lr schedule # y = [] # for _ in range(epochs): # scheduler.step() # y.append(optimizer.param_groups[0]['lr']) # plt.plot(y, '.-', label='LambdaLR') # plt.xlabel('epoch') # plt.ylabel('LR') # plt.tight_layout() # plt.savefig('LR.png', dpi=300) # Initialize distributed training if device.type != 'cpu' and torch.cuda.device_count( ) > 1 and torch.distributed.is_available(): dist.init_process_group( backend='nccl', # 'distributed backend' init_method= 'tcp://127.0.0.1:9999', # distributed training init method world_size=1, # number of nodes for distributed training rank=0) # distributed training node rank model = torch.nn.parallel.DistributedDataParallel( model, find_unused_parameters=True) model.yolo_layers = model.module.yolo_layers # move yolo layer indices to top level # Dataset dataset = LoadImagesAndLabels( train_path, img_size, batch_size, augment=False, hyp=hyp, # augmentation hyperparameters rect=opt.rect, # rectangular training cache_images=opt.cache_images, single_cls=opt.single_cls) # Dataloader batch_size = min(batch_size, len(dataset)) nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]) # number of workers dataloader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, num_workers=nw, shuffle=not opt. rect, # Shuffle=True unless rectangular training is used pin_memory=True, collate_fn=dataset.collate_fn) # Testloader testloader = torch.utils.data.DataLoader( LoadImagesAndLabels( test_path, imgsz_test, batch_size, hyp=hyp, # rect=True, rect=False, cache_images=opt.cache_images, single_cls=opt.single_cls), batch_size=batch_size, num_workers=nw, pin_memory=True, collate_fn=dataset.collate_fn) # Model parameters model.nc = nc # attach number of classes to model model.hyp = hyp # attach hyperparameters to model model.gr = 1.0 # giou loss ratio (obj_loss = 1.0 or giou) model.class_weights = labels_to_class_weights(dataset.labels, nc).to( device) # attach class weights # model.module_defs = parse_model_cfg(cfg) # model.module_list, model.routs = create_modules(model.module_defs, img_size) # model.yolo_layers = get_yolo_layers(model) # Model EMA ema = torch_utils.ModelEMA(model) # print("Before freeze Checking layers param state") # print(model.parameters()) # for name, param in model.named_parameters(): # print(name) # print(param.requires_grad) # Freezing the midasnet if midasnet_freeze == 'True': depth_lambda = 0 print("Freezing the midasnet") for name, param in model.named_parameters(): if not name.startswith('yolo'): param.requires_grad = False if yolo_freeze == 'True': print("Freezing the yolo branch") bbox_lambda = 0 for name, param in model.named_parameters(): if name.startswith('yolo'): param.requires_grad = False # print("After freeze Checking layers param state") # for name, param in model.named_parameters(): # print(name) # print(param.requires_grad) print_summary(model, 'YMP Model Summary', input_size=(3, opt.img_size[0], opt.img_size[0])) # Start training nb = len(dataloader) # number of batches n_burn = max(3 * nb, 500) # burn-in iterations, max(3 epochs, 500 iterations) maps = np.zeros(nc) # mAP per class # torch.autograd.set_detect_anomaly(True) results = ( 0, 0, 0, 0, 0, 0, 0 ) # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification' t0 = time.time() print('Image sizes %g - %g train, %g test' % (imgsz_min, imgsz_max, imgsz_test)) print('Using %g dataloader workers' % nw) print('Starting training for %g epochs...' % epochs) for epoch in range( start_epoch, epochs ): # epoch ------------------------------------------------------------------ model.train() # Update image weights (optional) if dataset.image_weights: w = model.class_weights.cpu().numpy() * (1 - maps)**2 # class weights image_weights = labels_to_image_weights(dataset.labels, nc=nc, class_weights=w) dataset.indices = random.choices(range(dataset.n), weights=image_weights, k=dataset.n) # rand weighted idx mloss = torch.zeros(4).to(device) # mean losses print(('\n' + '%17s' * 10) % ('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets', 'ImgSize', 'RmseGradMeanLoss', 'SSIM_meanLoss')) pbar = tqdm(enumerate(dataloader), total=nb) # progress bar for i, ( imgs, targets, paths, _, depth_targets, _, _ ) in pbar: # batch ------------------------------------------------------------- ni = i + nb * epoch # number integrated batches (since train start) imgs = imgs.to(device).float( ) / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0 targets = targets.to(device) # Burn-in if ni <= n_burn * 2: model.gr = np.interp( ni, [0, n_burn * 2], [0.0, 1.0]) # giou loss ratio (obj_loss = 1.0 or giou) if ni == n_burn: # burnin complete print_model_biases(model) for j, x in enumerate(optimizer.param_groups): # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 x['lr'] = np.interp( ni, [0, n_burn], [0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)]) if 'momentum' in x: x['momentum'] = np.interp(ni, [0, n_burn], [0.9, hyp['momentum']]) # Multi-Scale training if opt.multi_scale: if ni / accumulate % 1 == 0: # adjust img_size (67% - 150%) every 1 batch img_size = random.randrange(grid_min, grid_max + 1) * gs sf = img_size / max(imgs.shape[2:]) # scale factor if sf != 1: ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:] ] # new shape (stretched to 32-multiple) imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False) # Run model # print(f"DBG imgs.shape - {imgs.shape}") depth_preds, yolo_pred = model(imgs) yolo_pred = [item.to(device) for item in yolo_pred] # yolo_pred1,yolo_pred2,yolo_pred3 = yolo_pred # yolo_pred1 = yolo_pred1.to(device) # yolo_pred2 = yolo_pred2.to(device) # yolo_pred3 = yolo_pred3.to(device) # yolo_pred = [yolo_pred1,yolo_pred2,yolo_pred3] # print('yolo_pred1 Size:',yolo_pred1.size()) # print('yolo_pred2 Size:',yolo_pred2.size()) # print('yolo_pred3 Size:',yolo_pred3.size()) #depth_pred = depth_pred.to(device) #yolo_pred = yolo_pred.to(device) # Compute loss loss, loss_items = compute_loss(yolo_pred, targets, model) loss = bbox_lambda * loss if midasnet_freeze == 'False': # print(f"Calculating depth loss") rmse_gradient_mean_loss, ssim_mean_loss = [ depth_lambda * item for item in compute_depth_loss( depth_preds, depth_targets, batch_size, device) ] loss += rmse_gradient_mean_loss + ssim_mean_loss else: rmse_gradient_mean_loss = 0 ssim_mean_loss = 0 if not torch.isfinite(loss): print('WARNING: non-finite loss, ending training ', loss_items, rmse_gradient_mean_loss, ssim_mean_loss) return results # Scale loss by nominal batch_size of 64 loss *= batch_size / 64 # Compute gradient if mixed_precision: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() # Optimize accumulated gradient if ni % accumulate == 0: optimizer.step() optimizer.zero_grad() ema.update(model) # Print batch results mloss = (mloss * i + loss_items) / (i + 1) # update mean losses mem = '%.3gG' % (torch.cuda.memory_cached() / 1E9 if torch.cuda.is_available() else 0) # (GB) s = ('%10s' * 2 + '%17.3g' * 8) % ( '%g/%g' % (epoch, epochs - 1), mem, *mloss, len(targets), img_size, rmse_gradient_mean_loss, ssim_mean_loss) pbar.set_description(s) # Plot images with bounding boxes if ni < 1: f = 'train_batch%g.png' % i # filename plot_images(imgs=imgs, targets=targets, paths=paths, fname=f) if tb_writer: tb_writer.add_image(f, cv2.imread(f)[:, :, ::-1], dataformats='HWC') # tb_writer.add_graph(model, imgs) # add model to tensorboard # end batch ------------------------------------------------------------------------------------------------ # Update scheduler scheduler.step() # Process epoch results ema.update_attr(model) final_epoch = epoch + 1 == epochs if not opt.notest or final_epoch: # Calculate mAP is_coco = any([ x in data for x in ['coco.data', 'coco2014.data', 'coco2017.data'] ]) and model.nc == 80 results, maps = test.test(cfg, data, batch_size=batch_size, img_size=imgsz_test, model=ema.ema, save_json=final_epoch and is_coco, single_cls=opt.single_cls, dataloader=testloader, bbox_lambda=bbox_lambda, depth_lambda=depth_lambda) # Write epoch results with open(results_file, 'a') as f: f.write( s + '%10.3g' * 9 % results + '\n' ) # P, R, mAP, F1, test_losses=(GIoU, obj, cls), depthmap_loss, total_loss if len(opt.name) and opt.bucket: os.system('gsutil cp results.txt gs://%s/results/results%s.txt' % (opt.bucket, opt.name)) # Write Tensorboard results if tb_writer: tags = [ 'train/giou_loss', 'train/obj_loss', 'train/cls_loss', 'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/F1', 'val/giou_loss', 'val/obj_loss', 'val/cls_loss' ] for x, tag in zip(list(mloss[:-1]) + list(results), tags): tb_writer.add_scalar(tag, x, epoch) # Update best mAP fi = fitness(np.array(results).reshape( 1, -1)) # fitness_i = weighted combination of [P, R, mAP, F1] if fi > best_fitness: best_fitness = fi # Save training results save = (not opt.nosave) or (final_epoch and not opt.evolve) if save: with open(results_file, 'r') as f: # Create checkpoint chkpt = { 'epoch': epoch, 'best_fitness': best_fitness, 'training_results': f.read(), 'model': ema.ema.module.state_dict() if hasattr(model, 'module') else ema.ema.state_dict(), 'optimizer': None if final_epoch else optimizer.state_dict() } # Save last checkpoint torch.save(chkpt, last) # Save best checkpoint if (best_fitness == fi) and not final_epoch: torch.save(chkpt, best) # Save backup every 10 epochs (optional) # if epoch > 0 and epoch % 10 == 0: # torch.save(chkpt, wdir + 'backup%g.pt' % epoch) # Delete checkpoint del chkpt # end epoch ---------------------------------------------------------------------------------------------------- # end training n = opt.name if len(n): n = '_' + n if not n.isnumeric() else n fresults, flast, fbest = 'results%s.txt' % n, wdir + 'last%s.pt' % n, wdir + 'best%s.pt' % n for f1, f2 in zip([wdir + 'last.pt', wdir + 'best.pt', 'results.txt'], [flast, fbest, fresults]): if os.path.exists(f1): os.rename(f1, f2) # rename ispt = f2.endswith('.pt') # is *.pt strip_optimizer(f2) if ispt else None # strip optimizer os.system('gsutil cp %s gs://%s/weights' % ( f2, opt.bucket)) if opt.bucket and ispt else None # upload if not opt.evolve: plot_results() # save as results.png print('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) dist.destroy_process_group() if torch.cuda.device_count() > 1 else None torch.cuda.empty_cache() return results