def track(opt):
result_root = opt.output_root if opt.output_root != '' else '.'
mkdir_if_missing(result_root)
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
timer = Timer()
accs = []
n_frame = 0
logger.info('Starting tracking...')
dataloader = datasets.LoadVideo(opt.input_video, opt.img_size)
result_filename = os.path.join(result_root, 'results.txt')
frame_rate = dataloader.frame_rate
frame_dir = None if opt.output_format == 'text' else osp.join(
result_root, 'frame')
try:
ob = eval_seq(opt,
dataloader,
'mot',
result_filename,
save_dir=frame_dir,
show_image=False,
frame_rate=frame_rate)
o = ob[3][1:] #bd{
i = ob[4]
l_count = 0
l_count = ob[5]
ai = sum(i) / len(i)
ao = sum(o) / len(o)
print("\nAverage Inflow=", ai)
print("\nAverage Outflow=", ao)
if (ai > ao):
print(
"\n Congratulations!! The ROI is proving to be a hotspot !!!\n"
)
elif (ao > ai):
print(
"\n The ROI is losing people attention !!!\n"
)
else:
print(
"\n The ROI has a moderate people attention !!!\n"
) #bd}
print(
"\n The ROI has a current crowd of %d people !!!\n"
% l_count)
except Exception as e:
logger.info(e)
if opt.output_format == 'video':
output_video_path = osp.join(result_root, 'result.mp4')
cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(
osp.join(result_root, 'frame'), output_video_path)
os.system(cmd_str)
return o, i, l_count
def __init__(self, cfg, img_size=(416, 416)):
super(Darknet, self).__init__()
# 1、读取 cfg 文件，得到 module_defs（ 是一个包含多个 dict 的list，比如索引 082 对应的 dict 为
# <class 'dict'>: {'type': 'yolo', 'mask': '6,7,8', 'anchors': array([[ 10., 13.],
# [ 16., 30.],
# [ 33., 23.],
# [ 30., 61.],
# [ 62., 45.],
# [ 59., 119.],
# [116., 90.],
# [156., 198.],
# [373., 326.]]), 'classes': '80', 'num': '9', 'jitter': '.3', 'ignore_thresh': '.7', 'truth_thresh': '1', 'random': '1'}
# 内容上和 .cfg 文件本身完全一样
self.module_defs = parse_model_cfg(
cfg) # 注意这里除了包含索引 0-106 的所有层之外，还包含了[net]的信息，比如输入通道数channels=3，
# 这个在下一步 create_modules() 搭建conv层时会用到，所以还不能删；等进入 create_modules() 后，
# 才可以 module_defs.pop(0)；另外 get_yolo_layers() 也因此放在再下一步，也就是第三步，注意顺序不能颠倒
# 2、将 module_defs 转化为 module_list，并且找到所有 路由层 的索引，包括 residual模块的 shortcut、
# 有一个OrderDict类型的变量_module ，比如 key '82' 对应的 value 为 YOLOLayer()对象
# 这里根据 1 中每个 dict， 创建相应层的对象，放入 nn.ModuleList()类型的 module_list，
self.module_list, self.routs = create_modules(self.module_defs)
# 3、这里存的只是三个 yolo_layer 的索引，
# 通过遍历 module_defs 很容易得到，即 [82, 94, 106]
self.yolo_layers = get_yolo_layers(self)
def track(opt):
result_root = opt.output_root if opt.output_root != '' else '.'
mkdir_if_missing(result_root)
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
timer = Timer()
accs = []
n_frame = 0
logger.info('Starting tracking...')
dataloader = datasets.LoadVideo(opt.input_video, opt.img_size)
result_filename = os.path.join(result_root, 'results.txt')
frame_rate = dataloader.frame_rate
frame_dir = None if opt.output_format == 'text' else osp.join(
result_root, 'frame')
try:
eval_seq(opt,
dataloader,
'mot',
result_filename,
save_dir=frame_dir,
show_image=False,
frame_rate=frame_rate)
except Exception as e:
logger.info(e)
if opt.output_format == 'video':
output_video_path = osp.join(result_root, 'result.mp4')
cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(
osp.join(result_root, 'frame'), output_video_path)
os.system(cmd_str)
def __init__(self, cfg_path, img_size=416):
super(DarkRFBNet, self).__init__()
self.module_defs = parse_model_cfg(cfg_path)
self.module_defs[0]['cfg'] = cfg_path
self.module_defs[0]['height'] = img_size
self.hyperparams, self.module_list = create_modules(self.module_defs)
self.img_size = img_size
def track(opt):
if opt.AVA:
if opt.ov:
if opt.skip:
outPath = '../../output/trmot_ov/{}'.format(opt.skip)
else:
outPath = '../../output/trmot_ov'
elif opt.cpu:
if opt.skip:
outPath = '../../output/trmot_cpu/{}'.format(opt.skip)
else:
outPath = '../../output/trmot_cpu'
else:
if opt.skip:
outPath = '../../output/trmot_gpu/{}'.format(opt.skip)
else:
outPath = '../../output/trmot_gpu'
if opt.skip==0:
opt.skip=1
if not os.path.exists(outPath):
os.makedirs(outPath)
if opt.video_path!=0:
countFileName = '{}/{}.csv'.format(outPath, opt.video_path.split('/')[-1].split('.mp4')[0])
else:
countFileName = '{}/results_live_cam.csv'.format(outPath)
countFile = open(countFileName, 'w')
countFile.close()
result_root = opt.output_root if opt.output_root!='' else '.'
mkdir_if_missing(result_root)
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
timer = Timer()
accs = []
n_frame = 0
logger.info('Starting tracking...')
dataloader = datasets.LoadVideo(opt.video_path, opt.img_size)
result_filename = os.path.join(result_root, 'results.txt')
frame_rate = dataloader.frame_rate
frame_dir = None if opt.output_format=='text' else osp.join(result_root, 'frame')
try:
eval_seq(opt, dataloader, 'mot', result_filename, countFileName, save_dir=frame_dir, show_image=False, frame_rate=frame_rate)
except Exception as e:
logger.info(e)
if opt.output_format == 'video':
output_video_path = osp.join(result_root, 'result.mp4')
cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(osp.join(result_root, 'frame'), output_video_path)
os.system(cmd_str)
def __init__(self, batch_size=19):
cfg_path = "./cfg/network.cfg"
net_block = parse_model_cfg(cfg_path)[0] # [net]
self.n_classes = net_block["n_classes"]
self.train_list = read_txt(net_block["train_set"])
self.test_list = read_txt(net_block["test_set"])
self.total_list = read_txt(net_block["total_set"])
self.BATCH_SIZE = batch_size # 一次读入多少个样本
self.num_workers = 2 # 加载batch的线程数
def __init__(self, cfg_path, img_size=416):
super(YOLOv3, self).__init__()
self.module_defs = parse_model_cfg(cfg_path)
self.module_defs[0]['cfg'] = cfg_path
self.module_defs[0]['height'] = img_size
self.hyperparams, self.module_list = create_modules(self.module_defs)
self.img_size = img_size
self.loss_names = ['loss', 'xy', 'wh', 'conf', 'cls', 'nT']
self.losses = []
def main(opt, data_root='/data/MOT16/train', det_root=None, seqs=('MOT16-05',), exp_name='demo',
save_images=False, save_videos=False, show_image=True):
logger.setLevel(logging.INFO)
result_root = os.path.join(data_root, '..', 'results', exp_name)
mkdir_if_missing(result_root)
data_type = 'mot'
# Read config
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
accs = []
n_frame = 0
timer_avgs, timer_calls = [], []
for seq in seqs:
output_dir = os.path.join(data_root, '..','outputs', exp_name, seq) if save_images or save_videos else None
logger.info('start seq: {}'.format(seq))
dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size)
result_filename = os.path.join(result_root, '{}.txt'.format(seq))
meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read()
frame_rate = int(meta_info[meta_info.find('frameRate')+10:meta_info.find('\nseqLength')])
nf, ta, tc = eval_seq(opt, dataloader, data_type, result_filename,
save_dir=output_dir, show_image=show_image, frame_rate=frame_rate)
n_frame += nf
timer_avgs.append(ta)
timer_calls.append(tc)
# eval
logger.info('Evaluate seq: {}'.format(seq))
evaluator = Evaluator(data_root, seq, data_type)
accs.append(evaluator.eval_file(result_filename))
if save_videos:
output_video_path = osp.join(output_dir, '{}.mp4'.format(seq))
cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(output_dir, output_video_path)
os.system(cmd_str)
timer_avgs = np.asarray(timer_avgs)
timer_calls = np.asarray(timer_calls)
all_time = np.dot(timer_avgs, timer_calls)
avg_time = all_time / np.sum(timer_calls)
logger.info('Time elapsed: {:.2f} seconds, FPS: {:.2f}'.format(all_time, 1.0 / avg_time))
# get summary
metrics = mm.metrics.motchallenge_metrics
mh = mm.metrics.create()
summary = Evaluator.get_summary(accs, seqs, metrics)
strsummary = mm.io.render_summary(
summary,
formatters=mh.formatters,
namemap=mm.io.motchallenge_metric_names
)
print(strsummary)
sys.stdout.flush()
Evaluator.save_summary(summary, os.path.join(result_root, 'summary_{}.xlsx'.format(exp_name)))
def __init__(self, cfg, img_size=(416, 416), verbose=False):
super(YOLOV3_SPP, self).__init__()
self.input_size = [img_size] * 2 if isinstance(img_size, int) else img_size
self.module_defs = parse_model_cfg(cfg)
self.module_list, self.routs = self.create_modules(self.module_defs, img_size, cfg)
self.yolo_layers = self.get_index('YOLOLayer')
# YOLOV3_SPP Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346
self.version = np.array([0, 2, 5], dtype=np.int32) # (int32) version info: major, minor, revision
self.seen = np.array([0], dtype=np.int64) # (int64) number of images seen during training
self.info(verbose) if not ONNX_EXPORT else None # print model description
def __init__(self, cfg, img_size=(416, 416)):
super(Darknet, self).__init__()
self.module_defs = parse_model_cfg(cfg)
self.module_defs[0]['cfg'] = cfg
self.module_defs[0]['height'] = img_size
self.hyperparams, self.module_list = create_modules(self.module_defs)
self.yolo_layers = get_yolo_layers(self)
# Needed to write header when saving weights
self.header_info = np.zeros(5, dtype=np.int32) # First five are header values
self.seen = self.header_info[3] # number of images seen during training
def mask_cfg_and_converted(mask_replace_layer,
cfg='cfg/yolov3-voc.cfg',
weight_path='../weights/converted-voc.pt',
target='../weights/maskconverted-voc.pt'):
mask_cfg = '/'.join(cfg.split('/')[:-1]) + '/mask' + cfg.split('/')[-1]
origin_mdfs = parse_model_cfg(cfg)
mask_mdfs = []
mask_mdfs.append(origin_mdfs.pop(0))
for i, mdf in enumerate(origin_mdfs):
if str(i) in mask_replace_layer:
mdf['type'] = 'maskconvolutional'
mask_mdfs.append(mdf)
write_cfg(mask_cfg, mask_mdfs)
mask_weight = OrderedDict()
origin_weight = torch.load(weight_path)['model']
for k, v in origin_weight.items():
key_list = k.split('.')
idx = key_list[1]
if idx in mask_replace_layer and key_list[2] == 'Conv2d':
key_list[2] = 'Mask' + key_list[2]
key = '.'.join(key_list)
mask_weight[key] = v
mask_weight[key.replace('weight',
'selected_channels_mask')] = torch.ones(
v.size(1), dtype=torch.float32)
else:
key = '.'.join(key_list)
mask_weight[key] = v
model = Darknet(mask_cfg)
model.load_state_dict(mask_weight, strict=True)
if target is not None:
chkpt = {
'epoch': -1,
'best_fitness': None,
'training_results': None,
'model': model.state_dict(),
'optimizer': None
}
torch.save(chkpt, target)
return mask_cfg, model.state_dict()
def load_model(model_path, device, cfg):
# select device
print("device: %s" % device)
# load network
all_props = parse_model_cfg(cfg)
yolo_props = []
for item in all_props:
if item['type'] == 'yolo':
yolo_props.append(item)
# print(yolo_props[0])
model = YMPNet(yolo_props[0]).to(device)
model.inference = True
if model_path.endswith('.pt'): # pytorch format
# possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc.
midas_chkpt = torch.load(model_path, map_location=device)
# load model
try:
# yolo_chkpt['model'] = {k: v for k, v in yolo_chkpt['model'].items() if model.state_dict()[k].numel() == v.numel()}
# model.load_state_dict(yolo_chkpt['model'], strict=False)
# own_state = load_my_state_dict(chkpt, model)
# model.load_state_dict(own_state, strict=False)
model_dict = model.state_dict()
# 1. filter out unnecessary keys
# midas_weighs_dict = {k: v for k, v in midas_chkpt.items() if k in model_dict}
for k, v in midas_chkpt['model'].items():
if k in model_dict:
model_dict[k] = v
# 2. overwrite entries in the existing state dict
# model_dict.update(midas_weighs_dict)
# 3. load the new state dict
model.load_state_dict(model_dict)
print("Loaded Model weights successfully")
except KeyError as e:
s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \
"See https://github.com/ultralytics/yolov3/issues/657" % (opt.midas_weights, opt.cfg, opt.midas_weights)
raise KeyError(s) from e
return model
def __init__(self):
super(runModel, self).__init__()
self.keep_going = False
self.load_path = "weight/91acc_9category_1024_512_256.pt" # 继续训练时，要加载的参数文件
self.batch_size = 30 # 分批训练数据、每批数据量
self.learning_rate = 0.01 # 1e-2 # 学习率
self.num_epoches = 700 # 训练次数
cfg_path = "./cfg/network.cfg"
net_block = parse_model_cfg(cfg_path)[0] # [net]
self.n_classes = net_block["n_classes"] # 类别数
# 保存用于可视化观察参数变化的列表
self.loss_list, self.lr_list, self.acc_list = [], [], [
] # 记录损失值\学习率\准确率变化
self.layer_list = [1024, 512, 256]
# self.model = MyFC(self.layer_list, 42, self.n_classes)
self.model = classifier("./cfg/network.cfg")
self.train_loader = MyDataLoader(batch_size=self.batch_size).train()
self.test_loader = MyDataLoader(batch_size=self.batch_size).test()
self.best_acc = 0 # 最高准确类
self.writer = SummaryWriter(logdir='./log') # 记录训练日志
def track(opt):
result_root = opt.output_root if opt.output_root!='' else '.'
mkdir_if_missing(result_root)
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
logger.info('Starting tracking...')
dataloader = datasets.LoadVideo(opt.input_video, opt.img_size)
result_filename = os.path.join(result_root, 'results.txt')
frame_rate = dataloader.frame_rate
frame_dir = None if opt.output_format=='text' else osp.join(result_root, 'frame')
try:
eval_seq(opt, dataloader, 'mot', result_filename,
save_dir=frame_dir, show_image=True, frame_rate=frame_rate)
except Exception as e:
logger.info(e)
def __init__(self, args, video_path):
self.args = args
self.video_path = video_path
self.logger = get_logger("root")
use_cuda = args.use_cuda and torch.cuda.is_available()
if not use_cuda:
warnings.warn("Running in cpu mode which maybe very slow!",
UserWarning)
if args.display:
cv2.namedWindow("test", cv2.WINDOW_NORMAL)
cv2.resizeWindow("test", args.display_width, args.display_height)
if args.rtsp != "":
print("Using rtsp " + str(args.rtsp))
self.vdo = cv2.VideoCapture(args.rtsp)
elif args.cam != -1:
print("Using webcam " + str(args.cam))
self.vdo = cv2.VideoCapture(args.cam)
else:
self.vdo = cv2.VideoCapture()
frame_rate = 30
cfg_dict = parse_model_cfg(args.cfg)
self.width, self.height = int(cfg_dict[0]['width']), int(
cfg_dict[0]['height'])
args.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
self.tracker = JDETracker(args, frame_rate=frame_rate)
self.known_faces = {}
self.faceUtils = FaceUtils()
self.camUtils = CamUtils(args)
self.tmp_imgs = []
self.tmp_moves = []
self.result_imgs = queue.Queue()
self.is_running = True
print("Loading Control Done.")
def track(opt):
# set saving dir
result_root = opt.output_root if opt.output_root != '' else '.'
mkdir_if_missing(result_root)
# set configuration
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
logger.info('Starting tracking...')
# use camera to track
dataloader = datasets.LoadCamera(img_size=opt.img_size)
result_filename = os.path.join(result_root, 'results.txt')
try:
# start tracking
eval_seq(opt,
dataloader,
'mot',
result_filename,
save_dir=result_root,
show_image=True)
except Exception as e:
logger.info(e)
def prune(mask_cfg, progress_weights, mask_replace_layer, new_cfg_file,
new_weights):
only_in = mask_replace_layer[-3:]
mask_replace_layer = mask_replace_layer[:-2]
device_in = torch.device('cpu')
model = Darknet(mask_cfg)
chkpt = torch.load(progress_weights, map_location=device_in)
model.load_state_dict(chkpt['model'])
new_cfg = parse_model_cfg(mask_cfg)
for layer in mask_replace_layer[:-1]:
assert isinstance(model.module_list[int(layer)][0],
MaskConv2d), "Not a pruned model!"
tail_layer = mask_replace_layer[mask_replace_layer.index(layer) + 1]
assert isinstance(model.module_list[int(tail_layer)][0],
MaskConv2d), "Not a pruned model!"
in_channels_mask = model.module_list[int(
layer)][0].selected_channels_mask
out_channels_mask = model.module_list[int(
tail_layer)][0].selected_channels_mask
in_channels = int(torch.sum(in_channels_mask))
out_channels = int(torch.sum(out_channels_mask))
new_cfg[int(layer) + 1]['type'] = 'convolutional'
new_cfg[int(layer) + 1]['filters'] = str(out_channels)
new_conv = nn.Conv2d(
in_channels,
out_channels,
kernel_size=model.module_list[int(layer)][0].kernel_size,
stride=model.module_list[int(layer)][0].stride,
padding=model.module_list[int(layer)][0].padding,
bias=False)
thin_weight = model.module_list[int(layer)][0].weight[
out_channels_mask.bool()]
thin_weight = thin_weight[:, in_channels_mask.bool()]
assert new_conv.weight.numel() == thin_weight.numel(
), 'Do not match in shape!'
new_conv.weight.data.copy_(thin_weight.data)
new_batch = nn.BatchNorm2d(out_channels, momentum=0.1)
new_batch.weight.data.copy_(model.module_list[int(layer)][1].weight[
out_channels_mask.bool()].data)
new_batch.bias.data.copy_(model.module_list[int(layer)][1].bias[
out_channels_mask.bool()].data)
new_batch.running_mean.copy_(model.module_list[int(
layer)][1].running_mean[out_channels_mask.bool()].data)
new_batch.running_var.copy_(model.module_list[int(
layer)][1].running_var[out_channels_mask.bool()].data)
new_module = nn.Sequential()
new_module.add_module('Conv2d', new_conv)
new_module.add_module('BatchNorm2d', new_batch)
new_module.add_module('activation', model.module_list[int(layer)][2])
model.module_list[int(layer)] = new_module
for layer in only_in:
new_cfg[int(layer) + 1]['type'] = 'convolutional'
assert isinstance(model.module_list[int(layer)][0],
MaskConv2d), "Not a pruned model!"
in_channels_mask = model.module_list[int(
layer)][0].selected_channels_mask > 0.1
in_channels = int(torch.sum(in_channels_mask))
new_conv = nn.Conv2d(
in_channels,
out_channels=model.module_list[int(layer)][0].out_channels,
kernel_size=model.module_list[int(layer)][0].kernel_size,
stride=model.module_list[int(layer)][0].stride,
padding=model.module_list[int(layer)][0].padding,
bias=False)
new_conv.weight.data.copy_(model.module_list[int(layer)]
[0].weight[:, in_channels_mask.bool()].data)
new_module = nn.Sequential()
new_module.add_module('Conv2d', new_conv)
new_module.add_module('BatchNorm2d', model.module_list[int(layer)][1])
new_module.add_module('activation', model.module_list[int(layer)][2])
model.module_list[int(layer)] = new_module
write_cfg(new_cfg_file, new_cfg)
chkpt = {
'epoch': -1,
'best_fitness': None,
'training_results': None,
'model': model.state_dict(),
'optimizer': None
}
torch.save(chkpt, new_weights)
def __init__(self, cfg: str):
super(classifier, self).__init__()
# 解析网络对应的.cfg文件,得到模型定义
self.module_definitions = parse_model_cfg(cfg)
# 根据解析的网络结构一层一层去搭建
self.module_list = create_modules(self.module_definitions)
if i != len(args[1]) - 1:
next_item = args[1][i + 1]
else:
next_item = "1"
if "--" in curr_item and "--" in next_item:
key = curr_item.replace('-', '')
if key not in params: params[key] = "1"
elif "--" in curr_item and "--" not in next_item:
key = curr_item.replace('-', '')
if key not in params: params[key] = next_item
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
cfg_dict = parse_model_cfg(path=command_args.cfg)
command_args.img_size = [int(cfg_dict[0]["width"]), int(cfg_dict[0]["height"])]
timer = Timer()
logger.info('Starting tracking...')
dataloader = datasets.LoadVideo(command_args.input_video, command_args.img_size)
frame_rate = dataloader.frame_rate
orig_width = dataloader.vw
orig_height = dataloader.vh
width = dataloader.w
height = dataloader.h
video_writer = cv2.VideoWriter(os.path.join(command_args.output_dir, os.path.splitext(os.path.basename(command_args.input_video))[0] + ".mp4"),
cv2.VideoWriter_fourcc(*"mp4v"), float(frame_rate), (orig_width, orig_height))
tracker = JDETracker(opt=command_args, frame_rate=frame_rate)
def run(input_path, output_path, model_path):
"""Run MonoDepthNN to compute depth maps.
Args:
input_path (str): path to input folder
output_path (str): path to output folder
model_path (str): path to saved model
"""
print("initialize")
# select device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("device: %s" % device)
# load network
all_props = parse_model_cfg(cfg)
yolo_props = []
for item in all_props:
if item['type'] == 'yolo':
yolo_props.append(item)
# print(yolo_props[0])
model = PPENet(yolo_props[0]).to(device)
model.inference = True
if model_path.endswith('.pt'): # pytorch format
# possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc.
midas_chkpt = torch.load(model_path, map_location=device)
# load model
try:
# yolo_chkpt['model'] = {k: v for k, v in yolo_chkpt['model'].items() if model.state_dict()[k].numel() == v.numel()}
# model.load_state_dict(yolo_chkpt['model'], strict=False)
# own_state = load_my_state_dict(chkpt, model)
# model.load_state_dict(own_state, strict=False)
model_dict = model.state_dict()
# 1. filter out unnecessary keys
# midas_weighs_dict = {k: v for k, v in midas_chkpt.items() if k in model_dict}
for k, v in midas_chkpt['model'].items():
if k in model_dict:
model_dict[k] = v
# 2. overwrite entries in the existing state dict
# model_dict.update(midas_weighs_dict)
# 3. load the new state dict
model.load_state_dict(model_dict)
print("Loaded Model weights successfully")
except KeyError as e:
s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \
"See https://github.com/ultralytics/yolov3/issues/657" % (opt.midas_weights, opt.cfg, opt.midas_weights)
raise KeyError(s) from e
transform = Compose([
Resize(
384,
384,
resize_target=None,
keep_aspect_ratio=True,
ensure_multiple_of=32,
resize_method="upper_bound",
image_interpolation_method=cv2.INTER_CUBIC,
),
NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
PrepareForNet(),
])
model.to(device)
model.eval()
# get input
img_names = glob.glob(os.path.join(input_path, "*"))
num_images = len(img_names)
# create output folder
os.makedirs(output_path, exist_ok=True)
print("start processing")
for ind, img_name in enumerate(img_names):
print(" processing {} ({}/{})".format(img_name, ind + 1, num_images))
# input
img = midas_utils.read_image(img_name)
img_input = transform({"image": img})["image"]
# print(f"DBG run img.shape - {img.shape}")
# compute
with torch.no_grad():
sample = torch.from_numpy(img_input).to(device).unsqueeze(0)
prediction, _ = model(sample)
# print(f"DBG prediction.shape - {prediction.shape}")
# print(f"prediction.min() - {prediction.min()}")
# print(f"prediction.max() - {prediction.max()}")
prediction = (torch.nn.functional.interpolate(
prediction.unsqueeze(1),
size=img.shape[:2],
mode="bicubic",
align_corners=False,
).squeeze().cpu().numpy())
# output
filename = os.path.join(
output_path,
os.path.splitext(os.path.basename(img_name))[0])
midas_utils.write_depth(filename, prediction, bits=2)
print("finished")
def sort_confidence(preds):
"""分析错分样本的置信度"""
values, indexes = preds.topk(k=n_classes,
dim=-1) # 将置信度前三的从大到小排列得到v，每个置信度相应的标签为index
# values, indexes = values.tolist(), indexes.tolist()
# values = [round(v, 3) for v in values]
return values, indexes
if __name__ == '__main__':
cfg_path = "./cfg/network.cfg"
pt_file = "weight/91acc_9category_1024_512_256.pt"
# pt_file = "runs/2021-04-16/93acc_699epoch_9category.pt"
net_block = parse_model_cfg(cfg_path)[0] # [net]
n_classes = net_block["n_classes"]
batch_size = net_block["batch"]
model = classifier(cfg_path).cuda()
param_dict = torch.load(pt_file)
model.load_state_dict(param_dict["model_state"])
data_loader = MyDataLoader(batch_size=batch_size).test()
confidence_threshold = 0.98 # 置信度阈值，大于该值才预测为真，否则pass
# torch.save(model, './classification_model.pth')
model = torch.load('./classification_model.pth')
classes = [
'0-other', '1-OK', '2-palm', '3-up', '4-down', '5-right', '6-left',
'7-heart', '8-hush'
]
if n_classes == 8:
def test(
cfg,
data,
weights=None,
batch_size=16,
img_size=512,
conf_thres=0.001,
iou_thres=0.6, # for nms
save_json=False,
single_cls=False,
augment=False,
model=None,
dataloader=None,
bbox_lambda=None,
depth_lambda=None):
# Initialize/load model and set device
if model:
model.inference = True
if model is None:
device = torch_utils.select_device(opt.device, batch_size=batch_size)
verbose = opt.task == 'test'
# Remove previous
for f in glob.glob('test_batch*.png'):
os.remove(f)
# Initialize model
# model = Darknet(cfg, img_size)
all_props = parse_model_cfg(cfg)
yolo_props = []
for item in all_props:
if item['type'] == 'yolo':
yolo_props.append(item)
# print(yolo_props[0])
model = YMPNet(yolo_props[0], inference=True).to(device)
# model.module_defs = parse_model_cfg(cfg)
# model.module_list, model.routs = create_modules(model.module_defs, img_size)
# model.yolo_layers = get_yolo_layers(model)
# Load weights
attempt_download(weights)
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(
torch.load(weights, map_location=device)['model'])
# else: # darknet format
# load_darknet_weights(model, weights)
# Fuse
model.fuse()
model.to(device)
if device.type != 'cpu' and torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else: # called by train.py
device = next(model.parameters()).device # get model device
verbose = False
# Configure run
data = parse_data_cfg(data)
nc = 1 if single_cls else int(data['classes']) # number of classes
path = data['valid'] # path to test images
names = load_classes(data['names']) # class names
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
iouv = iouv[0].view(1) # comment for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if dataloader is None:
dataset = LoadImagesAndLabels(path,
img_size,
batch_size,
rect=True,
single_cls=opt.single_cls)
batch_size = min(batch_size, len(dataset))
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=min([
os.cpu_count(),
batch_size if batch_size > 1 else 0, 8
]),
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
# _ = model(torch.zeros((1, 3, img_size, img_size), device=device)) if device.type != 'cpu' else None # run once
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%17s' * 10) % ('Class', 'Images', 'Targets', 'P', 'R',
'[email protected]', 'F1', 'RmseGradientLoss',
'SSIMLoss', 'DLoss', 'TotalLoss')
p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
bbox_loss = torch.zeros(3, device=device)
rmse_gradient_loss = 0
ssim_loss = 0
depth_loss = 0
total_loss = 0
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths, shapes, depth_targets, _,
_) in enumerate(tqdm(dataloader, desc=s)):
imgs = imgs.to(
device).float() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
# print(f"DBG Imgs.shape {imgs.shape}")
nb, _, height, width = imgs.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Plot images with bounding boxes
f = 'test_batch%g.png' % batch_i # filename
if batch_i < 1 and not os.path.exists(f):
plot_images(imgs=imgs, targets=targets, paths=paths, fname=f)
# Disable gradients
with torch.no_grad():
# Run model
t = torch_utils.time_synchronized()
# inf_out, train_out = model(imgs, augment=augment) # inference and training outputs
depth_out, (inf_out, train_out) = model(
imgs) # inference and training outputs
# print("DBG1", type(inf_out), type(train_out))
t0 += torch_utils.time_synchronized() - t
# Compute loss
# if hasattr(model, 'hyp'): # if model has loss hyperparameters
# loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
bbox_loss += bbox_lambda * compute_loss(
train_out, targets, model)[1][:3] # GIoU, obj, cls
rmse_gradient_mean_loss, ssim_mean_loss = [
depth_lambda * item for item in compute_depth_loss(
depth_out, depth_targets, batch_size, device)
]
rmse_gradient_loss += rmse_gradient_mean_loss
ssim_loss += ssim_mean_loss
# depth_loss += depth_lambda * compute_depth_loss(depth_out, depth_targets, batch_size, device)
depth_loss = rmse_gradient_loss + ssim_loss
# print(f"DBG bbox_loss.shape - {bbox_loss.shape}")
# print(f"DBG bbox_loss.shape - {bbox_loss.shape}")
# Run NMS
t = torch_utils.time_synchronized()
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
iou_thres=iou_thres) # nms
t1 += torch_utils.time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box, shapes[si][0],
shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
'image_id': image_id,
'category_id': coco91class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score': round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero().view(
-1) # prediction indices
pi = (cls == pred[:,
5]).nonzero().view(-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(
1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero():
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
total_loss += sum(bbox_loss.tolist()) + depth_loss
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
if niou > 1:
p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(
1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]]
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# print(f"DBG bbox_loss: {bbox_loss}")
# print(f"DBG depth_loss: {depth_loss}")
# print(f"DBG total_loss: {total_loss}")
# print(f"DBG seen, nt.sum(), mp, mr, map, mf1 - {seen, nt.sum(), mp, mr, map, mf1}")
# Print results
pf = '%20s' + '%17.3g' * 10 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1, rmse_gradient_loss,
ssim_loss, depth_loss, total_loss))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Print speeds
if verbose or save_json:
t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (
img_size, img_size, batch_size) # tuple
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, *(bbox_loss.cpu() / len(dataloader)).tolist(),
depth_loss / len(dataloader), total_loss / len(dataloader)), maps
def main(opt,
data_root='/media/dh/data/MOT16/train',
det_root=None,
seqs=('MOT16-05', ),
exp_name='demo',
save_images=False,
save_videos=False,
show_image=True):
logger.setLevel(logging.INFO)
result_root = os.path.join(data_root, '..', 'results', exp_name)
mkdir_if_missing(result_root)
data_type = 'mot'
# Read config
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
accs = []
n_frame = 0
timer_avgs, timer_calls = [], []
for seq in seqs:
output_dir = os.path.join(data_root, '..', 'outputs', exp_name,
seq) if save_images or save_videos else None
logger.info('start seq: {}'.format(seq))
#dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size)
print(osp.join(data_root, seq))
dataloader = datasets.LoadVideo(osp.join(data_root, seq))
# print ("DATALOADER", dataloader.vw)
result_filename = os.path.join(result_root, '{}.csv'.format(seq))
#meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read()
#frame_rate = int(meta_info[meta_info.find('frameRate')+10:meta_info.find('\nseqLength')])
frame_rate = 30
nf, ta, tc = eval_seq(opt,
dataloader,
data_type,
result_filename,
save_dir=output_dir,
show_image=show_image,
frame_rate=frame_rate,
vw=dataloader.vw)
n_frame += nf
timer_avgs.append(ta)
timer_calls.append(tc)
# eval
logger.info('Evaluate seq: {}'.format(seq))
evaluator = Evaluator(data_root, seq, data_type)
accs.append(evaluator.eval_file(result_filename))
if save_videos:
output_video_path = osp.join(output_dir, '{}.mp4'.format(seq))
cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg -c:v copy {}'.format(
output_dir, output_video_path)
os.system(cmd_str)
timer_avgs = np.asarray(timer_avgs)
timer_calls = np.asarray(timer_calls)
all_time = np.dot(timer_avgs, timer_calls)
avg_time = all_time / np.sum(timer_calls)
logger.info('Time elapsed: {:.2f} seconds, FPS: {:.2f}'.format(
all_time, 1.0 / avg_time))
in_flw.append(count_up)}
# save results
write_results(result_filename, results, data_type)
return frame_id, timer.average_time, timer.calls,out_flw,in_flw,l #bd
def main(opt, data_root='/data/MOT16/train', det_root=None, seqs=('MOT16-05',), exp_name='demo',
save_images=False, save_videos=False, show_image=True):
logger.setLevel(logging.INFO)
result_root = os.path.join(data_root, '..', 'results', exp_name)
mkdir_if_missing(result_root)
data_type = 'mot'
# Read config
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
# run tracking
accs = []
n_frame = 0
timer_avgs, timer_calls = [], []
for seq in seqs:
output_dir = os.path.join(data_root, '..','outputs', exp_name, seq) if save_images or save_videos else None
logger.info('start seq: {}'.format(seq))
dataloader = datasets.LoadImages(osp.join(data_root, seq, 'img1'), opt.img_size)
result_filename = os.path.join(result_root, '{}.txt'.format(seq))
meta_info = open(os.path.join(data_root, seq, 'seqinfo.ini')).read()
frame_rate = int(meta_info[meta_info.find('frameRate')+10:meta_info.find('\nseqLength')])
nf, ta, tc = eval_seq(opt, dataloader, data_type, result_filename,
def track(opt):
result_root = opt.output_root if opt.output_root != '' else '.'
mkdir_if_missing(result_root)
cfg_dict = parse_model_cfg(opt.cfg)
opt.img_size = [int(cfg_dict[0]['width']), int(cfg_dict[0]['height'])]
batch_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
batch_list
gpu_used_list = []
fps_list = []
for batch in batch_list:
# run tracking
opt.batch_size = batch
timer = Timer()
accs = []
n_frame = 0
logger.info('Starting processing for batch_size: {}'.format(batch))
dataloader = datasets.LoadVideoBatches(opt.input_video, opt.batch_size,
opt.img_size)
result_filename = os.path.join(result_root, 'results.txt')
frame_rate = dataloader.frame_rate
frame_dir = None if opt.output_format == 'text' else osp.join(
result_root, 'frame')
# try:
fps, gpu_used = batch_size_effect_measure(opt,
dataloader,
'mot',
result_filename,
save_dir=None,
show_image=False,
frame_rate=frame_rate)
gpu_used_list.append(gpu_used)
fps_list.append(fps)
filename_for_analsis = "Batch_analysis_results/results_" + str(
opt.video_number) + ".csv"
with open(filename_for_analsis, 'w', newline='') as file:
writer = csv.writer(file)
writer.writerow(["Batch_size", "GPU_used", "fps"])
print("\n\n\n")
print("Batch_size GPU_used fps")
for i in range(len(batch_list)):
print(batch_list[i], " ", gpu_used_list[i], " ", fps_list[i])
writer.writerow([batch_list[i], gpu_used_list[i], fps_list[i]])
import matplotlib.pyplot as plt
# x axis values
x = batch_list
# corresponding y axis values
y = gpu_used_list
# plotting the points
plt.plot(x, y, label="GPU usage")
# naming the x axis
plt.xlabel('Batch size')
# naming the y axis
plt.ylabel('GPU Utilization (in MB)')
# giving a title to my graph
plt.title('GPU usage for varying batch sizes')
plt.legend(loc="upper right")
plt.grid()
# function to show the plot
graphname = "GPU_graphs/GPU_" + str(opt.video_number) + ".svg"
plt.savefig(graphname)
plt.clf()
plt.cla()
plt.close()
# x axis values
x = batch_list
# corresponding y axis values
y = fps_list
# plotting the points
plt.plot(x, y, label="Speed")
# naming the x axis
plt.xlabel('Batch size')
# naming the y axis
plt.ylabel('Speed (in FPS)')
# giving a title to my graph
plt.title('Speed for different batch sizes')
plt.legend(loc="upper right")
plt.grid()
# function to show the plot
graphname = "FPS_graphs/FPS_" + str(opt.video_number) + ".svg"
plt.savefig(graphname)
def __init__(self, cfg, img_size):
super(SSD, self).__init__()
self.module_defs = parse_model_cfg(cfg)
self.module_list, self.routs, self.num_cls = create_modules(
self.module_defs, img_size)
def train():
cfg = opt.cfg
data = opt.data
epochs = opt.epochs # 500200 batches at bs 64, 117263 images = 273 epochs
batch_size = opt.batch_size
accumulate = opt.accumulate # effective bs = batch_size * accumulate = 16 * 4 = 64
midas_weights = opt.midas_weights # initial training weights
yolo_weights = opt.yolo_weights
weights = opt.weights
imgsz_min, imgsz_max, imgsz_test = opt.img_size # img sizes (min, max, test)
init_train = opt.init_train
midasnet_freeze = opt.midasnet_freeze
yolo_freeze = opt.yolo_freeze
bbox_lambda = float(opt.bbox_lambda)
depth_lambda = float(opt.depth_lambda)
# Image Sizes
gs = 64 # (pixels) grid size
assert math.fmod(
imgsz_min,
gs) == 0, '--img-size %g must be a %g-multiple' % (imgsz_min, gs)
opt.multi_scale |= imgsz_min != imgsz_max # multi if different (min, max)
if opt.multi_scale:
if imgsz_min == imgsz_max:
imgsz_min //= 1.5
imgsz_max //= 0.667
grid_min, grid_max = imgsz_min // gs, imgsz_max // gs
imgsz_min, imgsz_max = grid_min * gs, grid_max * gs
img_size = imgsz_max # initialize with max size
# Configure run
init_seeds()
data_dict = parse_data_cfg(data)
train_path = data_dict['train']
test_path = data_dict['valid']
nc = 1 if opt.single_cls else int(
data_dict['classes']) # number of classes
hyp['cls'] *= nc / 80 # update coco-tuned hyp['cls'] to current dataset
# Remove previous results
for f in glob.glob('*_batch*.png') + glob.glob(results_file):
os.remove(f)
print(f"cfg - {cfg}")
print(f"data - {data}")
print(f"epochs - {epochs}")
print(f"batch_size - {batch_size}")
print(f"accumulate - {accumulate}")
print(f"yolo weights - {yolo_weights}")
print(f"midas weights - {midas_weights}")
print(
f"imgsz_min- {imgsz_min}, imgsz_max- {imgsz_max}, imgsz_test- {imgsz_test}"
)
print(f"opt.rect - {opt.rect}")
print(f"train_path - {train_path}")
print(f"test_path - {test_path}")
print(f"init_train - {init_train}")
print(f"weights - {weights}")
print(f"midasnet_freeze - {midasnet_freeze}")
print(f"yolo_freeze - {yolo_freeze}")
print(f"mixed_precision enabled - {mixed_precision}")
print(f"Init LR - {hyp['lr0']}")
# Initialize model
# model = MidasNetEncoder(cfg).to(device)
# print_summary(model,'MidasNet Encoder Model')
all_props = parse_model_cfg(cfg)
yolo_props = []
for item in all_props:
if item['type'] == 'yolo':
yolo_props.append(item)
# print(yolo_props[0])
model = PPENet(yolo_props[0]).to(device)
# print("\n\n****** Model Information *******\n\n")
# print(model)
# print_summary(model,'PPENet Model')
# for name, param in model.named_parameters():
# print(name)
# print(type(param))
# print(param.shape)
# print('param.requires_grad: ', param.requires_grad)
# print('=====')
# for name, child in model.named_children():
# print(name, child)
# model.info()
# Optimizer
pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
for k, v in dict(model.named_parameters()).items():
if '.bias' in k:
pg2 += [v] # biases
elif 'Conv2d.weight' in k:
pg1 += [v] # apply weight_decay
else:
pg0 += [v] # all else
if opt.adam:
# hyp['lr0'] *= 0.1 # reduce lr (i.e. SGD=5E-3, Adam=5E-4)
optimizer = optim.Adam(pg0, lr=hyp['lr0'])
# optimizer = AdaBound(pg0, lr=hyp['lr0'], final_lr=0.1)
else:
optimizer = optim.SGD(pg0,
lr=hyp['lr0'],
momentum=hyp['momentum'],
nesterov=True)
optimizer.add_param_group({
'params': pg1,
'weight_decay': hyp['weight_decay']
}) # add pg1 with weight_decay
optimizer.add_param_group({'params': pg2}) # add pg2 (biases)
del pg0, pg1, pg2
start_epoch = 0
best_fitness = 0.0
# attempt_download(midas_weights)
attempt_download(yolo_weights)
if midas_weights.endswith('.pt'): # pytorch format
# possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc.
midas_chkpt = torch.load(midas_weights, map_location=device)
# load model
try:
# yolo_chkpt['model'] = {k: v for k, v in yolo_chkpt['model'].items() if model.state_dict()[k].numel() == v.numel()}
# model.load_state_dict(yolo_chkpt['model'], strict=False)
# own_state = load_my_state_dict(chkpt, model)
# model.load_state_dict(own_state, strict=False)
model_dict = model.state_dict()
# 1. filter out unnecessary keys
midas_weighs_dict = {
k: v
for k, v in midas_chkpt.items() if k in model_dict
}
# 2. overwrite entries in the existing state dict
model_dict.update(midas_weighs_dict)
# 3. load the new state dict
model.load_state_dict(model_dict)
print("Loaded Midas Encoder weights successfully")
except KeyError as e:
s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \
"See https://github.com/ultralytics/yolov3/issues/657" % (opt.midas_weights, opt.cfg, opt.midas_weights)
raise KeyError(s) from e
# print("dbg Model dict")
# for key in model_dict.keys():
# if key.startswith('yolo'):
# if (not (key.endswith('num_batches_tracked') or ('custom_learner' in key) or key.endswith('running_mean') or key.endswith('running_var'))):
# print(f"{key}")
# print(f"{model_dict[key].size()}")
if yolo_weights.endswith('.pt'): # pytorch format
# possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc.
yolo_chkpt = torch.load(yolo_weights, map_location=device)
if init_train == 'True':
# print("Loading Yolo weights")
model_dict = model.state_dict()
# 1. filter out mapping keys of yolo_chkpt['model'] <- orig weights and yolo_weights_mapping <- model
for k, v in model_dict.items():
if k in yolo_weights_mapping.keys():
model_dict[k] = yolo_chkpt['model'][
yolo_weights_mapping[k]]
# 3. load the new state dict
model.load_state_dict(model_dict)
print("Loaded Yolo Decoder weights successfully")
if weights.endswith('.pt'): # pytorch format
# possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc.
chkpt = torch.load(weights, map_location=device)
# load model
try:
# print(f"model state dict {model.state_dict().keys()}")
# print(f"best weights {chkpt['model'].keys()}")
# print("model state dict")
# for k, v in model.state_dict().items():
# print(k)
# print(v.shape)
# print("best weights dict")
# for k,v in chkpt['model'].items():
# print(k)
# print(v.shape)
chkpt['model'] = {
k: v
for k, v in chkpt['model'].items()
if model.state_dict()[k].numel() == v.numel()
}
model.load_state_dict(chkpt['model'], strict=False)
except KeyError as e:
s = "%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s. " \
"See https://github.com/ultralytics/yolov3/issues/657" % (opt.weights, opt.cfg, opt.weights)
raise KeyError(s) from e
# load optimizer
if chkpt['optimizer'] is not None:
optimizer.load_state_dict(chkpt['optimizer'])
best_fitness = chkpt['best_fitness']
# load results
if chkpt.get('training_results') is not None:
with open(results_file, 'w') as file:
file.write(chkpt['training_results']) # write results.txt
start_epoch = chkpt['epoch'] + 1
del chkpt
# elif len(weights) > 0: # darknet format
# possible weights are '*.weights', 'yolov3-tiny.conv.15', 'darknet53.conv.74' etc.
# load_darknet_weights(model, weights)
# Mixed precision training https://github.com/NVIDIA/apex
if mixed_precision:
model, optimizer = amp.initialize(model,
optimizer,
opt_level='O1',
verbosity=0)
# Scheduler https://github.com/ultralytics/yolov3/issues/238
lf = lambda x: (
((1 + math.cos(x * math.pi / epochs)) / 2
)**1.0) * 0.95 + 0.05 # cosine https://arxiv.org/pdf/1812.01187.pdf
scheduler = lr_scheduler.LambdaLR(optimizer,
lr_lambda=lf,
last_epoch=start_epoch - 1)
# scheduler = lr_scheduler.MultiStepLR(optimizer, [round(epochs * x) for x in [0.8, 0.9]], 0.1, start_epoch - 1)
# Plot lr schedule
# y = []
# for _ in range(epochs):
# scheduler.step()
# y.append(optimizer.param_groups[0]['lr'])
# plt.plot(y, '.-', label='LambdaLR')
# plt.xlabel('epoch')
# plt.ylabel('LR')
# plt.tight_layout()
# plt.savefig('LR.png', dpi=300)
# Initialize distributed training
if device.type != 'cpu' and torch.cuda.device_count(
) > 1 and torch.distributed.is_available():
dist.init_process_group(
backend='nccl', # 'distributed backend'
init_method=
'tcp://127.0.0.1:9999', # distributed training init method
world_size=1, # number of nodes for distributed training
rank=0) # distributed training node rank
model = torch.nn.parallel.DistributedDataParallel(
model, find_unused_parameters=True)
model.yolo_layers = model.module.yolo_layers # move yolo layer indices to top level
# Dataset
dataset = LoadImagesAndLabels(
train_path,
img_size,
batch_size,
augment=False,
hyp=hyp, # augmentation hyperparameters
rect=opt.rect, # rectangular training
cache_images=opt.cache_images,
single_cls=opt.single_cls)
# Dataloader
batch_size = min(batch_size, len(dataset))
nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0,
8]) # number of workers
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=batch_size,
num_workers=nw,
shuffle=not opt.
rect, # Shuffle=True unless rectangular training is used
pin_memory=True,
collate_fn=dataset.collate_fn)
# Testloader
testloader = torch.utils.data.DataLoader(
LoadImagesAndLabels(
test_path,
imgsz_test,
batch_size,
hyp=hyp,
# rect=True,
rect=False,
cache_images=opt.cache_images,
single_cls=opt.single_cls),
batch_size=batch_size,
num_workers=nw,
pin_memory=True,
collate_fn=dataset.collate_fn)
# Model parameters
model.nc = nc # attach number of classes to model
model.hyp = hyp # attach hyperparameters to model
model.gr = 1.0 # giou loss ratio (obj_loss = 1.0 or giou)
model.class_weights = labels_to_class_weights(dataset.labels, nc).to(
device) # attach class weights
# model.module_defs = parse_model_cfg(cfg)
# model.module_list, model.routs = create_modules(model.module_defs, img_size)
# model.yolo_layers = get_yolo_layers(model)
# Model EMA
ema = torch_utils.ModelEMA(model)
# print("Before freeze Checking layers param state")
# print(model.parameters())
# for name, param in model.named_parameters():
# print(name)
# print(param.requires_grad)
# Freezing the midasnet
if midasnet_freeze == 'True':
depth_lambda = 0
print("Freezing the midasnet")
for name, param in model.named_parameters():
if not name.startswith('yolo'):
param.requires_grad = False
if yolo_freeze == 'True':
print("Freezing the yolo branch")
bbox_lambda = 0
for name, param in model.named_parameters():
if name.startswith('yolo'):
param.requires_grad = False
# print("After freeze Checking layers param state")
# for name, param in model.named_parameters():
# print(name)
# print(param.requires_grad)
print_summary(model,
'YMP Model Summary',
input_size=(3, opt.img_size[0], opt.img_size[0]))
# Start training
nb = len(dataloader) # number of batches
n_burn = max(3 * nb,
500) # burn-in iterations, max(3 epochs, 500 iterations)
maps = np.zeros(nc) # mAP per class
# torch.autograd.set_detect_anomaly(True)
results = (
0, 0, 0, 0, 0, 0, 0
) # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification'
t0 = time.time()
print('Image sizes %g - %g train, %g test' %
(imgsz_min, imgsz_max, imgsz_test))
print('Using %g dataloader workers' % nw)
print('Starting training for %g epochs...' % epochs)
for epoch in range(
start_epoch, epochs
): # epoch ------------------------------------------------------------------
model.train()
# Update image weights (optional)
if dataset.image_weights:
w = model.class_weights.cpu().numpy() * (1 -
maps)**2 # class weights
image_weights = labels_to_image_weights(dataset.labels,
nc=nc,
class_weights=w)
dataset.indices = random.choices(range(dataset.n),
weights=image_weights,
k=dataset.n) # rand weighted idx
mloss = torch.zeros(4).to(device) # mean losses
print(('\n' + '%17s' * 10) %
('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets',
'ImgSize', 'RmseGradMeanLoss', 'SSIM_meanLoss'))
pbar = tqdm(enumerate(dataloader), total=nb) # progress bar
for i, (
imgs, targets, paths, _, depth_targets, _, _
) in pbar: # batch -------------------------------------------------------------
ni = i + nb * epoch # number integrated batches (since train start)
imgs = imgs.to(device).float(
) / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
# Burn-in
if ni <= n_burn * 2:
model.gr = np.interp(
ni, [0, n_burn * 2],
[0.0, 1.0]) # giou loss ratio (obj_loss = 1.0 or giou)
if ni == n_burn: # burnin complete
print_model_biases(model)
for j, x in enumerate(optimizer.param_groups):
# bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
x['lr'] = np.interp(
ni, [0, n_burn],
[0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
if 'momentum' in x:
x['momentum'] = np.interp(ni, [0, n_burn],
[0.9, hyp['momentum']])
# Multi-Scale training
if opt.multi_scale:
if ni / accumulate % 1 == 0: # adjust img_size (67% - 150%) every 1 batch
img_size = random.randrange(grid_min, grid_max + 1) * gs
sf = img_size / max(imgs.shape[2:]) # scale factor
if sf != 1:
ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]
] # new shape (stretched to 32-multiple)
imgs = F.interpolate(imgs,
size=ns,
mode='bilinear',
align_corners=False)
# Run model
# print(f"DBG imgs.shape - {imgs.shape}")
depth_preds, yolo_pred = model(imgs)
yolo_pred = [item.to(device) for item in yolo_pred]
# yolo_pred1,yolo_pred2,yolo_pred3 = yolo_pred
# yolo_pred1 = yolo_pred1.to(device)
# yolo_pred2 = yolo_pred2.to(device)
# yolo_pred3 = yolo_pred3.to(device)
# yolo_pred = [yolo_pred1,yolo_pred2,yolo_pred3]
# print('yolo_pred1 Size:',yolo_pred1.size())
# print('yolo_pred2 Size:',yolo_pred2.size())
# print('yolo_pred3 Size:',yolo_pred3.size())
#depth_pred = depth_pred.to(device)
#yolo_pred = yolo_pred.to(device)
# Compute loss
loss, loss_items = compute_loss(yolo_pred, targets, model)
loss = bbox_lambda * loss
if midasnet_freeze == 'False':
# print(f"Calculating depth loss")
rmse_gradient_mean_loss, ssim_mean_loss = [
depth_lambda * item for item in compute_depth_loss(
depth_preds, depth_targets, batch_size, device)
]
loss += rmse_gradient_mean_loss + ssim_mean_loss
else:
rmse_gradient_mean_loss = 0
ssim_mean_loss = 0
if not torch.isfinite(loss):
print('WARNING: non-finite loss, ending training ', loss_items,
rmse_gradient_mean_loss, ssim_mean_loss)
return results
# Scale loss by nominal batch_size of 64
loss *= batch_size / 64
# Compute gradient
if mixed_precision:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
# Optimize accumulated gradient
if ni % accumulate == 0:
optimizer.step()
optimizer.zero_grad()
ema.update(model)
# Print batch results
mloss = (mloss * i + loss_items) / (i + 1) # update mean losses
mem = '%.3gG' % (torch.cuda.memory_cached() /
1E9 if torch.cuda.is_available() else 0) # (GB)
s = ('%10s' * 2 + '%17.3g' * 8) % (
'%g/%g' % (epoch, epochs - 1), mem, *mloss, len(targets),
img_size, rmse_gradient_mean_loss, ssim_mean_loss)
pbar.set_description(s)
# Plot images with bounding boxes
if ni < 1:
f = 'train_batch%g.png' % i # filename
plot_images(imgs=imgs, targets=targets, paths=paths, fname=f)
if tb_writer:
tb_writer.add_image(f,
cv2.imread(f)[:, :, ::-1],
dataformats='HWC')
# tb_writer.add_graph(model, imgs) # add model to tensorboard
# end batch ------------------------------------------------------------------------------------------------
# Update scheduler
scheduler.step()
# Process epoch results
ema.update_attr(model)
final_epoch = epoch + 1 == epochs
if not opt.notest or final_epoch: # Calculate mAP
is_coco = any([
x in data
for x in ['coco.data', 'coco2014.data', 'coco2017.data']
]) and model.nc == 80
results, maps = test.test(cfg,
data,
batch_size=batch_size,
img_size=imgsz_test,
model=ema.ema,
save_json=final_epoch and is_coco,
single_cls=opt.single_cls,
dataloader=testloader,
bbox_lambda=bbox_lambda,
depth_lambda=depth_lambda)
# Write epoch results
with open(results_file, 'a') as f:
f.write(
s + '%10.3g' * 9 % results + '\n'
) # P, R, mAP, F1, test_losses=(GIoU, obj, cls), depthmap_loss, total_loss
if len(opt.name) and opt.bucket:
os.system('gsutil cp results.txt gs://%s/results/results%s.txt' %
(opt.bucket, opt.name))
# Write Tensorboard results
if tb_writer:
tags = [
'train/giou_loss', 'train/obj_loss', 'train/cls_loss',
'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5',
'metrics/F1', 'val/giou_loss', 'val/obj_loss', 'val/cls_loss'
]
for x, tag in zip(list(mloss[:-1]) + list(results), tags):
tb_writer.add_scalar(tag, x, epoch)
# Update best mAP
fi = fitness(np.array(results).reshape(
1, -1)) # fitness_i = weighted combination of [P, R, mAP, F1]
if fi > best_fitness:
best_fitness = fi
# Save training results
save = (not opt.nosave) or (final_epoch and not opt.evolve)
if save:
with open(results_file, 'r') as f:
# Create checkpoint
chkpt = {
'epoch':
epoch,
'best_fitness':
best_fitness,
'training_results':
f.read(),
'model':
ema.ema.module.state_dict()
if hasattr(model, 'module') else ema.ema.state_dict(),
'optimizer':
None if final_epoch else optimizer.state_dict()
}
# Save last checkpoint
torch.save(chkpt, last)
# Save best checkpoint
if (best_fitness == fi) and not final_epoch:
torch.save(chkpt, best)
# Save backup every 10 epochs (optional)
# if epoch > 0 and epoch % 10 == 0:
# torch.save(chkpt, wdir + 'backup%g.pt' % epoch)
# Delete checkpoint
del chkpt
# end epoch ----------------------------------------------------------------------------------------------------
# end training
n = opt.name
if len(n):
n = '_' + n if not n.isnumeric() else n
fresults, flast, fbest = 'results%s.txt' % n, wdir + 'last%s.pt' % n, wdir + 'best%s.pt' % n
for f1, f2 in zip([wdir + 'last.pt', wdir + 'best.pt', 'results.txt'],
[flast, fbest, fresults]):
if os.path.exists(f1):
os.rename(f1, f2) # rename
ispt = f2.endswith('.pt') # is *.pt
strip_optimizer(f2) if ispt else None # strip optimizer
os.system('gsutil cp %s gs://%s/weights' % (
f2, opt.bucket)) if opt.bucket and ispt else None # upload
if not opt.evolve:
plot_results() # save as results.png
print('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1,
(time.time() - t0) / 3600))
dist.destroy_process_group() if torch.cuda.device_count() > 1 else None
torch.cuda.empty_cache()
return results
