def eval(eval_file_list, batch_size, data_args, model_path):
'''
评估脚本，用于评估训好模型
:param eval_file_list:要测试的图像列表
:param batch_size:batch的大小
:param data_args:数据的设置参数
:param model_path:模型的路径
:return:
'''
# 通过神经网络模型获取损失函数和额外层
cost, detect_out = vgg_ssd_net.net_conf(mode='eval')
# 检查模型模型路径是否正确
assert os.path.isfile(model_path), 'Invalid model.'
# 通过训练好的模型生成参数
parameters = paddle.parameters.Parameters.from_tar(gzip.open(model_path))
# 创建优化方法
optimizer = paddle.optimizer.Momentum()
# 创建训练器
trainer = paddle.trainer.SGD(cost=cost,
parameters=parameters,
extra_layers=[detect_out],
update_equation=optimizer)
# 定义数据层之间的关系
feeding = {'image': 0, 'bbox': 1}
# 生成要训练的数据
reader = paddle.batch(data_provider.test(data_args, eval_file_list),
batch_size=batch_size)
# 获取测试结果
result = trainer.test(reader=reader, feeding=feeding)
# 打印模型的测试信息
print "TestCost: %f, Detection mAP=%g" % \
(result.cost, result.metrics['detection_evaluator'])
def infer(eval_file_list, save_path, data_args, batch_size, model_path, threshold):
'''
预测图像
:param eval_file_list: 指定图像路径列表
:param save_path: 指定预测结果保存路径
:param data_args: 配置数据预处理所需参数
:param batch_size: 为每多少样本预测一次
:param model_path: 指模型的位置
:param threshold: 置信度阈值
:return:
'''
# 通过网络模型获取输出层
detect_out = vgg_ssd_net.net_conf(mode='infer')
# 检查模型路径是否正确
assert os.path.isfile(model_path), 'Invalid model.'
# 加载训练好的参数
parameters = paddle.parameters.Parameters.from_tar(gzip.open(model_path))
# 或预测器
inferer = paddle.inference.Inference(
output_layer=detect_out, parameters=parameters)
# 获取预测数据
reader = data_provider.infer(data_args, eval_file_list)
all_fname_list = [line.strip()
for line in open(eval_file_list).readlines()]
# 预测的数据
test_data = []
# 预测的图像路径list
fname_list = []
# 图像的宽
img_w = []
# 图像的高
img_h = []
# 图像的idx
idx = 0
'''按批处理推理，
bbox的坐标将根据图像大小进行缩放
'''
with open(save_path, 'w') as fout:
for img in reader():
test_data.append([img])
fname_list.append(all_fname_list[idx])
w, h = Image.open(os.path.join('../data', fname_list[-1])).size
img_w.append(w)
img_h.append(h)
# 当数据达到一个batch后，就开始预测
if len(test_data) == batch_size:
ret_res = _infer(inferer, test_data, threshold)
save_batch_res(ret_res, img_w, img_h, fname_list, fout)
# 预测后要清空之前的数据
test_data = []
fname_list = []
img_w = []
img_h = []
idx += 1
# 剩下没有达到一个batch的在在最后也要预测
if len(test_data) > 0:
ret_res = _infer(inferer, test_data, threshold)
save_batch_res(ret_res, img_w, img_h, fname_list, fout)
def train(train_file_list, dev_file_list, data_args, init_model_path):
optimizer = paddle.optimizer.Momentum(
momentum=cfg.TRAIN.MOMENTUM,
learning_rate=cfg.TRAIN.LEARNING_RATE,
regularization=paddle.optimizer.L2Regularization(
rate=cfg.TRAIN.L2REGULARIZATION),
learning_rate_decay_a=cfg.TRAIN.LEARNING_RATE_DECAY_A,
learning_rate_decay_b=cfg.TRAIN.LEARNING_RATE_DECAY_B,
learning_rate_schedule=cfg.TRAIN.LEARNING_RATE_SCHEDULE)
cost, detect_out = vgg_ssd_net.net_conf('train')
parameters = paddle.parameters.create(cost)
if not (init_model_path is None):
assert os.path.isfile(init_model_path), 'Invalid model.'
parameters.init_from_tar(gzip.open(init_model_path))
trainer = paddle.trainer.SGD(
cost=cost,
parameters=parameters,
extra_layers=[detect_out],
update_equation=optimizer)
feeding = {'image': 0, 'bbox': 1}
train_reader = paddle.batch(
data_provider.train(data_args, train_file_list),
batch_size=cfg.TRAIN.BATCH_SIZE) # generate a batch image each time
dev_reader = paddle.batch(
data_provider.test(data_args, dev_file_list),
batch_size=cfg.TRAIN.BATCH_SIZE)
def event_handler(event):
if isinstance(event, paddle.event.EndIteration):
if event.batch_id % 1 == 0:
print "\nPass %d, Batch %d, TrainCost %f, Detection mAP=%f" % \
(event.pass_id,
event.batch_id,
event.cost,
event.metrics['detection_evaluator'])
else:
sys.stdout.write('.')
sys.stdout.flush()
if isinstance(event, paddle.event.EndPass):
with gzip.open('checkpoints/params_pass_%05d.tar.gz' % \
event.pass_id, 'w') as f:
parameters.to_tar(f)
result = trainer.test(reader=dev_reader, feeding=feeding)
print "\nTest with Pass %d, TestCost: %f, Detection mAP=%g" % \
(event.pass_id,
result.cost,
result.metrics['detection_evaluator'])
trainer.train(
reader=train_reader,
event_handler=event_handler,
num_passes=cfg.TRAIN.NUM_PASS,
feeding=feeding)
def infer(eval_file_list, save_path, data_args, batch_size, model_path,
threshold):
detect_out = vgg_ssd_net.net_conf(mode='infer')
assert os.path.isfile(model_path), 'Invalid model.'
parameters = paddle.parameters.Parameters.from_tar(gzip.open(model_path))
inferer = paddle.inference.Inference(output_layer=detect_out,
parameters=parameters)
reader = data_provider.infer(data_args, eval_file_list)
all_fname_list = [
line.strip() for line in open(eval_file_list).readlines()
]
test_data = []
fname_list = []
img_w = []
img_h = []
idx = 0
"""Do inference batch by batch,
coords of bbox will be scaled based on image size
"""
with open(save_path, 'w') as fout:
f = open('people_num.txt', 'w')
for img in reader():
test_data.append([img])
fname_list.append(all_fname_list[idx])
w, h = Image.open(os.path.join('./data', fname_list[-1])).size
img_w.append(w)
img_h.append(h)
if len(test_data) == batch_size:
ret_res = _infer(inferer, test_data, threshold)
save_batch_res(ret_res, img_w, img_h, fname_list, fout)
people_num = 0
for det_res in ret_res:
img_idx = int(det_res[0])
label = int(det_res[1])
if label == 0:
people_num += 1
f.write(fname_list[img_idx] + '\t' + str(people_num))
f.write('\n')
test_data = []
fname_list = []
img_w = []
img_h = []
idx += 1
if len(test_data) > 0:
ret_res = _infer(inferer, test_data, threshold)
save_batch_res(ret_res, img_w, img_h, fname_list, fout)
def infer(eval_file_list, save_path, data_args, batch_size, model_path,
threshold):
detect_out = vgg_ssd_net.net_conf(mode="infer")
assert os.path.isfile(model_path), "Invalid model."
parameters = paddle.parameters.Parameters.from_tar(gzip.open(model_path))
inferer = paddle.inference.Inference(
output_layer=detect_out, parameters=parameters)
data_reader = reader.infer(data_args, eval_file_list)
all_fname_list = [
line.strip().split()[0] for line in open(eval_file_list).readlines()
]
test_data = []
fname_list = []
img_w = []
img_h = []
idx = 0
with open(save_path, "w") as fout:
for idx, img in enumerate(data_reader()):
if idx > 50: break
test_data.append([img])
fname_list.append(all_fname_list[idx])
w, h = Image.open(
os.path.join(data_args.data_dir, "resized_images",
fname_list[-1])).size
img_w.append(w)
img_h.append(h)
if len(test_data) == batch_size:
ret_res = _infer(inferer, test_data, threshold)
if ret_res is None: continue
save_batch_res(ret_res, img_w, img_h, fname_list, fout)
test_data = []
fname_list = []
img_w = []
img_h = []
if len(test_data) > 0:
ret_res = _infer(inferer, test_data, threshold)
save_batch_res(ret_res, img_w, img_h, fname_list, fout)
def eval(eval_file_list, batch_size, data_args, model_path):
cost, detect_out = vgg_ssd_net.net_conf(mode='eval')
assert os.path.isfile(model_path), 'Invalid model.'
parameters = paddle.parameters.Parameters.from_tar(gzip.open(model_path))
optimizer = paddle.optimizer.Momentum()
trainer = paddle.trainer.SGD(cost=cost,
parameters=parameters,
extra_layers=[detect_out],
update_equation=optimizer)
feeding = {'image': 0, 'bbox': 1}
reader = paddle.batch(data_provider.test(data_args, eval_file_list),
batch_size=batch_size)
result = trainer.test(reader=reader, feeding=feeding)
print "TestCost: %f, Detection mAP=%g" % \
(result.cost, result.metrics['detection_evaluator'])
def train(train_file_list, dev_file_list, data_args, init_model_path=None):
# 创建优化方法
optimizer = paddle.optimizer.Momentum(
momentum=cfg.TRAIN.MOMENTUM,
learning_rate=cfg.TRAIN.LEARNING_RATE,
regularization=paddle.optimizer.L2Regularization(
rate=cfg.TRAIN.L2REGULARIZATION),
learning_rate_decay_a=cfg.TRAIN.LEARNING_RATE_DECAY_A,
learning_rate_decay_b=cfg.TRAIN.LEARNING_RATE_DECAY_B,
learning_rate_schedule=cfg.TRAIN.LEARNING_RATE_SCHEDULE)
# 通过神经网络模型获取损失函数和额外层
cost, detect_out = vgg_ssd_net.net_conf('train')
# 通过损失函数创建训练参数
parameters = paddle.parameters.create(cost)
# 如果有训练好的模型，可以使用训练好的模型再训练
if not (init_model_path is None):
assert os.path.isfile(init_model_path), 'Invalid model.'
parameters.init_from_tar(gzip.open(init_model_path))
# 创建训练器
trainer = paddle.trainer.SGD(cost=cost,
parameters=parameters,
extra_layers=[detect_out],
update_equation=optimizer)
# 定义数据层之间的关系
feeding = {'image': 0, 'bbox': 1}
# 创建训练数据
train_reader = paddle.batch(data_provider.train(data_args,
train_file_list),
batch_size=cfg.TRAIN.BATCH_SIZE)
# 创建测试数据
dev_reader = paddle.batch(data_provider.test(data_args, dev_file_list),
batch_size=cfg.TRAIN.BATCH_SIZE)
# 定义训练事件
def event_handler(event):
if isinstance(event, paddle.event.EndIteration):
if event.batch_id % 100 == 0:
print "\nPass %d, Batch %d, TrainCost %f, Detection mAP=%f" % \
(event.pass_id,
event.batch_id,
event.cost,
event.metrics['detection_evaluator'])
else:
sys.stdout.write('.')
sys.stdout.flush()
if isinstance(event, paddle.event.EndPass):
with gzip.open('../models/params_pass.tar.gz', 'w') as f:
trainer.save_parameter_to_tar(f)
result = trainer.test(reader=dev_reader, feeding=feeding)
print "\nTest with Pass %d, TestCost: %f, Detection mAP=%g" % \
(event.pass_id,
result.cost,
result.metrics['detection_evaluator'])
# 开始训练
trainer.train(reader=train_reader,
event_handler=event_handler,
num_passes=cfg.TRAIN.NUM_PASS,
feeding=feeding)
def train(train_file_list,
data_args,
init_model_path,
save_dir,
dev_file_list=None):
optimizer = paddle.optimizer.Momentum(
momentum=cfg.TRAIN.MOMENTUM,
learning_rate=cfg.TRAIN.LEARNING_RATE,
regularization=paddle.optimizer.L2Regularization(
rate=cfg.TRAIN.L2REGULARIZATION),
learning_rate_decay_a=cfg.TRAIN.LEARNING_RATE_DECAY_A,
learning_rate_decay_b=cfg.TRAIN.LEARNING_RATE_DECAY_B,
learning_rate_schedule=cfg.TRAIN.LEARNING_RATE_SCHEDULE)
cost, detect_out = vgg_ssd_net.net_conf("train")
parameters = paddle.parameters.create(cost)
if init_model_path is not None:
assert os.path.isfile(init_model_path), "Invalid model."
parameters.init_from_tar(gzip.open(init_model_path))
trainer = paddle.trainer.SGD(cost=cost,
parameters=parameters,
extra_layers=[detect_out],
update_equation=optimizer)
feeding = {"image": 0, "bbox": 1}
train_reader = paddle.batch(
reader.train(data_args, train_file_list),
batch_size=cfg.TRAIN.BATCH_SIZE) # generate a batch image each time
if dev_file_list is not None:
dev_reader = paddle.batch(reader.test(data_args, dev_file_list),
batch_size=cfg.TRAIN.BATCH_SIZE)
def event_handler(event):
if isinstance(event, paddle.event.EndIteration):
if (event.batch_id + 1) % 1 == 0:
print("Pass %d, Batch %d, TrainCost %f, Detection mAP=%f" %
(event.pass_id, event.batch_id + 1, event.cost,
event.metrics["detection_evaluator"]))
sys.stdout.flush()
if isinstance(event, paddle.event.EndPass):
if not (event.pass_id + 1) % 20:
with gzip.open(
os.path.join(save_dir, "params_pass_%05d.tar.gz" %
event.pass_id), "w") as f:
trainer.save_parameter_to_tar(f)
if dev_file_list is not None:
result = trainer.test(reader=dev_reader, feeding=feeding)
print("Test with Pass %d, TestCost: %f, Detection mAP=%g" %
(event.pass_id, result.cost,
result.metrics["detection_evaluator"]))
trainer.train(reader=train_reader,
event_handler=event_handler,
num_passes=cfg.TRAIN.NUM_PASS,
feeding=feeding)