def train():
# check if set use_gpu=True in paddlepaddle cpu version
check_gpu(cfg.use_gpu)
devices_num = get_device_num() if cfg.use_gpu else 1
print("Found {} CUDA/CPU devices.".format(devices_num))
if cfg.debug or args.enable_ce:
fluid.default_startup_program().random_seed = 1000
fluid.default_main_program().random_seed = 1000
random.seed(0)
np.random.seed(0)
if not os.path.exists(cfg.model_save_dir):
os.makedirs(cfg.model_save_dir)
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id) if cfg.use_data_parallel else fluid.CUDAPlace(0)
with fluid.dygraph.guard(place):
if args.use_data_parallel:
strategy = fluid.dygraph.parallel.prepare_context()
model = YOLOv3(3, is_train=True)
if cfg.pretrain:
restore, _ = fluid.load_dygraph(cfg.pretrain)
model.block.set_dict(restore)
if cfg.finetune:
restore, _ = fluid.load_dygraph(cfg.finetune)
model.set_dict(restore, use_structured_name=True)
if args.use_data_parallel:
model = fluid.dygraph.parallel.DataParallel(model, strategy)
boundaries = cfg.lr_steps
gamma = cfg.lr_gamma
step_num = len(cfg.lr_steps)
learning_rate = cfg.learning_rate
values = [learning_rate * (gamma ** i) for i in range(step_num + 1)]
lr = fluid.dygraph.PiecewiseDecay(
boundaries=boundaries,
values=values,
begin=args.start_iter)
lr = fluid.layers.linear_lr_warmup(
learning_rate=lr,
warmup_steps=cfg.warm_up_iter,
start_lr=0.0,
end_lr=cfg.learning_rate,
)
optimizer = fluid.optimizer.Momentum(
learning_rate=lr,
regularization=fluid.regularizer.L2Decay(cfg.weight_decay),
momentum=cfg.momentum,
parameter_list=model.parameters()
)
start_time = time.time()
snapshot_loss = 0
snapshot_time = 0
total_sample = 0
input_size = cfg.input_size
shuffle = True
shuffle_seed = None
total_iter = cfg.max_iter - cfg.start_iter
mixup_iter = total_iter - cfg.no_mixup_iter
random_sizes = [cfg.input_size]
if cfg.random_shape:
random_sizes = [32 * i for i in range(10,20)]
train_reader = reader.train(
input_size,
batch_size=cfg.batch_size,
shuffle=shuffle,
shuffle_seed=shuffle_seed,
total_iter=total_iter * devices_num,
mixup_iter=mixup_iter * devices_num,
random_sizes=random_sizes,
use_multiprocess_reader=cfg.use_multiprocess_reader,
num_workers=cfg.worker_num)
if args.use_data_parallel:
train_reader = fluid.contrib.reader.distributed_batch_reader(train_reader)
smoothed_loss = SmoothedValue()
for iter_id, data in enumerate(train_reader()):
prev_start_time = start_time
start_time = time.time()
img = np.array([x[0] for x in data]).astype('float32')
img = to_variable(img)
gt_box = np.array([x[1] for x in data]).astype('float32')
gt_box = to_variable(gt_box)
gt_label = np.array([x[2] for x in data]).astype('int32')
gt_label = to_variable(gt_label)
gt_score = np.array([x[3] for x in data]).astype('float32')
gt_score = to_variable(gt_score)
loss = model(img, gt_box, gt_label, gt_score, None, None)
smoothed_loss.add_value(np.mean(loss.numpy()))
snapshot_loss += loss.numpy()
snapshot_time += start_time - prev_start_time
total_sample += 1
print("Iter {:d}, loss {:.6f}, time {:.5f}".format(
iter_id,
smoothed_loss.get_mean_value(),
start_time-prev_start_time))
if args.use_data_parallel:
loss = model.scale_loss(loss)
loss.backward()
model.apply_collective_grads()
loss.backward()
optimizer.minimize(loss)
model.clear_gradients()
save_parameters = (not args.use_data_parallel) or (
args.use_data_parallel and
fluid.dygraph.parallel.Env().local_rank == 0)
if save_parameters and iter_id > 1 and iter_id % cfg.snapshot_iter == 0:
fluid.save_dygraph(model.state_dict(), args.model_save_dir + "/yolov3_{}".format(iter_id))
def train():
if cfg.debug or args.enable_ce:
fluid.default_startup_program().random_seed = 1000
fluid.default_main_program().random_seed = 1000
random.seed(0)
np.random.seed(0)
if not os.path.exists(cfg.model_save_dir):
os.makedirs(cfg.model_save_dir)
model = YOLOv3()
model.build_model()
input_size = cfg.input_size
loss = model.loss()
loss.persistable = True
devices_num = get_device_num()
print("Found {} CUDA devices.".format(devices_num))
learning_rate = cfg.learning_rate
boundaries = cfg.lr_steps
gamma = cfg.lr_gamma
step_num = len(cfg.lr_steps)
values = [learning_rate * (gamma**i) for i in range(step_num + 1)]
optimizer = fluid.optimizer.Momentum(
learning_rate=exponential_with_warmup_decay(
learning_rate=learning_rate,
boundaries=boundaries,
values=values,
warmup_iter=cfg.warm_up_iter,
warmup_factor=cfg.warm_up_factor),
regularization=fluid.regularizer.L2Decay(cfg.weight_decay),
momentum=cfg.momentum)
optimizer.minimize(loss)
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id) if cfg.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if cfg.pretrain:
if not os.path.exists(cfg.pretrain):
print("Pretrain weights not found: {}".format(cfg.pretrain))
def if_exist(var):
return os.path.exists(os.path.join(cfg.pretrain, var.name)) \
and var.name.find('yolo_output') < 0
fluid.io.load_vars(exe, cfg.pretrain, predicate=if_exist)
build_strategy = fluid.BuildStrategy()
build_strategy.memory_optimize = False #gc and memory optimize may conflict
syncbn = cfg.syncbn
if (syncbn and devices_num <= 1) or num_trainers > 1:
print("Disable syncbn in single device")
syncbn = False
build_strategy.sync_batch_norm = syncbn
exec_strategy = fluid.ExecutionStrategy()
if cfg.use_gpu and num_trainers > 1:
dist_utils.prepare_for_multi_process(exe, build_strategy,
fluid.default_main_program())
exec_strategy.num_threads = 1
compile_program = fluid.compiler.CompiledProgram(fluid.default_main_program(
)).with_data_parallel(
loss_name=loss.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
random_sizes = [cfg.input_size]
if cfg.random_shape:
random_sizes = [32 * i for i in range(10, 20)]
total_iter = cfg.max_iter - cfg.start_iter
mixup_iter = total_iter - cfg.no_mixup_iter
shuffle = True
if args.enable_ce:
shuffle = False
shuffle_seed = None
# NOTE: yolov3 is a special model, if num_trainers > 1, each process
# trian the completed dataset.
# if num_trainers > 1: shuffle_seed = 1
train_reader = reader.train(
input_size,
batch_size=cfg.batch_size,
shuffle=shuffle,
shuffle_seed=shuffle_seed,
total_iter=total_iter * devices_num,
mixup_iter=mixup_iter * devices_num,
random_sizes=random_sizes,
use_multiprocess_reader=cfg.use_multiprocess_reader)
py_reader = model.py_reader
py_reader.decorate_paddle_reader(train_reader)
def save_model(postfix):
model_path = os.path.join(cfg.model_save_dir, postfix)
if os.path.isdir(model_path):
shutil.rmtree(model_path)
fluid.io.save_persistables(exe, model_path)
fetch_list = [loss]
py_reader.start()
smoothed_loss = SmoothedValue()
try:
start_time = time.time()
prev_start_time = start_time
snapshot_loss = 0
snapshot_time = 0
for iter_id in range(cfg.start_iter, cfg.max_iter):
prev_start_time = start_time
start_time = time.time()
losses = exe.run(compile_program,
fetch_list=[v.name for v in fetch_list])
smoothed_loss.add_value(np.mean(np.array(losses[0])))
snapshot_loss += np.mean(np.array(losses[0]))
snapshot_time += start_time - prev_start_time
lr = np.array(fluid.global_scope().find_var('learning_rate')
.get_tensor())
print("Iter {:d}, lr {:.6f}, loss {:.6f}, time {:.5f}".format(
iter_id, lr[0],
smoothed_loss.get_mean_value(), start_time - prev_start_time))
sys.stdout.flush()
if (iter_id + 1) % cfg.snapshot_iter == 0:
save_model("model_iter{}".format(iter_id))
print("Snapshot {} saved, average loss: {}, \
average time: {}".format(
iter_id + 1, snapshot_loss / float(cfg.snapshot_iter),
snapshot_time / float(cfg.snapshot_iter)))
if args.enable_ce and iter_id == cfg.max_iter - 1:
if devices_num == 1:
print("kpis\ttrain_cost_1card\t%f" %
(snapshot_loss / float(cfg.snapshot_iter)))
print("kpis\ttrain_duration_1card\t%f" %
(snapshot_time / float(cfg.snapshot_iter)))
else:
print("kpis\ttrain_cost_8card\t%f" %
(snapshot_loss / float(cfg.snapshot_iter)))
print("kpis\ttrain_duration_8card\t%f" %
(snapshot_time / float(cfg.snapshot_iter)))
snapshot_loss = 0
snapshot_time = 0
except fluid.core.EOFException:
py_reader.reset()
save_model('model_final')
def train():
if cfg.debug:
fluid.default_startup_program().random_seed = 1000
fluid.default_main_program().random_seed = 1000
random.seed(0)
np.random.seed(0)
if not os.path.exists(cfg.model_save_dir):
os.makedirs(cfg.model_save_dir)
model = YOLOv3()
model.build_model()
input_size = cfg.input_size
loss = model.loss()
loss.persistable = True
devices = os.getenv("CUDA_VISIBLE_DEVICES") or ""
devices_num = len(devices.split(","))
print("Found {} CUDA devices.".format(devices_num))
learning_rate = cfg.learning_rate
boundaries = cfg.lr_steps
gamma = cfg.lr_gamma
step_num = len(cfg.lr_steps)
values = [learning_rate * (gamma**i) for i in range(step_num + 1)]
optimizer = fluid.optimizer.Momentum(
learning_rate=exponential_with_warmup_decay(
learning_rate=learning_rate,
boundaries=boundaries,
values=values,
warmup_iter=cfg.warm_up_iter,
warmup_factor=cfg.warm_up_factor),
regularization=fluid.regularizer.L2Decay(cfg.weight_decay),
momentum=cfg.momentum)
optimizer.minimize(loss)
if cfg.use_gpu:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if cfg.pretrain:
if not os.path.exists(cfg.pretrain):
print("Pretrain weights not found: {}".format(cfg.pretrain))
def if_exist(var):
return os.path.exists(os.path.join(cfg.pretrain, var.name))
fluid.io.load_vars(exe, cfg.pretrain, predicate=if_exist)
build_strategy = fluid.BuildStrategy()
build_strategy.memory_optimize = True
build_strategy.sync_batch_norm = cfg.syncbn
compile_program = fluid.compiler.CompiledProgram(
fluid.default_main_program()).with_data_parallel(
loss_name=loss.name, build_strategy=build_strategy)
random_sizes = [cfg.input_size]
if cfg.random_shape:
random_sizes = [32 * i for i in range(10, 20)]
total_iter = cfg.max_iter - cfg.start_iter
mixup_iter = total_iter - cfg.no_mixup_iter
train_reader = reader.train(input_size,
batch_size=cfg.batch_size,
shuffle=True,
total_iter=total_iter * devices_num,
mixup_iter=mixup_iter * devices_num,
random_sizes=random_sizes,
use_multiprocessing=cfg.use_multiprocess)
py_reader = model.py_reader
py_reader.decorate_paddle_reader(train_reader)
def save_model(postfix):
model_path = os.path.join(cfg.model_save_dir, postfix)
if os.path.isdir(model_path):
shutil.rmtree(model_path)
fluid.io.save_persistables(exe, model_path)
fetch_list = [loss]
py_reader.start()
smoothed_loss = SmoothedValue()
try:
start_time = time.time()
prev_start_time = start_time
snapshot_loss = 0
snapshot_time = 0
for iter_id in range(cfg.start_iter, cfg.max_iter):
prev_start_time = start_time
start_time = time.time()
losses = exe.run(compile_program,
fetch_list=[v.name for v in fetch_list])
smoothed_loss.add_value(np.mean(np.array(losses[0])))
snapshot_loss += np.mean(np.array(losses[0]))
snapshot_time += start_time - prev_start_time
lr = np.array(
fluid.global_scope().find_var('learning_rate').get_tensor())
print("Iter {:d}, lr {:.6f}, loss {:.6f}, time {:.5f}".format(
iter_id, lr[0], smoothed_loss.get_mean_value(),
start_time - prev_start_time))
sys.stdout.flush()
if (iter_id + 1) % cfg.snapshot_iter == 0:
save_model("model_iter{}".format(iter_id))
print("Snapshot {} saved, average loss: {}, \
average time: {}".format(
iter_id + 1, snapshot_loss / float(cfg.snapshot_iter),
snapshot_time / float(cfg.snapshot_iter)))
snapshot_loss = 0
snapshot_time = 0
except fluid.core.EOFException:
py_reader.reset()
save_model('model_final')