def test_open_sync_batch_norm(self):
import paddle.fluid as fluid
import paddle.fluid.incubate.fleet.base.role_maker as role_maker
from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy
if not fluid.core.is_compiled_with_cuda():
# Operator "gen_nccl_id" has not been registered
return
data = fluid.layers.data(name='X', shape=[1], dtype='float32')
hidden = fluid.layers.fc(input=data, size=10)
loss = fluid.layers.mean(hidden)
optimizer = fluid.optimizer.AdamOptimizer()
role = role_maker.UserDefinedCollectiveRoleMaker(0, ['127.0.0.1:6170'])
fleet.init(role)
dist_strategy = DistributedStrategy()
dist_strategy.sync_batch_norm = True
dist_optimizer = fleet.distributed_optimizer(optimizer,
strategy=dist_strategy)
dist_optimizer.minimize(loss)
self.assertEqual(dist_strategy.exec_strategy.num_threads, 1)
def run_gpu_fleet_api_trainer(self, args):
assert args.update_method == "nccl2"
self.lr = args.lr
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 1
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.fuse_memory_size = 1 # MB
dist_strategy.fuse_laryer_size = 1
if args.use_local_sgd:
dist_strategy.use_local_sgd = True
if args.ut4grad_allreduce:
dist_strategy._ut4grad_allreduce = True
if args.sync_batch_norm:
dist_strategy.sync_batch_norm = True
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
print_to_err("gpu_fleet", "fleet.node_num:")
# "fleet.node_id:", fleet.node_id(),
# "fleet.trainer_num:", fleet.worker_num())
test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \
self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy)
trainer_prog = fleet._origin_program
dist_prog = fleet.main_program
device_id = int(os.getenv("FLAGS_selected_gpus", "0"))
place = fluid.CUDAPlace(device_id)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
eprint(type(self).__name__, "run worker startup program done.")
feed_var_list = [
var for var in trainer_prog.global_block().vars.values()
if var.is_data
]
eprint("feed_var_list:", feed_var_list)
# tmp add this code to pass python35 gcc8 CI
# Fixme(gongweibao, wangxi), need fix fleet api program order
if feed_var_list[0].name == 'label':
feed_var_list = feed_var_list[::-1]
feeder = fluid.DataFeeder(feed_var_list, place)
reader_generator = train_reader()
def get_data():
origin_batch = next(reader_generator)
if args.update_method != "local" and args.use_reader_alloc:
new_batch = []
for offset, item in enumerate(origin_batch):
if offset % 2 == args.trainer_id:
new_batch.append(item)
return new_batch
else:
return origin_batch
print_to_err(type(self).__name__, "begin to train on trainer")
out_losses = []
for i in six.moves.xrange(RUN_STEP):
loss, = exe.run(dist_prog,
fetch_list=[avg_cost.name],
feed=feeder.feed(get_data()))
out_losses.append(loss[0])
print_to_err(type(self).__name__, "run step %d finished" % i)
print_to_err(type(self).__name__, "trainer run finished")
if six.PY2:
print(pickle.dumps(out_losses))
else:
sys.stdout.buffer.write(pickle.dumps(out_losses))
if args.save_model:
model_save_dir = "/tmp"
if fleet.worker_index() == 0:
model_save_dir_fluid = os.path.join(model_save_dir,
"fluid_persistables")
model_save_dir_fleet = os.path.join(model_save_dir,
"fleet_persistables")
infer_save_dir_fluid = os.path.join(model_save_dir,
"fluid_infer")
infer_save_dir_fleet = os.path.join(model_save_dir,
"fleet_infer")
else:
model_save_dir_fluid = os.path.join(model_save_dir,
"fluid_persistables_2")
model_save_dir_fleet = os.path.join(model_save_dir,
"fleet_persistables_2")
infer_save_dir_fluid = os.path.join(model_save_dir,
"fluid_infer_2")
infer_save_dir_fleet = os.path.join(model_save_dir,
"fleet_infer_2")
fluid.io.save_persistables(exe, model_save_dir_fluid,
fleet._origin_program)
fleet.save_persistables(executor=exe, dirname=model_save_dir_fleet)
feeded_var_names = [var.name for var in feed_var_list]
fluid.io.save_inference_model(infer_save_dir_fluid,
feeded_var_names, [avg_cost], exe,
fleet._origin_program)
fleet.save_inference_model(exe, infer_save_dir_fleet,
feeded_var_names, [avg_cost])
def main():
role = role_maker.PaddleCloudRoleMaker(is_collective=True) # new line 3
fleet.init(role) # new line 4
env = os.environ
num_trainers = int(env.get('PADDLE_TRAINERS_NUM', 0))
assert num_trainers != 0, "multi-machine training process must be started using distributed.launch..."
trainer_id = int(env.get("PADDLE_TRAINER_ID", 0))
# set different seeds for different trainers
random.seed(trainer_id)
np.random.seed(trainer_id)
if FLAGS.enable_ce:
random.seed(0)
np.random.seed(0)
cfg = load_config(FLAGS.config)
merge_config(FLAGS.opt)
check_config(cfg)
# check if set use_gpu=True in paddlepaddle cpu version
check_gpu(cfg.use_gpu)
# check if paddlepaddle version is satisfied
check_version()
save_only = getattr(cfg, 'save_prediction_only', False)
if save_only:
raise NotImplementedError('The config file only support prediction,'
' training stage is not implemented now')
main_arch = cfg.architecture
assert cfg.use_gpu == True, "GPU must be supported for multi-machine training..."
devices_num = fluid.core.get_cuda_device_count()
if 'FLAGS_selected_gpus' in env:
device_id = int(env['FLAGS_selected_gpus'])
else:
device_id = 0
place = fluid.CUDAPlace(device_id) if cfg.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
lr_builder = create('LearningRate')
optim_builder = create('OptimizerBuilder')
# build program
startup_prog = fluid.Program()
train_prog = fluid.Program()
if FLAGS.enable_ce:
startup_prog.random_seed = 1000
train_prog.random_seed = 1000
with fluid.program_guard(train_prog, startup_prog):
with fluid.unique_name.guard():
model = create(main_arch)
if FLAGS.fp16:
assert (getattr(model.backbone, 'norm_type', None)
!= 'affine_channel'), \
'--fp16 currently does not support affine channel, ' \
' please modify backbone settings to use batch norm'
with mixed_precision_context(FLAGS.loss_scale, FLAGS.fp16) as ctx:
inputs_def = cfg['TrainReader']['inputs_def']
feed_vars, train_loader = model.build_inputs(**inputs_def)
train_fetches = model.train(feed_vars)
loss = train_fetches['loss']
if FLAGS.fp16:
loss *= ctx.get_loss_scale_var()
lr = lr_builder()
optimizer = optim_builder(lr)
dist_strategy = DistributedStrategy()
sync_bn = getattr(model.backbone, 'norm_type',
None) == 'sync_bn'
dist_strategy.sync_batch_norm = sync_bn
dist_strategy.nccl_comm_num = 1
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 3
exec_strategy.num_iteration_per_drop_scope = 30
dist_strategy.exec_strategy = exec_strategy
dist_strategy.fuse_all_reduce_ops = True
optimizer = fleet.distributed_optimizer(
optimizer, strategy=dist_strategy) # new line 5
optimizer.minimize(loss)
if FLAGS.fp16:
loss /= ctx.get_loss_scale_var()
if 'use_ema' in cfg and cfg['use_ema']:
global_steps = _decay_step_counter()
ema = ExponentialMovingAverage(cfg['ema_decay'],
thres_steps=global_steps)
ema.update()
# parse train fetches
train_keys, train_values, _ = parse_fetches(train_fetches)
train_values.append(lr)
if FLAGS.eval:
eval_prog = fluid.Program()
with fluid.program_guard(eval_prog, startup_prog):
with fluid.unique_name.guard():
model = create(main_arch)
inputs_def = cfg['EvalReader']['inputs_def']
feed_vars, eval_loader = model.build_inputs(**inputs_def)
fetches = model.eval(feed_vars)
eval_prog = eval_prog.clone(True)
eval_reader = create_reader(cfg.EvalReader, devices_num=1)
# When iterable mode, set set_sample_list_generator(eval_reader, place)
eval_loader.set_sample_list_generator(eval_reader)
# parse eval fetches
extra_keys = []
if cfg.metric == 'COCO':
extra_keys = ['im_info', 'im_id', 'im_shape']
if cfg.metric == 'VOC':
extra_keys = ['gt_bbox', 'gt_class', 'is_difficult']
if cfg.metric == 'WIDERFACE':
extra_keys = ['im_id', 'im_shape', 'gt_bbox']
eval_keys, eval_values, eval_cls = parse_fetches(
fetches, eval_prog, extra_keys)
exe.run(startup_prog)
compiled_train_prog = fleet.main_program
if FLAGS.eval:
compiled_eval_prog = fluid.CompiledProgram(eval_prog)
fuse_bn = getattr(model.backbone, 'norm_type', None) == 'affine_channel'
ignore_params = cfg.finetune_exclude_pretrained_params \
if 'finetune_exclude_pretrained_params' in cfg else []
start_iter = 0
if FLAGS.resume_checkpoint:
checkpoint.load_checkpoint(exe, train_prog, FLAGS.resume_checkpoint)
start_iter = checkpoint.global_step()
elif cfg.pretrain_weights and fuse_bn and not ignore_params:
checkpoint.load_and_fusebn(exe, train_prog, cfg.pretrain_weights)
elif cfg.pretrain_weights:
checkpoint.load_params(exe,
train_prog,
cfg.pretrain_weights,
ignore_params=ignore_params)
train_reader = create_reader(cfg.TrainReader,
(cfg.max_iters - start_iter) * devices_num,
cfg,
devices_num=devices_num)
# When iterable mode, set set_sample_list_generator(train_reader, place)
train_loader.set_sample_list_generator(train_reader)
# whether output bbox is normalized in model output layer
is_bbox_normalized = False
if hasattr(model, 'is_bbox_normalized') and \
callable(model.is_bbox_normalized):
is_bbox_normalized = model.is_bbox_normalized()
# if map_type not set, use default 11point, only use in VOC eval
map_type = cfg.map_type if 'map_type' in cfg else '11point'
train_stats = TrainingStats(cfg.log_iter, train_keys)
train_loader.start()
start_time = time.time()
end_time = time.time()
cfg_name = os.path.basename(FLAGS.config).split('.')[0]
save_dir = os.path.join(cfg.save_dir, cfg_name)
time_stat = deque(maxlen=cfg.log_iter)
best_box_ap_list = [0.0, 0] #[map, iter]
# use VisualDL to log data
if FLAGS.use_vdl:
assert six.PY3, "VisualDL requires Python >= 3.5"
from visualdl import LogWriter
vdl_writer = LogWriter(FLAGS.vdl_log_dir)
vdl_loss_step = 0
vdl_mAP_step = 0
for it in range(start_iter, cfg.max_iters):
start_time = end_time
end_time = time.time()
time_stat.append(end_time - start_time)
time_cost = np.mean(time_stat)
eta_sec = (cfg.max_iters - it) * time_cost
eta = str(datetime.timedelta(seconds=int(eta_sec)))
outs = exe.run(compiled_train_prog, fetch_list=train_values)
stats = {k: np.array(v).mean() for k, v in zip(train_keys, outs[:-1])}
# use vdl-paddle to log loss
if FLAGS.use_vdl:
if it % cfg.log_iter == 0:
for loss_name, loss_value in stats.items():
vdl_writer.add_scalar(loss_name, loss_value, vdl_loss_step)
vdl_loss_step += 1
train_stats.update(stats)
logs = train_stats.log()
if it % cfg.log_iter == 0 and trainer_id == 0:
strs = 'iter: {}, lr: {:.6f}, {}, time: {:.3f}, eta: {}'.format(
it, np.mean(outs[-1]), logs, time_cost, eta)
logger.info(strs)
# NOTE : profiler tools, used for benchmark
if FLAGS.is_profiler and it == 5:
profiler.start_profiler("All")
elif FLAGS.is_profiler and it == 10:
profiler.stop_profiler("total", FLAGS.profiler_path)
return
if (it > 0 and it % cfg.snapshot_iter == 0 or it == cfg.max_iters - 1) \
and trainer_id == 0:
save_name = str(it) if it != cfg.max_iters - 1 else "model_final"
if 'use_ema' in cfg and cfg['use_ema']:
exe.run(ema.apply_program)
checkpoint.save(exe, train_prog, os.path.join(save_dir, save_name))
if FLAGS.eval:
# evaluation
resolution = None
if 'Mask' in cfg.architecture:
resolution = model.mask_head.resolution
results = eval_run(exe,
compiled_eval_prog,
eval_loader,
eval_keys,
eval_values,
eval_cls,
cfg,
resolution=resolution)
box_ap_stats = eval_results(results, cfg.metric,
cfg.num_classes, resolution,
is_bbox_normalized,
FLAGS.output_eval, map_type,
cfg['EvalReader']['dataset'])
# use vdl_paddle to log mAP
if FLAGS.use_vdl:
vdl_writer.add_scalar("mAP", box_ap_stats[0], vdl_mAP_step)
vdl_mAP_step += 1
if box_ap_stats[0] > best_box_ap_list[0]:
best_box_ap_list[0] = box_ap_stats[0]
best_box_ap_list[1] = it
checkpoint.save(exe, train_prog,
os.path.join(save_dir, "best_model"))
logger.info("Best test box ap: {}, in iter: {}".format(
best_box_ap_list[0], best_box_ap_list[1]))
if 'use_ema' in cfg and cfg['use_ema']:
exe.run(ema.restore_program)
train_loader.reset()
def compress(args):
shuffle = True
if args.ce_test:
# set seed
seed = 111
paddle.seed(seed)
np.random.seed(seed)
random.seed(seed)
args.num_workers = 0
shuffle = False
env = os.environ
num_trainers = int(env.get('PADDLE_TRAINERS_NUM', 1))
use_data_parallel = num_trainers > 1
if use_data_parallel:
# Fleet step 1: initialize the distributed environment
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
train_reader = None
test_reader = None
if args.data == "mnist":
transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])])
train_dataset = paddle.vision.datasets.MNIST(
mode='train', backend="cv2", transform=transform)
val_dataset = paddle.vision.datasets.MNIST(
mode='test', backend="cv2", transform=transform)
class_dim = 10
image_shape = "1,28,28"
args.pretrained_model = False
elif args.data == "cifar10":
transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])])
train_dataset = paddle.vision.datasets.Cifar10(
mode="train", backend="cv2", transform=transform)
val_dataset = paddle.vision.datasets.Cifar10(
mode="test", backend="cv2", transform=transform)
class_dim = 10
image_shape = "3, 32, 32"
args.pretrained_model = False
elif args.data == "imagenet":
import imagenet_reader as reader
train_dataset = reader.ImageNetDataset(mode='train')
val_dataset = reader.ImageNetDataset(mode='val')
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list)
if args.use_gpu:
places = paddle.static.cuda_places()
else:
places = paddle.static.cpu_places()
place = places[0]
exe = paddle.static.Executor(place)
image = paddle.static.data(
name='image', shape=[None] + image_shape, dtype='float32')
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
batch_size_per_card = args.batch_size
batch_sampler = paddle.io.DistributedBatchSampler(
train_dataset,
batch_size=batch_size_per_card,
shuffle=shuffle,
drop_last=True)
train_loader = paddle.io.DataLoader(
train_dataset,
places=place,
batch_sampler=batch_sampler,
feed_list=[image, label],
return_list=False,
use_shared_memory=True,
num_workers=args.num_workers)
valid_loader = paddle.io.DataLoader(
val_dataset,
places=place,
feed_list=[image, label],
drop_last=False,
return_list=False,
use_shared_memory=True,
batch_size=args.batch_size_for_validation,
shuffle=False)
step_per_epoch = int(
np.ceil(len(train_dataset) * 1. / args.batch_size / num_trainers))
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
if args.data == 'cifar10':
label = paddle.reshape(label, [-1, 1])
cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label)
avg_cost = paddle.mean(x=cost)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
val_program = paddle.static.default_main_program().clone(for_test=True)
opt, learning_rate = create_optimizer(args, step_per_epoch)
# Fleet step 2: distributed strategy
if use_data_parallel:
dist_strategy = DistributedStrategy()
dist_strategy.sync_batch_norm = False
dist_strategy.exec_strategy = paddle.static.ExecutionStrategy()
dist_strategy.fuse_all_reduce_ops = False
train_program = paddle.static.default_main_program()
if args.pruning_strategy == 'gmp':
# GMP pruner step 0: define configs for GMP, no need to define configs for the base training.
configs = {
'stable_iterations': args.stable_epochs * step_per_epoch,
'pruning_iterations': args.pruning_epochs * step_per_epoch,
'tunning_iterations': args.tunning_epochs * step_per_epoch,
'resume_iteration': (args.last_epoch + 1) * step_per_epoch,
'pruning_steps': args.pruning_steps,
'initial_ratio': args.initial_ratio,
}
elif args.pruning_strategy == 'base':
configs = None
# GMP pruner step 1: initialize a pruner object by calling entry function.
pruner = create_unstructured_pruner(
train_program, args, place, configs=configs)
if use_data_parallel:
# Fleet step 3: decorate the origial optimizer and minimize it
opt = fleet.distributed_optimizer(opt, strategy=dist_strategy)
opt.minimize(avg_cost, no_grad_set=pruner.no_grad_set)
exe.run(paddle.static.default_startup_program())
if args.last_epoch > -1:
assert args.checkpoint is not None and os.path.exists(
args.checkpoint), "Please specify a valid checkpoint path."
paddle.fluid.io.load_persistables(
executor=exe, dirname=args.checkpoint, main_program=train_program)
elif args.pretrained_model:
assert os.path.exists(
args.
pretrained_model), "Pretrained model path {} doesn't exist".format(
args.pretrained_model)
def if_exist(var):
return os.path.exists(os.path.join(args.pretrained_model, var.name))
_logger.info("Load pretrained model from {}".format(
args.pretrained_model))
# NOTE: We are using fluid.io.load_vars() because the pretrained model is from an older version which requires this API.
# Please consider using paddle.static.load(program, model_path) when possible
paddle.fluid.io.load_vars(
exe, args.pretrained_model, predicate=if_exist)
def test(epoch, program):
acc_top1_ns = []
acc_top5_ns = []
_logger.info(
"The current sparsity of the inference model is {}%".format(
round(100 * UnstructuredPruner.total_sparse(
paddle.static.default_main_program()), 2)))
for batch_id, data in enumerate(valid_loader):
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program, feed=data, fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
_logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
def train(epoch, program):
train_reader_cost = 0.0
train_run_cost = 0.0
total_samples = 0
reader_start = time.time()
for batch_id, data in enumerate(train_loader):
train_reader_cost += time.time() - reader_start
train_start = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
program,
feed=data,
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
# GMP pruner step 2: step() to update ratios and other internal states of the pruner.
pruner.step()
train_run_cost += time.time() - train_start
total_samples += args.batch_size
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] lr: {:.6f} - loss: {}; acc_top1: {}; acc_top5: {}; avg_reader_cost: {:.5f} sec, avg_batch_cost: {:.5f} sec, avg_samples: {:.5f}, ips: {:.5f} images/sec".
format(epoch, batch_id,
learning_rate.get_lr(), loss_n, acc_top1_n,
acc_top5_n, train_reader_cost / args.log_period, (
train_reader_cost + train_run_cost
) / args.log_period, total_samples / args.log_period,
total_samples / (train_reader_cost + train_run_cost
)))
train_reader_cost = 0.0
train_run_cost = 0.0
total_samples = 0
learning_rate.step()
reader_start = time.time()
if use_data_parallel:
# Fleet step 4: get the compiled program from fleet
compiled_train_program = fleet.main_program
else:
compiled_train_program = paddle.static.CompiledProgram(
paddle.static.default_main_program())
for i in range(args.last_epoch + 1, args.num_epochs):
train(i, compiled_train_program)
# GMP pruner step 3: update params before summrizing sparsity, saving model or evaluation.
pruner.update_params()
_logger.info("The current sparsity of the pruned model is: {}%".format(
round(100 * UnstructuredPruner.total_sparse(
paddle.static.default_main_program()), 2)))
if (i + 1) % args.test_period == 0:
test(i, val_program)
if (i + 1) % args.model_period == 0:
if use_data_parallel:
fleet.save_persistables(executor=exe, dirname=args.model_path)
else:
paddle.fluid.io.save_persistables(
executor=exe, dirname=args.model_path)