def main(args):
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
config = get_config(args.config, overrides=args.override, show=True)
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id)
startup_prog = fluid.Program()
valid_prog = fluid.Program()
valid_dataloader, valid_fetchs = program.build(
config, valid_prog, startup_prog, is_train=False)
valid_prog = valid_prog.clone(for_test=True)
exe = fluid.Executor(place)
exe.run(startup_prog)
init_model(config, valid_prog, exe)
valid_reader = Reader(config, 'valid')()
valid_dataloader.set_sample_list_generator(valid_reader, place)
compiled_valid_prog = program.compile(config, valid_prog)
program.run(valid_dataloader, exe, compiled_valid_prog, valid_fetchs, -1,
'eval', config)
def distribute_train(args):
# 根据环境变量确定当前机器/进程在分布式训练中扮演的角色
# 然后使用 fleet api的 init()方法初始化这个节点
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
# 我们还可以进一步指定分布式的运行模式，通过 DistributeTranspilerConfig进行配置
# 如下，我们设置分布式运行模式为异步(async)，同时将参数进行切分，以分配到不同的节点
strategy = DistributeTranspilerConfig()
strategy.sync_mode = False
strategy.runtime_split_send_recv = True
ctr_model = CTR()
inputs = ctr_model.input_data(args)
avg_cost, auc_var = ctr_model.net(inputs, args)
# 配置分布式的optimizer，传入我们指定的strategy，构建program
optimizer = fluid.optimizer.Adam(args.learning_rate)
optimizer = fleet.distributed_optimizer(optimizer, strategy)
optimizer.minimize(avg_cost)
# 根据节点角色，分别运行不同的逻辑
if fleet.is_server():
# 初始化及运行参数服务器节点
fleet.init_server()
fleet.run_server()
elif fleet.is_worker():
# 初始化工作节点
fleet.init_worker()
exe = fluid.Executor(fluid.CPUPlace())
# 初始化含有分布式流程的fleet.startup_program
exe.run(fleet.startup_program)
dataset, file_list = get_dataset(inputs, args)
for epoch in range(args.epochs):
# 以文件为粒度进行shuffle
random.shuffle(file_list)
dataset.set_filelist(file_list)
# 训练节点运行的是经过分布式裁剪的fleet.mian_program
start_time = time.time()
exe.train_from_dataset(program=fleet.main_program,
dataset=dataset,
fetch_list=[auc_var],
fetch_info=["Epoch {} auc ".format(epoch)],
print_period=100,
debug=False)
end_time = time.time()
logger.info("epoch %d finished, use time=%d\n" %
((epoch), end_time - start_time))
# 默认使用0号节点保存模型
if args.save_model and fleet.is_first_worker():
model_path = os.path.join(str(args.model_path),
"epoch_" + str(epoch))
fleet.save_persistables(executor=exe, dirname=model_path)
fleet.stop_worker()
logger.info("Distribute Train Success!")
def main(args):
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
config = get_config(args.config, overrides=args.override, show=True)
# assign the place
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id)
# startup_prog is used to do some parameter init work,
# and train prog is used to hold the network
startup_prog = fluid.Program()
train_prog = fluid.Program()
train_dataloader, train_fetchs = program.build(config,
train_prog,
startup_prog,
is_train=True)
if config.validate:
valid_prog = fluid.Program()
valid_dataloader, valid_fetchs = program.build(config,
valid_prog,
startup_prog,
is_train=False)
# clone to prune some content which is irrelevant in valid_prog
valid_prog = valid_prog.clone(for_test=True)
# create the "Executor" with the statement of which place
exe = fluid.Executor(place=place)
# only run startup_prog once to init
exe.run(startup_prog)
# load model from checkpoint or pretrained model
init_model(config, train_prog, exe)
train_reader = Reader(config, 'train')()
train_dataloader.set_sample_list_generator(train_reader, place)
if config.validate:
valid_reader = Reader(config, 'valid')()
valid_dataloader.set_sample_list_generator(valid_reader, place)
compiled_valid_prog = program.compile(config, valid_prog)
compiled_train_prog = fleet.main_program
for epoch_id in range(config.epochs):
# 1. train with train dataset
program.run(train_dataloader, exe, compiled_train_prog, train_fetchs,
epoch_id, 'train')
# 2. validate with validate dataset
if config.validate and epoch_id % config.valid_interval == 0:
program.run(valid_dataloader, exe, compiled_valid_prog,
valid_fetchs, epoch_id, 'valid')
# 3. save the persistable model
if epoch_id % config.save_interval == 0:
model_path = os.path.join(config.model_save_dir,
config.ARCHITECTURE["name"])
save_model(train_prog, model_path, epoch_id)
def get_distributed_optimizer(optimizer):
"""
Get the default collective distributed optimizer under fleet.
"""
dist_strategy = DistributedStrategy()
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
optimizer = fleet.distributed_optimizer(optimizer, strategy=dist_strategy)
return optimizer
def default_exe_params(is_distributed, use_cuda, thread_num):
"""
Set the default execute parameters.
"""
gpu_id = 0
trainer_num = 1
trainer_id = 0
dist_strategy = None
places = None
if is_distributed:
if use_cuda:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
gpu_id = int(os.getenv("FLAGS_selected_gpus"))
trainer_num = fleet.worker_num()
trainer_id = fleet.worker_index()
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = 4
exec_strategy.num_iteration_per_drop_scope = 1
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.nccl_comm_num = 2
dist_strategy.fuse_all_reduce_ops = True
dist_strategy.forward_recompute = True
dist_strategy.use_amp = True
dist_strategy.amp_loss_scaling = 12800.0
places = fluid.cuda_places()
else:
print('Only gpu is supported for distributed mode at present.')
exit(-1)
else:
if use_cuda:
places = fluid.cuda_places()
else:
places = fluid.cpu_places(thread_num)
os.environ['CPU_NUM'] = str(thread_num)
if use_cuda:
exe = fluid.Executor(fluid.CUDAPlace(gpu_id))
else:
exe = fluid.Executor(fluid.CPUPlace())
return {
'exe': exe,
'trainer_num': trainer_num,
'trainer_id': trainer_id,
'gpu_id': gpu_id,
'dist_strategy': dist_strategy,
'places': places
}
def test_ps_rolemaker(self):
os.environ["TRAINING_ROLE"] = "PSERVER"
os.environ["POD_IP"] = "127.0.0.1"
ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
ro.generate_role()
self.assertFalse(ro.is_worker())
self.assertTrue(ro.is_server())
self.assertEqual(ro.worker_num(), 2)
def append_additional_args(self, FLAGS):
"""
append addtional args from the existing args
"""
#dataset_dir and train_dir is defined in padllecloud, cannot be set by user
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
return super(PaddleCloudFleetTrainer, self).append_additional_args(FLAGS)
def train(args):
"""
Train main function.
"""
if args.is_distributed:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
dev_count = fluid.core.get_cuda_device_count()
gpu_id = int(os.getenv("FLAGS_selected_gpus"))
trainers_num = fleet.worker_num()
trainer_id = fleet.worker_index()
else:
dev_count = 1
gpu_id = 0
trainers_num = 1
trainer_id = 0
place = fluid.CUDAPlace(gpu_id)
task = tasks.create_task(args)
model = models.create_model(args, place)
train_generator = task.reader.data_generator(input_file=args.train_file,
num_epochs=args.num_epochs,
num_part=trainers_num,
part_id=trainer_id,
phase="train")
valid_generator = task.reader.data_generator(
input_file=args.valid_file,
num_part=dev_count,
part_id=gpu_id,
phase="distributed_valid" if args.is_distributed else "valid")
# run training
model_timer = Timer()
for step, data in enumerate(train_generator(), 1):
model_timer.start()
metrics = task.train_step(model, data)
model_timer.pause()
if step % args.log_steps == 0:
time_cost = model_timer.pass_time
current_epoch, current_file_index, total_file = task.reader.get_train_progress(
)
print(
f"[train][{current_epoch}] progress: {current_file_index}/{total_file} "
f"step: {step}, time: {time_cost:.3f}, "
f"speed: {args.log_steps / time_cost:.3f} steps/s")
print("\tcurrent lr:", metrics.pop('scheduled_lr'))
print("\t" + task.show_metrics(metrics))
model_timer.reset()
if step % args.validation_steps == 0:
evaluate(task, model, valid_generator, args, dev_count, gpu_id)
if step % args.save_steps == 0:
save_path = f"{args.save_path}/step_{step}"
model.save(save_path, is_checkpoint=True)
def test_tr_rolemaker(self):
os.environ["TRAINING_ROLE"] = "TRAINER"
os.environ["PADDLE_TRAINER_ID"] = "0"
ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
ro.generate_role()
self.assertTrue(ro.is_worker())
self.assertFalse(ro.is_server())
self.assertEqual(ro.worker_num(), 2)
def main(args):
log.info("start")
worker_num = int(os.getenv("PADDLE_TRAINERS_NUM", "0"))
num_devices = int(os.getenv("CPU_NUM", 10))
model = Metapath2vecModel(config=args)
pyreader = model.pyreader
loss = model.forward()
# init fleet
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
train_steps = math.ceil(args.num_nodes * args.epochs / args.batch_size /
num_devices / worker_num)
log.info("Train step: %s" % train_steps)
real_batch_size = args.batch_size * args.walk_len * args.win_size
if args.optimizer == "sgd":
args.lr *= real_batch_size
optimization(args.lr, loss, train_steps, args.optimizer)
# init and run server or worker
if fleet.is_server():
fleet.init_server(args.warm_start_from_dir)
fleet.run_server()
if fleet.is_worker():
log.info("start init worker done")
fleet.init_worker()
#just the worker, load the sample
log.info("init worker done")
exe = F.Executor(F.CPUPlace())
exe.run(fleet.startup_program)
log.info("Startup done")
dataset = m2vGraph(args)
log.info("Build graph done.")
data_generator = multiprocess_data_generator(args, dataset)
cur_time = time.time()
for idx, _ in enumerate(data_generator()):
log.info("iter %s: %s s" % (idx, time.time() - cur_time))
cur_time = time.time()
if idx == 100:
break
pyreader.decorate_tensor_provider(data_generator)
pyreader.start()
compiled_prog = build_complied_prog(fleet.main_program, loss)
train_prog(exe, compiled_prog, loss, pyreader, args, train_steps)
def prepare_cpumulti_env(self, is_local):
"""
:param is_local:
:return:
"""
if is_local:
self.is_fleet = False
else:
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
self.is_fleet = True
logging.debug("init fleet cpu multi")
def test_distributed_basic(self):
checker = acp._get_checker()
fs = HDFSClient(checker.hdfs_home, None)
fs.delete(checker.hdfs_checkpoint_path)
self._reset_generator()
logger.info("begin test_distributed_basic")
fs = LocalFS()
save_dir = "./run_save_0"
fs.delete(save_dir)
#basic
exe, main_prog, startup_prog = self._generate()
compiled, data_loader, optimizer, loss, image, label = \
self._init_env(exe, main_prog, startup_prog, minimize=False)
#fleet
os.environ["TRAINING_ROLE"] = "TRAINER"
os.environ["PADDLE_TRAINER_ID"] = "0"
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:6070"
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
with fluid.program_guard(main_prog, startup_prog):
dist_optimizer = fleet.distributed_optimizer(optimizer)
dist_optimizer.minimize(loss)
exe.run(startup_prog)
o = None
i = 0
name = None
for i in acp.train_epoch_range(3, 0):
o = acp._get_train_epoch_range()
name = o.name
logger.info("_run_save_0 name:{} epoch_no:{}".format(o.name, i))
for data in data_loader():
fetch = exe.run(fleet.main_program,
feed=data,
fetch_list=[loss])
self.assertEqual(len(o._exe_status), 1)
o = acp._get_train_epoch_range()
assert o == None, "now train epoch must not exits now"
self.assertEqual(i, 2)
fs.delete(save_dir)
logger.info("end test_distributed_basic")
def main(args):
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
config = get_config(args.config, overrides=args.override, show=True)
place = env.place()
startup_prog = fluid.Program()
train_prog = fluid.Program()
train_dataloader, train_fetchs = program.build(
config, train_prog, startup_prog, is_train=True)
if config.validate:
valid_prog = fluid.Program()
valid_dataloader, valid_fetchs = program.build(
config, valid_prog, startup_prog, is_train=False)
valid_prog = valid_prog.clone(for_test=True)
exe = fluid.Executor(place)
exe.run(startup_prog)
init_model(config, train_prog, exe)
train_reader = Reader(config, 'train')()
train_dataloader.set_sample_list_generator(train_reader, place)
if config.validate:
valid_reader = Reader(config, 'valid')()
valid_dataloader.set_sample_list_generator(valid_reader, place)
compiled_valid_prog = program.compile(config, valid_prog)
compiled_train_prog = fleet.main_program
for epoch_id in range(config.epochs):
program.run(train_dataloader, exe, compiled_train_prog, train_fetchs,
epoch_id, 'train')
if config.validate and epoch_id % config.valid_interval == 0:
program.run(valid_dataloader, exe, compiled_valid_prog,
valid_fetchs, epoch_id, 'valid')
if epoch_id % config.save_interval == 0:
model_path = os.path.join(config.model_save_dir,
config.architecture)
save_model(train_prog, model_path, epoch_id)
def run_nccl_trainer(self, args):
"""run fleet api"""
assert args.update_method == "nccl"
import paddle.fluid as fluid
import six
from paddle.fluid.incubate.fleet.collective import fleet
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = args.run_params['num_threads']
#dist_strategy = DistributedStrategy()
#dist_strategy.exec_strategy = exec_strategy
#dist_strategy.fuse_memory_size = 1 # MB
#dist_strategy.fuse_laryer_size = 1
if args.role.upper() != "TRAINER":
raise ValueError("args role must be TRAINER")
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
strategy = DistributeTranspilerConfig()
avg_cost = self.net(args)
losses = self.do_training(fleet,args)
losses = "" if not losses else losses
print(losses)
def run_nccl_trainer(self, args):
"""
run nccl trainer, used for gpu case.
Args:
args (ArgumentParser): run args to config dist fleet.
"""
assert args.update_method == "nccl"
from paddle.fluid.incubate.fleet.collective import fleet
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = args.run_params['num_threads']
#dist_strategy = DistributedStrategy()
#dist_strategy.exec_strategy = exec_strategy
#dist_strategy.fuse_memory_size = 1 # MB
#dist_strategy.fuse_laryer_size = 1
if args.role.upper() != "TRAINER":
raise ValueError("args role must be TRAINER")
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
avg_cost = self.net(args)
losses = self.do_training(fleet, args)
losses = "" if not losses else losses
print(losses)
def _test_check_point(self, fs, dir_path):
file_name = "persistables"
os.environ["TRAINING_ROLE"] = "TRAINER"
os.environ["PADDLE_TRAINER_ID"] = "0"
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:6070"
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label],
place=fluid.CPUPlace())
predict = fluid.layers.fc(input=image, size=10, act='softmax')
loss = fluid.layers.cross_entropy(input=predict, label=label)
avg_loss = fluid.layers.mean(loss)
optimizer = fluid.optimizer.AdamOptimizer(learning_rate=0.001)
dist_optimizer = fleet.distributed_optimizer(optimizer)
dist_optimizer.minimize(avg_loss)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
status = TrainStatus(2)
fleet.save_check_point(exe, dir_path, train_status=status, fs=fs)
n1 = fleet._get_last_checkpoint_no(dir_path, fs=fs)
status2 = fleet.load_check_point(exe, dir_path, trainer_id=0, fs=fs)
self.assertEqual(status2, status)
fleet.save_check_point(exe, dir_path, train_status=status, fs=fs)
n2 = fleet._get_last_checkpoint_no(dir_path, fs=fs)
self.assertEqual(n2, n1 + 1)
fleet.clean_redundant_check_points(dir_path, fs=fs)
def test_traing_role(self):
os.environ["TRAINING_ROLE"] = "TEST"
ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
self.assertRaises(ValueError, ro.generate_role)
def __init__(self):
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
cfleet.init(role)
def compress(args):
shuffle = True
if args.ce_test:
# set seed
seed = 111
paddle.seed(seed)
np.random.seed(seed)
random.seed(seed)
args.num_workers = 0
shuffle = False
env = os.environ
num_trainers = int(env.get('PADDLE_TRAINERS_NUM', 1))
use_data_parallel = num_trainers > 1
if use_data_parallel:
# Fleet step 1: initialize the distributed environment
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
train_reader = None
test_reader = None
if args.data == "mnist":
transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])])
train_dataset = paddle.vision.datasets.MNIST(
mode='train', backend="cv2", transform=transform)
val_dataset = paddle.vision.datasets.MNIST(
mode='test', backend="cv2", transform=transform)
class_dim = 10
image_shape = "1,28,28"
args.pretrained_model = False
elif args.data == "cifar10":
transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])])
train_dataset = paddle.vision.datasets.Cifar10(
mode="train", backend="cv2", transform=transform)
val_dataset = paddle.vision.datasets.Cifar10(
mode="test", backend="cv2", transform=transform)
class_dim = 10
image_shape = "3, 32, 32"
args.pretrained_model = False
elif args.data == "imagenet":
import imagenet_reader as reader
train_dataset = reader.ImageNetDataset(mode='train')
val_dataset = reader.ImageNetDataset(mode='val')
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(args.model,
model_list)
if args.use_gpu:
places = paddle.static.cuda_places()
else:
places = paddle.static.cpu_places()
place = places[0]
exe = paddle.static.Executor(place)
image = paddle.static.data(
name='image', shape=[None] + image_shape, dtype='float32')
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
batch_size_per_card = args.batch_size
batch_sampler = paddle.io.DistributedBatchSampler(
train_dataset,
batch_size=batch_size_per_card,
shuffle=shuffle,
drop_last=True)
train_loader = paddle.io.DataLoader(
train_dataset,
places=place,
batch_sampler=batch_sampler,
feed_list=[image, label],
return_list=False,
use_shared_memory=True,
num_workers=args.num_workers)
valid_loader = paddle.io.DataLoader(
val_dataset,
places=place,
feed_list=[image, label],
drop_last=False,
return_list=False,
use_shared_memory=True,
batch_size=args.batch_size_for_validation,
shuffle=False)
step_per_epoch = int(
np.ceil(len(train_dataset) * 1. / args.batch_size / num_trainers))
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
if args.data == 'cifar10':
label = paddle.reshape(label, [-1, 1])
cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label)
avg_cost = paddle.mean(x=cost)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
val_program = paddle.static.default_main_program().clone(for_test=True)
opt, learning_rate = create_optimizer(args, step_per_epoch)
# Fleet step 2: distributed strategy
if use_data_parallel:
dist_strategy = DistributedStrategy()
dist_strategy.sync_batch_norm = False
dist_strategy.exec_strategy = paddle.static.ExecutionStrategy()
dist_strategy.fuse_all_reduce_ops = False
train_program = paddle.static.default_main_program()
if args.pruning_strategy == 'gmp':
# GMP pruner step 0: define configs for GMP, no need to define configs for the base training.
configs = {
'stable_iterations': args.stable_epochs * step_per_epoch,
'pruning_iterations': args.pruning_epochs * step_per_epoch,
'tunning_iterations': args.tunning_epochs * step_per_epoch,
'resume_iteration': (args.last_epoch + 1) * step_per_epoch,
'pruning_steps': args.pruning_steps,
'initial_ratio': args.initial_ratio,
}
elif args.pruning_strategy == 'base':
configs = None
# GMP pruner step 1: initialize a pruner object by calling entry function.
pruner = create_unstructured_pruner(
train_program, args, place, configs=configs)
if use_data_parallel:
# Fleet step 3: decorate the origial optimizer and minimize it
opt = fleet.distributed_optimizer(opt, strategy=dist_strategy)
opt.minimize(avg_cost, no_grad_set=pruner.no_grad_set)
exe.run(paddle.static.default_startup_program())
if args.last_epoch > -1:
assert args.checkpoint is not None and os.path.exists(
args.checkpoint), "Please specify a valid checkpoint path."
paddle.fluid.io.load_persistables(
executor=exe, dirname=args.checkpoint, main_program=train_program)
elif args.pretrained_model:
assert os.path.exists(
args.
pretrained_model), "Pretrained model path {} doesn't exist".format(
args.pretrained_model)
def if_exist(var):
return os.path.exists(os.path.join(args.pretrained_model, var.name))
_logger.info("Load pretrained model from {}".format(
args.pretrained_model))
# NOTE: We are using fluid.io.load_vars() because the pretrained model is from an older version which requires this API.
# Please consider using paddle.static.load(program, model_path) when possible
paddle.fluid.io.load_vars(
exe, args.pretrained_model, predicate=if_exist)
def test(epoch, program):
acc_top1_ns = []
acc_top5_ns = []
_logger.info(
"The current sparsity of the inference model is {}%".format(
round(100 * UnstructuredPruner.total_sparse(
paddle.static.default_main_program()), 2)))
for batch_id, data in enumerate(valid_loader):
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program, feed=data, fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".
format(epoch, batch_id,
np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
_logger.info("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch,
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
def train(epoch, program):
train_reader_cost = 0.0
train_run_cost = 0.0
total_samples = 0
reader_start = time.time()
for batch_id, data in enumerate(train_loader):
train_reader_cost += time.time() - reader_start
train_start = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
program,
feed=data,
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
# GMP pruner step 2: step() to update ratios and other internal states of the pruner.
pruner.step()
train_run_cost += time.time() - train_start
total_samples += args.batch_size
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] lr: {:.6f} - loss: {}; acc_top1: {}; acc_top5: {}; avg_reader_cost: {:.5f} sec, avg_batch_cost: {:.5f} sec, avg_samples: {:.5f}, ips: {:.5f} images/sec".
format(epoch, batch_id,
learning_rate.get_lr(), loss_n, acc_top1_n,
acc_top5_n, train_reader_cost / args.log_period, (
train_reader_cost + train_run_cost
) / args.log_period, total_samples / args.log_period,
total_samples / (train_reader_cost + train_run_cost
)))
train_reader_cost = 0.0
train_run_cost = 0.0
total_samples = 0
learning_rate.step()
reader_start = time.time()
if use_data_parallel:
# Fleet step 4: get the compiled program from fleet
compiled_train_program = fleet.main_program
else:
compiled_train_program = paddle.static.CompiledProgram(
paddle.static.default_main_program())
for i in range(args.last_epoch + 1, args.num_epochs):
train(i, compiled_train_program)
# GMP pruner step 3: update params before summrizing sparsity, saving model or evaluation.
pruner.update_params()
_logger.info("The current sparsity of the pruned model is: {}%".format(
round(100 * UnstructuredPruner.total_sparse(
paddle.static.default_main_program()), 2)))
if (i + 1) % args.test_period == 0:
test(i, val_program)
if (i + 1) % args.model_period == 0:
if use_data_parallel:
fleet.save_persistables(executor=exe, dirname=args.model_path)
else:
paddle.fluid.io.save_persistables(
executor=exe, dirname=args.model_path)
def main():
role = role_maker.PaddleCloudRoleMaker(is_collective=True) # new line 3
fleet.init(role) # new line 4
env = os.environ
num_trainers = int(env.get('PADDLE_TRAINERS_NUM', 0))
assert num_trainers != 0, "multi-machine training process must be started using distributed.launch..."
trainer_id = int(env.get("PADDLE_TRAINER_ID", 0))
# set different seeds for different trainers
random.seed(trainer_id)
np.random.seed(trainer_id)
if FLAGS.enable_ce:
random.seed(0)
np.random.seed(0)
cfg = load_config(FLAGS.config)
merge_config(FLAGS.opt)
check_config(cfg)
# check if set use_gpu=True in paddlepaddle cpu version
check_gpu(cfg.use_gpu)
# check if paddlepaddle version is satisfied
check_version()
save_only = getattr(cfg, 'save_prediction_only', False)
if save_only:
raise NotImplementedError('The config file only support prediction,'
' training stage is not implemented now')
main_arch = cfg.architecture
assert cfg.use_gpu == True, "GPU must be supported for multi-machine training..."
devices_num = fluid.core.get_cuda_device_count()
if 'FLAGS_selected_gpus' in env:
device_id = int(env['FLAGS_selected_gpus'])
else:
device_id = 0
place = fluid.CUDAPlace(device_id) if cfg.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
lr_builder = create('LearningRate')
optim_builder = create('OptimizerBuilder')
# build program
startup_prog = fluid.Program()
train_prog = fluid.Program()
if FLAGS.enable_ce:
startup_prog.random_seed = 1000
train_prog.random_seed = 1000
with fluid.program_guard(train_prog, startup_prog):
with fluid.unique_name.guard():
model = create(main_arch)
if FLAGS.fp16:
assert (getattr(model.backbone, 'norm_type', None)
!= 'affine_channel'), \
'--fp16 currently does not support affine channel, ' \
' please modify backbone settings to use batch norm'
with mixed_precision_context(FLAGS.loss_scale, FLAGS.fp16) as ctx:
inputs_def = cfg['TrainReader']['inputs_def']
feed_vars, train_loader = model.build_inputs(**inputs_def)
train_fetches = model.train(feed_vars)
loss = train_fetches['loss']
if FLAGS.fp16:
loss *= ctx.get_loss_scale_var()
lr = lr_builder()
optimizer = optim_builder(lr)
dist_strategy = DistributedStrategy()
sync_bn = getattr(model.backbone, 'norm_type',
None) == 'sync_bn'
dist_strategy.sync_batch_norm = sync_bn
dist_strategy.nccl_comm_num = 1
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 3
exec_strategy.num_iteration_per_drop_scope = 30
dist_strategy.exec_strategy = exec_strategy
dist_strategy.fuse_all_reduce_ops = True
optimizer = fleet.distributed_optimizer(
optimizer, strategy=dist_strategy) # new line 5
optimizer.minimize(loss)
if FLAGS.fp16:
loss /= ctx.get_loss_scale_var()
if 'use_ema' in cfg and cfg['use_ema']:
global_steps = _decay_step_counter()
ema = ExponentialMovingAverage(cfg['ema_decay'],
thres_steps=global_steps)
ema.update()
# parse train fetches
train_keys, train_values, _ = parse_fetches(train_fetches)
train_values.append(lr)
if FLAGS.eval:
eval_prog = fluid.Program()
with fluid.program_guard(eval_prog, startup_prog):
with fluid.unique_name.guard():
model = create(main_arch)
inputs_def = cfg['EvalReader']['inputs_def']
feed_vars, eval_loader = model.build_inputs(**inputs_def)
fetches = model.eval(feed_vars)
eval_prog = eval_prog.clone(True)
eval_reader = create_reader(cfg.EvalReader, devices_num=1)
# When iterable mode, set set_sample_list_generator(eval_reader, place)
eval_loader.set_sample_list_generator(eval_reader)
# parse eval fetches
extra_keys = []
if cfg.metric == 'COCO':
extra_keys = ['im_info', 'im_id', 'im_shape']
if cfg.metric == 'VOC':
extra_keys = ['gt_bbox', 'gt_class', 'is_difficult']
if cfg.metric == 'WIDERFACE':
extra_keys = ['im_id', 'im_shape', 'gt_bbox']
eval_keys, eval_values, eval_cls = parse_fetches(
fetches, eval_prog, extra_keys)
exe.run(startup_prog)
compiled_train_prog = fleet.main_program
if FLAGS.eval:
compiled_eval_prog = fluid.CompiledProgram(eval_prog)
fuse_bn = getattr(model.backbone, 'norm_type', None) == 'affine_channel'
ignore_params = cfg.finetune_exclude_pretrained_params \
if 'finetune_exclude_pretrained_params' in cfg else []
start_iter = 0
if FLAGS.resume_checkpoint:
checkpoint.load_checkpoint(exe, train_prog, FLAGS.resume_checkpoint)
start_iter = checkpoint.global_step()
elif cfg.pretrain_weights and fuse_bn and not ignore_params:
checkpoint.load_and_fusebn(exe, train_prog, cfg.pretrain_weights)
elif cfg.pretrain_weights:
checkpoint.load_params(exe,
train_prog,
cfg.pretrain_weights,
ignore_params=ignore_params)
train_reader = create_reader(cfg.TrainReader,
(cfg.max_iters - start_iter) * devices_num,
cfg,
devices_num=devices_num)
# When iterable mode, set set_sample_list_generator(train_reader, place)
train_loader.set_sample_list_generator(train_reader)
# whether output bbox is normalized in model output layer
is_bbox_normalized = False
if hasattr(model, 'is_bbox_normalized') and \
callable(model.is_bbox_normalized):
is_bbox_normalized = model.is_bbox_normalized()
# if map_type not set, use default 11point, only use in VOC eval
map_type = cfg.map_type if 'map_type' in cfg else '11point'
train_stats = TrainingStats(cfg.log_iter, train_keys)
train_loader.start()
start_time = time.time()
end_time = time.time()
cfg_name = os.path.basename(FLAGS.config).split('.')[0]
save_dir = os.path.join(cfg.save_dir, cfg_name)
time_stat = deque(maxlen=cfg.log_iter)
best_box_ap_list = [0.0, 0] #[map, iter]
# use VisualDL to log data
if FLAGS.use_vdl:
assert six.PY3, "VisualDL requires Python >= 3.5"
from visualdl import LogWriter
vdl_writer = LogWriter(FLAGS.vdl_log_dir)
vdl_loss_step = 0
vdl_mAP_step = 0
for it in range(start_iter, cfg.max_iters):
start_time = end_time
end_time = time.time()
time_stat.append(end_time - start_time)
time_cost = np.mean(time_stat)
eta_sec = (cfg.max_iters - it) * time_cost
eta = str(datetime.timedelta(seconds=int(eta_sec)))
outs = exe.run(compiled_train_prog, fetch_list=train_values)
stats = {k: np.array(v).mean() for k, v in zip(train_keys, outs[:-1])}
# use vdl-paddle to log loss
if FLAGS.use_vdl:
if it % cfg.log_iter == 0:
for loss_name, loss_value in stats.items():
vdl_writer.add_scalar(loss_name, loss_value, vdl_loss_step)
vdl_loss_step += 1
train_stats.update(stats)
logs = train_stats.log()
if it % cfg.log_iter == 0 and trainer_id == 0:
strs = 'iter: {}, lr: {:.6f}, {}, time: {:.3f}, eta: {}'.format(
it, np.mean(outs[-1]), logs, time_cost, eta)
logger.info(strs)
# NOTE : profiler tools, used for benchmark
if FLAGS.is_profiler and it == 5:
profiler.start_profiler("All")
elif FLAGS.is_profiler and it == 10:
profiler.stop_profiler("total", FLAGS.profiler_path)
return
if (it > 0 and it % cfg.snapshot_iter == 0 or it == cfg.max_iters - 1) \
and trainer_id == 0:
save_name = str(it) if it != cfg.max_iters - 1 else "model_final"
if 'use_ema' in cfg and cfg['use_ema']:
exe.run(ema.apply_program)
checkpoint.save(exe, train_prog, os.path.join(save_dir, save_name))
if FLAGS.eval:
# evaluation
resolution = None
if 'Mask' in cfg.architecture:
resolution = model.mask_head.resolution
results = eval_run(exe,
compiled_eval_prog,
eval_loader,
eval_keys,
eval_values,
eval_cls,
cfg,
resolution=resolution)
box_ap_stats = eval_results(results, cfg.metric,
cfg.num_classes, resolution,
is_bbox_normalized,
FLAGS.output_eval, map_type,
cfg['EvalReader']['dataset'])
# use vdl_paddle to log mAP
if FLAGS.use_vdl:
vdl_writer.add_scalar("mAP", box_ap_stats[0], vdl_mAP_step)
vdl_mAP_step += 1
if box_ap_stats[0] > best_box_ap_list[0]:
best_box_ap_list[0] = box_ap_stats[0]
best_box_ap_list[1] = it
checkpoint.save(exe, train_prog,
os.path.join(save_dir, "best_model"))
logger.info("Best test box ap: {}, in iter: {}".format(
best_box_ap_list[0], best_box_ap_list[1]))
if 'use_ema' in cfg and cfg['use_ema']:
exe.run(ema.restore_program)
train_loader.reset()
def main(args):
cfg = XiaoduHiConfig()
cfg.scene_sensor_algo = 'yolov4'
wae_ndarray = np.load(os.path.join(args.wae_dir, 'raw_wae.npy'))
start_epoch = 0
train_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(train_program, startup_program):
attention_ctrl = AttentionController(
inputs_type=args.inputs_type,
num_actions=wae_ndarray.shape[0],
act_tr_dim=wae_ndarray.shape[1],
act_emb_ndarray=wae_ndarray,
num_frames=cfg.ob_window_len,
tokens_per_frame=cfg.tokens_per_frame,
visual_token_dim=cfg.visual_token_dim,
model_dim=args.model_dim,
num_decoder_blocks=args.num_decoder_blocks,
num_heads=args.num_heads,
ffn_dim=args.ffn_dim,
dropout=args.dropout,
normalize_before=args.normalize_before,
frame_emb_trainable=args.frame_emb_trainable,
trigger_loss_coef=args.trigger_loss_coef,
obj_loss_coef=args.obj_loss_coef,
act_loss_coef=args.act_loss_coef,
use_last_act_loss=args.use_last_act_loss,
mode='train')
preds = attention_ctrl.predict()
test_program = train_program.clone(for_test=True)
optimizer = fluid.optimizer.AdamOptimizer(
learning_rate=args.lr,
regularization=fluid.regularizer.L2Decay(
regularization_coeff=0.1))
if args.distributed_training:
optimizer = fleet.distributed_optimizer(optimizer)
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
optimizer.minimize(attention_ctrl.loss)
if args.distributed_training:
place = fluid.CUDAPlace(int(os.environ.get('FLAGS_selected_gpus', 0)))
else:
place = fluid.CUDAPlace(args.gpu)
exe = fluid.Executor(place)
exe.run(startup_program)
if args.inputs_type.startswith('inst_crop') and \
args.inputs_type != 'inst_crop_wo_crop':
fluid.io.load_vars(
exe, MobileNetV2_Pretrained,
main_program=train_program,
predicate=lambda v: os.path.exists(
os.path.join(MobileNetV2_Pretrained, v.name)))
print('Loaded weights from {}'.format(MobileNetV2_Pretrained))
if args.init_params is not None:
base = os.path.basename(args.init_params)
if base.startswith('epoch_'):
start_epoch = int(base[len('epoch_'):]) + 1
tb_state = os.path.join(args.init_params, 'tb_state.txt')
if os.path.exists(tb_state):
global _update_step
global _eval_step
with open(tb_state, 'r') as f:
update_step, eval_step = f.readline().split(' ')
_update_step = int(update_step)
_eval_step = int(eval_step)
fluid.io.load_vars(
exe, args.init_params,
main_program=train_program,
predicate=lambda v: os.path.exists(
os.path.join(args.init_params, v.name)))
print('Loaded weights from {}'.format(args.init_params))
if args.distributed_training:
train_worker_gpus = [int(os.environ.get('FLAGS_selected_gpus', 0))]
test_worker_gpus = train_worker_gpus
else:
train_worker_gpus = convert_gpu_ids(args.data_worker_gpus_for_train)
test_worker_gpus = convert_gpu_ids(args.data_worker_gpus_for_test)
if not os.path.exists(args.save):
os.makedirs(args.save)
if not args.use_decord:
train_dataloader = XiaoduHiDataloaderv2(
attention_ctrl.feed_list, [place], args.yolov4_model_dir,
args.video_tracking_dir, args.train_dataset,
full_neg_txt=args.full_neg_train,
batch_size=args.bs,
num_workers=args.data_workers_for_train,
worker_gpus=train_worker_gpus,
roi_feat_resolution=cfg.roi_feat_resolution,
ob_window_len=cfg.ob_window_len,
interval=cfg.interval,
tokens_per_frame=cfg.tokens_per_frame,
visual_token_dim=cfg.visual_token_dim,
augment=False,
resample_negs_per_epoch=True)
test_dataloader = XiaoduHiDataloaderv2(
attention_ctrl.feed_list, [place], args.yolov4_model_dir,
args.video_tracking_dir, args.test_dataset,
full_neg_txt=args.full_neg_test,
batch_size=args.bs,
num_workers=args.data_workers_for_test,
worker_gpus=test_worker_gpus,
roi_feat_resolution=cfg.roi_feat_resolution,
ob_window_len=cfg.ob_window_len,
interval=cfg.interval,
tokens_per_frame=cfg.tokens_per_frame,
visual_token_dim=cfg.visual_token_dim,
augment=False,
resample_negs_per_epoch=False,
for_test=True)
test_dataloader.save_to_txt(
os.path.join(args.save, 'eval_data.txt'), dt=200)
else:
train_dataloader = XiaoduHiDecordLoader(
attention_ctrl.feed_list, [place],
args.yolov4_model_dir, args.decord_ds_pkl,
decord_readers=args.decord_readers,
yolov4_detectors=args.decord_detectors,
post_workers=args.decord_post_workers,
batch_size=args.bs,
detector_gpus=train_worker_gpus,
roi_feat_resolution=cfg.roi_feat_resolution,
tokens_per_frame=cfg.tokens_per_frame,
visual_token_dim=cfg.visual_token_dim,
for_test=False)
test_dataloader = XiaoduHiDecordLoader(
attention_ctrl.feed_list, [place],
args.yolov4_model_dir, args.decord_ds_pkl,
decord_readers=args.decord_readers,
yolov4_detectors=args.decord_detectors,
post_workers=args.decord_post_workers,
batch_size=args.bs,
detector_gpus=test_worker_gpus,
roi_feat_resolution=cfg.roi_feat_resolution,
tokens_per_frame=cfg.tokens_per_frame,
visual_token_dim=cfg.visual_token_dim,
for_test=True)
train_dataloader.start_workers()
test_dataloader.start_workers()
train_log = os.path.join(args.save, 'loss.csv')
eval_log = os.path.join(args.save, 'eval.txt')
with open(os.path.join(args.save, 'args.txt'), 'w') as f:
f.write(str(args))
tb_writer = SummaryWriter(
logdir=os.path.join(args.save, 'logdir'),
purge_step=None if _update_step == 0 else _update_step)
worker_index = None if not args.distributed_training \
else fleet.worker_index()
# if worker_index == 0:
# eval_model(exe, test_program, preds, attention_ctrl.act_loss,
# test_dataloader, -1, log_file=eval_log,
# tb_writer=tb_writer, worker_index=worker_index)
for epoch_id in range(start_epoch, args.epochs):
print('--------------- Epoch %d ---------------' % epoch_id)
train_epoch(exe, train_program, attention_ctrl, train_dataloader,
log_file=train_log, tb_writer=tb_writer,
worker_index=worker_index)
save_dir = os.path.join(args.save, 'epoch_{}'.format(epoch_id))
shutil.rmtree(save_dir, ignore_errors=True)
os.mkdir(save_dir)
fluid.io.save_params(exe, save_dir, main_program=train_program)
if epoch_id > 0 and epoch_id % args.run_eval_after_epochs == 0:
eval_model(exe, test_program, preds, attention_ctrl.act_loss,
test_dataloader, epoch_id, log_file=eval_log,
tb_writer=tb_writer)
tb_state = os.path.join(save_dir, 'tb_state.txt')
with open(tb_state, 'w') as f:
f.write('{} {}'.format(_update_step, _eval_step))
if epoch_id % args.run_eval_after_epochs != 0:
eval_model(exe, test_program, preds, attention_ctrl.act_loss,
test_dataloader, epoch_id, log_file=eval_log,
tb_writer=tb_writer, worker_index=worker_index)
train_dataloader.stop_workers()
test_dataloader.stop_workers()
def run_gpu_fleet_api_trainer(self, args):
assert args.update_method == "nccl2"
self.lr = args.lr
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 1
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.fuse_memory_size = 1 # MB
dist_strategy.fuse_laryer_size = 1
if args.use_local_sgd:
dist_strategy.use_local_sgd = True
if args.ut4grad_allreduce:
dist_strategy._ut4grad_allreduce = True
if args.sync_batch_norm:
dist_strategy.sync_batch_norm = True
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
print_to_err("gpu_fleet", "fleet.node_num:")
# "fleet.node_id:", fleet.node_id(),
# "fleet.trainer_num:", fleet.worker_num())
test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \
self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy)
trainer_prog = fleet._origin_program
dist_prog = fleet.main_program
device_id = int(os.getenv("FLAGS_selected_gpus", "0"))
place = fluid.CUDAPlace(device_id)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
eprint(type(self).__name__, "run worker startup program done.")
feed_var_list = [
var for var in trainer_prog.global_block().vars.values()
if var.is_data
]
eprint("feed_var_list:", feed_var_list)
# tmp add this code to pass python35 gcc8 CI
# Fixme(gongweibao, wangxi), need fix fleet api program order
if feed_var_list[0].name == 'label':
feed_var_list = feed_var_list[::-1]
feeder = fluid.DataFeeder(feed_var_list, place)
reader_generator = train_reader()
def get_data():
origin_batch = next(reader_generator)
if args.update_method != "local" and args.use_reader_alloc:
new_batch = []
for offset, item in enumerate(origin_batch):
if offset % 2 == args.trainer_id:
new_batch.append(item)
return new_batch
else:
return origin_batch
print_to_err(type(self).__name__, "begin to train on trainer")
out_losses = []
for i in six.moves.xrange(RUN_STEP):
loss, = exe.run(dist_prog,
fetch_list=[avg_cost.name],
feed=feeder.feed(get_data()))
out_losses.append(loss[0])
print_to_err(type(self).__name__, "run step %d finished" % i)
print_to_err(type(self).__name__, "trainer run finished")
if six.PY2:
print(pickle.dumps(out_losses))
else:
sys.stdout.buffer.write(pickle.dumps(out_losses))
if args.save_model:
model_save_dir = "/tmp"
if fleet.worker_index() == 0:
model_save_dir_fluid = os.path.join(model_save_dir,
"fluid_persistables")
model_save_dir_fleet = os.path.join(model_save_dir,
"fleet_persistables")
infer_save_dir_fluid = os.path.join(model_save_dir,
"fluid_infer")
infer_save_dir_fleet = os.path.join(model_save_dir,
"fleet_infer")
else:
model_save_dir_fluid = os.path.join(model_save_dir,
"fluid_persistables_2")
model_save_dir_fleet = os.path.join(model_save_dir,
"fleet_persistables_2")
infer_save_dir_fluid = os.path.join(model_save_dir,
"fluid_infer_2")
infer_save_dir_fleet = os.path.join(model_save_dir,
"fleet_infer_2")
fluid.io.save_persistables(exe, model_save_dir_fluid,
fleet._origin_program)
fleet.save_persistables(executor=exe, dirname=model_save_dir_fleet)
feeded_var_names = [var.name for var in feed_var_list]
fluid.io.save_inference_model(infer_save_dir_fluid,
feeded_var_names, [avg_cost], exe,
fleet._origin_program)
fleet.save_inference_model(exe, infer_save_dir_fleet,
feeded_var_names, [avg_cost])
def train(args):
print("pretraining start")
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
with open(args.task_group_json) as f:
task_group = json.load(f)
exec_strategy = fluid.ExecutionStrategy()
if args.use_fast_executor:
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = 4 if args.use_amp else 2
exec_strategy.num_iteration_per_drop_scope = min(1, args.skip_steps)
node_nums = int(os.getenv("PADDLE_NODES_NUM"))
print("args.is_distributed:", args.is_distributed)
num_trainers = 1
trainer_id = 0
if args.is_distributed:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
trainer_id = fleet.worker_index()
current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT")
worker_endpoints = fleet.worker_endpoints()
trainers_num = len(worker_endpoints)
print("worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"
.format(worker_endpoints, trainers_num, current_endpoint, trainer_id))
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.remove_unnecessary_lock = False # not useful
dist_strategy.fuse_all_reduce_ops = True if args.use_fuse else False
dist_strategy.nccl_comm_num = args.nccl_comm_num
if args.use_hierarchical_allreduce \
and trainers_num > args.hierarchical_allreduce_inter_nranks:
dist_strategy.use_hierarchical_allreduce = args.use_hierarchical_allreduce
dist_strategy.hierarchical_allreduce_inter_nranks = \
args.hierarchical_allreduce_inter_nranks
assert dist_strategy.use_hierarchical_allreduce > 1
assert trainers_num % dist_strategy.hierarchical_allreduce_inter_nranks == 0
dist_strategy.hierarchical_allreduce_exter_nranks = \
trainers_num / dist_strategy.hierarchical_allreduce_inter_nranks
if args.use_amp:
dist_strategy.use_amp = True
dist_strategy.amp_loss_scaling = args.init_loss_scaling
if args.use_recompute:
dist_strategy.forward_recompute = True
dist_strategy.enable_sequential_execution=True
trainer_id = fleet.worker_index()
current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT")
worker_endpoints = fleet.worker_endpoints()
trainers_num = len(worker_endpoints)
print("worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"
.format(worker_endpoints,trainers_num, current_endpoint, trainer_id))
else:
dist_strategy=None
gpu_id=0
gpus = fluid.core.get_cuda_device_count()
if args.is_distributed:
gpus = os.getenv("FLAGS_selected_gpus").split(",")
gpu_id = int(gpus[0])
if args.use_cuda:
place = fluid.CUDAPlace(gpu_id)
dev_count = len(gpus)
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
print("Device count %d, gpu_id:%d" % (dev_count, gpu_id))
train_program = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_pyreader, fetch_vars = create_model(
pyreader_name='train_reader', ernie_config=ernie_config, task_group=task_group)
graph_vars = fetch_vars["graph_vars"]
checkpoints = fetch_vars["checkpoints"]
total_loss = graph_vars[-1]
if args.use_recompute:
dist_strategy.recompute_checkpoints = checkpoints
scheduled_lr, loss_scaling = optimization(
loss=total_loss,
warmup_steps=args.warmup_steps,
num_train_steps=args.num_train_steps,
learning_rate=args.learning_rate,
train_program=train_program,
startup_prog=startup_prog,
weight_decay=args.weight_decay,
scheduler=args.lr_scheduler,
use_fp16=args.use_amp,
use_dynamic_loss_scaling=args.use_dynamic_loss_scaling,
init_loss_scaling=args.init_loss_scaling,
incr_every_n_steps=args.incr_every_n_steps,
decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
incr_ratio=args.incr_ratio,
decr_ratio=args.decr_ratio,
dist_strategy=dist_strategy)
origin_train_program = train_program
if args.is_distributed:
#raped by fleet, need to assign fleet's modified train_grogram back
train_program = fleet.main_program
origin_train_program = fleet._origin_program
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_pyreader, fetch_vars = create_model(
pyreader_name='test_reader', ernie_config=ernie_config, task_group=task_group)
graph_vars = fetch_vars["graph_vars"]
total_loss = graph_vars[-1]
test_prog = test_prog.clone(for_test=True)
exe = fluid.Executor(place)
exe.run(startup_prog)
if args.init_checkpoint and args.init_checkpoint != "":
#init_checkpoint(exe, args.init_checkpoint, origin_train_program, args.use_amp)
init_pretraining_params(exe, args.init_checkpoint, origin_train_program, args.use_amp)
data_reader = ErnieDataReader(
task_group,
False,
batch_size=args.batch_size,
vocab_path=args.vocab_path,
voc_size=ernie_config['vocab_size'],
epoch=args.epoch,
max_seq_len=args.max_seq_len,
generate_neg_sample=args.generate_neg_sample,
hack_old_trainset=args.hack_old_data)
#only fleet
train_exe = exe
predict = predict_wrapper(
args,
exe,
ernie_config,
task_group,
test_prog=test_prog,
pyreader=test_pyreader,
fetch_list=[var.name for var in graph_vars])
train_pyreader.set_batch_generator(data_reader.data_generator())
train_pyreader.start()
steps = 112000
time_begin = time.time()
node_nums = int(os.getenv("PADDLE_NODES_NUM"))
while True:#steps < args.num_train_steps:
try:
steps += 1#node_nums
skip_steps = args.skip_steps# * node_nums
fetch_list = []
if trainer_id == 0 and steps % skip_steps == 0:
fetch_list = [var.name for var in graph_vars] + [scheduled_lr.name]
if args.use_amp:
fetch_list.append(loss_scaling.name)
outputs = train_exe.run(fetch_list=fetch_list, program=train_program)
time_end = time.time()
used_time = time_end - time_begin
if outputs:
each_mask_lm_cost, lm_w = outputs[:2]
if args.use_amp:
each_total_constract_loss, each_total_cost, np_lr, l_scaling = outputs[-4:]
else:
each_total_constract_loss, each_total_cost, np_lr = outputs[-3:]
acc_list =[]
index = 2
for task in task_group:
each_task_acc = outputs[index]
task_w = outputs[index + 1]
acc = np.sum(each_task_acc * task_w) / np.sum(task_w)
acc_list.append("%s acc: %f" % (task["task_name"], acc))
index += 2
print("feed_queue size", train_pyreader.queue.size())
epoch, current_file_index, total_file, current_file, mask_type = data_reader.get_progress()
if args.use_amp:
print("current learning_rate:%f, loss scaling:%f" % (np_lr[0], l_scaling[0]))
else:
print("current learning_rate:%f" % np_lr[0])
print(
"epoch: %d, progress: %d/%d, step: %d, constract_loss: %f, loss: %f, "
"ppl: %f, %s, speed: %f steps/s, file: %s, mask_type: %s"
% (epoch, current_file_index, total_file, steps,
np.mean(each_total_constract_loss), np.mean(each_total_cost),
np.exp(np.sum(each_mask_lm_cost * lm_w) / np.sum(lm_w)),
", ".join(acc_list), skip_steps / used_time,
current_file, mask_type))
time_begin = time.time()
elif steps % skip_steps == 0:
epoch, current_file_index, total_file, current_file, mask_type = data_reader.get_progress(
)
print("feed_queue size", train_pyreader.queue.size())
print("epoch: %d, progress: %d/%d, step: %d, "
"speed: %f steps/s, file: %s, mask_type: %s"
% (epoch, current_file_index, total_file, steps,
skip_steps / used_time, current_file, mask_type))
time_begin = time.time()
if not trainer_id == 0:
continue
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, origin_train_program)
if steps % args.validation_steps == 0:
valid_list = predict()
print("[validation_set] epoch: %d, step: %d, %s" % \
(epoch, steps, ", ".join(valid_list)))
except fluid.core.EOFException:
train_pyreader.reset()
break
def train(args):
"""
Train main function.
"""
if args.is_distributed:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
dev_count = fluid.core.get_cuda_device_count()
gpu_id = int(os.getenv("FLAGS_selected_gpus"))
trainers_num = fleet.worker_num()
trainer_id = fleet.worker_index()
else:
dev_count = 1
gpu_id = 0
trainers_num = 1
trainer_id = 0
place = fluid.CUDAPlace(gpu_id)
task = tasks.create_task(args)
model = models.create_model(args, place)
train_generator = task.get_data_loader(model,
input_file=args.train_file,
num_epochs=args.num_epochs,
num_part=trainers_num,
part_id=trainer_id,
phase="train")
valid_generator = task.get_data_loader(
model,
input_file=args.valid_file,
num_part=dev_count,
part_id=gpu_id,
phase="distributed_valid" if args.is_distributed else "valid")
# run training
timer = Timer()
timer.start()
if args.Model.model == 'NSPModel':
best_metrics = 0.0
else:
best_metrics = 10000
shuffledatafile()
for step, data in enumerate(train_generator(), args.start_step + 1):
outputs = task.train_step(model, data)
timer.pause()
if step % args.log_steps == 0:
time_cost = timer.pass_time
current_epoch, current_file_index, total_file = task.reader.get_train_progress(
)
print(
f"[train][{current_epoch}] progress: {current_file_index}/{total_file} "
f"step: {step}, time: {time_cost:.3f}, "
f"speed: {args.log_steps / time_cost:.3f} steps/s")
print(f"\tcurrent lr: {outputs.pop('scheduled_lr'):.7f}")
metrics = task.get_metrics(outputs)
print("\t" + ", ".join(f"{k}: {v:.4f}"
for k, v in metrics.items()))
timer.reset()
if step % args.validation_steps == 0:
# shuffledatafile()
metrics = evaluate(task, model, valid_generator, args, dev_count,
gpu_id, step)
if args.Model.model == 'NSPModel' and metrics[
'nsp_acc'] > best_metrics:
best_metrics = metrics['nsp_acc']
save_path = f"{args.save_path}/step_{step}_{best_metrics}"
model.save(save_path, is_checkpoint=True)
elif args.Model.model == 'Plato' and metrics['loss'] < best_metrics:
best_metrics = metrics['loss']
save_path = f"{args.save_path}/step_{step}_{best_metrics}"
model.save(save_path, is_checkpoint=True)
# if step % args.save_steps == 0 and trainer_id == 0:
# save_path = f"{args.save_path}/step_{step}"
# model.save(save_path, is_checkpoint=True)
# with open(save_path + ".finish", "w") as f:
# pass
timer.start()
def run_gpu_fleet_api_trainer(self, args):
assert args.update_method == "nccl2"
self.lr = args.lr
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 1
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.fuse_memory_size = 1 # MB
dist_strategy.fuse_laryer_size = 1
if args.use_local_sgd:
dist_strategy.use_local_sgd = True
if args.ut4grad_allreduce:
dist_strategy._ut4grad_allreduce = True
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
print_to_err("gpu_fleet", "fleet.node_num:")
# "fleet.node_id:", fleet.node_id(),
# "fleet.trainer_num:", fleet.worker_num())
test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \
self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy)
trainer_prog = fleet._origin_program
dist_prog = fleet.main_program
device_id = int(os.getenv("FLAGS_selected_gpus", "0"))
place = fluid.CUDAPlace(device_id)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
eprint(type(self).__name__, "run worker startup program done.")
feed_var_list = [
var for var in trainer_prog.global_block().vars.values()
if var.is_data
]
feeder = fluid.DataFeeder(feed_var_list, place)
reader_generator = train_reader()
def get_data():
origin_batch = next(reader_generator)
if args.update_method != "local" and args.use_reader_alloc:
new_batch = []
for offset, item in enumerate(origin_batch):
if offset % 2 == args.trainer_id:
new_batch.append(item)
return new_batch
else:
return origin_batch
print_to_err(type(self).__name__, "begin to train on trainer")
out_losses = []
for i in six.moves.xrange(RUN_STEP):
loss, = exe.run(dist_prog,
fetch_list=[avg_cost.name],
feed=feeder.feed(get_data()))
out_losses.append(loss[0])
print_to_err(type(self).__name__, "run step %d finished" % i)
print_to_err(type(self).__name__, "trainer run finished")
if six.PY2:
print(pickle.dumps(out_losses))
else:
sys.stdout.buffer.write(pickle.dumps(out_losses))
dataset.set_pipe_command(pipe_command)
dataset.set_batch_size(32)
dataset.set_thread(10)
dataset.set_hdfs_config("hdfs://192.168.48.87:9000", "root,")
optimizer = fluid.optimizer.SGD(0.0001)
#optimizer.minimize(avg_cost)
exe = fluid.Executor(fluid.CPUPlace())
input_folder = "hdfs:"
output = sp.check_output(
"hdfs dfs -ls /train_data | awk '{if(NR>1) print $8}'", shell=True)
train_filelist = [
"{}{}".format(input_folder, f)
for f in output.decode('ascii').strip().split('\n')
]
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
config = DistributeTranspilerConfig()
config.sync_mode = False
optimizer = fleet.distributed_optimizer(optimizer, config)
optimizer.minimize(avg_cost)
if fleet.is_server():
fleet.init_server()
fleet.run_server()
elif fleet.is_worker():
place = fluid.CPUPlace()
exe = fluid.Executor(place)
fleet.init_worker()
def _test_checkpoint(self, fs, dir_path):
file_name = "persistables"
os.environ["TRAINING_ROLE"] = "TRAINER"
os.environ["PADDLE_TRAINER_ID"] = "0"
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:6070"
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label],
place=fluid.CPUPlace())
predict = fluid.layers.fc(input=image, size=10, act='softmax')
loss = fluid.layers.cross_entropy(input=predict, label=label)
avg_loss = fluid.layers.mean(loss)
optimizer = fluid.optimizer.AdamOptimizer(learning_rate=0.001)
dist_optimizer = fleet.distributed_optimizer(optimizer)
dist_optimizer.minimize(avg_loss)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
status = ExeTrainStatus()
status.epoch_no = 2
_, n1 = fleet.save_checkpoint(exe,
dir_path,
trainer_id=0,
train_status=status,
fs=fs)
status2 = ExeTrainStatus()
fleet.load_checkpoint(exe,
dir_path,
trainer_id=0,
fs=fs,
train_status=status2)
self.assertEqual(status2, status)
_, n2 = fleet.save_checkpoint(exe,
dir_path,
trainer_id=0,
train_status=status,
fs=fs,
remain_all_checkpoint=False)
self.assertEqual(n2, n1 + 1)
c = CheckpointSaver(fs)
cp_nos = c.get_checkpoint_no(dir_path)
assert len(cp_nos) == 1 # cleanup all others
# unnormal
# test remain_all_checkpoint
fleet.save_checkpoint(exe,
dir_path,
trainer_id=0,
train_status=status,
fs=fs,
remain_all_checkpoint=False)
# can't save under a file
fs = LocalFS()
cache_path = "./.load_cache"
fs.touch(cache_path)
try:
fleet.save_checkpoint(exe,
dir_path,
trainer_id=0,
train_status=status,
fs=fs,
cache_path=cache_path)
self.assertFalse(True)
except:
pass
# can't load under a file
try:
fleet.load_checkpoint(exe,
dir_path,
trainer_id=0,
train_status=status2,
fs=fs,
cache_path=cache_path)
self.assertFalse(True)
except:
pass
fs.delete(cache_path)
def main(args):
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
config = get_config(args.config, overrides=args.override, show=True)
# assign the place
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id)
# startup_prog is used to do some parameter init work,
# and train prog is used to hold the network
startup_prog = fluid.Program()
train_prog = fluid.Program()
best_top1_acc = 0.0 # best top1 acc record
if not config.get('use_ema'):
train_dataloader, train_fetchs = program.build(config,
train_prog,
startup_prog,
is_train=True)
else:
train_dataloader, train_fetchs, ema = program.build(config,
train_prog,
startup_prog,
is_train=True)
if config.validate:
valid_prog = fluid.Program()
valid_dataloader, valid_fetchs = program.build(config,
valid_prog,
startup_prog,
is_train=False)
# clone to prune some content which is irrelevant in valid_prog
valid_prog = valid_prog.clone(for_test=True)
# create the "Executor" with the statement of which place
exe = fluid.Executor(place)
# Parameter initialization
exe.run(startup_prog)
# load model from 1. checkpoint to resume training, 2. pretrained model to finetune
init_model(config, train_prog, exe)
train_reader = Reader(config, 'train')()
train_dataloader.set_sample_list_generator(train_reader, place)
if config.validate:
valid_reader = Reader(config, 'valid')()
valid_dataloader.set_sample_list_generator(valid_reader, place)
compiled_valid_prog = program.compile(config, valid_prog)
compiled_train_prog = fleet.main_program
vdl_writer = LogWriter(args.vdl_dir) if args.vdl_dir else None
for epoch_id in range(config.epochs):
# 1. train with train dataset
program.run(train_dataloader, exe, compiled_train_prog, train_fetchs,
epoch_id, 'train', vdl_writer)
if int(os.getenv("PADDLE_TRAINER_ID", 0)) == 0:
# 2. validate with validate dataset
if config.validate and epoch_id % config.valid_interval == 0:
if config.get('use_ema'):
logger.info(logger.coloring("EMA validate start..."))
with train_fetchs('ema').apply(exe):
top1_acc = program.run(valid_dataloader, exe,
compiled_valid_prog,
valid_fetchs, epoch_id, 'valid')
logger.info(logger.coloring("EMA validate over!"))
top1_acc = program.run(valid_dataloader, exe,
compiled_valid_prog, valid_fetchs,
epoch_id, 'valid')
if top1_acc > best_top1_acc:
best_top1_acc = top1_acc
message = "The best top1 acc {:.5f}, in epoch: {:d}".format(
best_top1_acc, epoch_id)
logger.info("{:s}".format(logger.coloring(message, "RED")))
if epoch_id % config.save_interval == 0:
model_path = os.path.join(config.model_save_dir,
config.ARCHITECTURE["name"])
save_model(train_prog, model_path,
"best_model_in_epoch_" + str(epoch_id))
# 3. save the persistable model
if epoch_id % config.save_interval == 0:
model_path = os.path.join(config.model_save_dir,
config.ARCHITECTURE["name"])
save_model(train_prog, model_path, epoch_id)
def main(args):
with open(args.config, 'r') as f:
config = json.load(f)
logging.info('Load data ...')
if len(args.dataset.split(',')) > 1:
# for large pretraining dataset, ZINC15 and ChEMBL
# directly load the processed npz files
train_data_list = []
for ds in args.dataset.split(','):
# use processed data.npz
train_data_list.extend(
load_data(os.path.join(args.root, ds, 'processed')))
# dataset = MoleculeDataset(
# args.root, ds,
# add_symmetry=False,
# add_self_loop=False)
# data_list = dataset.get_data_list()
# processed_dir = os.path.join(args.root, ds, 'processed')
# os.makedirs(processed_dir, exist_ok=True)
# save_data_list_to_npz(
# data_list, os.path.join(processed_dir, 'data.npz'))
# logging.info('Processed {}'.format(ds))
# train_data_list.extend(data_list)
else:
if args.dataset == 'mutag':
train_data_list, _ = load_mutag_dataset(
os.path.join(args.root, args.dataset, 'raw'))
elif args.dataset == 'ptc_mr':
train_data_list, _ = load_ptc_mr_dataset(
os.path.join(args.root, args.dataset, 'raw'))
else:
raise ValueError('Unsupported dataset')
if args.is_fleet:
train_data_list = [
x for i, x in enumerate(train_data_list)
if i % fleet.worker_num() == fleet.worker_index()
]
logging.info("Data loaded.")
logging.info("Train Examples: %s" % len(train_data_list))
sys.stdout.flush()
if args.emb_dir is not None:
os.makedirs(args.emb_dir, exist_ok=True)
train_prog = F.Program()
test_prog = F.Program()
startup_prog = F.Program()
with F.program_guard(train_prog, startup_prog):
with F.unique_name.guard():
agent = create_model(args, config)
test_prog = train_prog.clone(for_test=True)
opt = F.optimizer.Adam(learning_rate=args.lr)
if args.is_fleet:
dist_strategy = DistributedStrategy()
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
opt = fleet.distributed_optimizer(opt, strategy=dist_strategy)
opt.minimize(agent.loss)
place = F.CUDAPlace(0) if args.use_cuda else F.CPUPlace()
exe = F.Executor(place)
exe.run(startup_prog)
if (not args.dont_save_emb) and \
(not args.is_fleet or fleet.worker_index() == 0):
save_embedding(args, exe, test_prog, agent, train_data_list, -1)
for epoch_id in range(args.max_epoch):
train(args, exe, train_prog, agent, train_data_list, epoch_id)
if not args.is_fleet or fleet.worker_index() == 0:
F.io.save_params(exe, '%s/epoch%s' % (args.model_dir, epoch_id),
train_prog)
if not args.dont_save_emb:
save_embedding(args, exe, test_prog, agent, train_data_list,
epoch_id)
def main(args):
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
if args.use_cuda:
dev_list = fluid.cuda_places()
place = dev_list[0]
dev_count = len(dev_list)
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
exe = fluid.Executor(place)
reader = reader_ce.ClassifyReader(vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
total_num=args.train_data_size,
do_lower_case=args.do_lower_case,
in_tokens=args.in_tokens,
random_seed=args.random_seed,
tokenizer=args.tokenizer,
for_cn=args.for_cn,
task_id=args.task_id)
if not (args.do_train or args.do_val or args.do_test):
raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
"least one of them must be True.")
if args.do_test:
assert args.test_save is not None
startup_prog = fluid.Program()
if args.random_seed is not None:
startup_prog.random_seed = args.random_seed
if args.predict_batch_size == None:
args.predict_batch_size = args.batch_size
if args.do_train:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
dev_count = fleet.worker_num()
train_data_generator = reader.data_generator(
input_file=args.train_set,
batch_size=args.batch_size,
epoch=args.epoch,
dev_count=1,
trainer_id=fleet.worker_index(),
trainer_num=fleet.worker_num(),
shuffle=True,
phase="train")
num_train_examples = reader.get_num_examples(args.train_set)
if args.in_tokens:
max_train_steps = args.epoch * num_train_examples // (
args.batch_size // args.max_seq_len) // dev_count
else:
max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
warmup_steps = int(max_train_steps * args.warmup_proportion)
log.info("Device count: %d" % dev_count)
log.info("Num train examples: %d" % num_train_examples)
log.info("Max train steps: %d" % max_train_steps)
log.info("Num warmup steps: %d" % warmup_steps)
train_program = fluid.Program()
# use fleet api
exec_strategy = fluid.ExecutionStrategy()
if args.use_fast_executor:
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = dev_count
if args.is_distributed:
exec_strategy.num_threads = 3
exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.nccl_comm_num = 1
if args.is_distributed:
dist_strategy.nccl_comm_num = 2
dist_strategy.use_hierarchical_allreduce = True
if args.use_mix_precision:
dist_strategy.use_amp = True
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_pyreader, graph_vars = create_model(
args,
pyreader_name='train_reader',
ernie_config=ernie_config)
scheduled_lr = optimization(
loss=graph_vars["loss"],
warmup_steps=warmup_steps,
num_train_steps=max_train_steps,
learning_rate=args.learning_rate,
train_program=train_program,
startup_prog=startup_prog,
weight_decay=args.weight_decay,
scheduler=args.lr_scheduler,
use_dynamic_loss_scaling=args.use_dynamic_loss_scaling,
incr_every_n_steps=args.incr_every_n_steps,
decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
incr_ratio=args.incr_ratio,
decr_ratio=args.decr_ratio,
dist_strategy=dist_strategy)
if args.verbose:
if args.in_tokens:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program,
batch_size=args.batch_size // args.max_seq_len)
else:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program, batch_size=args.batch_size)
log.info("Theoretical memory usage in training: %.3f - %.3f %s" %
(lower_mem, upper_mem, unit))
if args.do_val or args.do_test:
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_pyreader, graph_vars = create_model(
args,
pyreader_name='test_reader',
ernie_config=ernie_config,
is_prediction=True)
test_prog = test_prog.clone(for_test=True)
train_program = fleet.main_program
exe = fluid.Executor(place)
exe.run(startup_prog)
if args.do_train:
if args.init_checkpoint and args.init_pretraining_params:
log.warning(
"WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
"both are set! Only arg 'init_checkpoint' is made valid.")
if args.init_checkpoint:
init_checkpoint(exe,
args.init_checkpoint,
main_program=startup_prog)
elif args.init_pretraining_params:
init_pretraining_params(exe,
args.init_pretraining_params,
main_program=startup_prog)
elif args.do_val or args.do_test:
if not args.init_checkpoint:
raise ValueError("args 'init_checkpoint' should be set if"
"only doing validation or testing!")
init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
if args.do_train:
train_exe = exe
train_pyreader.decorate_tensor_provider(train_data_generator)
else:
train_exe = None
test_exe = exe
# if args.do_val or args.do_test:
# if args.use_multi_gpu_test:
# test_exe = fluid.ParallelExecutor(
# use_cuda=args.use_cuda,
# main_program=test_prog,
# share_vars_from=train_exe)
current_epoch = 0
steps = 0
if args.do_train:
train_pyreader.start()
if warmup_steps > 0:
graph_vars["learning_rate"] = scheduled_lr
ce_info = []
time_begin = time.time()
last_epoch = 0
while True:
try:
steps += 1
# log.info("step: %d" % steps)
if fleet.worker_index() != 0:
train_exe.run(fetch_list=[], program=train_program)
continue
if steps % args.skip_steps != 0:
train_exe.run(fetch_list=[], program=train_program)
else:
outputs = evaluate(train_exe,
train_program,
train_pyreader,
graph_vars,
"train",
metric=args.metric)
if args.verbose:
verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
)
verbose += "learning rate: %f" % (
outputs["learning_rate"]
if warmup_steps > 0 else args.learning_rate)
log.info(verbose)
current_example, current_epoch = reader.get_train_progress(
)
time_end = time.time()
used_time = time_end - time_begin
log.info(
"epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
"ave acc: %f, speed: %f steps/s" %
(current_epoch, current_example * dev_count,
num_train_examples, steps, outputs["loss"],
outputs["accuracy"], args.skip_steps / used_time))
ce_info.append(
[outputs["loss"], outputs["accuracy"], used_time])
time_begin = time.time()
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(exe, save_path,
fleet._origin_program)
# if steps % args.validation_steps == 0 or last_epoch != current_epoch:
if steps % args.validation_steps == 0:
# evaluate dev set
if args.do_val:
evaluate_wrapper(args, reader, exe, test_prog,
test_pyreader, graph_vars,
current_epoch, steps)
if args.do_test:
predict_wrapper(args, reader, exe, test_prog,
test_pyreader, graph_vars,
current_epoch, steps)
if last_epoch != current_epoch:
last_epoch = current_epoch
except fluid.core.EOFException:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(exe, save_path,
fleet._origin_program)
train_pyreader.reset()
break
# final eval on dev set
if args.do_val:
evaluate_wrapper(args, reader, exe, test_prog, test_pyreader,
graph_vars, current_epoch, steps)
# final eval on test set
if args.do_test:
predict_wrapper(args, reader, exe, test_prog, test_pyreader,
graph_vars, current_epoch, steps)
# final eval on dianostic, hack for glue-ax
if args.diagnostic:
test_pyreader.decorate_tensor_provider(
reader.data_generator(args.diagnostic,
batch_size=args.batch_size,
epoch=1,
dev_count=1,
shuffle=False))
log.info("Final diagnostic")
qids, preds, probs = predict(test_exe, test_prog, test_pyreader,
graph_vars)
assert len(qids) == len(preds), '{} v.s. {}'.format(
len(qids), len(preds))
with open(args.diagnostic_save, 'w') as f:
for id, s, p in zip(qids, preds, probs):
f.write('{}\t{}\t{}\n'.format(id, s, p))
log.info("Done final diagnostic, saving to {}".format(
args.diagnostic_save))
