def eval_():
# set args
dev = "GPU"
compute_type = str(args_opt.dtype).lower()
ckpt_dir = str(args_opt.ckpt_path)
total_batch = int(args_opt.batch_size)
# init context
if args_opt.mode == "GRAPH":
mode = context.GRAPH_MODE
else:
mode = context.PYNATIVE_MODE
context.set_context(mode=mode, device_target=dev, save_graphs=False)
# create dataset
dataset = create_dataset(dataset_path=args_opt.dataset_path,
do_train=False,
repeat_num=1,
batch_size=total_batch,
target=dev,
dtype=compute_type)
# define net
net = resnet(class_num=1001, dtype=compute_type)
# load checkpoint
param_dict = load_checkpoint(ckpt_dir)
load_param_into_net(net, param_dict)
net.set_train(False)
# define loss, model
loss = CrossEntropySmooth(sparse=True,
reduction='mean',
smooth_factor=0.1,
num_classes=1001)
# define model
model = Model(net,
loss_fn=loss,
metrics={'top_1_accuracy', 'top_5_accuracy'})
# eval model
print("========START EVAL RESNET50 ON GPU ========")
res = model.eval(dataset)
print("result:", res, "ckpt=", ckpt_dir)
# create dataset
dataset = create_dataset(dataset_path=args_opt.dataset_path,
do_train=True,
repeat_num=1,
batch_size=total_batch,
target=dev,
dtype=compute_type)
step_size = dataset.get_dataset_size()
if (print_per_steps > step_size or print_per_steps < 1):
print("Arg: print_per_steps should lessequal to dataset_size ",
step_size)
print("Change to default: 20")
print_per_steps = 20
# define net
net = resnet(class_num=1001, dtype=compute_type)
# init weight
for _, cell in net.cells_and_names():
if isinstance(cell, nn.Conv2d):
cell.weight.set_data(
weight_init.initializer(weight_init.XavierUniform(),
cell.weight.shape, cell.weight.dtype))
if isinstance(cell, nn.Dense):
cell.weight.set_data(
weight_init.initializer(weight_init.TruncatedNormal(),
cell.weight.shape, cell.weight.dtype))
# init lr
lr = get_liner_lr(lr_init=0,
lr_end=0,
def train():
# set args
dev = "GPU"
epoch_size = int(args_opt.epoch_size)
total_batch = int(args_opt.batch_size)
print_per_steps = int(args_opt.print_per_steps)
compute_type = str(args_opt.dtype).lower()
ckpt_save_dir = str(args_opt.ckpt_path)
save_ckpt = bool(args_opt.save_ckpt)
device_num = 1
# init context
if args_opt.mode == "GRAPH":
mode = context.GRAPH_MODE
all_reduce_fusion_config = [85, 160]
else:
mode = context.PYNATIVE_MODE
all_reduce_fusion_config = [30, 90, 160]
context.set_context(mode=mode, device_target=dev, save_graphs=False)
if args_opt.run_distribute:
init()
device_num = get_group_size()
context.set_auto_parallel_context(
device_num=device_num,
parallel_mode=ParallelMode.DATA_PARALLEL,
gradients_mean=True,
all_reduce_fusion_config=all_reduce_fusion_config)
ckpt_save_dir = ckpt_save_dir + "ckpt_" + str(get_rank()) + "/"
# create dataset
dataset = create_dataset(dataset_path=args_opt.dataset_path,
do_train=True,
repeat_num=1,
batch_size=total_batch,
target=dev,
dtype=compute_type,
device_num=device_num)
step_size = dataset.get_dataset_size()
if (print_per_steps > step_size or print_per_steps < 1):
print("Arg: print_per_steps should lessequal to dataset_size ",
step_size)
print("Change to default: 20")
print_per_steps = 20
# define net
net = resnet(class_num=1001, dtype=compute_type)
# init weight
for _, cell in net.cells_and_names():
if isinstance(cell, nn.Conv2d):
cell.weight.set_data(
weight_init.initializer(weight_init.XavierUniform(),
cell.weight.shape, cell.weight.dtype))
if isinstance(cell, nn.Dense):
cell.weight.set_data(
weight_init.initializer(weight_init.TruncatedNormal(),
cell.weight.shape, cell.weight.dtype))
# init lr
lr = get_liner_lr(lr_init=0,
lr_end=0,
lr_max=0.8,
warmup_epochs=0,
total_epochs=epoch_size,
steps_per_epoch=step_size)
lr = Tensor(lr)
# define opt
decayed_params = []
no_decayed_params = []
for param in net.trainable_params():
if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name:
decayed_params.append(param)
else:
no_decayed_params.append(param)
# define loss, model
loss = CrossEntropySmooth(sparse=True,
reduction='mean',
smooth_factor=0.1,
num_classes=1001)
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr,
0.9, 1e-4)
loss_scale = FixedLossScaleManager(1024, drop_overflow_update=False)
model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'})
# Mixed precision
if compute_type == "fp16":
if mode == context.PYNATIVE_MODE:
opt = MomentumWeightDecay(
filter(lambda x: x.requires_grad, net.get_parameters()), lr,
0.9, 1e-4, 1024)
else:
opt = Momentum(
filter(lambda x: x.requires_grad, net.get_parameters()), lr,
0.9, 1e-4, 1024)
model = Model(net,
loss_fn=loss,
optimizer=opt,
loss_scale_manager=loss_scale,
metrics={'acc'},
amp_level="O2",
keep_batchnorm_fp32=False)
# define callbacks
if mode == context.PYNATIVE_MODE:
print_per_steps = 1
time_cb = MyTimeMonitor(total_batch, print_per_steps, step_size, mode)
cb = [time_cb]
if save_ckpt:
config_ck = CheckpointConfig(save_checkpoint_steps=5 * step_size,
keep_checkpoint_max=5)
ckpt_cb = ModelCheckpoint(prefix="resnet_benchmark",
directory=ckpt_save_dir,
config=config_ck)
cb += [ckpt_cb]
# train model
print("========START RESNET50 GPU BENCHMARK========")
if mode == context.GRAPH_MODE:
model.train(int(epoch_size * step_size / print_per_steps),
dataset,
callbacks=cb,
sink_size=print_per_steps)
else:
model.train(epoch_size, dataset, callbacks=cb)
all_reduce_fusion_config=[85, 160])
# create dataset
dataset = create_dataset(dataset_path=args_opt.dataset_path,
do_train=True,
repeat_num=1,
batch_size=total_batch,
target=dev)
step_size = dataset.get_dataset_size()
if (print_per_steps > step_size or print_per_steps < 1):
print("Arg: print_per_steps should lessequal to dataset_size ",
step_size)
print("Change to default: 20")
print_per_steps = 20
# define net
net = resnet(class_num=1001)
# init weight
for _, cell in net.cells_and_names():
if isinstance(cell, nn.Conv2d):
cell.weight.set_data(
weight_init.initializer(weight_init.XavierUniform(),
cell.weight.shape, cell.weight.dtype))
if isinstance(cell, nn.Dense):
cell.weight.set_data(
weight_init.initializer(weight_init.TruncatedNormal(),
cell.weight.shape, cell.weight.dtype))
# init lr
lr = get_liner_lr(lr_init=0,
lr_end=0,