def _test_static(self, python_func, paddle_api, kwarg, place):
scheduler = paddle_api(**kwarg)
adam = paddle.optimizer.Adam(learning_rate=scheduler)
main_prog = paddle.static.Program()
start_prog = paddle.static.Program()
with paddle.static.program_guard(main_prog, start_prog):
x = paddle.static.data(name='x', shape=[3, 4, 5])
loss = paddle.mean(x)
adam.minimize(loss)
lr_var = adam._global_learning_rate()
test_prog = main_prog.clone()
num = 0
exe = paddle.static.Executor(place)
exe.run(start_prog)
for epoch in range(5):
for batch_id in range(2):
out = exe.run(
main_prog,
feed={'x': np.random.randn(3, 4, 5).astype('float32')},
fetch_list=lr_var.name)
self.assertEqual(out, np.array(python_func(num, **kwarg)))
scheduler.step()
num += 1
for epoch in range(5):
for batch_id in range(2):
out = exe.run(
test_prog,
feed={'x': np.random.randn(3, 4, 5).astype('float32')},
fetch_list=lr_var.name)
self.assertEqual(out, np.array(python_func(num, **kwarg)))
scheduler.step()
num += 1
if isinstance(place, paddle.CPUPlace):
compiled_train_prog = paddle.static.CompiledProgram(
main_prog).with_data_parallel(loss_name=loss.name,
places=fluid.cpu_places(4))
for epoch in range(5):
python_result = python_func(num, **kwarg)
for batch_id in range(2):
_ = exe.run(compiled_train_prog,
feed={
'x':
np.random.randn(12, 4, 5).astype('float32')
},
fetch_list=lr_var.name)
scopes = compiled_train_prog._executor.local_scopes()
out = np.array(scopes[0].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
out = np.array(scopes[1].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
out = np.array(scopes[2].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
out = np.array(scopes[3].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
scheduler.step()
num += 1
compiled_test_prog = paddle.static.CompiledProgram(
test_prog).with_data_parallel(
loss_name=loss.name,
share_vars_from=compiled_train_prog,
places=fluid.cpu_places(4))
for epoch in range(5):
python_result = python_func(num, **kwarg)
for batch_id in range(2):
_ = exe.run(compiled_test_prog,
feed={
'x':
np.random.randn(12, 4, 5).astype('float32')
},
fetch_list=lr_var.name)
scopes = compiled_test_prog._executor.local_scopes()
out = np.array(scopes[0].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
out = np.array(scopes[1].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
out = np.array(scopes[2].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
out = np.array(scopes[3].var(lr_var.name).get_tensor())
self.assertEqual(out, np.array(python_result))
scheduler.step()
num += 1
ernie = ErnieForTokenClassification.from_pretrained(
"ernie-1.0", num_classes=len(label_vocab))
model = ErnieCrfForTokenClassification(ernie)
metric = ChunkEvaluator(label_list=label_vocab.keys(), suffix=True)
optimizer = paddle.optimizer.AdamW(learning_rate=2e-5,
parameters=model.parameters())
step = 0
for epoch in range(args.epochs):
for input_ids, token_type_ids, lengths, labels in train_loader:
loss = model(input_ids,
token_type_ids,
lengths=lengths,
labels=labels)
avg_loss = paddle.mean(loss)
avg_loss.backward()
optimizer.step()
optimizer.clear_grad()
step += 1
print("[TRAIN] Epoch:%d - Step:%d - Loss: %f" %
(epoch, step, avg_loss))
evaluate(model, metric, dev_loader)
paddle.save(model.state_dict(),
os.path.join(args.save_dir, 'model_%d' % step))
preds = predict(model, test_loader, test_ds, label_vocab)
file_path = "ernie_crf_results.txt"
with open(file_path, "w", encoding="utf8") as fout:
fout.write("\n".join(preds))
def test_accuracy(self):
image = paddle.static.data(name='image',
shape=[None, 1, 28, 28],
dtype='float32')
label = paddle.static.data(name='label',
shape=[None, 1],
dtype='int64')
model = MobileNet()
out = model.net(input=image, class_dim=10)
cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label)
avg_cost = paddle.mean(x=cost)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
optimizer = paddle.optimizer.Momentum(
momentum=0.9,
learning_rate=0.01,
weight_decay=paddle.regularizer.L2Decay(4e-5))
optimizer.minimize(avg_cost)
main_prog = paddle.static.default_main_program()
val_prog = main_prog.clone(for_test=True)
place = paddle.CUDAPlace(
0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace()
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
def transform(x):
return np.reshape(x, [1, 28, 28])
train_dataset = paddle.vision.datasets.MNIST(mode='train',
backend='cv2',
transform=transform)
test_dataset = paddle.vision.datasets.MNIST(mode='test',
backend='cv2',
transform=transform)
train_loader = paddle.io.DataLoader(train_dataset,
places=place,
feed_list=[image, label],
drop_last=True,
batch_size=64,
return_list=False)
valid_loader = paddle.io.DataLoader(test_dataset,
places=place,
feed_list=[image, label],
batch_size=64,
return_list=False)
def sample_generator_creator():
def __reader__():
for data in test_dataset:
image, label = data
yield image, label
return __reader__
def train(program):
iter = 0
for data in train_loader():
cost, top1, top5 = exe.run(
program,
feed=data,
fetch_list=[avg_cost, acc_top1, acc_top5])
iter += 1
if iter % 100 == 0:
print(
'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'.
format(iter, cost, top1, top5))
def test(program, outputs=[avg_cost, acc_top1, acc_top5]):
iter = 0
result = [[], [], []]
for data in valid_loader():
cost, top1, top5 = exe.run(program,
feed=data,
fetch_list=outputs)
iter += 1
if iter % 100 == 0:
print(
'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'.
format(iter, cost, top1, top5))
result[0].append(cost)
result[1].append(top1)
result[2].append(top5)
print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format(
np.mean(result[0]), np.mean(result[1]), np.mean(result[2])))
return np.mean(result[1]), np.mean(result[2])
train(main_prog)
top1_1, top5_1 = test(val_prog)
paddle.fluid.io.save_inference_model(
dirname='./test_quant_post',
feeded_var_names=[image.name, label.name],
target_vars=[avg_cost, acc_top1, acc_top5],
main_program=val_prog,
executor=exe,
model_filename='model',
params_filename='params')
quant_post_static(exe,
'./test_quant_post',
'./test_quant_post_inference',
sample_generator=sample_generator_creator(),
model_filename='model',
params_filename='params',
batch_nums=10)
quant_post_prog, feed_target_names, fetch_targets = paddle.fluid.io.load_inference_model(
dirname='./test_quant_post_inference',
executor=exe,
model_filename='__model__',
params_filename='__params__')
top1_2, top5_2 = test(quant_post_prog, fetch_targets)
print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1))
print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def create_loss(self, raw_pred, label):
loss = paddle.nn.functional.log_loss(input=raw_pred,
label=paddle.cast(
label, "float32"))
loss = paddle.mean(loss)
return loss
# Define the model netword and its loss
model = ErnieForTokenClassification.from_pretrained(
"ernie-1.0", num_classes=len(label_vocab))
if trainer_num > 1:
model = paddle.DataParallel(model)
metric = ChunkEvaluator(label_list=label_vocab.keys(), suffix=True)
loss_fn = paddle.nn.loss.CrossEntropyLoss(ignore_index=ignore_label)
optimizer = paddle.optimizer.AdamW(learning_rate=2e-5,
parameters=model.parameters())
step = 0
for epoch in range(args.epochs):
for input_ids, token_type_ids, length, labels in train_loader:
logits = model(input_ids, token_type_ids)
loss = paddle.mean(loss_fn(logits, labels))
loss.backward()
optimizer.step()
optimizer.clear_grad()
step += 1
print("[TRAIN] Epoch:%d - Step:%d - Loss: %f" %
(epoch, step, loss))
evaluate(model, metric, dev_loader)
model_to_save = model._layers if isinstance(
model, paddle.DataParallel) else model
model_to_save.save_pretrained(
os.path.join(args.save_dir, 'model_%d' % step))
if rank == 0:
preds = predict(model, test_loader, test_ds, label_vocab)
file_path = "ernie_results.txt"
def forward(self, logits, label):
return paddle.mean(F.softmax_with_cross_entropy(logits, label))
import paddle img = paddle.randn(shape=[100]) img.stop_gradient = False img[10] = img[10] + 10.0 out = img * 2 rlt = paddle.mean(out) rlt.backward()
import os
import paddle
import paddle.static as static
paddle.enable_static()
os.environ['CPU_NUM'] = str(2)
places = static.cpu_places()
data = static.data(name="x", shape=[None, 1], dtype="float32")
hidden = static.nn.fc(input=data, size=10)
loss = paddle.mean(hidden)
paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
build_strategy = static.BuildStrategy()
build_strategy.enable_inplace = True
build_strategy.memory_optimize = True
build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
program = static.CompiledProgram(static.default_main_program())
program = program.with_data_parallel(loss_name=loss.name,
build_strategy=build_strategy,
places=places)
def adaptation_loss(t_var, s_var):
hint_loss = paddle.mean(
paddle.nn.functional.square_error_cost(s_var, t_var))
return hint_loss
def train_resnet(self, enable_amp=True, use_data_loader=False):
seed = 90
batch_size = train_parameters["batch_size"]
batch_num = 1
paddle.seed(seed)
paddle.framework.random._manual_program_seed(seed)
resnet = ResNet(use_cudnn=True)
optimizer = optimizer_setting(
train_parameters, parameter_list=resnet.parameters())
np.random.seed(seed)
train_reader = paddle.batch(
paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size)
dy_param_init_value = {}
for param in resnet.parameters():
dy_param_init_value[param.name] = param.numpy()
program = None
scaler = paddle.amp.GradScaler(
enable=enable_amp, init_loss_scaling=2.**10)
if use_data_loader:
train_reader = paddle.batch(
reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True)
train_loader = fluid.io.DataLoader.from_generator(
capacity=4,
use_double_buffer=True,
iterable=True,
return_list=True)
train_loader.set_sample_list_generator(train_reader)
train_reader = train_loader
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num:
break
if use_data_loader:
img, label = data
else:
dy_x_data = np.array(
[x[0].reshape(3, 224, 224) for x in data]).astype('float32')
if len(np.array([x[1]
for x in data]).astype('int64')) != batch_size:
continue
y_data = np.array([x[1] for x in data]).astype('int64').reshape(
-1, 1)
img = paddle.to_tensor(dy_x_data)
label = paddle.to_tensor(y_data)
label.stop_gradient = True
with paddle.amp.auto_cast(enable=enable_amp):
out = resnet(img)
loss = paddle.nn.functional.cross_entropy(input=out, label=label)
avg_loss = paddle.mean(x=loss)
dy_out = avg_loss.numpy()
scaled_loss = scaler.scale(avg_loss)
scaled_loss.backward()
scaler.minimize(optimizer, scaled_loss)
dy_grad_value = {}
for param in resnet.parameters():
if param.trainable:
np_array = np.array(param._grad_ivar().value().get_tensor())
dy_grad_value[param.name + fluid.core.grad_var_suffix(
)] = np_array
resnet.clear_gradients()
dy_param_value = {}
for param in resnet.parameters():
dy_param_value[param.name] = param.numpy()
if use_data_loader:
train_reader._reset()
return dy_out, dy_param_value, dy_grad_value
def skip_gram_word2vec(dict_size, embedding_size, is_sparse=False, neg_num=5):
datas = []
input_word = paddle.static.data(name="input_word",
shape=[None, 1],
dtype='int64')
true_word = paddle.static.data(name='true_label',
shape=[None, 1],
dtype='int64')
neg_word = paddle.static.data(name="neg_label",
shape=[None, neg_num],
dtype='int64')
datas.append(input_word)
datas.append(true_word)
datas.append(neg_word)
py_reader = fluid.layers.create_py_reader_by_data(capacity=64,
feed_list=datas,
name='py_reader',
use_double_buffer=True)
words = fluid.layers.read_file(py_reader)
words[0] = paddle.reshape(words[0], [-1])
words[1] = paddle.reshape(words[1], [-1])
init_width = 0.5 / embedding_size
input_emb = paddle.static.nn.embedding(
input=words[0],
is_sparse=is_sparse,
size=[dict_size, embedding_size],
param_attr=paddle.ParamAttr(name='emb',
initializer=paddle.nn.initializer.Uniform(
-init_width, init_width)))
true_emb_w = paddle.static.nn.embedding(
input=words[1],
is_sparse=is_sparse,
size=[dict_size, embedding_size],
param_attr=paddle.ParamAttr(
name='emb_w',
initializer=paddle.nn.initializer.Constant(value=0.0)))
true_emb_b = paddle.static.nn.embedding(
input=words[1],
is_sparse=is_sparse,
size=[dict_size, 1],
param_attr=paddle.ParamAttr(
name='emb_b',
initializer=paddle.nn.initializer.Constant(value=0.0)))
neg_word_reshape = paddle.reshape(words[2], shape=[-1])
neg_word_reshape.stop_gradient = True
neg_emb_w = paddle.static.nn.embedding(
input=neg_word_reshape,
is_sparse=is_sparse,
size=[dict_size, embedding_size],
param_attr=paddle.ParamAttr(name='emb_w', learning_rate=1.0))
neg_emb_w_re = paddle.reshape(neg_emb_w,
shape=[-1, neg_num, embedding_size])
neg_emb_b = paddle.static.nn.embedding(
input=neg_word_reshape,
is_sparse=is_sparse,
size=[dict_size, 1],
param_attr=paddle.ParamAttr(name='emb_b', learning_rate=1.0))
neg_emb_b_vec = paddle.reshape(neg_emb_b, shape=[-1, neg_num])
true_logits = paddle.add(
paddle.mean(paddle.multiply(input_emb, true_emb_w),
axis=1,
keepdim=True), true_emb_b)
input_emb_re = paddle.reshape(input_emb, shape=[-1, 1, embedding_size])
neg_matmul = fluid.layers.matmul(input_emb_re,
neg_emb_w_re,
transpose_y=True)
neg_matmul_re = paddle.reshape(neg_matmul, shape=[-1, neg_num])
neg_logits = paddle.add(neg_matmul_re, neg_emb_b_vec)
#nce loss
label_ones = fluid.layers.fill_constant_batch_size_like(true_logits,
shape=[-1, 1],
value=1.0,
dtype='float32')
label_zeros = fluid.layers.fill_constant_batch_size_like(
true_logits, shape=[-1, neg_num], value=0.0, dtype='float32')
true_xent = fluid.layers.sigmoid_cross_entropy_with_logits(
true_logits, label_ones)
neg_xent = fluid.layers.sigmoid_cross_entropy_with_logits(
neg_logits, label_zeros)
cost = paddle.add(paddle.sum(true_xent, axis=1),
paddle.sum(neg_xent, axis=1))
avg_cost = paddle.mean(cost)
return avg_cost, py_reader
def obj_loss(self, sobj, tobj):
obj_mask = paddle.cast(tobj > 0., dtype="float32")
obj_mask.stop_gradient = True
loss = paddle.mean(
ops.sigmoid_cross_entropy_with_logits(sobj, obj_mask))
return loss
def obj_weighted_cls(self, scls, tcls, tobj):
loss = ops.sigmoid_cross_entropy_with_logits(scls, F.sigmoid(tcls))
weighted_loss = paddle.mean(paddle.multiply(loss, F.sigmoid(tobj)))
return weighted_loss
def train(to_static):
"""
Tests model decorated by `dygraph_to_static_output` in static mode. For users, the model is defined in dygraph mode and trained in static mode.
"""
paddle.disable_static(place)
np.random.seed(SEED)
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
train_reader = paddle.batch(reader_decorator(
paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True)
data_loader = paddle.io.DataLoader.from_generator(capacity=5,
iterable=True)
data_loader.set_sample_list_generator(train_reader)
resnet = ResNet()
optimizer = optimizer_setting(parameter_list=resnet.parameters())
for epoch in range(epoch_num):
total_loss = 0.0
total_acc1 = 0.0
total_acc5 = 0.0
total_sample = 0
for batch_id, data in enumerate(data_loader()):
start_time = time.time()
img, label = data
pred = resnet(img)
loss = paddle.nn.functional.cross_entropy(input=pred, label=label)
avg_loss = paddle.mean(x=loss)
acc_top1 = paddle.metric.accuracy(input=pred, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=pred, label=label, k=5)
avg_loss.backward()
optimizer.minimize(avg_loss)
resnet.clear_gradients()
total_loss += avg_loss
total_acc1 += acc_top1
total_acc5 += acc_top5
total_sample += 1
end_time = time.time()
if batch_id % 2 == 0:
print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \
( epoch, batch_id, total_loss.numpy() / total_sample, \
total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time))
if batch_id == 10:
if to_static:
paddle.jit.save(resnet, MODEL_SAVE_PREFIX)
else:
paddle.fluid.dygraph.save_dygraph(resnet.state_dict(),
DY_STATE_DICT_SAVE_PATH)
# avoid dataloader throw abort signaal
data_loader._reset()
break
paddle.enable_static()
return total_loss.numpy()
def create_loss(self, hit_prob):
loss = -paddle.sum(paddle.log(hit_prob), axis=-1)
avg_cost = paddle.mean(x=loss)
return avg_cost
def create_loss(self, pred, label):
cost = paddle.nn.functional.cross_entropy(input=pred, label=label)
avg_cost = paddle.mean(x=cost)
return avg_cost
def compress(args):
############################################################################################################
# 1. quantization configs
############################################################################################################
quant_config = {
# weight quantize type, default is 'channel_wise_abs_max'
'weight_quantize_type': 'channel_wise_abs_max',
# activation quantize type, default is 'moving_average_abs_max'
'activation_quantize_type': 'moving_average_abs_max',
# weight quantize bit num, default is 8
'weight_bits': 8,
# activation quantize bit num, default is 8
'activation_bits': 8,
# ops of name_scope in not_quant_pattern list, will not be quantized
'not_quant_pattern': ['skip_quant'],
# ops of type in quantize_op_types, will be quantized
'quantize_op_types': ['conv2d', 'depthwise_conv2d', 'mul'],
# data type after quantization, such as 'uint8', 'int8', etc. default is 'int8'
'dtype': 'int8',
# window size for 'range_abs_max' quantization. defaulf is 10000
'window_size': 10000,
# The decay coefficient of moving average, default is 0.9
'moving_rate': 0.9,
}
if args.data == "mnist":
train_dataset = paddle.vision.datasets.MNIST(mode='train')
val_dataset = paddle.vision.datasets.MNIST(mode='test')
class_dim = 10
image_shape = "1,28,28"
elif args.data == "imagenet":
import imagenet_reader as reader
train_dataset = reader.ImageNetDataset(mode='train')
val_dataset = reader.ImageNetDataset(mode='val')
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(
args.model, model_list)
image = paddle.static.data(name='image',
shape=[None] + image_shape,
dtype='float32')
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label)
avg_cost = paddle.mean(x=cost)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
train_prog = paddle.static.default_main_program()
val_program = paddle.static.default_main_program().clone(for_test=True)
place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace()
############################################################################################################
# 2. quantization transform programs (training aware)
# Make some quantization transforms in the graph before training and testing.
# According to the weight and activation quantization type, the graph will be added
# some fake quantize operators and fake dequantize operators.
############################################################################################################
val_program = quant_aware(val_program,
place,
quant_config,
scope=None,
for_test=True)
compiled_train_prog = quant_aware(train_prog,
place,
quant_config,
scope=None,
for_test=False)
opt = create_optimizer(args)
opt.minimize(avg_cost)
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
assert os.path.exists(
args.pretrained_model), "pretrained_model doesn't exist"
if args.pretrained_model:
paddle.static.load(train_prog, args.pretrained_model, exe)
places = paddle.static.cuda_places(
) if args.use_gpu else paddle.static.cpu_places()
train_loader = paddle.io.DataLoader(train_dataset,
places=places,
feed_list=[image, label],
drop_last=True,
batch_size=args.batch_size,
return_list=False,
use_shared_memory=True,
shuffle=True,
num_workers=4)
valid_loader = paddle.io.DataLoader(val_dataset,
places=place,
feed_list=[image, label],
drop_last=False,
return_list=False,
batch_size=args.batch_size,
use_shared_memory=True,
shuffle=False)
def test(epoch, program):
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in valid_loader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
program, feed=data, fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}"
.format(epoch, batch_id, np.mean(acc_top1_n),
np.mean(acc_top5_n), end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info(
"Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
epoch, np.mean(np.array(acc_top1_ns)),
np.mean(np.array(acc_top5_ns))))
return np.mean(np.array(acc_top1_ns))
def train(epoch, compiled_train_prog):
batch_id = 0
for data in train_loader():
start_time = time.time()
loss_n, acc_top1_n, acc_top5_n = exe.run(
compiled_train_prog,
feed=data,
fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name])
end_time = time.time()
loss_n = np.mean(loss_n)
acc_top1_n = np.mean(acc_top1_n)
acc_top5_n = np.mean(acc_top5_n)
if batch_id % args.log_period == 0:
_logger.info(
"epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {}; time: {}"
.format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n,
end_time - start_time))
batch_id += 1
build_strategy = paddle.static.BuildStrategy()
build_strategy.memory_optimize = False
build_strategy.enable_inplace = False
build_strategy.fuse_all_reduce_ops = False
build_strategy.sync_batch_norm = False
exec_strategy = paddle.static.ExecutionStrategy()
compiled_train_prog = compiled_train_prog.with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
############################################################################################################
# train loop
############################################################################################################
best_acc1 = 0.0
best_epoch = 0
for i in range(args.num_epochs):
train(i, compiled_train_prog)
acc1 = test(i, val_program)
paddle.static.save(program=val_program,
model_path=os.path.join(args.checkpoint_dir,
str(i)))
if acc1 > best_acc1:
best_acc1 = acc1
best_epoch = i
paddle.static.save(program=val_program,
model_path=os.path.join(args.checkpoint_dir,
'best_model'))
if os.path.exists(os.path.join(args.checkpoint_dir, 'best_model')):
paddle.static.load(executor=exe,
model_path=os.path.join(args.checkpoint_dir,
'best_model'),
program=val_program)
############################################################################################################
# 3. Freeze the graph after training by adjusting the quantize
# operators' order for the inference.
# The dtype of float_program's weights is float32, but in int8 range.
############################################################################################################
float_program, int8_program = convert(val_program, place, quant_config, \
scope=None, \
save_int8=True)
print("eval best_model after convert")
final_acc1 = test(best_epoch, float_program)
############################################################################################################
# 4. Save inference model
############################################################################################################
model_path = os.path.join(
quantization_model_save_dir, args.model,
'act_' + quant_config['activation_quantize_type'] + '_w_' +
quant_config['weight_quantize_type'])
float_path = os.path.join(model_path, 'float')
if not os.path.isdir(model_path):
os.makedirs(model_path)
paddle.fluid.io.save_inference_model(dirname=float_path,
feeded_var_names=[image.name],
target_vars=[out],
executor=exe,
main_program=float_program,
model_filename=float_path + '/model',
params_filename=float_path +
'/params')
def create_loss(predict, label_input):
cost = F.square_error_cost(predict,
paddle.cast(x=label_input, dtype='float32'))
avg_cost = paddle.mean(cost)
return avg_cost
def train(model):
# 开启0号GPU训练
use_gpu = True
paddle.set_device('gpu:0') if use_gpu else paddle.set_device('cpu')
print('start training ... ')
model.train()
epoch_num = 5
opt = paddle.optimizer.Momentum(learning_rate=0.001,
momentum=0.9,
parameters=model.parameters())
# 使用Paddle自带的数据读取器
train_loader = paddle.batch(paddle.dataset.mnist.train(), batch_size=10)
valid_loader = paddle.batch(paddle.dataset.mnist.test(), batch_size=10)
for epoch in range(epoch_num):
for batch_id, data in enumerate(train_loader()):
# 调整输入数据形状和类型
x_data = np.array([item[0] for item in data],
dtype='float32').reshape(-1, 1, 28, 28)
y_data = np.array([item[1] for item in data],
dtype='int64').reshape(-1, 1)
# 将numpy.ndarray转化成Tensor
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
# 计算模型输出
logits = model(img)
# 计算损失函数
loss = F.softmax_with_cross_entropy(logits, label)
avg_loss = paddle.mean(loss)
if batch_id % 1000 == 0:
print("epoch: {}, batch_id: {}, loss is: {}".format(
epoch, batch_id, avg_loss.numpy()))
avg_loss.backward()
opt.step()
opt.clear_grad()
model.eval()
accuracies = []
losses = []
for batch_id, data in enumerate(valid_loader()):
# 调整输入数据形状和类型
x_data = np.array([item[0] for item in data],
dtype='float32').reshape(-1, 1, 28, 28)
y_data = np.array([item[1] for item in data],
dtype='int64').reshape(-1, 1)
# 将numpy.ndarray转化成Tensor
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
# 计算模型输出
logits = model(img)
pred = F.softmax(logits)
# 计算损失函数
loss = F.softmax_with_cross_entropy(logits, label)
acc = paddle.metric.accuracy(pred, label)
accuracies.append(acc.numpy())
losses.append(loss.numpy())
print("[validation] accuracy/loss: {}/{}".format(
np.mean(accuracies), np.mean(losses)))
model.train()
# 保存模型参数
paddle.save(model.state_dict(), 'mnist.pdparams')
def train_iter(self, optimizers=None):
self.set_requires_grad(self.nets['discriminator'], False)
optimizers['optimG'].clear_grad()
l_total = 0
self.output = self.nets['generator'](self.lq)
self.visual_items['output'] = self.output
# pixel loss
if self.pixel_criterion:
l_pix = self.pixel_criterion(self.output, self.gt)
l_total += l_pix
self.losses['loss_pix'] = l_pix
if self.perceptual_criterion:
l_g_percep, l_g_style = self.perceptual_criterion(
self.output, self.gt)
# l_total += l_pix
if l_g_percep is not None:
l_total += l_g_percep
self.losses['loss_percep'] = l_g_percep
if l_g_style is not None:
l_total += l_g_style
self.losses['loss_style'] = l_g_style
# gan loss (relativistic gan)
real_d_pred = self.nets['discriminator'](self.gt).detach()
fake_g_pred = self.nets['discriminator'](self.output)
l_g_real = self.gan_criterion(real_d_pred - paddle.mean(fake_g_pred),
False,
is_disc=False)
l_g_fake = self.gan_criterion(fake_g_pred - paddle.mean(real_d_pred),
True,
is_disc=False)
l_g_gan = (l_g_real + l_g_fake) / 2
l_total += l_g_gan
self.losses['l_g_gan'] = l_g_gan
l_total.backward()
optimizers['optimG'].step()
self.set_requires_grad(self.nets['discriminator'], True)
optimizers['optimD'].clear_grad()
# real
fake_d_pred = self.nets['discriminator'](self.output).detach()
real_d_pred = self.nets['discriminator'](self.gt)
l_d_real = self.gan_criterion(
real_d_pred - paddle.mean(fake_d_pred), True, is_disc=True) * 0.5
# fake
fake_d_pred = self.nets['discriminator'](self.output.detach())
l_d_fake = self.gan_criterion(
fake_d_pred - paddle.mean(real_d_pred.detach()),
False,
is_disc=True) * 0.5
(l_d_real + l_d_fake).backward()
optimizers['optimD'].step()
self.losses['l_d_real'] = l_d_real
self.losses['l_d_fake'] = l_d_fake
self.losses['out_d_real'] = paddle.mean(real_d_pred.detach())
self.losses['out_d_fake'] = paddle.mean(fake_d_pred.detach())
def train(args):
# 使用 GPU训练
if paddle.is_compiled_with_cuda():
paddle.set_device("gpu:0")
# 创建多进程的游戏环境
envs = MultipleEnvironments(args.world, args.stage, args.action_type, args.num_processes)
# 固定初始化状态
paddle.seed(123)
# 创建模型
model = Model(envs.num_states, envs.num_actions)
# 创建保存模型的文件夹
if not os.path.isdir(args.saved_path):
os.makedirs(args.saved_path)
paddle.save(model.state_dict(), "{}/model_{}_{}.pdparams".format(args.saved_path, args.world, args.stage))
# 为游戏评估单独开一个进程
mp = _mp.get_context("spawn")
process = mp.Process(target=eval, args=(args, envs.num_states, envs.num_actions))
process.start()
# 创建优化方法
clip_grad = paddle.nn.ClipGradByNorm(clip_norm=0.5)
optimizer = paddle.optimizer.Adam(parameters=model.parameters(), learning_rate=args.lr, grad_clip=clip_grad)
# 刚开始给每个进程的游戏执行初始化
[agent_conn.send(("reset", None)) for agent_conn in envs.agent_conns]
# 获取游戏初始的界面
curr_states = [agent_conn.recv() for agent_conn in envs.agent_conns]
curr_states = paddle.to_tensor(np.concatenate(curr_states, 0), dtype='float32')
curr_episode = 0
while curr_episode < 400:
curr_episode += 1
old_log_policies, actions, values, states, rewards, dones = [], [], [], [], [], []
for _ in range(args.num_local_steps):
states.append(curr_states)
# 执行预测
logits, value = model(curr_states)
# 计算每个动作的概率值
policy = F.softmax(logits)
# 根据每个标签的概率随机生成符合概率的标签
old_m = Categorical(policy)
action = old_m.sample([1]).squeeze()
# 记录预测数据
actions.append(action)
values.append(value.squeeze())
# 计算类别的概率的对数
old_log_policy = old_m.log_prob(paddle.unsqueeze(action, axis=1))
old_log_policy = paddle.squeeze(old_log_policy)
old_log_policies.append(old_log_policy)
# 向各个进程游戏发送动作
[agent_conn.send(("step", int(act[0]))) for agent_conn, act in zip(envs.agent_conns, action)]
# 将多进程的游戏数据打包
state, reward, done, info = zip(*[agent_conn.recv() for agent_conn in envs.agent_conns])
# 进行数据转换
state = paddle.to_tensor(np.concatenate(state, 0), dtype='float32')
# 转换为tensor数据
reward = paddle.to_tensor(reward, dtype='float32')
done = paddle.to_tensor(done, dtype='float32')
# 记录预测数据
rewards.append(reward)
dones.append(done)
curr_states = state
# 根据上面最后的图像预测
_, next_value, = model(curr_states)
next_value = next_value.squeeze()
old_log_policies = paddle.concat(old_log_policies).detach().squeeze()
actions = paddle.concat(actions).squeeze()
values = paddle.concat(values).squeeze().detach()
states = paddle.concat(states).squeeze()
gae = 0.0
R = []
for value, reward, done in list(zip(values, rewards, dones))[::-1]:
gae = gae * args.gamma * args.tau
gae = gae + reward + args.gamma * next_value.detach() * (1.0 - done) - value.detach()
next_value = value
R.append(gae + value)
R = R[::-1]
R = paddle.concat(R).detach()
advantages = R - values
for i in range(args.num_epochs):
indice = paddle.randperm(args.num_local_steps * args.num_processes)
for j in range(args.batch_size):
batch_indices = indice[
int(j * (args.num_local_steps * args.num_processes / args.batch_size)): int((j + 1) * (
args.num_local_steps * args.num_processes / args.batch_size))]
# 根据拿到的图像执行预测
logits, value = model(paddle.gather(states, batch_indices))
# 计算每个动作的概率值
new_policy = F.softmax(logits)
# 计算类别的概率的对数
new_m = Categorical(new_policy)
new_log_policy = new_m.log_prob(paddle.unsqueeze(paddle.gather(actions, batch_indices), axis=1))
new_log_policy = paddle.squeeze(new_log_policy)
# 计算actor损失
ratio = paddle.exp(new_log_policy - paddle.gather(old_log_policies, batch_indices))
advantage = paddle.gather(advantages, batch_indices)
actor_loss = paddle.clip(ratio, 1.0 - args.epsilon, 1.0 + args.epsilon) * advantage
actor_loss = paddle.concat([paddle.unsqueeze(ratio * advantage, axis=0), paddle.unsqueeze(actor_loss, axis=0)])
actor_loss = -paddle.mean(paddle.min(actor_loss, axis=0))
# 计算critic损失
critic_loss = F.smooth_l1_loss(paddle.gather(R, batch_indices), value.squeeze())
entropy_loss = paddle.mean(new_m.entropy())
# 计算全部损失
total_loss = actor_loss + critic_loss - args.beta * entropy_loss
# 计算梯度
total_loss.backward()
optimizer.step()
optimizer.clear_grad()
paddle.save(model.state_dict(), "{}/model_{}_{}.pdparams".format(args.saved_path, args.world, args.stage))
print("Episode: {}. Total loss: {:.4f}".format(curr_episode, total_loss.numpy()[0]))
if os.path.exists("{}/model_{}_{}_finish.pdparams".format(args.saved_path, args.world, args.stage)):
process.kill()
envs.close()
print('顺利通关，world=%d, stage=%d' % (args.world, args.stage))
break
process.kill()
envs.close()
def forward(self, *args, **kwargs):
return paddle.mean(self.loss_fn(*args, **kwargs))
fleet.init(is_collective=True)
# 3. create layer & optimizer
layer = paddle.nn.Linear(10, 10)
adam = paddle.optimizer.Adam(learning_rate=0.001,
parameters=layer.parameters())
# 4. get data_parallel model using fleet
adam = fleet.distributed_optimizer(adam)
dp_layer = fleet.distributed_model(layer)
# 5. run layer
for step in range(1):
inputs = paddle.randn([10, 10], 'float32')
outputs = dp_layer(inputs)
loss = paddle.mean(outputs)
print("step:{}\tloss:{}".format(step, loss.numpy()))
loss = dp_layer.scale_loss(loss)
loss.backward()
dp_layer.apply_collective_grads()
adam.step()
adam.clear_grad()
print("origin lr:", adam.get_lr())
lr_list = [0.2, 0.3, 0.4, 0.5, 0.6]
for i in range(5):
adam.set_lr(lr_list[i])
lr = adam.get_lr()
assert (lr == lr_list[i])
def variation_loss(image, ksize=1):
dh = image[:, :, :-ksize, :] - image[:, :, ksize:, :]
dw = image[:, :, :, :-ksize] - image[:, :, :, ksize:]
return (paddle.mean(paddle.abs(dh)) + paddle.mean(paddle.abs(dw)))
def train(args):
config = parse_config(args.config)
train_config = merge_configs(config, 'train', vars(args))
valid_config = merge_configs(config, 'valid', vars(args))
print_configs(train_config, 'Train')
use_data_parallel = args.use_data_parallel
paddle.disable_static(paddle.CUDAPlace(0))
place = paddle.CUDAPlace(paddle.distributed.ParallelEnv().dev_id) \
if use_data_parallel else paddle.CUDAPlace(0)
if use_data_parallel:
paddle.distributed.init_parallel_env()
video_model = TSN_ResNet(train_config)
if use_data_parallel:
video_model = paddle.DataParallel(video_model)
pre_state_dict = paddle.load(args.pretrain)
#if paddle.distributed.parallel.Env().local_rank == 0:
video_model = init_model(video_model, pre_state_dict)
optimizer = create_optimizer(train_config.TRAIN, video_model.parameters())
bs_denominator = 1
if args.use_gpu:
# check number of GPUs
gpus = os.getenv("CUDA_VISIBLE_DEVICES", "")
if gpus == "":
pass
else:
gpus = gpus.split(",")
num_gpus = len(gpus)
bs_denominator = num_gpus
bs_train_single = int(train_config.TRAIN.batch_size / bs_denominator)
bs_val_single = int(valid_config.VALID.batch_size / bs_denominator)
train_dataset = TSN_UCF101_Dataset(train_config, 'train')
val_dataset = TSN_UCF101_Dataset(valid_config, 'valid')
train_sampler = DistributedBatchSampler(
train_dataset,
batch_size=bs_train_single,
shuffle=train_config.TRAIN.use_shuffle,
drop_last=True)
train_loader = DataLoader(train_dataset,
batch_sampler=train_sampler,
places=place,
num_workers=train_config.TRAIN.num_workers,
return_list=True)
val_sampler = DistributedBatchSampler(val_dataset,
batch_size=bs_val_single)
val_loader = DataLoader(val_dataset,
batch_sampler=val_sampler,
places=place,
num_workers=valid_config.VALID.num_workers,
return_list=True)
# resume training the model
if args.resume is not None:
model_state, opt_state = paddle.load(args.resume)
video_model.set_dict(model_state)
optimizer.set_dict(opt_state)
reader_cost_averager = TimeAverager()
batch_cost_averager = TimeAverager()
for epoch in range(1, train_config.TRAIN.epoch + 1):
epoch_start = time.time()
video_model.train()
total_loss = 0.0
total_acc1 = 0.0
total_acc5 = 0.0
total_sample = 0
batch_start = time.time()
for batch_id, data in enumerate(train_loader):
reader_cost_averager.record(time.time() - batch_start)
imgs = paddle.to_tensor(data[0], place=paddle.CUDAPinnedPlace())
labels = paddle.to_tensor(data[1], place=paddle.CUDAPinnedPlace())
labels.stop_gradient = True
outputs = video_model(imgs)
loss = F.cross_entropy(input=outputs,
label=labels,
ignore_index=-1)
avg_loss = paddle.mean(loss)
acc_top1 = paddle.metric.accuracy(input=outputs, label=labels, k=1)
acc_top5 = paddle.metric.accuracy(input=outputs, label=labels, k=5)
dy_out = avg_loss.numpy()[0]
if use_data_parallel:
# (data_parallel step5/6)
avg_loss = video_model.scale_loss(avg_loss)
avg_loss.backward()
video_model.apply_collective_grads()
else:
avg_loss.backward()
optimizer.minimize(avg_loss)
optimizer.step()
optimizer.clear_grad()
total_loss += dy_out
total_acc1 += acc_top1.numpy()[0]
total_acc5 += acc_top5.numpy()[0]
total_sample += 1
batch_cost_averager.record(time.time() - batch_start,
num_samples=bs_train_single)
if batch_id % args.log_interval == 0:
print(
'TRAIN Epoch: {}, iter: {}, loss={:.6f}, acc1 {:.6f}, acc5 {:.6f}, batch_cost: {:.5f} sec, reader_cost: {:.5f} sec, ips: {:.5f} samples/sec'
.format(epoch, batch_id, total_loss / total_sample,
total_acc1 / total_sample,
total_acc5 / total_sample,
batch_cost_averager.get_average(),
reader_cost_averager.get_average(),
batch_cost_averager.get_ips_average()))
batch_cost_averager.reset()
reader_cost_averager.reset()
batch_start = time.time()
train_epoch_cost = time.time() - epoch_start
print(
'TRAIN End, Epoch {}, avg_loss= {:.6f}, avg_acc1= {:.6f}, avg_acc5= {:.6f}, epoch_cost: {:.5f} sec'
.format(epoch, total_loss / total_sample,
total_acc1 / total_sample, total_acc5 / total_sample,
train_epoch_cost))
# save model's and optimizer's parameters which used for resuming the training stage
save_parameters = (not use_data_parallel) or (
use_data_parallel
and paddle.distributed.ParallelEnv().local_rank == 0)
if save_parameters:
model_path_pre = "_tsn"
if not os.path.isdir(args.checkpoint):
os.makedirs(args.checkpoint)
model_path = os.path.join(
args.checkpoint,
"_" + model_path_pre + "_epoch{}".format(epoch))
paddle.save(video_model.state_dict(), model_path)
paddle.save(optimizer.state_dict(), model_path)
if args.validate:
video_model.eval()
val_acc = val(epoch, video_model, val_loader, valid_config, args)
# save the best parameters in trainging stage
if epoch == 1:
best_acc = val_acc
else:
if val_acc > best_acc:
best_acc = val_acc
if paddle.distributed.ParallelEnv().local_rank == 0:
if not os.path.isdir(args.weights):
os.makedirs(args.weights)
paddle.save(video_model.state_dict(),
args.weights + "/final")
else:
if paddle.distributed.parallel.Env().local_rank == 0:
if not os.path.isdir(args.weights):
os.makedirs(args.weights)
paddle.save(video_model.state_dict(), args.weights + "/final")
logger.info('[TRAIN] training finished')
def get_bert_encoding(bert_config,
model_bert,
tokenizer,
input_sequence,
input_schema,
bert_input_version='v1',
max_seq_length=512,
num_out_layers_n=1,
num_out_layers_h=1):
if bert_input_version == 'v1':
nlu_t, hds = prepare_input(tokenizer, input_sequence, input_schema,
max_seq_length)
elif bert_input_version == 'v2':
nlu_t, hds, max_seq_length = prepare_input_v2(tokenizer,
input_sequence,
input_schema)
wemb_n, wemb_h, l_n, l_hpu, l_hs, nlu_tt, t_to_tt_idx, tt_to_t_idx, t_to_tt_idx_hds = get_wemb_bert(
bert_config, model_bert, tokenizer, nlu_t, hds, max_seq_length,
num_out_layers_n, num_out_layers_h)
t_to_tt_idx = t_to_tt_idx[0]
assert len(t_to_tt_idx) == len(input_sequence)
assert sum(len(t_to_tt_idx_hds1)
for t_to_tt_idx_hds1 in t_to_tt_idx_hds) == len(
input_schema.column_names_embedder_input)
assert list(wemb_h.shape)[0] == len(
input_schema.column_names_embedder_input)
utterance_states = []
for i in range(len(t_to_tt_idx)):
start = t_to_tt_idx[i]
if i == len(t_to_tt_idx) - 1:
end = l_n[0]
else:
end = t_to_tt_idx[i + 1]
utterance_states.append(
paddle.mean(wemb_n[:, start:end, :], axis=[0, 1]))
assert len(utterance_states) == len(input_sequence)
schema_token_states = []
cnt = -1
for t_to_tt_idx_hds1 in t_to_tt_idx_hds:
for t_to_tt_idx_hds11 in t_to_tt_idx_hds1:
cnt += 1
schema_token_states1 = []
for i in range(len(t_to_tt_idx_hds11)):
start = t_to_tt_idx_hds11[i]
if i == len(t_to_tt_idx_hds11) - 1:
end = l_hpu[cnt]
else:
end = t_to_tt_idx_hds11[i + 1]
schema_token_states1.append(
paddle.mean(wemb_h[cnt, start:end, :], axis=0))
assert len(schema_token_states1) == len(
input_schema.column_names_embedder_input[cnt].split())
schema_token_states.append(schema_token_states1)
assert len(schema_token_states) == len(
input_schema.column_names_embedder_input)
return utterance_states, schema_token_states
def test_accuracy(self):
image = paddle.static.data(
name='image', shape=[None, 1, 28, 28], dtype='float32')
image.stop_gradient = False
label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
model = MobileNet()
out = model.net(input=image, class_dim=10)
cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label)
avg_cost = paddle.mean(x=cost)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
optimizer = paddle.optimizer.Momentum(
momentum=0.9,
learning_rate=0.01,
weight_decay=paddle.regularizer.L2Decay(4e-5))
optimizer.minimize(avg_cost)
main_prog = paddle.static.default_main_program()
val_prog = main_prog.clone(for_test=True)
place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda(
) else paddle.CPUPlace()
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
def transform(x):
return np.reshape(x, [1, 28, 28])
train_dataset = paddle.vision.datasets.MNIST(
mode='train', backend='cv2', transform=transform)
test_dataset = paddle.vision.datasets.MNIST(
mode='test', backend='cv2', transform=transform)
train_loader = paddle.io.DataLoader(
train_dataset,
places=place,
feed_list=[image, label],
drop_last=True,
return_list=False,
batch_size=64)
valid_loader = paddle.io.DataLoader(
test_dataset,
places=place,
feed_list=[image, label],
batch_size=64,
return_list=False)
def train(program):
iter = 0
for data in train_loader():
cost, top1, top5 = exe.run(
program,
feed=data,
fetch_list=[avg_cost, acc_top1, acc_top5])
iter += 1
if iter % 100 == 0:
print(
'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'.
format(iter, cost, top1, top5))
def test(program):
iter = 0
result = [[], [], []]
for data in valid_loader():
cost, top1, top5 = exe.run(
program,
feed=data,
fetch_list=[avg_cost, acc_top1, acc_top5])
iter += 1
if iter % 100 == 0:
print('eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'.
format(iter, cost, top1, top5))
result[0].append(cost)
result[1].append(top1)
result[2].append(top5)
print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format(
np.mean(result[0]), np.mean(result[1]), np.mean(result[2])))
return np.mean(result[1]), np.mean(result[2])
train(main_prog)
top1_1, top5_1 = test(main_prog)
config = {
'weight_quantize_type': 'channel_wise_abs_max',
'activation_quantize_type': 'moving_average_abs_max',
'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'],
}
quant_train_prog_pact = quant_aware(
main_prog,
place,
config,
for_test=False,
act_preprocess_func=pact,
optimizer_func=get_optimizer,
executor=exe)
quant_eval_prog = quant_aware(val_prog, place, config, for_test=True)
train(quant_train_prog_pact)
quant_eval_prog, int8_prog = convert(
quant_eval_prog, place, config, save_int8=True)
top1_2, top5_2 = test(quant_eval_prog)
# values before quantization and after quantization should be close
print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1))
print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def forward(self, inputs, label=None):
inputs = paddle.reshape(inputs, [-1, inputs.shape[2], inputs.shape[3], inputs.shape[4]])
googLeNet_part1 = self.googLeNet_part1(inputs)
googleNet_b3d, before3d = self.before3d(googLeNet_part1)
if len(self.out_3d) == self.seg_num:
self.out_3d[:self.seg_num - 1] = self.out_3d[1:]
self.out_3d[self.seg_num - 1] = before3d
self.out_3d[self.seg_num - 2].stop_gradient = True
# for input_old in self.out_3d[:self.seg_num - 1]:
# input_old.stop_gradient = True
else:
while len(self.out_3d) < self.seg_num:
self.out_3d.append(before3d)
y_out_3d = self.out_3d[0]
for i in range(len(self.out_3d) - 1):
y_out_3d = paddle.concat(x=[y_out_3d,self.out_3d[i+1]], axis=0)
y_out_3d = paddle.reshape(y_out_3d, [-1, self.seg_num, y_out_3d.shape[1], y_out_3d.shape[2], y_out_3d.shape[3]])
y_out_3d = paddle.reshape(y_out_3d, [y_out_3d.shape[0], y_out_3d.shape[2], y_out_3d.shape[1], y_out_3d.shape[3], y_out_3d.shape[4]])
out_final_3d = self.res3d(y_out_3d)
out_final_3d = paddle.reshape(out_final_3d, [-1, out_final_3d.shape[1]])
out_final_3d = paddle.nn.functional.dropout(out_final_3d, p=0.5)
out_final_3d = paddle.reshape(out_final_3d, [-1, self.seg_num, out_final_3d.shape[1]])
out_final_3d = paddle.mean(out_final_3d, axis=1)
googLeNet_part2 = self.googLeNet_part2(googleNet_b3d)
googLeNet_part3 = self.googLeNet_part3(googLeNet_part2)
googLeNet_part3 = paddle.nn.functional.dropout(googLeNet_part3, p=0.6)
out_final_2d = paddle.reshape(googLeNet_part3, [-1, googLeNet_part3.shape[1]])
out_final_2d = paddle.reshape(out_final_2d, [-1, self.seg_num, out_final_2d.shape[1]])
out_final_2d = paddle.mean(out_final_2d, axis=1)
out_final = paddle.concat(x=[out_final_2d,out_final_3d], axis=1)
out_final = self.out(out_final)
out_final = paddle.nn.functional.softmax(out_final)
if label is not None:
acc = paddle.metric.accuracy(input=out_final, label=label)
return out_final, acc
else:
return out_final
def create_loss(self, pred, label):
cost = paddle.nn.functional.log_loss(input=pred,
label=paddle.cast(
label, dtype="float32"))
avg_cost = paddle.mean(x=cost)
return avg_cost
def forward(self, x):
mean = paddle.mean(x, axis=1, keepdim=True)
var = paddle.mean(paddle.square(x - mean), axis=1, keepdim=True)
tmp = (x - mean) / paddle.sqrt(var + self.eps)
return tmp
