def test_all_positive(self):
for dtype in self._dtypes:
x = 1 + 10 * np.random.random([13, 3, 3]).astype(dtype)
for place in self._places:
with dg.guard(place):
y = paddle.abs(paddle.to_tensor(x))
self.assertTrue(np.allclose(np.abs(x), y.numpy()))
def test_backward_accumulator_with_init_grad(self):
for dtype in self._dtypes:
x = np.random.random([
10,
]).astype(dtype)
y_grad = np.random.random([
10,
]).astype(dtype)
z_grad = np.random.random([
10,
]).astype(dtype)
self._places = [paddle.CPUPlace()]
for place in self._places:
with dg.guard(place):
x_tensor = paddle.to_tensor(x, stop_gradient=False)
y_tensor = x_tensor**2
z_tensor = y_tensor**3
y_grad_tensor = paddle.to_tensor(y_grad)
z_grad_tensor = paddle.to_tensor(z_grad)
paddle.autograd.backward([y_tensor, z_tensor],
[y_grad_tensor, z_grad_tensor])
y = x**2
z = x**3
x_grad = 2 * x_tensor * (y_grad_tensor + 3 * y_tensor *
y_tensor * z_grad_tensor)
self.assertTrue(
np.allclose(x_grad.numpy(), x_tensor.grad.numpy()))
def test_textcnn_siamese_infer(self):
textcnn_config = {
"vocab_size": TestTextSimilarity.tokenizer.size(),
"emb_dim" : 512,
"num_filters": 256,
"num_channels":1,
"win_size_list": [3],
"is_sparse": True,
"use_cudnn": True,
"triplet_margin": 1.1,
}
infer_config = {
"best_model_save_path": os.path.join(TestTextSimilarity.test_output_dir, "textcnn_best"),
"batch_size": 32,
"max_seq_len": 300,
"print_step": 200,
}
topk=10
infer_res_path = os.path.join(TestTextSimilarity.test_output_dir, "textcnn_infer_res.txt")
with D.guard():
textcnn_siamese = TextCNNSiameseModel()
textcnn_siamese.build(**textcnn_config)
textcnn_siamese.load_model(infer_config["best_model_save_path"])
text_emb_list = textcnn_siamese.batch_infer(TestTextSimilarity.src_text_ids)[0]
self.distance_rank(
text_emb_list,
TestTextSimilarity.src_text_list,
TestTextSimilarity.src_label_list,
infer_res_path,
topk)
def test_textcnn_siamese_train(self):
textcnn_config = {
"vocab_size": TestTextSimilarity.tokenizer.size(),
"emb_dim" : 512,
"num_filters": 256,
"num_channels":1,
"win_size_list": [3],
"is_sparse": True,
"use_cudnn": True,
"triplet_margin": 1.1,
}
run_config = {
"model_save_path": os.path.join(TestTextSimilarity.test_output_dir, "textcnn"),
"best_model_save_path": os.path.join(TestTextSimilarity.test_output_dir, "textcnn_best"),
"epochs": 15,
"batch_size": 32,
"max_seq_len": 300,
"print_step": 200,
"learning_rate": 5e-5,
"load_best_model": False,
}
with D.guard():
textcnn_siamese = TextCNNSiameseModel()
textcnn_siamese.build(**textcnn_config)
best_acc = textcnn_siamese.train(
TestTextSimilarity.train_data, TestTextSimilarity.eval_data,
**run_config)
logging.info("ernie siamese est train score: {}".format(best_acc))
def functional_imperative(self, place):
with dg.guard(place):
x_var = dg.to_variable(self.input)
w_var = dg.to_variable(self.weight)
y_var = F.row_conv(x_var, w_var, act=self.act)
y_np = y_var.numpy()
return y_np
def test_ernie_siamese_infer(self):
ernie_config = {
"pretrain_dir_or_url": "ernie-1.0",
"triplet_margin": 1.1,
}
infer_config = {
"best_model_save_path": os.path.join(TestTextSimilarity.test_output_dir, "ernie_best"),
"batch_size": 32,
"max_seq_len": 300,
"print_step": 200,
}
topk=10
infer_res_path = os.path.join(TestTextSimilarity.test_output_dir, "ernie_infer_res.txt")
with D.guard():
ernie_siamese = ErnieSiameseModel()
ernie_siamese.build(**ernie_config)
ernie_siamese.load_model(infer_config["best_model_save_path"])
text_emb_list = ernie_siamese.batch_infer(TestTextSimilarity.src_text_ids)[0]
self.distance_rank(
text_emb_list,
TestTextSimilarity.src_text_list,
TestTextSimilarity.src_label_list,
infer_res_path,
topk)
def synthesize(config):
pprint(vars(config))
# Get checkpoint directory path.
run_dir = os.path.join("runs", config.model, config.name)
checkpoint_dir = os.path.join(run_dir, "checkpoint")
# Configurate device.
place = fluid.CUDAPlace(0) if config.use_gpu else fluid.CPUPlace()
with dg.guard(place):
# Fix random seed.
seed = config.seed
random.seed(seed)
np.random.seed(seed)
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
print("Random Seed: ", seed)
# Build model.
model = WaveFlow(config, checkpoint_dir)
model.build(training=False)
# Obtain the current iteration.
if config.checkpoint is None:
if config.iteration is None:
iteration = io.load_latest_checkpoint(checkpoint_dir)
else:
iteration = config.iteration
else:
iteration = int(config.checkpoint.split('/')[-1].split('-')[-1])
# Run model inference.
model.infer(iteration)
def check_identity(self, place):
with dg.guard(place):
x_var = paddle.to_tensor(self.x)
y_var = F.glu(x_var, self.dim)
y_np = y_var.numpy()
np.testing.assert_allclose(y_np, self.out)
def train_gru(num_class, vocab_size, data):
logging.warning("gru train start")
gru_config = {
"num_class": num_class,
"vocab_size": vocab_size,
"emb_dim" : 512,
"gru_dim" : 256,
"fc_hid_dim": 512,
"is_sparse": True,
"bi_direction": True,
}
run_config = {
"epochs": 2,
"batch_size": 32,
"max_seq_len": 100,
"print_step": 1,
"learning_rate": 5e-4,
"load_best_model": False,
}
start_time = time.time()
class GRUModel(ClassificationModel):
def build(self, **model_config):
self.model = GRUClassifier(**model_config)
self.built = True
with D.guard():
gru_model = GRUModel()
gru_model.build(**gru_config)
best_acc = gru_model.train(
data, data,
label_encoder=None,
**run_config)
logging.warning("gru best train score: {}, cost time: {}s".format(best_acc, time.time()- start_time))
def test_textcnn(self):
textcnn_config = {
"num_class": TestDygraphModels.label_encoder.size(),
"vocab_size": TestDygraphModels.tokenizer.size(),
"emb_dim" : 512,
"num_filters": 256,
"fc_hid_dim": 512,
"num_channels":1,
"win_size_list": [3],
"is_sparse": True,
"use_cudnn": True,
}
run_config = {
"model_save_path": os.path.join(TestDygraphModels.test_output_dir, "textcnn"),
"best_model_save_path": os.path.join(TestDygraphModels.test_output_dir, "textcnn_best"),
"epochs": 2,
"batch_size": 32,
"max_seq_len": 300,
"print_step": 200,
"learning_rate": 5e-4,
}
with D.guard():
textcnn_model = TextCNN(**textcnn_config)
best_acc = self.model_train_infer(textcnn_model, run_config)
logging.info("textcnn best train score: {}".format(best_acc))
def benchmark(config):
pprint(vars(config))
# Get checkpoint directory path.
run_dir = os.path.join("runs", config.model, config.name)
checkpoint_dir = os.path.join(run_dir, "checkpoint")
# Configurate device.
place = fluid.CUDAPlace(0) if config.use_gpu else fluid.CPUPlace()
with dg.guard(place):
# Fix random seed.
seed = config.seed
random.seed(seed)
np.random.seed(seed)
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
print("Random Seed: ", seed)
# Build model.
model = WaveFlow(config, checkpoint_dir)
model.build(training=False)
# Run model inference.
model.benchmark()
def check_identity(self, place):
with dg.guard(place):
x_var = dg.to_variable(self.x)
y_var = fluid.nets.glu(x_var, self.dim)
y_np = y_var.numpy()
np.testing.assert_allclose(y_np, self.out)
def test_ernie_parallelized(self):
ernie_config = {
"pretrain_dir_or_url": "ernie-1.0",
"num_labels": TestDygraphModelsParallelized.label_encoder.size(),
}
run_config = {
"model_save_path": os.path.join(TestDygraphModelsParallelized.test_output_dir, "ernie"),
"best_model_save_path": os.path.join(TestDygraphModelsParallelized.test_output_dir, "ernie_best"),
"epochs": 2,
"batch_size": 32,
"max_seq_len": 300,
"print_step": 100,
"learning_rate": 5e-5,
"load_best_model": False,
}
start_time = time.time()
class ErnieClassificationModel(ClassificationModel):
@model_parallelized(TestDygraphModelsParallelized.strategy)
def build(self, **model_config):
self.model = ErnieSequenceClassificationCustomized.from_pretrained(**model_config)
self.built = True
place = F.CUDAPlace(D.ParallelEnv().dev_id)
with D.guard(place):
ernie_classification_model = ErnieClassificationModel()
ernie_classification_model.build(**ernie_config)
best_acc = ernie_classification_model.train(
TestDygraphModelsParallelized.train_data, TestDygraphModelsParallelized.eval_data,
label_encoder=TestDygraphModelsParallelized.label_encoder,
**run_config)
logging.warning("ernie parallelized best train score: {}, cost time: {}s".format(best_acc, time.time()- start_time))
def run_gelu_op(approximate):
with dg.guard():
x = paddle.to_tensor(x_np)
x.stop_gradient = False
y = F.gelu(x, approximate=approximate)
x_grad = paddle.grad([y], [x], [paddle.to_tensor(y_g_np)])[0]
return y.numpy(), x_grad.numpy()
def infer(self, infer_file, output_file, batch_size=32):
"""推理，和check的区别是infer是针对文件的预测，而check是针对单条文本的预测
Args:
infer_file: 待预测文件，必须是两列以上，其中第二列为文本
output_file: 预测文件，第一列为模型预测文件
"""
def line_processor(line):
"""每行文本的处理函数
"""
parts = line.strip("\n").split("\t")
text = parts[1]
return (self.tokenizer.encode(text)[0], text)
# batch_infer预测时，data_iter可带上其标签信息，这里把标签信息替换成物料文本
# 方便之后结果输出
infer_data_iter = get_data(infer_file, read_func=line_processor)
with D.guard():
pred_logits, text_list = batch_infer(self.model, infer_data_iter, batch_size=batch_size)
pred_label_id = np.argmax(pred_logits, axis=-1)
pred_label_name = [self.label_encoder.inverse_transform(x) for x in pred_label_id]
with codecs.open(output_file, "w", 'gb18030') as wf:
for label, text in zip(pred_label_name, text_list):
wf.write("%s\t%s\n" % (label, text))
def _test_equivalence(self, place):
place = fluid.CPUPlace()
result1 = self.fluid_layer(place)
result2 = self.functional(place)
with dg.guard(place):
result3 = self.paddle_nn_layer()
np.testing.assert_array_almost_equal(result1, result2)
np.testing.assert_array_almost_equal(result2, result3)
def _test_case(self, place):
x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float64)
y_ref = erf(x)
with dg.guard(place) as g:
x_var = dg.to_variable(x)
y_var = fluid.layers.erf(x_var)
y_test = y_var.numpy()
self.assertTrue(np.allclose(y_ref, y_test))
def compare(self, x, y):
for place in self._places:
with dg.guard(place):
x_var = dg.to_variable(x)
y_var = dg.to_variable(y)
result = paddle.complex.matmul(x_var, y_var)
np_result = np.matmul(x, y)
self.assertTrue(np.allclose(result.numpy(), np_result))
def compare_op_by_basic_api(self, x, y):
for place in self._places:
with dg.guard(place):
var_x = dg.to_variable(x)
var_y = dg.to_variable(y)
self.assert_check((var_x + var_y).numpy(), x + y, place)
self.assert_check((var_x - var_y).numpy(), x - y, place)
self.assert_check((var_x * var_y).numpy(), x * y, place)
self.assert_check((var_x / var_y).numpy(), x / y, place)
def init_infer(self, model_type, model_dir, model_conf):
"""预测阶段初始化
"""
self.label_encoder = LabelEncoder(
os.path.join(model_dir, model_conf["label_id_path"]))
with D.guard():
self.model, self.tokenizer = self.init_model(
model_type, model_dir, model_conf, self.label_encoder.size())
def test_conj_api_real_number(self):
for dtype in self._dtypes:
input = rand([2, 20, 2, 3]).astype(dtype)
for place in self._places:
with dg.guard(place):
var_x = paddle.to_tensor(input)
result = paddle.conj(var_x).numpy()
target = np.conj(input)
self.assertTrue(np.array_equal(result, target))
def test_complex_x(self):
input = rand([2, 20, 2, 3]).astype(
self._dtype) + 1j * rand([2, 20, 2, 3]).astype(self._dtype)
for place in self._places:
with dg.guard(place):
var_x = dg.to_variable(input)
result = cpx.trace(var_x, offset=1, axis1=0, axis2=2).numpy()
target = np.trace(input, offset=1, axis1=0, axis2=2)
self.assertTrue(np.allclose(result, target))
def test_complex_x(self):
input = rand([2, 10, 10]).astype(self._dtype) + 1j * rand(
[2, 10, 10]).astype(self._dtype)
for place in self._places:
with dg.guard(place):
var_x = dg.to_variable(input)
result = cpx.sum(var_x, dim=[1, 2]).numpy()
target = np.sum(input, axis=(1, 2))
self.assertTrue(np.allclose(result, target))
def test_dygraph(self):
for place in self.places:
paddle.disable_static(place=place)
with dygraph.guard():
rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0)
out_np = rrelu(paddle.to_tensor(self.x_np))
self.assertTrue(
check_output(self.x_np, out_np.numpy(), self.lower_0,
self.upper_0))
paddle.enable_static()
def test_case(self):
a = np.random.randn(10, 10).astype(np.float64)
b = np.random.randn(10, 10).astype(np.float64)
place = fluid.CPUPlace()
with dg.guard(place):
a_var = dg.to_variable(a)
b_var = dg.to_variable(b)
c_var = paddle.kron(a_var, b_var)
np.testing.assert_allclose(c_var.numpy(), np.kron(a, b))
def test_case6(self):
x_np = np.random.randn(2, 3, 4) + 1j * np.random.randn(2, 3, 4)
x_np_slice = x_np[0][1:3][0:4:2]
for place in self._places:
with dg.guard(place):
x_var = dg.to_variable(x_np)
x_var_slice = x_var[0][1:3][0:4:2]
np.testing.assert_allclose(x_var_slice.numpy(), x_np_slice)
def _test_case1_gpu(self, approximate):
x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float32)
y_ref = gelu(x, approximate)
place = fluid.CUDAPlace(0)
with dg.guard(place) as g:
x_var = dg.to_variable(x)
y_var = fluid.layers.gelu(x_var, approximate)
y_test = y_var.numpy()
self.assertTrue(np.allclose(y_ref, y_test, rtol=1e-05, atol=1e-08))
def _test_case1_cpu(self):
x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float32)
y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2)))
place = fluid.CPUPlace()
with dg.guard(place) as g:
x_var = dg.to_variable(x)
y_var = fluid.layers.gelu(x_var)
y_test = y_var.numpy()
self.assertTrue(np.allclose(y_ref, y_test, rtol=1e-05, atol=1e-08))
def train(config):
use_gpu = config.use_gpu
# Get the rank of the current training process.
rank = dg.parallel.Env().local_rank
nranks = dg.parallel.Env().nranks
parallel = nranks > 1
if rank == 0:
# Print the whole config setting.
pprint(vars(config))
# Make checkpoint directory.
run_dir = os.path.join("runs", config.model, config.name)
checkpoint_dir = os.path.join(run_dir, "checkpoint")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
# Create tensorboard logger.
vdl = LogWriter(os.path.join(run_dir, "logs")) \
if rank == 0 else None
# Configurate device
place = fluid.CUDAPlace(rank) if use_gpu else fluid.CPUPlace()
with dg.guard(place):
# Fix random seed.
seed = config.seed
random.seed(seed)
np.random.seed(seed)
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
print("Random Seed: ", seed)
# Build model.
model = WaveFlow(config, checkpoint_dir, parallel, rank, nranks, vdl)
iteration = model.build()
while iteration < config.max_iterations:
# Run one single training step.
model.train_step(iteration)
iteration += 1
if iteration % config.test_every == 0:
# Run validation step.
model.valid_step(iteration)
if rank == 0 and iteration % config.save_every == 0:
# Save parameters.
model.save(iteration)
# Close TensorBoard.
if rank == 0:
vdl.close()
def test_identity(self):
data = np.random.random(
(2, 3, 4, 5)).astype("float32") + 1J * np.random.random(
(2, 3, 4, 5)).astype("float32")
perm = [3, 2, 0, 1]
np_trans = np.transpose(data, perm)
for place in self._places:
with dg.guard(place):
var = dg.to_variable(data)
trans = paddle.complex.transpose(var, perm=perm)
self.assertTrue(np.allclose(trans.numpy(), np_trans))
