def create_parameter(self,
dtype,
in_dim,
param_attr=None,
use_bias=True,
bias_attr=None,
transpose_weight=False):
if param_attr is None:
stddev = math.sqrt(2.0 / (in_dim + self.nclasses))
param_attr = ParamAttr(initializer=Normal(scale=stddev))
weight_shape = [self.shard_dim, in_dim
] if transpose_weight else [in_dim, self.shard_dim]
weight = self._layer_helper.create_parameter(shape=weight_shape,
dtype=dtype,
attr=param_attr,
is_bias=False)
# avoid allreducing gradients for distributed parameters
weight.is_distributed = True
# avoid broadcasting distributed parameters in startup program
default_startup_program().global_block().vars[
weight.name].is_distributed = True
bias = None
if use_bias:
bias = self._layer_helper.create_parameter(shape=[self.shard_dim],
attr=bias_attr,
dtype=dtype,
is_bias=True)
bias.is_distributed = True
default_startup_program().global_block().vars[
bias.name].is_distributed = True
return weight, bias
def infer():
args = parse_args()
num_layers = args.num_layers
src_vocab_size = args.vocab_size
tar_vocab_size = args.vocab_size
batch_size = args.batch_size
init_scale = args.init_scale
max_grad_norm = args.max_grad_norm
hidden_size = args.hidden_size
attr_init = args.attr_init
latent_size = 32
if args.enable_ce:
fluid.default_main_program().random_seed = 102
framework.default_startup_program().random_seed = 102
model = VAE(hidden_size,
latent_size,
src_vocab_size,
tar_vocab_size,
batch_size,
num_layers=num_layers,
init_scale=init_scale,
attr_init=attr_init)
beam_size = args.beam_size
trans_res = model.build_graph(mode='sampling', beam_size=beam_size)
# clone from default main program and use it as the validation program
main_program = fluid.default_main_program()
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
dir_name = args.reload_model
print("dir name", dir_name)
dir_name = os.path.join(dir_name, "checkpoint")
fluid.load(main_program, dir_name, exe)
vocab, tar_id2vocab = get_vocab(args.dataset_prefix)
infer_output = np.ones((batch_size, 1), dtype='int64') * BOS_ID
fetch_outs = exe.run(feed={'tar': infer_output},
fetch_list=[trans_res.name],
use_program_cache=False)
with io.open(args.infer_output_file, 'w', encoding='utf-8') as out_file:
for line in fetch_outs[0]:
end_id = -1
if EOS_ID in line:
end_id = np.where(line == EOS_ID)[0][0]
new_line = [tar_id2vocab[e[0]] for e in line[1:end_id]]
out_file.write(space_tok.join(new_line))
out_file.write(line_tok)
def test_forward(self):
data = layers.data(name='X', shape=[1], dtype='float32')
data.stop_gradient = False
cond = ConditionalBlock(inputs=[data])
out = layers.create_tensor(dtype='float32')
with cond.block():
hidden = layers.fc(input=data, size=10)
layers.assign(hidden, out)
cpu = core.CPUPlace()
exe = Executor(cpu)
exe.run(default_startup_program())
x = numpy.random.random(size=(10, 1)).astype('float32')
outs = exe.run(feed={'X': x}, fetch_list=[out])[0]
print(outs)
loss = layers.mean(out)
append_backward(loss=loss)
outs = exe.run(feed={'X': x},
fetch_list=[
default_main_program().block(0).var(data.name +
"@GRAD")
])[0]
print(outs)
def train_loop(main_program):
exe.run(framework.default_startup_program())
PASS_NUM = 100
for pass_id in range(PASS_NUM):
for batch_id, data in enumerate(train_reader()):
# train a mini-batch
outs = exe.run(program=main_program,
feed=func_feed(feeding, data),
fetch_list=[avg_cost])
out = np.array(outs[0])
if (batch_id + 1) % 10 == 0:
avg_cost_set = []
for test_data in test_reader():
avg_cost_np = exe.run(
program=test_program,
feed=func_feed(feeding, test_data),
fetch_list=[avg_cost])
avg_cost_set.append(avg_cost_np[0])
break # test only 1 segment for speeding up CI
# get test avg_cost
test_avg_cost = np.array(avg_cost_set).mean()
if test_avg_cost < 6.0:
# if avg_cost less than 6.0, we think our code is good.
if save_dirname is not None:
fluid.io.save_inference_model(save_dirname, [
"user_id", "gender_id", "age_id", "job_id",
"movie_id", "category_id", "movie_title"
], [scale_infer], exe)
return
if math.isnan(float(out[0])):
sys.exit("got NaN loss, training failed.")
def test_forward(self):
data = layers.data(name='X', shape=[1], dtype='float32')
data.stop_gradient = False
cond = layers.ConditionalBlock(inputs=[data])
out = layers.create_tensor(dtype='float32')
with cond.block():
hidden = layers.fc(input=data, size=10)
layers.assign(hidden, out)
cpu = core.CPUPlace()
exe = Executor(cpu)
exe.run(default_startup_program())
x = numpy.random.random(size=(10, 1)).astype('float32')
outs = exe.run(feed={'X': x}, fetch_list=[out])[0]
print outs
loss = layers.mean(out)
append_backward(loss=loss)
outs = exe.run(
feed={'X': x},
fetch_list=[
default_main_program().block(0).var(data.name + "@GRAD")
])[0]
print outs
def check_switch(self, value):
x = layers.fill_constant(shape=[1], dtype='float32', value=value)
zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0)
one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0)
two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0)
three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0)
result = layers.create_global_var(shape=[1],
value=-1.0,
dtype='float32',
persistable=True)
with layers.Switch() as switch:
with switch.case(layers.less_than(x, zero_var)):
layers.assign(zero_var, result)
with switch.case(layers.less_than(x, one_var)):
layers.assign(one_var, result)
with switch.case(layers.less_than(x, two_var)):
layers.assign(two_var, result)
with switch.default():
layers.assign(three_var, result)
cpu = core.CPUPlace()
exe = Executor(cpu)
exe.run(default_startup_program())
out = exe.run(feed={}, fetch_list=[result])[0][0]
return out
def load():
""" load model for predict """
config = model_config()
config.vocab_size = len(open(config.vocab_path).readlines())
final_score, final_ids, final_index = knowledge_seq2seq(config)
final_score.persistable = True
final_ids.persistable = True
final_index.persistable = True
main_program = fluid.default_main_program()
if config.use_gpu:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
fluid.io.load_params(executor=exe, dirname=config.model_path, main_program=main_program)
processors = KnowledgeCorpus(
data_dir=config.data_dir,
data_prefix=config.data_prefix,
vocab_path=config.vocab_path,
min_len=config.min_len,
max_len=config.max_len)
# load dict
id_dict_array = load_id2str_dict(config.vocab_path)
model_handle = [exe, place, final_score, final_ids, final_index, processors, id_dict_array]
return model_handle
def test_network_gradient(self):
static_input_grad, loss = self.build_graph()
self.exe.run(framework.default_startup_program())
actual_gradients, actual_lod = self.fetch_value(static_input_grad)
static_input_shape = self.static_input_tensor.get_dims()
numeric_gradients = np.zeros(
shape=static_input_shape).astype('float32')
# calculate numeric gradients
tensor_size = np.product(static_input_shape)
for i in xrange(tensor_size):
origin = self.static_input_tensor.get_float_element(i)
x_pos = origin + self._delta
self.static_input_tensor.set_float_element(i, x_pos)
y_pos = self.fetch_value(loss)[0][0]
x_neg = origin - self._delta
self.static_input_tensor.set_float_element(i, x_neg)
y_neg = self.fetch_value(loss)[0][0]
self.static_input_tensor.set_float_element(i, origin)
numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2
self.assertTrue(np.allclose(actual_gradients, numeric_gradients,
0.001))
self.assertTrue(np.allclose(actual_lod,
self.static_input_tensor.lod()))
def train(config):
""" model training """
config.vocab_size = len(open(config.vocab_path).readlines())
bow_loss, kl_loss, nll_loss, final_loss= knowledge_seq2seq(config)
bow_loss.persistable = True
kl_loss.persistable = True
nll_loss.persistable = True
final_loss.persistable = True
main_program = fluid.default_main_program()
inference_program = fluid.default_main_program().clone(for_test=True)
fluid.clip.set_gradient_clip(
clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=config.grad_clip))
optimizer = fluid.optimizer.Adam(learning_rate=config.lr)
if config.stage == 0:
print("stage 0")
optimizer.minimize(bow_loss)
else:
print("stage 1")
optimizer.minimize(final_loss)
opt_var_name_list = optimizer.get_opti_var_name_list()
if config.use_gpu:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
param_list = main_program.block(0).all_parameters()
param_name_list = [p.name for p in param_list]
init_model(config, param_name_list, place)
processors = KnowledgeCorpus(
data_dir=config.data_dir,
data_prefix=config.data_prefix,
vocab_path=config.vocab_path,
min_len=config.min_len,
max_len=config.max_len)
train_generator = processors.data_generator(
batch_size=config.batch_size,
phase="train",
shuffle=True)
valid_generator = processors.data_generator(
batch_size=config.batch_size,
phase="dev",
shuffle=False)
model_handle = [exe, place, bow_loss, kl_loss, nll_loss, final_loss]
train_loop(config,
train_generator, valid_generator,
main_program, inference_program,
model_handle, param_name_list, opt_var_name_list)
def minimize(self,
losses,
scopes=None,
startup_programs=None,
parameter_list=None,
no_grad_set=None):
if isinstance(losses, list):
raise ValueError("need implement later")
self._optimizer.minimize(losses, startup_programs, parameter_list,
no_grad_set)
fleet._origin_main_program = default_main_program().clone(
for_test=False)
fleet._origin_startup_program = default_startup_program().clone(
for_test=False)
compiled_config = public.CompileTimeStrategy(
fleet._origin_main_program, fleet._origin_startup_program,
self._strategy, fleet._role_maker)
fleet.compiled_config = compiled_config
fleet.main_program, fleet.startup_program = \
self._build_trainer_programs(compiled_config) if fleet.is_worker() \
else self._build_pserver_programs(compiled_config)
def check_switch(self, value):
x = layers.fill_constant(shape=[1], dtype='float32', value=value)
zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0)
one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0)
two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0)
three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0)
result = layers.create_global_var(
shape=[1], value=-1.0, dtype='float32', persistable=True)
with layers.Switch() as switch:
with switch.case(layers.less_than(x, zero_var)):
layers.assign(zero_var, result)
with switch.case(layers.less_than(x, one_var)):
layers.assign(one_var, result)
with switch.case(layers.less_than(x, two_var)):
layers.assign(two_var, result)
with switch.default():
layers.assign(three_var, result)
cpu = core.CPUPlace()
exe = Executor(cpu)
exe.run(default_startup_program())
out = exe.run(feed={}, fetch_list=[result])[0][0]
return out
def test_step_out(self):
static_step_outs = self.build_graph(only_forward=True)
self.exe.run(framework.default_startup_program())
expected_outs, expected_lods = self.get_expected_static_step_outs()
for i in xrange(self._max_sequence_len):
step_out, lod = self.fetch_value(static_step_outs[i])
self.assertTrue(np.allclose(step_out, expected_outs[i]))
self.assertTrue(np.allclose(lod, expected_lods[i]))
def test_step_out(self):
static_step_outs = self.build_graph(only_forward=True)
self.exe.run(framework.default_startup_program())
expected_outs, expected_lods = self.get_expected_static_step_outs()
for i in xrange(self._max_sequence_len):
step_out, lod = self.fetch_value(static_step_outs[i])
self.assertTrue(np.allclose(step_out, expected_outs[i]))
self.assertTrue(np.allclose(lod, expected_lods[i]))
def from_func_spec(func_spec, input_spec, class_instance):
"""
Builds the main_program with specialized inputs and returns outputs
of program as fetch_list.
Args:
func_spec(FunctionSpec): A FunctionSpec instance for decorated function.
input_spec(list[InputSpec]):
"""
# Transforms dygraph function into static function and caches it.
dygraph_function = func_spec.dygraph_function
static_func = convert_to_static(dygraph_function)
main_program, startup_program = framework.Program(), framework.Program()
# Note: The random seed should be synchronized into cached program
# if set in `fluid.dygraph_guard` because some ops rely on it, such as
# `fluid.layers.dropout`.
main_program.random_seed = framework.default_main_program().random_seed
startup_program.random_seed = framework.default_startup_program(
).random_seed
with framework.program_guard(main_program, startup_program):
with _switch_declarative_mode_guard_(is_declarative=True):
# 1. Adds `fluid.data` layers for input if needed
inputs = func_spec.to_static_inputs_with_spec(input_spec,
main_program)
if class_instance:
inputs = tuple([class_instance] + list(inputs))
# 2. Gets all ParamBases and buffered VarBases in the function
all_parameters_and_buffers = list(
get_parameters(class_instance).values()) + list(
get_buffers(class_instance).values())
# 3. Builds program only once and returns the output Variables.
with param_guard(get_parameters(
class_instance, False)), param_guard(
get_buffers(class_instance, False)):
try:
outputs = static_func(*inputs)
except BaseException as e:
# NOTE: If e is raised in compile time, e should be attached to ERROR_DATA here.
attach_error_data(e)
raise
if not isinstance(outputs,
(tuple, list)) and outputs is not None:
outputs = [outputs]
main_program = update_op_callstack_with_origin_info(main_program)
return ConcreteProgram(
inputs=inputs,
outputs=outputs,
parameters=all_parameters_and_buffers,
function=dygraph_function,
main_program=main_program,
startup_program=startup_program)
def test_manual_seed(self):
local_program = Program()
local_main_prog = default_main_program()
local_start_prog = default_startup_program()
self.assertEqual(0, local_program.random_seed)
self.assertEqual(0, local_main_prog.random_seed)
self.assertEqual(0, local_start_prog.random_seed)
manual_seed(102)
global_program1 = Program()
global_program2 = Program()
global_main_prog = default_main_program()
global_start_prog = default_startup_program()
self.assertEqual(102, global_program1.random_seed)
self.assertEqual(102, global_program2.random_seed)
self.assertEqual(102, global_main_prog.random_seed)
self.assertEqual(102, global_start_prog.random_seed)
def test_run(self):
inputs = fluid.data(name='inputs',
shape=[None, self.input_size],
dtype='float32')
pre_hidden = fluid.data(name='pre_hidden',
shape=[None, self.hidden_size],
dtype='float32')
pre_cell = fluid.data(name='pre_cell',
shape=[None, self.hidden_size],
dtype='float32')
cell = LSTMCell(self.hidden_size)
lstm_hidden_new, lstm_states_new = cell(inputs, [pre_hidden, pre_cell])
lstm_unit = contrib.layers.rnn_impl.BasicLSTMUnit(
"basicLSTM", self.hidden_size, None, None, None, None, 1.0,
"float32")
lstm_hidden, lstm_cell = lstm_unit(inputs, pre_hidden, pre_cell)
if core.is_compiled_with_cuda():
place = core.CUDAPlace(0)
else:
place = core.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
inputs_np = np.random.uniform(
-0.1, 0.1, (self.batch_size, self.input_size)).astype('float32')
pre_hidden_np = np.random.uniform(
-0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32')
pre_cell_np = np.random.uniform(
-0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32')
param_names = [[
"LSTMCell/BasicLSTMUnit_0.w_0", "basicLSTM/BasicLSTMUnit_0.w_0"
], ["LSTMCell/BasicLSTMUnit_0.b_0", "basicLSTM/BasicLSTMUnit_0.b_0"]]
for names in param_names:
param = np.array(fluid.global_scope().find_var(
names[0]).get_tensor())
param = np.random.uniform(-0.1, 0.1,
size=param.shape).astype('float32')
fluid.global_scope().find_var(names[0]).get_tensor().set(
param, place)
fluid.global_scope().find_var(names[1]).get_tensor().set(
param, place)
out = exe.run(feed={
'inputs': inputs_np,
'pre_hidden': pre_hidden_np,
'pre_cell': pre_cell_np
},
fetch_list=[lstm_hidden_new, lstm_hidden])
self.assertTrue(np.allclose(out[0], out[1], rtol=1e-4, atol=0))
def main(args):
task_name = args.task_name.lower()
processor = reader.MatchProcessor(data_dir=args.data_dir,
task_name=task_name,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case)
num_labels = len(processor.get_labels())
infer_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='test',
epoch=1,
shuffle=False)
num_test_examples = processor.get_num_examples(phase='test')
main_program = fluid.default_main_program()
feed_order, loss, probs, accuracy, num_seqs = create_model(
args, num_labels=num_labels, is_prediction=True)
if args.use_cuda:
place = fluid.CUDAPlace(0)
dev_count = fluid.core.get_cuda_device_count()
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
exe = fluid.Executor(place)
exe.run(framework.default_startup_program())
if args.init_checkpoint:
init_pretraining_params(exe, args.init_checkpoint, main_program)
feed_list = [
main_program.global_block().var(var_name) for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
out_scores = open(args.output, 'w')
for batch_id, data in enumerate(infer_data_generator()):
results = exe.run(fetch_list=[probs],
feed=feeder.feed(data),
return_numpy=True)
for elem in results[0]:
out_scores.write(str(elem[1]) + '\n')
out_scores.close()
if args.save_inference_model_path:
model_path = args.save_inference_model_path
fluid.io.save_inference_model(model_path,
feed_order,
probs,
exe,
main_program=main_program)
def main():
rnn_out = encoder_decoder()
label = layers.data(name="target_language_next_word",
shape=[1],
dtype='int64',
lod_level=1)
cost = layers.cross_entropy(input=rnn_out, label=label)
avg_cost = fluid.layers.mean(cost)
optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4)
optimizer.minimize(avg_cost)
# fluid.memory_optimize(fluid.default_main_program())
fluid.release_memory(fluid.default_main_program())
# fix the order of training data
train_data = paddle.batch(paddle.dataset.wmt14.train(dict_size),
batch_size=batch_size)
# train_data = paddle.batch(
# paddle.reader.shuffle(
# paddle.dataset.wmt14.train(dict_size), buf_size=1000),
# batch_size=batch_size)
place = core.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
feed_order = [
'src_word_id', 'target_language_word', 'target_language_next_word'
]
feed_list = [
fluid.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
batch_id = 0
for pass_id in xrange(10):
for data in train_data():
outs = exe.run(fluid.default_main_program(),
feed=feeder.feed(data),
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if batch_id > 2:
exit(0)
if math.isnan(float(avg_cost_val)):
sys.exit("got NaN loss, training failed.")
batch_id += 1
def decode_main(use_cuda, is_sparse):
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
context = encoder(is_sparse)
translation_ids, translation_scores = decoder_decode(context, is_sparse)
exe = Executor(place)
exe.run(framework.default_startup_program())
init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64')
init_scores_data = np.array([1. for _ in range(batch_size)],
dtype='float32')
init_ids_data = init_ids_data.reshape((batch_size, 1))
init_scores_data = init_scores_data.reshape((batch_size, 1))
init_recursive_seq_lens = [1] * batch_size
init_recursive_seq_lens = [
init_recursive_seq_lens, init_recursive_seq_lens
]
init_ids = fluid.create_lod_tensor(init_ids_data, init_recursive_seq_lens,
place)
init_scores = fluid.create_lod_tensor(init_scores_data,
init_recursive_seq_lens, place)
train_data = paddle.batch(paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
feed_order = ['src_word_id']
feed_list = [
framework.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
for data in train_data():
feed_dict = feeder.feed([[x[0]] for x in data])
feed_dict['init_ids'] = init_ids
feed_dict['init_scores'] = init_scores
result_ids, result_scores = exe.run(
framework.default_main_program(),
feed=feed_dict,
fetch_list=[translation_ids, translation_scores],
return_numpy=False)
print(result_ids.recursive_sequence_lengths())
break
def main(args):
task_name = args.task_name.lower()
processors = {
'match': reader.MatchProcessor,
}
processor = processors[task_name](data_dir=args.data_dir,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case)
num_labels = len(processor.get_labels())
infer_data_generator = processor.data_generator(
batch_size=args.batch_size,
phase='dev',
epoch=args.epoch,
shuffle=False)
main_program = fluid.default_main_program()
feed_order, loss, probs, accuracy, num_seqs = create_model(
args,
num_labels=num_labels)
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(framework.default_startup_program())
if args.init_checkpoint:
init_pretraining_params(exe, args.init_checkpoint, main_program)
feed_list = [
main_program.global_block().var(var_name) for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
label_list = []
for batch_id, data in enumerate(infer_data_generator()):
results = exe.run(
fetch_list=[probs],
feed=feeder.feed(data),
return_numpy=True)
for elem in results[0]:
label_list.append(str(elem[1]))
return label_list
def load_model():
"""
load model function
"""
main_program = fluid.default_main_program()
#place = fluid.CPUPlace()
place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)
exe.run(framework.default_startup_program())
path = "./models/inference_model"
[inference_program, feed_dict, fetch_targets] = \
fluid.io.load_inference_model(dirname=path, executor=exe)
model_handle = [exe, inference_program, feed_dict, fetch_targets, place]
return model_handle
def train():
startup_program = fluid.default_startup_program()
main_program = fluid.default_main_program()
raw_data = reader.raw_data('fra.txt', num_samples=num_samples)
train_data = raw_data[0]
data_vars = raw_data[1]
model = BaseModel(hidden_size=latent_dim,
src_vocab_size=data_vars['num_encoder_tokens'],
tar_vocab_size=data_vars['num_decoder_tokens'],
batch_size=batch_size,
batch_first=True)
loss = model.build_graph()
optimizer = fluid.optimizer.Adam(learning_rate=0.001)
optimizer.minimize(loss)
place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)
exe.run(framework.default_startup_program())
ce_ppl = []
for epoch_id in range(num_epochs):
print("epoch ", epoch_id)
train_data_iter = reader.get_data_iter(train_data, batch_size)
total_loss = 0
word_count = 0.0
for batch_id, batch in enumerate(train_data_iter):
input_data_feed, word_num = prepare_input(batch, epoch_id=epoch_id)
fetch_outs = exe.run(feed=input_data_feed,
fetch_list=[loss.name],
use_program_cache=True)
cost_train = np.array(fetch_outs[0])
total_loss += cost_train * batch_size
word_count += word_num
if batch_id > 0 and batch_id % batch_size == 0:
print(" ppl", batch_id, np.exp(total_loss / word_count))
ce_ppl.append(np.exp(total_loss / word_count))
total_loss = 0.0
word_count = 0.0
def main():
rnn_out = encoder_decoder()
label = layers.data(
name="target_language_next_word", shape=[1], dtype='int64', lod_level=1)
cost = layers.cross_entropy(input=rnn_out, label=label)
avg_cost = fluid.layers.mean(cost)
optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4)
optimizer.minimize(avg_cost)
# fluid.memory_optimize(fluid.default_main_program())
fluid.release_memory(fluid.default_main_program())
# fix the order of training data
train_data = paddle.batch(
paddle.dataset.wmt14.train(dict_size), batch_size=batch_size)
# train_data = paddle.batch(
# paddle.reader.shuffle(
# paddle.dataset.wmt14.train(dict_size), buf_size=1000),
# batch_size=batch_size)
place = core.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
batch_id = 0
for pass_id in xrange(10):
for data in train_data():
word_data = to_lodtensor(map(lambda x: x[0], data), place)
trg_word = to_lodtensor(map(lambda x: x[1], data), place)
trg_word_next = to_lodtensor(map(lambda x: x[2], data), place)
outs = exe.run(fluid.default_main_program(),
feed={
'src_word_id': word_data,
'target_language_word': trg_word,
'target_language_next_word': trg_word_next
},
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if batch_id > 2:
exit(0)
if math.isnan(float(avg_cost_val)):
sys.exit("got NaN loss, training failed.")
batch_id += 1
def __init__(self):
self._inputs = []
self._outputs = []
# Always set program to default_main_program. Because once `__call__` is called,
# it means layers(or Ops) are added into default_main_program switched by outer
# `with` statement.
self._main_program = framework.default_main_program()
self._startup_program = framework.default_startup_program()
self._func_cache = FunctionCache()
# Stores the entry function of Net or Model.
self._forward_func = None
self._feed_name_to_idx = {}
self._is_repeated = False
# Indicates whether the function call is still building program.
# Because user can call recursively when `Net` has sub class in
# `forward()`.
self._in_build_process = True
def decode_main(use_cuda, is_sparse):
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
context = encoder(is_sparse)
translation_ids, translation_scores = decode(context, is_sparse)
exe = Executor(place)
exe.run(framework.default_startup_program())
init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64')
init_scores_data = np.array(
[1. for _ in range(batch_size)], dtype='float32')
init_ids_data = init_ids_data.reshape((batch_size, 1))
init_scores_data = init_scores_data.reshape((batch_size, 1))
init_lod = [1] * batch_size
init_lod = [init_lod, init_lod]
init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place)
init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
feed_order = ['src_word_id']
feed_list = [
framework.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
for data in train_data():
feed_dict = feeder.feed(map(lambda x: [x[0]], data))
feed_dict['init_ids'] = init_ids
feed_dict['init_scores'] = init_scores
result_ids, result_scores = exe.run(
framework.default_main_program(),
feed=feed_dict,
fetch_list=[translation_ids, translation_scores],
return_numpy=False)
print result_ids.lod()
break
def train_loop(main_program):
exe.run(framework.default_startup_program())
feed_list = [
main_program.global_block().var(var_name) for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
for pass_id in range(1):
for batch_id, data in enumerate(train_reader()):
outs = exe.run(main_program,
feed=feeder.feed(data),
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if batch_id > 3:
break
def _transpile(self, config):
if not isinstance(config, DistributeTranspilerConfig):
raise TypeError(
"config must be an instance of DistributeTranspilerConfig")
if not config.sync_mode:
config.runtime_split_send_recv = True
# _origin_program is a deep copy for default_main_program, for inference
self._origin_program = default_main_program().clone(for_test=False)
self._transpile_config = config
if config.geo_sgd_mode:
self._transpiler = GeoSgdTranspiler(config)
else:
self._transpiler = OriginTranspiler(config)
if self.is_worker():
self._transpiler.transpile(
trainer_id=fleet.worker_index(),
pservers=fleet.server_endpoints(to_string=True),
trainers=fleet.worker_num(),
sync_mode=config.sync_mode)
if isinstance(self._role_maker, MPISymetricRoleMaker):
config.wait_port = False
self.main_program = self._transpiler.get_trainer_program(
wait_port=config.wait_port)
self.startup_program = default_startup_program()
if self._transpile_config.geo_sgd_mode:
self.vars_info = self._transpiler._get_vars_info()
self.startup_program = self._transpiler.trainer_startup_program
else:
self._transpiler.transpile(
trainer_id=fleet.worker_index(),
pservers=fleet.server_endpoints(to_string=True),
trainers=fleet.worker_num(),
sync_mode=config.sync_mode,
current_endpoint=self.server_endpoints()[self.server_index()])
self.main_program, self.startup_program = \
self._transpiler.get_pserver_programs(
self.server_endpoints()[self.server_index()])
def train(use_cuda, save_dirname=None):
[avg_cost, prediction] = seq_to_seq_net()
optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4)
optimizer.minimize(avg_cost)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
feed_order = ['source_sequence', 'target_sequence', 'label_sequence']
feed_list = [
framework.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
batch_id = 0
for pass_id in xrange(2):
for data in train_data():
outs = exe.run(framework.default_main_program(),
feed=feeder.feed(data),
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if math.isnan(float(avg_cost_val[0])):
sys.exit("got NaN loss, training failed.")
if batch_id > 3:
if save_dirname is not None:
fluid.io.save_inference_model(
save_dirname, ['source_sequence',
'target_sequence'], [prediction], exe)
return
batch_id += 1
def train(use_cuda, save_dirname=None):
[avg_cost, prediction] = seq_to_seq_net()
optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4)
optimizer.minimize(avg_cost)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
feed_order = ['source_sequence', 'target_sequence', 'label_sequence']
feed_list = [
framework.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
batch_id = 0
for pass_id in xrange(2):
for data in train_data():
outs = exe.run(framework.default_main_program(),
feed=feeder.feed(data),
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if math.isnan(float(avg_cost_val[0])):
sys.exit("got NaN loss, training failed.")
if batch_id > 3:
if save_dirname is not None:
fluid.io.save_inference_model(
save_dirname, ['source_sequence',
'target_sequence'], [prediction], exe)
return
batch_id += 1
def test_network_gradient(self):
static_input_grad, loss = self.build_graph()
self.exe.run(framework.default_startup_program())
actual_gradients, actual_lod = self.fetch_value(static_input_grad)
static_input_shape = self.static_input_tensor.get_dims()
numeric_gradients = np.zeros(shape=static_input_shape).astype('float32')
# calculate numeric gradients
tensor_size = np.product(static_input_shape)
for i in xrange(tensor_size):
origin = self.static_input_tensor.get_float_element(i)
x_pos = origin + self._delta
self.static_input_tensor.set_float_element(i, x_pos)
y_pos = self.fetch_value(loss)[0][0]
x_neg = origin - self._delta
self.static_input_tensor.set_float_element(i, x_neg)
y_neg = self.fetch_value(loss)[0][0]
self.static_input_tensor.set_float_element(i, origin)
numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2
self.assertTrue(np.allclose(actual_gradients, numeric_gradients, 0.001))
self.assertTrue(np.allclose(actual_lod, self.static_input_tensor.lod()))
def _transpile(self, config):
if not isinstance(config, DistributeTranspilerConfig):
raise ValueError(
"config must be an instance of DistributeTranspilerConfig")
if not config.sync_mode:
config.runtime_split_send_recv = True
self._transpile_config = config
self._transpiler = OriginTranspiler(config)
self._transpiler.transpile(
trainer_id=fleet.worker_index(),
pservers=fleet.server_endpoints(to_string=True),
trainers=fleet.worker_num(),
sync_mode=config.sync_mode)
if self.is_worker():
self.main_program = self._transpiler.get_trainer_program()
self.startup_program = default_startup_program()
else:
self.main_program, self.startup_program = \
self._transpiler.get_pserver_programs(self.server_endpoints()[self.server_index()])
def decode_main(use_cuda, is_sparse):
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
context = encoder(is_sparse)
translation_ids, translation_scores = decoder_decode(context, is_sparse)
exe = Executor(place)
exe.run(framework.default_startup_program())
init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64')
init_scores_data = np.array([1. for _ in range(batch_size)],
dtype='float32')
init_ids_data = init_ids_data.reshape((batch_size, 1))
init_scores_data = init_scores_data.reshape((batch_size, 1))
init_lod = [i for i in range(batch_size)] + [batch_size]
init_lod = [init_lod, init_lod]
train_data = paddle.batch(paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
for _, data in enumerate(train_data()):
init_ids = set_init_lod(init_ids_data, init_lod, place)
init_scores = set_init_lod(init_scores_data, init_lod, place)
src_word_data = to_lodtensor(map(lambda x: x[0], data), place)
result_ids, result_scores = exe.run(
framework.default_main_program(),
feed={
'src_word_id': src_word_data,
'init_ids': init_ids,
'init_scores': init_scores
},
fetch_list=[translation_ids, translation_scores],
return_numpy=False)
print result_ids.lod()
break
def train_loop(main_program):
exe.run(framework.default_startup_program())
batch_id = 0
for pass_id in xrange(1):
for data in train_data():
word_data = to_lodtensor(map(lambda x: x[0], data), place)
trg_word = to_lodtensor(map(lambda x: x[1], data), place)
trg_word_next = to_lodtensor(map(lambda x: x[2], data), place)
outs = exe.run(main_program,
feed={
'src_word_id': word_data,
'target_language_word': trg_word,
'target_language_next_word': trg_word_next
},
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if batch_id > 3:
break
batch_id += 1
def train_loop(main_program):
exe.run(framework.default_startup_program())
feed_list = [
main_program.global_block().var(var_name) for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
PASS_NUM = 100
for pass_id in range(PASS_NUM):
for batch_id, data in enumerate(train_reader()):
# train a mini-batch
outs = exe.run(program=main_program,
feed=feeder.feed(data),
fetch_list=[avg_cost])
out = np.array(outs[0])
if (batch_id + 1) % 10 == 0:
avg_cost_set = []
for test_data in test_reader():
avg_cost_np = exe.run(program=test_program,
feed=feeder.feed(test_data),
fetch_list=[avg_cost])
avg_cost_set.append(avg_cost_np[0])
break # test only 1 segment for speeding up CI
# get test avg_cost
test_avg_cost = np.array(avg_cost_set).mean()
if test_avg_cost < 6.0:
# if avg_cost less than 6.0, we think our code is good.
if save_dirname is not None:
fluid.io.save_inference_model(save_dirname, [
"user_id", "gender_id", "age_id", "job_id",
"movie_id", "category_id", "movie_title"
], [scale_infer], exe)
return
if math.isnan(float(out[0])):
sys.exit("got NaN loss, training failed.")
def from_func_spec(func_spec):
"""
Builds the main_program with specialized inputs and returns outputs
of program as fetch_list.
"""
# Transforms dygraph function into static function and caches it.
dygaph_function = func_spec.dyfunc
static_func = convert_function_with_cache(dygaph_function)
main_program, startup_program = framework.Program(), framework.Program(
)
# Note: The random seed should be synchronized into cached program
# if set in `fluid.dygrap_guard` because some ops rely on it, such as
# `fluid.layers.dropout`.
main_program.random_seed = framework.default_main_program().random_seed
startup_program.random_seed = framework.default_startup_program(
).random_seed
with framework.program_guard(main_program, startup_program):
# 1. Adds `fluid.data` layers for input if needed
inputs = func_spec.to_static_inputs(main_program)
# 2. Gets all ParamBases in the function
all_parameters = list(func_spec.parameters().values())
# 3. Builds program only once and returns the output Variables.
with param_guard(func_spec.parameters(False)):
outputs = static_func(*inputs)
if not isinstance(outputs, (tuple, list)):
outputs = [outputs] if outputs else []
return ConcreteProgram(inputs=inputs,
outputs=outputs,
parameters=all_parameters,
func=dygaph_function,
main_program=main_program,
startup_program=startup_program)
def test_ptb_rnn_cpu_bfloat16(self):
seed = 90
hidden_size = 10
vocab_size = 500
num_layers = 1
num_steps = 3
init_scale = 0.1
batch_size = 4
batch_num = 100
with new_program_scope():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
ptb_model = PtbModel("ptb_model",
hidden_size=hidden_size,
vocab_size=vocab_size,
num_layers=num_layers,
num_steps=num_steps,
init_scale=init_scale)
place = self.set_place()
exe = fluid.Executor(place)
sgd = SGDOptimizer(learning_rate=1e-3)
x = fluid.layers.data(name="x",
shape=[-1, num_steps],
dtype='int64')
y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32')
init_hidden = fluid.layers.data(name="init_hidden",
shape=[1],
dtype='float32')
init_cell = fluid.layers.data(name="init_cell",
shape=[1],
dtype='float32')
static_loss, static_last_hidden, static_last_cell = ptb_model(
x, y, init_hidden, init_cell)
sgd = paddle.static.amp.bf16.decorate_bf16(
sgd,
amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16(
custom_fp32_list={'transpose2', 'concat'}),
use_bf16_guard=False,
use_pure_bf16=True)
sgd.minimize(static_loss, framework.default_startup_program())
out = exe.run(framework.default_startup_program())
for i in range(batch_num):
x_data = np.arange(12).reshape(4, 3).astype('int64')
y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
x_data = x_data.reshape((-1, num_steps, 1))
y_data = y_data.reshape((-1, 1))
#TODO investigate initializing model with "float32" instead of "uint16" as it was before
# slice_op PR(datatypes in model graph are different than datatypes during runtime because of that)
init_hidden_data = np.zeros(
(num_layers, batch_size, hidden_size), dtype='uint16')
init_cell_data = np.zeros(
(num_layers, batch_size, hidden_size), dtype='uint16')
fetch_list = [
static_loss, static_last_hidden, static_last_cell
]
out = exe.run(fluid.default_main_program(),
feed={
"x": x_data,
"y": y_data,
"init_hidden": init_hidden_data,
"init_cell": init_cell_data
},
fetch_list=fetch_list)
# get value before save
main_program = framework.default_main_program()
base_map = {}
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
t = np.array(fluid.global_scope().find_var(
var.name).get_tensor())
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
# set var to zero
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
ten = fluid.global_scope().find_var(var.name).get_tensor()
ten.set(np.zeros_like(np.array(ten)), place)
new_t = np.array(fluid.global_scope().find_var(
var.name).get_tensor())
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
new_t = np.array(fluid.global_scope().find_var(
var.name).get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
def main():
args = parse_args()
print(args)
num_layers = args.num_layers
src_vocab_size = args.vocab_size
tar_vocab_size = args.vocab_size
batch_size = args.batch_size
init_scale = args.init_scale
max_grad_norm = args.max_grad_norm
hidden_size = args.hidden_size
attr_init = args.attr_init
latent_size = 32
main_program = fluid.Program()
startup_program = fluid.Program()
if args.enable_ce:
fluid.default_main_program().random_seed = 123
framework.default_startup_program().random_seed = 123
# Training process
with fluid.program_guard(main_program, startup_program):
with fluid.unique_name.guard():
model = VAE(hidden_size,
latent_size,
src_vocab_size,
tar_vocab_size,
batch_size,
num_layers=num_layers,
init_scale=init_scale,
attr_init=attr_init)
loss, kl_loss, rec_loss = model.build_graph()
# clone from default main program and use it as the validation program
main_program = fluid.default_main_program()
inference_program = fluid.default_main_program().clone(
for_test=True)
clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=max_grad_norm)
learning_rate = fluid.layers.create_global_var(
name="learning_rate",
shape=[1],
value=float(args.learning_rate),
dtype="float32",
persistable=True)
opt_type = args.optimizer
if opt_type == "sgd":
optimizer = fluid.optimizer.SGD(learning_rate, grad_clip=clip)
elif opt_type == "adam":
optimizer = fluid.optimizer.Adam(learning_rate, grad_clip=clip)
else:
print("only support [sgd|adam]")
raise Exception("opt type not support")
optimizer.minimize(loss)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = Executor(place)
exe.run(startup_program)
train_program = fluid.compiler.CompiledProgram(main_program)
dataset_prefix = args.dataset_prefix
print("begin to load data")
raw_data = reader.raw_data(dataset_prefix, args.max_len)
print("finished load data")
train_data, valid_data, test_data, _ = raw_data
anneal_r = 1.0 / (args.warm_up * len(train_data) / args.batch_size)
def prepare_input(batch, kl_weight=1.0, lr=None):
src_ids, src_mask = batch
res = {}
src_ids = src_ids.reshape((src_ids.shape[0], src_ids.shape[1]))
in_tar = src_ids[:, :-1]
label_tar = src_ids[:, 1:]
in_tar = in_tar.reshape((in_tar.shape[0], in_tar.shape[1]))
label_tar = label_tar.reshape(
(label_tar.shape[0], label_tar.shape[1], 1))
res['src'] = src_ids
res['tar'] = in_tar
res['label'] = label_tar
res['src_sequence_length'] = src_mask
res['tar_sequence_length'] = src_mask - 1
res['kl_weight'] = np.array([kl_weight]).astype(np.float32)
if lr is not None:
res['learning_rate'] = np.array([lr]).astype(np.float32)
return res, np.sum(src_mask), np.sum(src_mask - 1)
# get train epoch size
def eval(data):
eval_data_iter = reader.get_data_iter(data, batch_size, mode='eval')
total_loss = 0.0
word_count = 0.0
batch_count = 0.0
for batch_id, batch in enumerate(eval_data_iter):
input_data_feed, src_word_num, dec_word_sum = prepare_input(batch)
fetch_outs = exe.run(inference_program,
feed=input_data_feed,
fetch_list=[loss.name],
use_program_cache=False)
cost_train = np.array(fetch_outs[0])
total_loss += cost_train * batch_size
word_count += dec_word_sum
batch_count += batch_size
nll = total_loss / batch_count
ppl = np.exp(total_loss / word_count)
return nll, ppl
def train():
ce_time = []
ce_ppl = []
max_epoch = args.max_epoch
kl_w = args.kl_start
lr_w = args.learning_rate
best_valid_nll = 1e100 # +inf
best_epoch_id = -1
decay_cnt = 0
max_decay = args.max_decay
decay_factor = 0.5
decay_ts = 2
steps_not_improved = 0
for epoch_id in range(max_epoch):
start_time = time.time()
if args.enable_ce:
train_data_iter = reader.get_data_iter(train_data,
batch_size,
args.sort_cache,
args.cache_num,
enable_ce=True)
else:
train_data_iter = reader.get_data_iter(train_data, batch_size,
args.sort_cache,
args.cache_num)
total_loss = 0
total_rec_loss = 0
total_kl_loss = 0
word_count = 0.0
batch_count = 0.0
batch_times = []
for batch_id, batch in enumerate(train_data_iter):
batch_start_time = time.time()
kl_w = min(1.0, kl_w + anneal_r)
kl_weight = kl_w
input_data_feed, src_word_num, dec_word_sum = prepare_input(
batch, kl_weight, lr_w)
fetch_outs = exe.run(
program=train_program,
feed=input_data_feed,
fetch_list=[loss.name, kl_loss.name, rec_loss.name],
use_program_cache=False)
cost_train = np.array(fetch_outs[0])
kl_cost_train = np.array(fetch_outs[1])
rec_cost_train = np.array(fetch_outs[2])
total_loss += cost_train * batch_size
total_rec_loss += rec_cost_train * batch_size
total_kl_loss += kl_cost_train * batch_size
word_count += dec_word_sum
batch_count += batch_size
batch_end_time = time.time()
batch_time = batch_end_time - batch_start_time
batch_times.append(batch_time)
if batch_id > 0 and batch_id % 200 == 0:
print("-- Epoch:[%d]; Batch:[%d]; Time: %.4f s; "
"kl_weight: %.4f; kl_loss: %.4f; rec_loss: %.4f; "
"nll: %.4f; ppl: %.4f" %
(epoch_id, batch_id, batch_time, kl_w,
total_kl_loss / batch_count, total_rec_loss /
batch_count, total_loss / batch_count,
np.exp(total_loss / word_count)))
ce_ppl.append(np.exp(total_loss / word_count))
end_time = time.time()
epoch_time = end_time - start_time
ce_time.append(epoch_time)
print(
"\nTrain epoch:[%d]; Epoch Time: %.4f; avg_time: %.4f s/step\n"
% (epoch_id, epoch_time, sum(batch_times) / len(batch_times)))
val_nll, val_ppl = eval(valid_data)
print("dev ppl", val_ppl)
test_nll, test_ppl = eval(test_data)
print("test ppl", test_ppl)
if val_nll < best_valid_nll:
best_valid_nll = val_nll
steps_not_improved = 0
best_nll = test_nll
best_ppl = test_ppl
best_epoch_id = epoch_id
save_path = os.path.join(args.model_path,
"epoch_" + str(best_epoch_id),
"checkpoint")
print("save model {}".format(save_path))
fluid.save(main_program, save_path)
else:
steps_not_improved += 1
if steps_not_improved == decay_ts:
old_lr = lr_w
lr_w *= decay_factor
steps_not_improved = 0
new_lr = lr_w
print('-----\nchange lr, old lr: %f, new lr: %f\n-----' %
(old_lr, new_lr))
dir_name = args.model_path + "/epoch_" + str(best_epoch_id)
fluid.load(main_program, dir_name, exe)
decay_cnt += 1
if decay_cnt == max_decay:
break
print('\nbest testing nll: %.4f, best testing ppl %.4f\n' %
(best_nll, best_ppl))
if args.enable_ce:
card_num = get_cards()
_ppl = 0
_time = 0
try:
_time = ce_time[-1]
_ppl = ce_ppl[-1]
except:
print("ce info error")
print("kpis\ttrain_duration_card%s\t%s" % (card_num, _time))
print("kpis\ttrain_ppl_card%s\t%f" % (card_num, _ppl))
with profile_context(args.profile):
train()
def testLoadStaticModel(self):
# static mode
a = fluid.data(name="a", shape=[10, 10])
conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10])
fc_out1 = fluid.layers.fc(a, 10)
fc_out2 = fluid.layers.fc(a, 20)
conv_out_1 = fluid.layers.conv2d(conv_in,
num_filters=10,
filter_size=5,
act="relu")
conv_out_2 = fluid.layers.conv2d(conv_in,
num_filters=10,
filter_size=5,
act="relu")
conv3d_in = fluid.data(name='conv3d_in',
shape=[None, 3, 12, 32, 32],
dtype='float32')
conv3d_out_1 = fluid.layers.conv3d(input=conv3d_in,
num_filters=2,
filter_size=3,
act="relu")
conv3d_out_2 = fluid.layers.conv3d(input=conv3d_in,
num_filters=2,
filter_size=3,
act="relu")
batchnorm_in = fluid.data(name="batchnorm_in",
shape=[None, 10],
dtype='float32')
batchnorm_out_1 = fluid.layers.batch_norm(batchnorm_in)
batchnorm_out_2 = fluid.layers.batch_norm(batchnorm_in)
emb_in = fluid.data(name='emb_in', shape=[None, 10], dtype='int64')
emb_out_1 = fluid.embedding(emb_in, [1000, 100])
emb_out_2 = fluid.embedding(emb_in, [2000, 200])
layernorm = fluid.data(name="ln", shape=[None, 10], dtype='float32')
layernorm_1 = fluid.layers.layer_norm(layernorm)
layernorm_2 = fluid.layers.layer_norm(layernorm)
nce_in = fluid.data(name="nce_in", shape=[None, 100], dtype='float32')
nce_label = fluid.data(name="nce_label",
shape=[None, 10],
dtype='int64')
nce_out_1 = fluid.layers.nce(nce_in, nce_label, 10000)
nce_out_2 = fluid.layers.nce(nce_in, nce_label, 10000)
prelu_in = fluid.data(name="prelu_in",
shape=[None, 5, 10, 10],
dtype='float32')
prelu_out_1 = fluid.layers.prelu(prelu_in, "channel")
prelu_out_2 = fluid.layers.prelu(prelu_in, "channel")
bilinear_tensor_pro_x = fluid.data("t1",
shape=[None, 5],
dtype="float32")
bilinear_tensor_pro_y = fluid.data("t2",
shape=[None, 4],
dtype="float32")
bilinear_tensor_pro_out_1 = fluid.layers.bilinear_tensor_product(
x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000)
bilinear_tensor_pro_out_2 = fluid.layers.bilinear_tensor_product(
x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000)
conv2d_trans_in = fluid.data(name="conv2d_trans_in",
shape=[None, 10, 10, 10])
conv2d_trans_out_1 = fluid.layers.conv2d_transpose(conv2d_trans_in,
num_filters=10,
filter_size=5,
act="relu")
conv2d_trans_out_2 = fluid.layers.conv2d_transpose(conv2d_trans_in,
num_filters=10,
filter_size=5,
act="relu")
conv3d_trans_in = fluid.data(name='conv3d_trans_in',
shape=[None, 3, 12, 32, 32],
dtype='float32')
conv3d_trans_out_1 = fluid.layers.conv3d_transpose(
input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu")
conv3d_trans_out_2 = fluid.layers.conv3d_transpose(
input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu")
groupnorm_in = fluid.data(name='groupnorm_in',
shape=[None, 8, 32, 32],
dtype='float32')
groupnorm_out1 = fluid.layers.group_norm(input=groupnorm_in, groups=4)
groupnorm_out2 = fluid.layers.group_norm(input=groupnorm_in, groups=4)
'''
spec_norm = fluid.data(name='spec_norm', shape=[2, 8, 32, 32], dtype='float32')
spe_norm_out_1 = fluid.layers.spectral_norm(weight=spec_norm, dim=1, power_iters=2)
spe_norm_out_2 = fluid.layers.spectral_norm(weight=spec_norm, dim=1, power_iters=2)
'''
nodes_vector = fluid.data(name='vectors',
shape=[None, 10, 5],
dtype='float32')
edge_set = fluid.data(name='edge_set',
shape=[None, 10, 2],
dtype='float32')
tree_conv_out1 = fluid.contrib.layers.tree_conv(
nodes_vector, edge_set, 6, 1, 2)
tree_conv_out2 = fluid.contrib.layers.tree_conv(
nodes_vector, edge_set, 6, 1, 2)
para1 = fluid.layers.create_parameter([100, 100],
'float32',
name="weight_test_1")
para2 = fluid.layers.create_parameter([20, 200],
'float32',
name="weight_test_2")
para_list = fluid.default_main_program().list_vars()
exe = fluid.Executor(fluid.CPUPlace(
) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0))
out = exe.run(framework.default_startup_program())
fluid.save(framework.default_main_program(), "./test_1")
para_dict = fluid.load_program_state("./test_1")
new_dict = {}
for k, v in para_dict.items():
#print( k, v.shape )
if k.startswith("fc"):
name = k.replace("fc", "linear", 1)
new_dict[name] = v
else:
new_dict[k] = v
with fluid.dygraph.guard():
class MyTest(fluid.dygraph.Layer):
def __init__(self):
super(MyTest, self).__init__()
self.linear1 = Linear(10, 10)
self.lienar2 = Linear(10, 20)
self.conv2d_1 = Conv2D(num_channels=10,
num_filters=10,
filter_size=5,
act="relu")
self.conv2d_2 = Conv2D(num_channels=10,
num_filters=10,
filter_size=5,
act="relu")
self.conv3d_1 = Conv3D(num_channels=3,
num_filters=2,
filter_size=3,
act="relu")
self.conv3d_2 = Conv3D(num_channels=3,
num_filters=2,
filter_size=3,
act="relu")
self.batch_norm_1 = BatchNorm(10)
self.batch_norm_2 = BatchNorm(10)
self.emb1 = Embedding([1000, 100])
self.emb2 = Embedding([2000, 200])
self.layer_norm_1 = LayerNorm([10])
self.layer_norm_2 = LayerNorm(10)
self.nce1 = NCE(10000, 100)
self.nce2 = NCE(10000, 100)
self.prelu1 = PRelu("channel", channel=5)
self.prelu2 = PRelu("channel", channel=5)
self.group_norm1 = GroupNorm(8, 4)
self.gourp_norm2 = GroupNorm(8, 4)
self.w_1 = self.create_parameter([100, 100],
dtype='float32',
attr="weight_test_1")
self.w_2 = self.create_parameter([20, 200],
dtype='float32',
attr="weight_test_2")
my_test = MyTest()
my_test.set_dict(new_dict, use_structured_name=False)
for k, v in my_test.state_dict().items():
self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name]))
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle.fluid.core as core
from paddle.fluid.framework import Program, default_startup_program
main_program = default_startup_program()
class TestOperator(unittest.TestCase):
def test_error_type(self):
block = main_program.create_block()
try:
block.append_op()
self.assertFail()
except ValueError as v_err:
self.assertEqual(
v_err.message,
"`type` to initilized an Operator can not be None.")
try:
block.append_op(type="no_such_op")
self.assertFail()
