def train(use_cuda, train_program, params_dirname):
train_reader = paddle.batch(
paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
def event_handler(event):
if isinstance(event, fluid.EndStepEvent):
outs = trainer.test(
reader=test_reader,
feed_order=['firstw', 'secondw', 'thirdw', 'forthw', 'nextw'])
avg_cost = outs[0]
print("loss= ", avg_cost)
if avg_cost < 10.0:
trainer.save_params(params_dirname)
trainer.stop()
if math.isnan(avg_cost):
sys.exit("got NaN loss, training failed.")
trainer = fluid.Trainer(
train_func=train_program,
optimizer=fluid.optimizer.SGD(learning_rate=0.001),
place=place)
trainer.train(
reader=train_reader,
num_epochs=1,
event_handler=event_handler,
feed_order=['firstw', 'secondw', 'thirdw', 'forthw', 'nextw'])
def get_model(args):
# Input data
images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE)
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# Train program
predict = cnn_model(images)
cost = fluid.layers.cross_entropy(input=predict, label=label)
avg_cost = fluid.layers.mean(x=cost)
# Evaluator
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
batch_acc = fluid.layers.accuracy(
input=predict, label=label, total=batch_size_tensor)
# inference program
inference_program = fluid.default_main_program().clone()
# Optimization
opt = fluid.optimizer.AdamOptimizer(
learning_rate=0.001, beta1=0.9, beta2=0.999)
# Reader
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=args.batch_size)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=args.batch_size)
return avg_cost, inference_program, opt, train_reader, test_reader, batch_acc
def train(use_cuda, train_program, params_dirname):
BATCH_SIZE = 128
train_reader = paddle.batch(
paddle.reader.shuffle(
cifar10_small_test_set.train10(batch_size=10), buf_size=128 * 10),
batch_size=BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE)
def event_handler(event):
if isinstance(event, fluid.EndStepEvent):
avg_cost, accuracy = trainer.test(
reader=test_reader, feed_order=['pixel', 'label'])
print('Loss {0:2.2}, Acc {1:2.2}'.format(avg_cost, accuracy))
if accuracy > 0.01: # Low threshold for speeding up CI
if params_dirname is not None:
trainer.save_params(params_dirname)
return
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
trainer = fluid.Trainer(
train_func=train_program,
place=place,
optimizer=fluid.optimizer.Adam(learning_rate=0.001))
trainer.train(
reader=train_reader,
num_epochs=1,
event_handler=event_handler,
feed_order=['pixel', 'label'])
def get_model(args):
if args.data_set == "cifar10":
classdim = 10
if args.data_format == 'NCHW':
data_shape = [3, 32, 32]
else:
data_shape = [32, 32, 3]
else:
classdim = 102
if args.data_format == 'NCHW':
data_shape = [3, 224, 224]
else:
data_shape = [224, 224, 3]
# Input data
images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# Train program
net = vgg16_bn_drop(images)
predict = fluid.layers.fc(input=net, size=classdim, act='softmax')
cost = fluid.layers.cross_entropy(input=predict, label=label)
avg_cost = fluid.layers.mean(x=cost)
# Evaluator
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
batch_acc = fluid.layers.accuracy(
input=predict, label=label, total=batch_size_tensor)
# inference program
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
inference_program = fluid.io.get_inference_program(
target_vars=[batch_acc, batch_size_tensor])
# Optimization
optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
# data reader
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.cifar.train10()
if args.data_set == 'cifar10' else paddle.dataset.flowers.train(),
buf_size=5120),
batch_size=args.batch_size)
test_reader = paddle.batch(
paddle.dataset.cifar.test10()
if args.data_set == 'cifar10' else paddle.dataset.flowers.test(),
batch_size=args.batch_size)
return avg_cost, inference_program, optimizer, train_reader, test_reader, batch_acc
def test_accuracy(executor, inference_program, feed_target_names,
fetch_targets):
if args.data_set == "cifar10":
data_shape = [3, 32, 32]
elif args.data_set == "imagenet":
data_shape = [3, 224, 224]
else:
raise ValueError("%s dataset is not supported" % data_set)
test_reader = paddle.batch(
paddle.dataset.cifar.test10()
if args.data_set == "cifar10" else paddle.dataset.flowers.test(),
batch_size=args.inf_batch_size)
test_num = 0
correct_num = 0
for test_data in test_reader():
test_image = np.array(
map(lambda x: x[0].reshape(data_shape), test_data)).astype(
"float32")
test_label = np.array(map(lambda x: x[1], test_data)).astype("int64")
test_label = test_label.reshape([-1, 1])
results = executor.run(program=inference_program,
feed={feed_target_names[0]: test_image},
fetch_list=fetch_targets)
prediction = np.argmax(results[0], axis=1).reshape([-1, 1])
correct_num += np.sum(prediction == test_label)
test_num += test_label.size
print("{0} out of {1} predictions are correct.".format(correct_num,
test_num))
print("Test accuray is {0}.".format(float(correct_num) / float(test_num)))
def train(use_cuda, train_program, params_dirname):
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
optimizer = fluid.optimizer.Adam(learning_rate=0.001)
trainer = fluid.Trainer(
train_func=train_program, place=place, optimizer=optimizer)
def event_handler(event):
if isinstance(event, fluid.EndEpochEvent):
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
avg_cost, acc = trainer.test(
reader=test_reader, feed_order=['img', 'label'])
print("avg_cost: %s" % avg_cost)
print("acc : %s" % acc)
if acc > 0.2: # Smaller value to increase CI speed
trainer.save_params(params_dirname)
else:
print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format(
event.epoch + 1, avg_cost, acc))
if math.isnan(avg_cost):
sys.exit("got NaN loss, training failed.")
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=500),
batch_size=BATCH_SIZE)
trainer.train(
num_epochs=1,
event_handler=event_handler,
reader=train_reader,
feed_order=['img', 'label'])
def event_handler(event):
if isinstance(event, fluid.EndEpochEvent):
test_reader = paddle.batch(
paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE)
avg_cost, acc = trainer.test(
reader=test_reader, feed_order=['words', 'label'])
print("avg_cost: %s" % avg_cost)
print("acc : %s" % acc)
if acc > 0.2: # Smaller value to increase CI speed
trainer.save_params(params_dirname)
trainer.stop()
else:
print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format(
event.epoch + 1, avg_cost, acc))
if math.isnan(avg_cost):
sys.exit("got NaN loss, training failed.")
elif isinstance(event, fluid.EndStepEvent):
print("Step {0}, Epoch {1} Metrics {2}".format(
event.step, event.epoch, map(np.array, event.metrics)))
if event.step == 1: # Run 2 iterations to speed CI
trainer.save_params(params_dirname)
trainer.stop()
def train(use_cuda, is_sparse, is_local=True):
EPOCH_NUM = 1
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
feed_order = [
'src_word_id', 'target_language_word', 'target_language_next_word'
]
def event_handler(event):
if isinstance(event, fluid.EndStepEvent):
print('pass_id=' + str(event.epoch) + ' batch=' + str(event.step))
if event.step == 10:
trainer.stop()
trainer = fluid.Trainer(
train_func=partial(train_program, is_sparse),
optimizer=fluid.optimizer.Adagrad(
learning_rate=1e-4,
regularization=fluid.regularizer.L2DecayRegularizer(
regularization_coeff=0.1)),
place=place)
trainer.train(
reader=train_reader,
num_epochs=EPOCH_NUM,
event_handler=event_handler,
feed_order=feed_order)
def test_main(self):
main = fluid.Program()
startup = fluid.Program()
startup.random_seed = 1
with fluid.scope_guard(fluid.core.Scope()):
with fluid.program_guard(main, startup):
data = fluid.layers.data(
name='image', shape=[3, 224, 224], dtype='float32')
label = fluid.layers.data(
name='label', shape=[1], dtype='int64')
out = Lenet(data, class_dim=102)
loss = fluid.layers.cross_entropy(input=out, label=label)
loss = fluid.layers.mean(loss)
opt = fluid.optimizer.Momentum(
learning_rate=0.1,
momentum=0.9,
regularization=fluid.regularizer.L2Decay(1e-4))
opt.minimize(loss)
place = fluid.CUDAPlace(0)
feeder = fluid.DataFeeder(place=place, feed_list=[data, label])
reader = feeder.decorate_reader(
paddle.batch(
flowers.train(), batch_size=16), multi_devices=True)
exe = fluid.Executor(place)
exe.run(startup)
pe = fluid.ParallelExecutor(
use_cuda=True, loss_name=loss.name, main_program=main)
for batch_id, data in enumerate(reader()):
loss_np = np.array(pe.run(feed=data, fetch_list=[loss.name])[0])
print batch_id, loss_np
if batch_id == 2:
break
def event_handler(event):
if isinstance(event, fluid.EndEpochEvent):
test_reader = paddle.batch(
paddle.dataset.conll05.test(), batch_size=BATCH_SIZE)
avg_cost_set = trainer.test(
reader=test_reader, feed_order=feed_order)
# get avg cost
avg_cost = np.array(avg_cost_set).mean()
print("avg_cost: %s" % avg_cost)
if float(avg_cost) < 100.0: # Large value to increase CI speed
trainer.save_params(params_dirname)
else:
print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1,
float(avg_cost)))
if math.isnan(float(avg_cost)):
sys.exit("got NaN loss, training failed.")
elif isinstance(event, fluid.EndStepEvent):
print("Step {0}, Epoch {1} Metrics {2}".format(
event.step, event.epoch, map(np.array, event.metrics)))
if event.step == 1: # Run 2 iterations to speed CI
trainer.save_params(params_dirname)
trainer.stop()
def test_fetch_op(self):
tst_reader = paddle.batch(flowers.test(use_xmap=False), batch_size=16)
tst_reader_iter = tst_reader()
iters = 3
train_inputs = []
for i in range(iters):
train_inputs.append(tst_reader_iter.next())
self.parallel_exe(train_inputs, seed=1)
def test_ifelse(self):
prog = Program()
startup_prog = Program()
with program_guard(prog, startup_prog):
image = layers.data(name='x', shape=[784], dtype='float32')
label = layers.data(name='y', shape=[1], dtype='int64')
limit = layers.fill_constant_batch_size_like(
input=label, dtype='int64', shape=[1], value=5.0)
cond = layers.less_than(x=label, y=limit)
ie = layers.IfElse(cond)
with ie.true_block():
true_image = ie.input(image)
hidden = layers.fc(input=true_image, size=100, act='tanh')
prob = layers.fc(input=hidden, size=10, act='softmax')
ie.output(prob)
with ie.false_block():
false_image = ie.input(image)
hidden = layers.fc(input=false_image, size=200, act='tanh')
prob = layers.fc(input=hidden, size=10, act='softmax')
ie.output(prob)
prob = ie()
loss = layers.cross_entropy(input=prob[0], label=label)
avg_loss = layers.mean(loss)
optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9)
optimizer.minimize(avg_loss, startup_prog)
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=8192),
batch_size=200)
place = core.CPUPlace()
exe = Executor(place)
exe.run(kwargs['startup_program'])
PASS_NUM = 100
for pass_id in range(PASS_NUM):
for data in train_reader():
x_data = np.array(map(lambda x: x[0], data)).astype("float32")
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape((y_data.shape[0], 1))
outs = exe.run(kwargs['main_program'],
feed={'x': x_data,
'y': y_data},
fetch_list=[avg_loss])
print outs[0]
if outs[0] < 1.0:
return
self.assertFalse(True)
def train(use_cuda, train_program, params_dirname):
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
optimizer = fluid.optimizer.SGD(learning_rate=0.01)
trainer = fluid.Trainer(
train_func=train_program, place=place, optimizer=optimizer)
feed_order = [
'word_data', 'ctx_n2_data', 'ctx_n1_data', 'ctx_0_data', 'ctx_p1_data',
'ctx_p2_data', 'verb_data', 'mark_data', 'target'
]
#embedding_param = fluid.global_scope().find_var(
# EMBEDDING_NAME).get_tensor()
#embedding_param.set(
# load_parameter(conll05.get_embedding(), WORD_DICT_LEN, WORD_DIM),
# place)
def event_handler(event):
if isinstance(event, fluid.EndEpochEvent):
test_reader = paddle.batch(
paddle.dataset.conll05.test(), batch_size=BATCH_SIZE)
avg_cost_set = trainer.test(
reader=test_reader, feed_order=feed_order)
# get avg cost
avg_cost = np.array(avg_cost_set).mean()
print("avg_cost: %s" % avg_cost)
if float(avg_cost) < 100.0: # Large value to increase CI speed
trainer.save_params(params_dirname)
else:
print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1,
float(avg_cost)))
if math.isnan(float(avg_cost)):
sys.exit("got NaN loss, training failed.")
elif isinstance(event, fluid.EndStepEvent):
print("Step {0}, Epoch {1} Metrics {2}".format(
event.step, event.epoch, map(np.array, event.metrics)))
if event.step == 1: # Run 2 iterations to speed CI
trainer.save_params(params_dirname)
trainer.stop()
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.conll05.test(), buf_size=8192),
batch_size=BATCH_SIZE)
trainer.train(
num_epochs=1,
event_handler=event_handler,
reader=train_reader,
feed_order=feed_order)
def setUpClass(cls):
# Convert mnist to recordio file
with fluid.program_guard(fluid.Program(), fluid.Program()):
reader = paddle.batch(mnist.train(), batch_size=4)
feeder = fluid.DataFeeder(
feed_list=[ # order is image and label
fluid.layers.data(
name='image', shape=[784]),
fluid.layers.data(
name='label', shape=[1], dtype='int64'),
],
place=fluid.CPUPlace())
fluid.recordio_writer.convert_reader_to_recordio_file(
MNIST_RECORDIO_FILE, reader, feeder)
def iter_infer(self, input, feeding=None):
from data_feeder import DataFeeder
feeder = DataFeeder(self.__data_types__, feeding)
batch_size = len(input)
def __reader_impl__():
for each_sample in input:
yield each_sample
reader = paddle.batch(__reader_impl__, batch_size=batch_size)
self.__gradient_machine__.start()
for data_batch in reader():
yield self.__gradient_machine__.forwardTest(feeder(data_batch))
self.__gradient_machine__.finish()
def main():
rnn_out = encoder_decoder()
label = layers.data(
name="target_language_next_word", shape=[1], dtype='int64', lod_level=1)
cost = layers.cross_entropy(input=rnn_out, label=label)
avg_cost = fluid.layers.mean(cost)
optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4)
optimizer.minimize(avg_cost)
# fluid.memory_optimize(fluid.default_main_program())
fluid.release_memory(fluid.default_main_program())
# fix the order of training data
train_data = paddle.batch(
paddle.dataset.wmt14.train(dict_size), batch_size=batch_size)
# train_data = paddle.batch(
# paddle.reader.shuffle(
# paddle.dataset.wmt14.train(dict_size), buf_size=1000),
# batch_size=batch_size)
place = core.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
batch_id = 0
for pass_id in xrange(10):
for data in train_data():
word_data = to_lodtensor(map(lambda x: x[0], data), place)
trg_word = to_lodtensor(map(lambda x: x[1], data), place)
trg_word_next = to_lodtensor(map(lambda x: x[2], data), place)
outs = exe.run(fluid.default_main_program(),
feed={
'src_word_id': word_data,
'target_language_word': trg_word,
'target_language_next_word': trg_word_next
},
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if batch_id > 2:
exit(0)
if math.isnan(float(avg_cost_val)):
sys.exit("got NaN loss, training failed.")
batch_id += 1
def setUpClass(cls):
reader = paddle.batch(
wmt16.train(ModelHyperParams.src_vocab_size,
ModelHyperParams.trg_vocab_size),
batch_size=transformer_model.batch_size)
with fluid.recordio_writer.create_recordio_writer(
WMT16_RECORDIO_FILE) as writer:
for batch in reader():
for tensor in prepare_batch_input(
batch, ModelHyperParams.src_pad_idx,
ModelHyperParams.trg_pad_idx, ModelHyperParams.n_head):
t = fluid.LoDTensor()
t.set(tensor, fluid.CPUPlace())
writer.append_tensor(t)
writer.complete_append_tensor()
def decode_main(use_cuda, is_sparse):
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
context = encoder(is_sparse)
translation_ids, translation_scores = decode(context, is_sparse)
exe = Executor(place)
exe.run(framework.default_startup_program())
init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64')
init_scores_data = np.array(
[1. for _ in range(batch_size)], dtype='float32')
init_ids_data = init_ids_data.reshape((batch_size, 1))
init_scores_data = init_scores_data.reshape((batch_size, 1))
init_lod = [1] * batch_size
init_lod = [init_lod, init_lod]
init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place)
init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
feed_order = ['src_word_id']
feed_list = [
framework.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
for data in train_data():
feed_dict = feeder.feed(map(lambda x: [x[0]], data))
feed_dict['init_ids'] = init_ids
feed_dict['init_scores'] = init_scores
result_ids, result_scores = exe.run(
framework.default_main_program(),
feed=feed_dict,
fetch_list=[translation_ids, translation_scores],
return_numpy=False)
print result_ids.lod()
break
def train(use_cuda, train_program, params_dirname):
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
optimizer = fluid.optimizer.Adagrad(learning_rate=0.002)
word_dict = paddle.dataset.imdb.word_dict()
trainer = fluid.Trainer(
train_func=partial(train_program, word_dict),
place=place,
optimizer=optimizer)
def event_handler(event):
if isinstance(event, fluid.EndEpochEvent):
test_reader = paddle.batch(
paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE)
avg_cost, acc = trainer.test(
reader=test_reader, feed_order=['words', 'label'])
print("avg_cost: %s" % avg_cost)
print("acc : %s" % acc)
if acc > 0.2: # Smaller value to increase CI speed
trainer.save_params(params_dirname)
trainer.stop()
else:
print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format(
event.epoch + 1, avg_cost, acc))
if math.isnan(avg_cost):
sys.exit("got NaN loss, training failed.")
elif isinstance(event, fluid.EndStepEvent):
print("Step {0}, Epoch {1} Metrics {2}".format(
event.step, event.epoch, map(np.array, event.metrics)))
if event.step == 1: # Run 2 iterations to speed CI
trainer.save_params(params_dirname)
trainer.stop()
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.imdb.train(word_dict), buf_size=25000),
batch_size=BATCH_SIZE)
trainer.train(
num_epochs=1,
event_handler=event_handler,
reader=train_reader,
feed_order=['words', 'label'])
def setUp(self):
self.batch_size = 64
# Convert mnist to recordio file
with fluid.program_guard(fluid.Program(), fluid.Program()):
reader = paddle.batch(mnist.train(), batch_size=self.batch_size)
feeder = fluid.DataFeeder(
feed_list=[ # order is image and label
fluid.layers.data(
name='image', shape=[784]),
fluid.layers.data(
name='label', shape=[1], dtype='int64'),
],
place=fluid.CPUPlace())
self.num_batch = fluid.recordio_writer.convert_reader_to_recordio_file(
'./mnist_0.recordio', reader, feeder)
copyfile('./mnist_0.recordio', './mnist_1.recordio')
copyfile('./mnist_0.recordio', './mnist_2.recordio')
def event_handler(event):
if isinstance(event, fluid.EndEpochEvent):
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
avg_cost, acc = trainer.test(
reader=test_reader, feed_order=['img', 'label'])
print("avg_cost: %s" % avg_cost)
print("acc : %s" % acc)
if acc > 0.2: # Smaller value to increase CI speed
trainer.save_params(params_dirname)
else:
print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format(
event.epoch + 1, avg_cost, acc))
if math.isnan(avg_cost):
sys.exit("got NaN loss, training failed.")
def train(use_cuda, train_program, params_dirname):
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
optimizer = fluid.optimizer.SGD(learning_rate=0.2)
trainer = fluid.Trainer(
train_func=train_program, place=place, optimizer=optimizer)
feed_order = [
'user_id', 'gender_id', 'age_id', 'job_id', 'movie_id', 'category_id',
'movie_title', 'score'
]
def event_handler(event):
if isinstance(event, fluid.EndStepEvent):
test_reader = paddle.batch(
paddle.dataset.movielens.test(), batch_size=BATCH_SIZE)
avg_cost_set = trainer.test(
reader=test_reader, feed_order=feed_order)
# get avg cost
avg_cost = np.array(avg_cost_set).mean()
print("avg_cost: %s" % avg_cost)
if float(avg_cost) < 4: # Smaller value to increase CI speed
trainer.save_params(params_dirname)
trainer.stop()
else:
print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1,
float(avg_cost)))
if math.isnan(float(avg_cost)):
sys.exit("got NaN loss, training failed.")
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.movielens.train(), buf_size=8192),
batch_size=BATCH_SIZE)
trainer.train(
num_epochs=1,
event_handler=event_handler,
reader=train_reader,
feed_order=feed_order)
def train(use_cuda, save_dirname=None):
[avg_cost, prediction] = seq_to_seq_net()
optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4)
optimizer.minimize(avg_cost)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.wmt14.train(dict_size), buf_size=1000),
batch_size=batch_size)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
exe.run(framework.default_startup_program())
feed_order = ['source_sequence', 'target_sequence', 'label_sequence']
feed_list = [
framework.default_main_program().global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
batch_id = 0
for pass_id in xrange(2):
for data in train_data():
outs = exe.run(framework.default_main_program(),
feed=feeder.feed(data),
fetch_list=[avg_cost])
avg_cost_val = np.array(outs[0])
print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
" avg_cost=" + str(avg_cost_val))
if math.isnan(float(avg_cost_val[0])):
sys.exit("got NaN loss, training failed.")
if batch_id > 3:
if save_dirname is not None:
fluid.io.save_inference_model(
save_dirname, ['source_sequence',
'target_sequence'], [prediction], exe)
return
batch_id += 1
def event_handler(event):
if isinstance(event, fluid.EndStepEvent):
test_reader = paddle.batch(
paddle.dataset.movielens.test(), batch_size=BATCH_SIZE)
avg_cost_set = trainer.test(
reader=test_reader, feed_order=feed_order)
# get avg cost
avg_cost = np.array(avg_cost_set).mean()
print("avg_cost: %s" % avg_cost)
if float(avg_cost) < 4: # Smaller value to increase CI speed
trainer.save_params(params_dirname)
trainer.stop()
else:
print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1,
float(avg_cost)))
if math.isnan(float(avg_cost)):
sys.exit("got NaN loss, training failed.")
redis_port = 6379
redis_password = ""
r = redis.StrictRedis(host=redis_host,
port=redis_port,
password=redis_password)
# reader generation
reader = fluid.layers.py_reader(capacity=64,
shapes=[(-1, 64, 8, 8), (-1, 1)],
dtypes=['float32', 'int64'])
samples = r.keys()
train_data = data_generater(samples, r)
reader.decorate_paddle_reader(
paddle.batch(paddle.reader.shuffle(train_data, buf_size=5000),
batch_size=64))
conv1, label = fluid.layers.read_file(reader)
# train program
place = fluid.CUDAPlace(0)
model = ResNet(layers=50)
predicts = model.net(conv1, 10)
cost = fluid.layers.cross_entropy(input=predicts, label=label)
accuracy = fluid.layers.accuracy(input=predicts, label=label)
loss = fluid.layers.mean(cost)
optimizer = fluid.optimizer.Adam(learning_rate=0.001)
optimizer.minimize(loss)
exe = fluid.Executor(place)
def train(nn_type,
use_cuda,
save_dirname=None,
model_filename=None,
params_filename=None):
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
startup_program = fluid.default_startup_program()
main_program = fluid.default_main_program()
if args.enable_ce:
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
startup_program.random_seed = 90
main_program.random_seed = 90
else:
train_reader = paddle.batch(
paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500),
batch_size=BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
img = fluid.data(name='img', shape=[None, 1, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
if nn_type == 'softmax_regression':
net_conf = softmax_regression
elif nn_type == 'multilayer_perceptron':
net_conf = multilayer_perceptron
else:
net_conf = convolutional_neural_network
prediction, avg_loss, acc = net_conf(img, label)
test_program = main_program.clone(for_test=True)
optimizer = fluid.optimizer.Adam(learning_rate=0.001) # 使用Adam优化器
optimizer.minimize(avg_loss)
def train_test(train_test_program, train_test_feed, train_test_reader):
acc_set = []
avg_loss_set = []
for test_data in train_test_reader():
acc_np, avg_loss_np = exe.run(
program=train_test_program,
feed=train_test_feed.feed(test_data),
fetch_list=[acc, avg_loss])
acc_set.append(float(acc_np))
avg_loss_set.append(float(avg_loss_np))
# get test acc and loss
acc_val_mean = numpy.array(acc_set).mean()
avg_loss_val_mean = numpy.array(avg_loss_set).mean()
return avg_loss_val_mean, acc_val_mean
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[img, label], place=place)
exe.run(startup_program)
epochs = [epoch_id for epoch_id in range(PASS_NUM)]
lists = []
step = 0
for epoch_id in epochs:
for step_id, data in enumerate(train_reader()):
metrics = exe.run(
main_program,
feed=feeder.feed(data),
fetch_list=[avg_loss, acc])
if step % 100 == 0:
print("Pass %d, Epoch %d, Cost %f" % (step, epoch_id,
metrics[0]))
step += 1
# test for epoch
avg_loss_val, acc_val = train_test(
train_test_program=test_program,
train_test_reader=test_reader,
train_test_feed=feeder)
print("Test with Epoch %d, avg_cost: %s, acc: %s" %
(epoch_id, avg_loss_val, acc_val))
lists.append((epoch_id, avg_loss_val, acc_val))
if save_dirname is not None:
fluid.io.save_inference_model(
save_dirname, ["img"], [prediction],
exe,
model_filename=model_filename,
params_filename=params_filename)
if args.enable_ce:
print("kpis\ttrain_cost\t%f" % metrics[0])
print("kpis\ttest_cost\t%s" % avg_loss_val)
print("kpis\ttest_acc\t%s" % acc_val)
# find the best pass
best = sorted(lists, key=lambda list: float(list[1]))[0]
print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1]))
print('The classification accuracy is %.2f%%' % (float(best[2]) * 100))
def train(word_dict,
net_method,
use_cuda,
seed,
quality,
save_dirname=None):
BATCH_SIZE = 128
PASS_NUM = 100
dict_dim = len(word_dict)
class_dim = 2
target_val_acc = quality
# Seed for batch producer
random.seed(seed)
# Seed for weight initialization
fluid.default_startup_program().random_seed = seed
# Setup input features and label as data layers
data = fluid.layers.data(
name="words", shape=[1], dtype="int64", lod_level=1)
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
cost, acc_out, prediction = net_method(
data, label, input_dim=dict_dim, class_dim=class_dim)
# Initialize a test program for obtaining test accuracy and cost
test_program = fluid.default_main_program().clone(for_test=True)
# Setup Adam optimizer
adam = fluid.optimizer.Adam(learning_rate=0.0005) #Learning rate of 5e-4 works for conv models and 2e-3 for LSTM model
optimize_ops, params_grads = adam.minimize(cost)
# Create reader to iterate over training set
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.imdb.train(word_dict), buf_size=25000),
batch_size=BATCH_SIZE)
# Setup place and executor for runtime
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[data, label], place=place)
# Create reader to iterate over validation set
test_reader = paddle.batch(
paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE)
def train_loop(main_program):
exe.run(fluid.default_startup_program())
for pass_id in xrange(PASS_NUM):
train_loss_set = []
train_acc_set = []
# Calculate average training loss and accuracy
# across all mini-batches in the training set
for batch_id, data in enumerate(train_reader()):
cost_val, acc_val = exe.run(main_program,
feed=feeder.feed(data),
fetch_list=[cost, acc_out])
train_loss_set.append(float(cost_val))
train_acc_set.append(float(acc_val))
train_loss = np.array(train_loss_set).mean()
train_acc = np.array(train_acc_set).mean() * 100
# Calculate average valication loss and accuracy
# across all mini-batches in the validation set
acc_set = []
avg_loss_set = []
for tid, test_data in enumerate(test_reader()):
avg_loss_np, acc_np = exe.run(
program=test_program,
feed=feeder.feed(test_data),
fetch_list=[cost, acc_out])
acc_set.append(float(acc_np))
avg_loss_set.append(float(avg_loss_np))
acc_val = np.array(acc_set).mean() * 100
avg_loss_val = np.array(avg_loss_set).mean()
print("Epoch =", pass_id, ", train-accuracy =", train_acc, ", train-loss =", train_loss, ", validation-accuracy =", acc_val, ", validation-loss =", avg_loss_val)
if acc_val > target_val_acc:
## Exit the program on reaching desired accuracy value
break
train_loop(fluid.default_main_program())
avg_cost_clip = prog_clip.block(0).var(avg_cost.name)
p_g = fluid.backward.append_backward(loss=avg_cost)
p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip)
with fluid.program_guard(main_program=prog_clip):
fluid.clip.set_gradient_clip(
fluid.clip.GradientClipByGlobalNorm(clip_norm=CLIP))
p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip)
grad_list = [elem[1] for elem in p_g]
grad_clip_list = [elem[1] for elem in p_g_clip]
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=8192),
batch_size=BATCH_SIZE)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[image, label], place=place)
exe.run(fluid.default_startup_program())
count = 0
for data in train_reader():
count += 1
if count > 5:
break
out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list)
out_clip = exe.run(prog_clip,
feed=feeder.feed(data),
def train(use_cuda, save_dirname, is_local=True):
scale_infer, avg_cost = model()
# test program
test_program = fluid.default_main_program().clone(for_test=True)
sgd_optimizer = SGDOptimizer(learning_rate=0.2)
sgd_optimizer.minimize(avg_cost)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.movielens.train(), buf_size=8192),
batch_size=BATCH_SIZE)
test_reader = paddle.batch(paddle.dataset.movielens.test(),
batch_size=BATCH_SIZE)
feed_order = [
'user_id', 'gender_id', 'age_id', 'job_id', 'movie_id', 'category_id',
'movie_title', 'score'
]
def train_loop(main_program):
exe.run(framework.default_startup_program())
feed_list = [
main_program.global_block().var(var_name)
for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
PASS_NUM = 100
for pass_id in range(PASS_NUM):
for batch_id, data in enumerate(train_reader()):
# train a mini-batch
outs = exe.run(program=main_program,
feed=feeder.feed(data),
fetch_list=[avg_cost])
out = np.array(outs[0])
if (batch_id + 1) % 10 == 0:
avg_cost_set = []
for test_data in test_reader():
avg_cost_np = exe.run(program=test_program,
feed=feeder.feed(test_data),
fetch_list=[avg_cost])
avg_cost_set.append(avg_cost_np[0])
break # test only 1 segment for speeding up CI
# get test avg_cost
test_avg_cost = np.array(avg_cost_set).mean()
if test_avg_cost < 6.0:
# if avg_cost less than 6.0, we think our code is good.
if save_dirname is not None:
fluid.io.save_inference_model(
save_dirname, [
"user_id", "gender_id", "age_id", "job_id",
"movie_id", "category_id", "movie_title"
], [scale_infer], exe)
return
if math.isnan(float(out[0])):
sys.exit("got NaN loss, training failed.")
if is_local:
train_loop(fluid.default_main_program())
else:
port = os.getenv("PADDLE_PSERVER_PORT", "6174")
pserver_ips = os.getenv("PADDLE_PSERVER_IPS") # ip,ip...
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist) # ip:port,ip:port...
trainers = int(os.getenv("PADDLE_TRAINERS"))
current_endpoint = os.getenv("POD_IP") + ":" + port
trainer_id = int(os.getenv("PADDLE_TRAINER_ID"))
training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER")
t = fluid.DistributeTranspiler()
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
if training_role == "PSERVER":
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint,
pserver_prog)
exe.run(pserver_startup)
exe.run(pserver_prog)
elif training_role == "TRAINER":
train_loop(t.get_trainer_program())
def test_se_resnext_float32(self):
seed = 90
batch_size = train_parameters["batch_size"]
batch_num = 1
epoch_num = 1
with fluid.dygraph.guard():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
se_resnext = SeResNeXt("se_resnext")
optimizer = optimizer_setting(train_parameters)
np.random.seed(seed)
import random
random.seed = seed
batch_py_reader = fluid.io.PyReader(capacity=1)
batch_py_reader.decorate_sample_list_generator(
paddle.batch(self.reader_decorator(
paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True),
places=fluid.CPUPlace())
dy_param_init_value = {}
for param in se_resnext.parameters():
dy_param_init_value[param.name] = param.numpy()
for epoch_id in range(epoch_num):
for batch_id, data in enumerate(batch_py_reader()):
if batch_id >= batch_num and batch_num != -1:
break
img = data[0]
label = data[1]
label.stop_gradient = True
label.stop_gradient = True
out = se_resnext(img)
loss = fluid.layers.cross_entropy(input=out, label=label)
avg_loss = fluid.layers.mean(x=loss)
dy_out = avg_loss.numpy()
if batch_id == 0:
for param in se_resnext.parameters():
if param.name not in dy_param_init_value:
dy_param_init_value[param.name] = param.numpy()
avg_loss.backward()
#dy_grad_value = {}
#for param in se_resnext.parameters():
# if param.trainable:
# np_array = np.array(param._ivar._grad_ivar().value()
# .get_tensor())
# dy_grad_value[param.name + core.grad_var_suffix()] = np_array
optimizer.minimize(avg_loss)
se_resnext.clear_gradients()
dy_param_value = {}
for param in se_resnext.parameters():
dy_param_value[param.name] = param.numpy()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
exe = fluid.Executor(fluid.CPUPlace(
) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0))
se_resnext = SeResNeXt("se_resnext")
optimizer = optimizer_setting(train_parameters)
np.random.seed(seed)
import random
random.seed = seed
train_reader = paddle.batch(
paddle.dataset.flowers.train(use_xmap=False),
batch_size=batch_size,
drop_last=True)
img = fluid.layers.data(name='pixel',
shape=[3, 224, 224],
dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
out = se_resnext(img)
loss = fluid.layers.cross_entropy(input=out, label=label)
avg_loss = fluid.layers.mean(x=loss)
optimizer.minimize(avg_loss)
# initialize params and fetch them
static_param_init_value = {}
static_param_name_list = []
static_grad_name_list = []
for param in se_resnext.parameters():
static_param_name_list.append(param.name)
for param in se_resnext.parameters():
if param.trainable:
static_grad_name_list.append(param.name +
core.grad_var_suffix())
out = exe.run(fluid.default_startup_program(),
fetch_list=static_param_name_list)
for i in range(len(static_param_name_list)):
static_param_init_value[static_param_name_list[i]] = out[i]
for epoch_id in range(epoch_num):
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num and batch_num != -1:
break
static_x_data = np.array([
x[0].reshape(3, 224, 224) for x in data
]).astype('float32')
y_data = np.array([x[1]
for x in data]).astype('int64').reshape(
[batch_size, 1])
fetch_list = [avg_loss.name]
fetch_list.extend(static_param_name_list)
fetch_list.extend(static_grad_name_list)
out = exe.run(fluid.default_main_program(),
feed={
"pixel": static_x_data,
"label": y_data
},
fetch_list=fetch_list)
static_param_value = {}
static_grad_value = {}
static_out = out[0]
param_start_pos = 1
grad_start_pos = len(
static_param_name_list) + param_start_pos
for i in range(
param_start_pos,
len(static_param_name_list) + param_start_pos):
static_param_value[static_param_name_list[
i - param_start_pos]] = out[i]
for i in range(grad_start_pos,
len(static_grad_name_list) +
grad_start_pos):
static_grad_value[static_grad_name_list[
i - grad_start_pos]] = out[i]
self.assertTrue(np.allclose(static_out, dy_out))
self.assertEqual(len(dy_param_init_value),
len(static_param_init_value))
for key, value in six.iteritems(static_param_init_value):
self.assertTrue(np.allclose(value, dy_param_init_value[key]))
self.assertTrue(np.isfinite(value.all()))
self.assertFalse(np.isnan(value.any()))
# FIXME(Yancey1989): np.array(_ivar.value().get_tensor()) leads to memory lake
#self.assertEqual(len(dy_grad_value), len(static_grad_value))
#for key, value in six.iteritems(static_grad_value):
# self.assertTrue(np.allclose(value, dy_grad_value[key]))
# self.assertTrue(np.isfinite(value.all()))
# self.assertFalse(np.isnan(value.any()))
self.assertEqual(len(dy_param_value), len(static_param_value))
for key, value in six.iteritems(static_param_value):
self.assertTrue(np.allclose(value, dy_param_value[key]))
self.assertTrue(np.isfinite(value.all()))
self.assertFalse(np.isnan(value.any()))
def local_train(learning_rate, batch_size, num_passes, model_save_dir='model'):
class_dim = 102
image_shape = [3, 224, 224]
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
out = mobile_net(image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=0.9,
regularization=fluid.regularizer.L2Decay(5 * 1e-5))
# optimizer = fluid.optimizer.SGD(learning_rate=learning_rate)
opts = optimizer.minimize(avg_cost)
b_size_var = fluid.layers.create_tensor(dtype='int64')
b_acc_var = fluid.layers.accuracy(input=out, label=label, total=b_size_var)
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
inference_program = fluid.io.get_inference_program(
target_vars=[b_acc_var, b_size_var])
place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(
args.device_id)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
train_reader = paddle.batch(paddle.dataset.flowers.train(),
batch_size=batch_size)
test_reader = paddle.batch(paddle.dataset.flowers.test(),
batch_size=batch_size)
feeder = fluid.DataFeeder(place=place, feed_list=[image, label])
train_pass_acc_evaluator = fluid.average.WeightedAverage()
test_pass_acc_evaluator = fluid.average.WeightedAverage()
train_proc = fluid.default_main_program()
for pass_id in range(num_passes):
start = time.clock()
train_pass_acc_evaluator.reset()
for batch_id, data in enumerate(train_reader()):
loss, acc, size = exe.run(
train_proc,
feed=feeder.feed(data),
fetch_list=[avg_cost, b_acc_var, b_size_var])
train_pass_acc_evaluator.add(value=acc, weight=size)
print("Pass {0}, batch {1}, loss {2}, acc {3}".format(
pass_id, batch_id, loss[0], acc[0]))
test_pass_acc_evaluator.reset()
for data in test_reader():
acc, size = exe.run(inference_program,
feed=feeder.feed(data),
fetch_list=[b_acc_var, b_size_var])
test_pass_acc_evaluator.add(value=acc, weight=size)
print("End pass {0}, train_acc {1}, test_acc {2}, cost {3} second".
format(pass_id, train_pass_acc_evaluator.eval(),
test_pass_acc_evaluator.eval(),
time.clock() - start))
def evaluate():
place = fluid.CUDAPlace(0) if cfg.use_cuda else fluid.CPUPlace()
inference_scope = fluid.Scope()
test_files = [
os.path.join(cfg.evaluate_file_path, x) for x in os.listdir(cfg.evaluate_file_path)
]
dataset = CriteoDataset()
test_reader = paddle.batch(dataset.test(test_files), batch_size=cfg.batch_size)
with fluid.dygraph.guard(place):
if cfg.train_model == 'drnn':
model = DRNN()
elif cfg.train_model == 'dnn':
model = DNN()
elif cfg.train_model == 'fcdnn':
model = FCDNN()
model_path = os.path.join(cfg.save_path, model.name, model.name + "_epoch_" + str(cfg.test_epoch))
model_dict, optimizer_dict = fluid.dygraph.load_dygraph(model_path)
model.set_dict(model_dict)
logger.info("load model {} finished.".format(model_path))
model.eval()
logger.info('Begin evaluate model.')
run_index = 0
infer_auc = 0.0
L = []
for batch_id, data in enumerate(test_reader()):
dense_feature, sparse_feature, label = zip(*data)
sparse_feature = np.array(sparse_feature, dtype=np.int64)
dense_feature = np.array(dense_feature, dtype=np.float32)
label = np.array(label, dtype=np.int64)
sparse_feature, dense_feature, label = [
to_variable(i)
for i in [sparse_feature, dense_feature, label]
]
avg_cost, auc_var = model(dense_feature, sparse_feature, label)
run_index += 1
infer_auc += auc_var.numpy().item()
L.append(avg_cost.numpy() / cfg.batch_size)
if batch_id % cfg.log_interval == 0:
logger.info("TEST --> batch: {} loss: {} auc: {}".format(
batch_id, avg_cost.numpy() / cfg.batch_size, infer_auc / run_index))
infer_loss = np.mean(L)
infer_auc = infer_auc / run_index
infer_result = {}
infer_result['loss'] = infer_loss
infer_result['auc'] = infer_auc
if not os.path.isdir(cfg.log_dir):
os.makedirs(cfg.log_dir)
log_path = os.path.join(cfg.log_dir, model.name + '_infer_result.log')
logger.info(str(infer_result))
with open(log_path, 'w+') as f:
f.write(str(infer_result))
logger.info("Done.")
return infer_result
def eval(args):
# parameters from arguments
seg_num = args.seg_num
class_dim = args.class_dim
num_layers = args.num_layers
batch_size = args.batch_size
test_model = args.test_model
if test_model == None:
print('Please specify the test model ...')
return
image_shape = [int(m) for m in args.image_shape.split(",")]
image_shape = [seg_num] + image_shape
# model definition
model = TSN_ResNet(layers=num_layers, seg_num=seg_num)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
out = model.net(input=image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
# for test
inference_program = fluid.default_main_program().clone(for_test=True)
if args.with_mem_opt:
fluid.memory_optimize(fluid.default_main_program())
place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
def is_parameter(var):
if isinstance(var, Parameter):
return isinstance(var, Parameter)
if test_model is not None:
vars = filter(is_parameter, inference_program.list_vars())
fluid.io.load_vars(exe, test_model, vars=vars)
# reader
test_reader = paddle.batch(reader.test(seg_num),
batch_size=batch_size / 16)
feeder = fluid.DataFeeder(place=place, feed_list=[image, label])
fetch_list = [avg_cost.name, acc_top1.name, acc_top5.name]
# test
cnt = 0
pass_id = 0
test_info = [[], [], []]
for batch_id, data in enumerate(test_reader()):
t1 = time.time()
loss, acc1, acc5 = exe.run(inference_program,
fetch_list=fetch_list,
feed=feeder.feed(data))
t2 = time.time()
period = t2 - t1
loss = np.mean(loss)
acc1 = np.mean(acc1)
acc5 = np.mean(acc5)
test_info[0].append(loss * len(data))
test_info[1].append(acc1 * len(data))
test_info[2].append(acc5 * len(data))
cnt += len(data)
if batch_id % 10 == 0:
print(
"[TEST] Pass: {0}\ttestbatch: {1}\tloss: {2}\tacc1: {3}\tacc5: {4}\ttime: {5}"
.format(pass_id, batch_id, '%.6f' % loss, acc1, acc5,
"%2.2f sec" % period))
sys.stdout.flush()
test_loss = np.sum(test_info[0]) / cnt
test_acc1 = np.sum(test_info[1]) / cnt
test_acc5 = np.sum(test_info[2]) / cnt
print("+ End pass: {0}, test_loss: {1}, test_acc1: {2}, test_acc5: {3}".
format(pass_id, '%.3f' % test_loss, '%.3f' % test_acc1,
'%.3f' % test_acc5))
sys.stdout.flush()
#导入必要的库
import paddle.fluid as fluid
import paddle
import numpy as np
import os
# 从paddle接口中获取房价数据训练集
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.uci_housing.train(), #获取uci_housing训练数据
buf_size=500), #在buf_size的空间内进行乱序
batch_size=20) #batch_size:每个批次读入的训练数据量
# 从paddle接口中获取房价数据测试集
test_reader = paddle.batch(
paddle.reader.shuffle(paddle.dataset.uci_housing.test(),
buf_size=500), #获取uci_housing的测试数据
batch_size=20) #batch_size:每个批次读入的测试数据量
#定义一个简单的线性网络，网络有1层。输入层为x，输出大小为1，激活函数是relu的全连接层
#定义张量变量x，表示13维的特征值
x = fluid.layers.data(name='x', shape=[13], dtype='float32')
#定义一个简单的线性网络
net = fluid.layers.fc(input=x, size=1, act=None)
# 定义损失函数
y = fluid.layers.data(name='y', shape=[1], dtype='float32') #定义张量y,表示目标值
cost = fluid.layers.square_error_cost(input=net, label=y) #求一个batch的损失值
avg_cost = fluid.layers.mean(cost) #对损失值求平均值
# 定义优化方法
optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.001)
opts = optimizer.minimize(avg_cost)
# 创建一个使用CPU的解释器
def freeze_program(self, use_cuda, seed):
def build_program(main, startup, is_test):
main.random_seed = seed
startup.random_seed = seed
with fluid.unique_name.guard():
with fluid.program_guard(main, startup):
img = fluid.layers.data(name='image',
shape=[1, 28, 28],
dtype='float32')
label = fluid.layers.data(name='label',
shape=[1],
dtype='int64')
loss = conv_net(img, label)
if not is_test:
opt = fluid.optimizer.Adam(learning_rate=0.001)
opt.minimize(loss)
return [img, label], loss
main = fluid.Program()
startup = fluid.Program()
test_program = fluid.Program()
import random
random.seed(0)
np.random.seed(0)
feeds, loss = build_program(main, startup, False)
build_program(test_program, startup, True)
test_program = test_program.clone(for_test=True)
quant_transpiler = QuantizeTranspiler()
quant_transpiler.training_transpile(main)
quant_transpiler.training_transpile(test_program)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
iters = 5
batch_size = 8
class_num = 10
exe.run(startup)
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=500),
batch_size=batch_size)
test_reader = paddle.batch(paddle.dataset.mnist.test(),
batch_size=batch_size)
feeder = fluid.DataFeeder(feed_list=feeds, place=place)
with fluid.program_guard(main):
for _ in range(iters):
data = next(train_reader())
loss_v = exe.run(program=main,
feed=feeder.feed(data),
fetch_list=[loss])
with fluid.program_guard(test_program):
test_data = next(test_reader())
w_var = fluid.framework._get_var('conv2d_1.w_0.quantized',
test_program)
# Testing during training
test_loss1, w_quant = exe.run(program=test_program,
feed=feeder.feed(test_data),
fetch_list=[loss, w_var])
# Freeze program for inference, but the weight of fc/conv is still float type.
quant_transpiler.freeze_program(test_program, place)
test_loss2, = exe.run(program=test_program,
feed=feeder.feed(test_data),
fetch_list=[loss])
self.assertAlmostEqual(test_loss1, test_loss2, delta=5e-3)
w_freeze = np.array(
fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())
# fail: -432.0 != -433.0, this is due to the calculation precision
#self.assertAlmostEqual(np.sum(w_freeze), np.sum(w_quant))
# Convert parameter to 8-bit.
quant_transpiler.convert_to_int8(test_program, place)
# Save the 8-bit parameter and model file.
fluid.io.save_inference_model('model_8bit', ['image', 'label'],
[loss], exe, test_program)
# Test whether the 8-bit parameter and model file can be loaded successfully.
[infer, feed,
fetch] = fluid.io.load_inference_model('model_8bit', exe)
# Check the loaded 8-bit weight.
w_8bit = np.array(fluid.global_scope().find_var(
'conv2d_1.w_0.int8').get_tensor())
self.assertEqual(w_8bit.dtype, np.int8)
self.assertEqual(np.sum(w_8bit), np.sum(w_freeze))
def do_training(self, fleet, args):
"""
training_from_pyreader
Args:
fleet (DistributedTranspiler): DistributedTranspiler inherited base class Fleet
args (ArgumentParser): run args to config dist fleet.
Returns:
list
"""
if args.run_params.get("increment", False):
global DATA_PATH
DATA_PATH = DATA_PATH + '.increment'
exe = fluid.Executor(fluid.CPUPlace())
fleet.init_worker()
exe.run(fleet.startup_program)
train_generator = py_reader1.CriteoDataset(1000001)
file_list = [str(DATA_PATH)] * 2
train_reader = paddle.batch(
train_generator.train(file_list, args.trainers, args.current_id),
batch_size=4)
self.pyreader.decorate_paddle_reader(train_reader)
if os.getenv("PADDLE_COMPATIBILITY_CHECK", False):
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = int(2)
build_strategy = fluid.BuildStrategy()
build_strategy.async_mode = self.async_mode
if args.run_params["sync_mode"] == "async":
build_strategy.memory_optimize = False
if args.run_params['cpu_num'] > 1:
build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
else:
build_strategy = self.strategy.get_build_strategy()
if args.run_params["sync_mode"] == "async":
build_strategy.memory_optimize = False
self.strategy.set_build_strategy(build_strategy)
exec_strategy = self.strategy.get_execute_strategy()
compiled_prog = fluid.compiler.CompiledProgram(
fleet.main_program).with_data_parallel(
loss_name=self.avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
# Notice: py_reader should use try & catch EOFException method to enter the dataset
# reader.start() must declare in advance
self.pyreader.start()
train_info = []
batch_id = 0
try:
while True:
avg_cost = exe.run(program=compiled_prog,
fetch_list=[self.avg_cost.name])
avg_cost = np.mean(avg_cost)
train_info.append(avg_cost)
batch_id += 1
if batch_id == args.run_params.get("total_batch_size", 5):
if params["is_first_trainer"]:
if params["is_pyreader_train"]:
model_path = str(params["model_path"] + "/final" +
"_pyreader")
fleet.save_persistables(
executor=fluid.Executor(fluid.CPUPlace()),
dirname=model_path)
elif params["is_dataset_train"]:
model_path = str(params["model_path"] + '/final' +
"_dataset")
fleet.save_persistables(
executor=fluid.Executor(fluid.CPUPlace()),
dirname=model_path)
else:
raise ValueError(
"Program must has Date feed method: is_pyreader_train / is_dataset_train"
)
break
except fluid.core.EOFException:
self.pyreader.reset()
fleet.stop_worker()
return train_info
def do_compress(args):
train_program = fluid.default_main_program()
startup_program = fluid.default_startup_program()
dataset = reader.Dataset(args)
with fluid.program_guard(train_program, startup_program):
train_program.random_seed = args.random_seed
startup_program.random_seed = args.random_seed
with fluid.unique_name.guard():
train_ret = creator.create_model(args,
dataset.vocab_size,
dataset.num_labels,
mode='train')
test_program = train_program.clone()
optimizer = fluid.optimizer.Adam(learning_rate=args.base_learning_rate)
# init executor
if args.use_cuda:
place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
dev_count = fluid.core.get_cuda_device_count()
else:
dev_count = min(multiprocessing.cpu_count(), args.cpu_num)
if (dev_count < args.cpu_num):
print(
"WARNING: The total CPU NUM in this machine is %d, which is less than cpu_num parameter you set. "
"Change the cpu_num from %d to %d" %
(dev_count, args.cpu_num, dev_count))
os.environ['CPU_NUM'] = str(dev_count)
place = fluid.CPUPlace()
train_reader = paddle.batch(dataset.file_reader(args.train_data),
batch_size=args.batch_size)
test_reader = paddle.batch(dataset.file_reader(args.test_data),
batch_size=args.batch_size)
exe = fluid.Executor(place)
exe.run(startup_program)
if args.init_checkpoint:
utils.init_checkpoint(exe, args.init_checkpoint + '.pdckpt',
train_program)
train_feed_list = [('words', train_ret['words'].name),
("targets", train_ret["targets"].name)]
train_fetch_list = [('loss', train_ret['avg_cost'].name)]
test_feed_list = [('words', train_ret['words'].name),
("targets", train_ret["targets"].name)]
test_fetch_list = [('f1_score', train_ret['f1_score'].name)]
print(train_ret['crf_decode'].name)
com_pass = Compressor(place,
fluid.global_scope(),
train_program=train_program,
train_reader=train_reader,
train_feed_list=train_feed_list,
train_fetch_list=train_fetch_list,
eval_program=test_program,
eval_reader=test_reader,
eval_feed_list=test_feed_list,
eval_fetch_list=test_fetch_list,
teacher_programs=[],
train_optimizer=optimizer,
distiller_optimizer=None)
com_pass.config(args.compress_config)
com_pass.run()
def main(dict_path):
word_dict = load_vocab(dict_path)
word_dict["<unk>"] = len(word_dict)
dict_dim = len(word_dict)
print("The dictionary size is : %d" % dict_dim)
data, label, prediction, avg_cost = conv_net(dict_dim)
sgd_optimizer = fluid.optimizer.SGD(learning_rate=conf.learning_rate)
optimize_ops, params_grads = sgd_optimizer.minimize(avg_cost)
batch_size_var = fluid.layers.create_tensor(dtype='int64')
batch_acc_var = fluid.layers.accuracy(input=prediction,
label=label,
total=batch_size_var)
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
inference_program = fluid.io.get_inference_program(
target_vars=[batch_acc_var, batch_size_var])
# The training data set.
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.imdb.train(word_dict), buf_size=51200),
batch_size=conf.batch_size)
# The testing data set.
test_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.imdb.test(word_dict), buf_size=51200),
batch_size=conf.batch_size)
if conf.use_gpu:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[data, label], place=place)
train_pass_acc_evaluator = fluid.average.WeightedAverage()
test_pass_acc_evaluator = fluid.average.WeightedAverage()
def test(exe):
test_pass_acc_evaluator.reset()
for batch_id, data in enumerate(test_reader()):
input_seq = to_lodtensor(map(lambda x: x[0], data), place)
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
b_acc, b_size = exe.run(inference_program,
feed={
"words": input_seq,
"label": y_data
},
fetch_list=[batch_acc_var, batch_size_var])
test_pass_acc_evaluator.add(value=b_acc, weight=b_size)
test_acc = test_pass_acc_evaluator.eval()
return test_acc
def train_loop(exe, train_program, trainer_id):
total_time = 0.
for pass_id in xrange(conf.num_passes):
train_pass_acc_evaluator.reset()
start_time = time.time()
total_samples = 0
with profiler.profiler("CPU",
'total',
profile_path='./profile_res_%d' %
trainer_id) as prof:
for batch_id, data in enumerate(train_reader()):
batch_start = time.time()
cost_val, acc_val, size_val = exe.run(
train_program,
feed=feeder.feed(data),
fetch_list=[avg_cost, batch_acc_var, batch_size_var])
train_pass_acc_evaluator.add(value=acc_val,
weight=size_val)
total_samples += float(size_val)
if batch_id and batch_id % conf.log_period == 0:
print(
"Pass id: %d, batch id: %d, cost: %f, pass_acc: %f, speed: %f, time: %f"
%
(pass_id, batch_id, cost_val,
train_pass_acc_evaluator.eval(), float(size_val) /
(time.time() - batch_start),
time.time() - batch_start))
end_time = time.time()
total_time += (end_time - start_time)
pass_test_acc = test(exe)
print("Pass id: %d, test_acc: %f, speed: %f" %
(pass_id, pass_test_acc, total_samples /
(end_time - start_time)))
print("Total train time: %f" % (total_time))
if args.local:
print("run as local mode")
exe.run(fluid.default_startup_program())
train_loop(exe, fluid.default_main_program(), 0)
else:
pserver_ips = os.getenv(
"PADDLE_INIT_PSERVERS") # all pserver endpoints
eplist = []
port = os.getenv("PADDLE_INIT_PORT")
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist)
print("pserver endpoints: ", pserver_endpoints)
trainers = int(os.getenv("TRAINERS")) # total trainer count
print("trainers total: ", trainers)
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID", "0"))
current_endpoint = os.getenv(
"POD_IP") + ":" + port # current pserver endpoint
training_role = os.getenv(
"TRAINING_ROLE",
"TRAINER") # get the training role: trainer/pserver
t = fluid.DistributeTranspiler()
t.transpile(optimize_ops,
params_grads,
trainer_id,
pservers=pserver_endpoints,
trainers=trainers)
if training_role == "PSERVER":
if not current_endpoint:
print("need env SERVER_ENDPOINT")
exit(1)
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint,
pserver_prog)
print("starting server side startup")
exe.run(pserver_startup)
print("starting parameter server...")
exe.run(pserver_prog)
elif training_role == "TRAINER":
trainer_prog = t.get_trainer_program()
exe.run(fluid.default_startup_program())
train_loop(exe, trainer_prog, trainer_id)
else:
print("environment var TRAINER_ROLE should be TRAINER os PSERVER")
def train(conf_dict, data_reader, use_cuda=False):
"""
Training of p classification model
"""
label_dict_len = data_reader.get_dict_size('label_dict')
# input layer
word = fluid.layers.data(name='word_data', shape=[1], dtype='int64', lod_level=1)
postag = fluid.layers.data(name='token_pos', shape=[1], dtype='int64', lod_level=1)
arc_relation = fluid.layers.data(name="arc_relation", shape=[1], dtype='int64', lod_level=1)
arc_head = fluid.layers.data(name="arc_head", shape=[1], dtype="int64", lod_level=1)
# label
target = fluid.layers.data(name='target', shape=[label_dict_len], dtype='float32', lod_level=0)
# NN: embedding + lstm + pooling
feature_out = p_model3.db_lstm(data_reader, word, postag, arc_relation, arc_head, conf_dict)
# loss function for multi-label classification
class_cost = fluid.layers.sigmoid_cross_entropy_with_logits(x=feature_out, label=target)
avg_cost = fluid.layers.mean(class_cost)
acc = fluid.layers.accuracy(input=feature_out, label=target)
test_program =fluid.default_main_program().clone(for_test=True)
# optimization method
sgd_optimizer = fluid.optimizer.AdamOptimizer(learning_rate=2e-3, )
sgd_optimizer.minimize(avg_cost)
train_batch_reader = paddle.batch(paddle.reader.shuffle(data_reader.get_train_reader(), buf_size=8192),
batch_size=conf_dict['batch_size'])
test_batch_reader = paddle.batch(paddle.reader.shuffle(data_reader.get_test_reader(), buf_size=8192),
batch_size=conf_dict['batch_size'])
def train_test(train_test_program, train_test_feed, train_test_reader):
acc_set = []
avg_loss_set = []
for test_data in train_test_reader():
avg_loss_np, acc_np = exe.run(
program=train_test_program,
feed=train_test_feed.feed(test_data),
fetch_list=[avg_cost, acc])
acc_set.append(float(acc_np))
avg_loss_set.append(float(avg_loss_np))
# get test acc and loss
acc_val_mean = np.array(acc_set).mean()
avg_loss_val_mean = np.array(avg_loss_set).mean()
return avg_loss_val_mean, acc_val_mean
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, postag, arc_relation, arc_head, target], place=place)
exe = fluid.Executor(place)
save_dirname = conf_dict['p_model_save_dir']
def train_loop(main_program, trainer_id=0):
"""start train"""
exe.run(fluid.default_startup_program())
start_time = time.time()
batch_id = 0
lists = []
best_epoch = 100000
for pass_id in six.moves.xrange(conf_dict['pass_num']):
pass_start_time = time.time()
cost_sum, cost_counter = 0, 0
for data in tqdm(train_batch_reader()):
cost = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost])
cost = cost[0]
cost_sum += cost
cost_counter += 1
if batch_id % 10 == 0 and batch_id != 0:
print("batch %d finished, second per batch: %02f" % (batch_id, (time.time() - start_time) / batch_id), file=sys.stderr)
# cost expected, training over
if float(cost) < 0.01:
pass_avg_cost = cost_sum / cost_counter if cost_counter > 0 else 0.0
print("%d pass end, cost time: %02f, avg_cost: %f" % (pass_id, time.time() - pass_start_time,
pass_avg_cost), file=sys.stderr)
save_path = os.path.join(save_dirname, 'final')
fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'arc_relation', 'arc_head'],
[feature_out], exe, params_filename='params')
return
batch_id = batch_id + 1
#test for epoch
avg_loss_val, acc_val = train_test(train_test_program=test_program,train_test_reader=test_batch_reader,
train_test_feed=feeder)
if best_epoch > avg_loss_val:
best_epoch = avg_loss_val
save_path = os.path.join(save_dirname, 'final')
fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'arc_relation', 'arc_head'],
[feature_out], exe, params_filename='params')
print("Test with Epoch %d, avg_cost: %s, acc: %s" % (pass_id, avg_loss_val, acc_val))
lists.append((pass_id, avg_loss_val, acc_val))
# save the model once each pass ends
pass_avg_cost = cost_sum / cost_counter if cost_counter > 0 else 0.0
print("Train with Epoch %d, cost time: %02f, avg_cost: %f" % (pass_id, time.time() - pass_start_time, pass_avg_cost),
file=sys.stderr)
save_path = os.path.join(save_dirname, 'pass_%04d-%f' % (pass_id, pass_avg_cost))
fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'arc_relation', 'arc_head'],
[feature_out], exe, params_filename='params')
# find the best pass
best = sorted(lists, key=lambda list: float(list[1]))[0]
print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1]))
print('The classification accuracy is %.2f%%' % (float(best[2]) * 100))
return
train_loop(fluid.default_main_program())
划分训练集与验证集，乱序，生成数据列表
'''
#每次生成数据列表前，首先清空train.txt和eval.txt
with open(train_list_path, 'w') as f:
f.seek(0)
f.truncate()
with open(eval_list_path, 'w') as f:
f.seek(0)
f.truncate()
#生成数据列表
get_data_list(target_path, train_list_path, eval_list_path)
'''
构造数据提供器
'''
train_reader = paddle.batch(custom_reader(train_list_path),
batch_size=batch_size,
drop_last=True)
eval_reader = paddle.batch(custom_reader(eval_list_path),
batch_size=batch_size,
drop_last=True)
# # **二、模型配置**
#
# 
#
# VGG的核心是五组卷积操作，每两组之间做Max-Pooling空间降维。同一组内采用多次连续的3X3卷积，卷积核的数目由较浅组的64增多到最深组的512，同一组内的卷积核数目是一样的。卷积之后接两层全连接层，之后是分类层。由于每组内卷积层的不同，有11、13、16、19层这几种模型，上图展示一个16层的网络结构。
#
#
# In[ ]:
def train(topology,
train_data_dir=None,
test_data_dir=None,
word_dict_path=None,
label_dict_path=None,
model_save_dir="models",
use_cuda=False,
window_size=5,
learning_rate=0.001,
batch_size=64,
num_passes=10):
"""
train window_net model or sentence_net model
:params train_data_path: path of training data, if this parameter
is not specified, Brown Corpus will be used to run this example
:type train_data_path: str
:params test_data_path: path of testing data, if this parameter
is not specified, Brown Corpus will be used to run this example
:type test_data_path: str
:params word_dict_path: path of word dictionary data, if this parameter
is not specified, a default dictionary file will be used to run this example
:type word_dict_path: str
:params label_dict_path: path of label dictionary data, if this parameter
is not specified, a default dictionary file will be used to run this example
:type label_dict_path: str
:params use_cuda: whether use the cuda
:type use_cuda: bool
:params window_size: size of window width
:type window_size: int
:params num_pass: train pass number
:type num_pass: int
"""
if not os.path.exists(model_save_dir):
os.mkdir(model_save_dir)
use_default_data = (train_data_dir is None)
if use_default_data:
logger.info(("No training data are provided, "
"use Brown corpus to train the model."))
logger.info("downloading Brown corpus...")
train_data_dir, test_data_dir, word_dict_path, label_dict_path = load_default_data(
)
logger.info("please wait to build the word dictionary ...")
if word_dict_path is None or not os.path.exists(word_dict_path):
logger.info(("word dictionary is not given, the dictionary "
"is automatically built from the training data."))
# build the word dictionary to map the original string-typed
# words into integer-typed index
build_dict(data_dir=train_data_dir,
save_path=word_dict_path,
use_col=0,
cutoff_fre=1,
insert_extra_words=["<UNK>"])
logger.info("the word dictionary path is %s" % word_dict_path)
if not os.path.exists(label_dict_path):
logger.info(("label dictionary is not given, the dictionary "
"is automatically built from the training data."))
# build the label dictionary to map the original string-typed
# label into integer-typed index
build_dict(data_dir=train_data_dir,
save_path=label_dict_path,
use_col=1,
cutoff_fre=10,
insert_extra_words=["<UNK>"])
logger.info("the label dictionary path is %s" % label_dict_path)
# get index info
word_dict = load_dict(word_dict_path)
lbl_dict = load_dict(label_dict_path)
class_num = len(lbl_dict)
logger.info("class number is : %d." % (len(lbl_dict)))
# get train data reader
train_reader = paddle.batch(paddle.reader.shuffle(reader.train_reader(
train_data_dir, word_dict, lbl_dict, window_size),
buf_size=51200),
batch_size=batch_size)
# get test data reader
if test_data_dir is not None:
# here, because training and testing data share a same format,
# we still use the reader.train_reader to read the testing data.
test_reader = paddle.batch(reader.train_reader(test_data_dir,
word_dict, lbl_dict,
window_size),
batch_size=batch_size)
else:
test_reader = None
# get size of word dictionary
dict_dim = len(word_dict) + 1
logger.info("length of word dictionary is : %d." % (dict_dim))
# define the input layers
data = fluid.layers.data(name="words",
shape=[1],
dtype="int64",
lod_level=1)
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
# return the network result
cost, acc, prediction = topology(data,
label,
dict_dim,
class_num=class_num)
# create optimizer
sgd_optimizer = fluid.optimizer.Adam(learning_rate=learning_rate)
sgd_optimizer.minimize(cost)
# create trainer
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[data, label], place=place)
# initialize training network
exe.run(fluid.default_startup_program())
prog = fluid.default_main_program()
# begin training network
for pass_id in range(num_passes):
## running the train data
data_size, data_count, total_acc, total_cost = 0, 0, 0.0, 0.0
for i, data_ in enumerate(train_reader()):
avg_cost_np, avg_acc_np = exe.run(prog,
feed=feeder.feed(data_),
fetch_list=[cost, acc])
data_size = len(data_)
total_acc += data_size * avg_acc_np
total_cost += data_size * avg_cost_np
data_count += data_size
if (i + 1) % 1000 == 0:
logger.info(
"pass_id: %d, batch %d, avg_acc: %f, avg_cost: %f" %
(pass_id, i + 1, total_acc / data_count,
total_cost / data_count))
avg_cost = total_cost / data_count
avg_acc = total_acc / data_count
logger.info("Train result -- pass_id: %d, avg_acc: %f, avg_cost: %f" %
(pass_id, avg_acc, avg_cost))
## running the test data
if test_reader is not None:
data_size, data_count, total_acc, total_cost = 0, 0, 0.0, 0.0
for i, data in enumerate(test_reader()):
avg_cost_np, avg_acc_np, prediction_np = exe.run(
prog,
feed=feeder.feed(data),
fetch_list=[cost, acc, prediction])
data_size = len(data)
total_acc += data_size * avg_acc_np
total_cost += data_size * avg_cost_np
data_count += data_size
avg_cost = total_cost / data_count
avg_acc = total_acc / data_count
logger.info(
"Test result -- pass_id: %d, avg_acc: %f, avg_cost: %f" %
(pass_id, avg_acc, avg_cost))
## save inference model
epoch_model = model_save_dir + "/" + args.nn_type + "_epoch" + str(
pass_id % 5)
logger.info("Saving inference model at %s" % (epoch_model))
##prediction is the topology return value
##if we use the prediction value as the infer result
fluid.io.save_inference_model(epoch_model, ["words"], prediction, exe)
logger.info("Training has finished.")
def test_mnist_sort_gradient_float32(self):
seed = 90
epoch_num = 1
with fluid.dygraph.guard():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
backward_strategy = fluid.dygraph.BackwardStrategy()
backward_strategy.sort_sum_gradient = True
mnist2 = MNIST("mnist")
sgd2 = SGDOptimizer(learning_rate=1e-3)
train_reader2 = paddle.batch(paddle.dataset.mnist.train(),
batch_size=128,
drop_last=True)
mnist2.train()
dy_param_init_value2 = {}
for epoch in range(epoch_num):
for batch_id, data in enumerate(train_reader2()):
dy_x_data2 = np.array([
x[0].reshape(1, 28, 28) for x in data
]).astype('float32')
y_data2 = np.array([x[1] for x in data
]).astype('int64').reshape(128, 1)
img2 = to_variable(dy_x_data2)
label2 = to_variable(y_data2)
label2.stop_gradient = True
cost2 = mnist2(img2)
loss2 = fluid.layers.cross_entropy(cost2, label2)
avg_loss2 = fluid.layers.mean(loss2)
dy_out2 = avg_loss2.numpy()
if epoch == 0 and batch_id == 0:
for param in mnist2.parameters():
dy_param_init_value2[param.name] = param.numpy()
avg_loss2.backward(backward_strategy)
sgd2.minimize(avg_loss2)
mnist2.clear_gradients()
dy_param_value2 = {}
for param in mnist2.parameters():
dy_param_value2[param.name] = param.numpy()
if batch_id == 20:
break
with new_program_scope():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
exe = fluid.Executor(fluid.CPUPlace(
) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0))
mnist = MNIST("mnist")
sgd = SGDOptimizer(learning_rate=1e-3)
train_reader = paddle.batch(paddle.dataset.mnist.train(),
batch_size=128,
drop_last=True)
img = fluid.layers.data(name='pixel',
shape=[1, 28, 28],
dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
cost = mnist(img)
loss = fluid.layers.cross_entropy(cost, label)
avg_loss = fluid.layers.mean(loss)
sgd.minimize(avg_loss)
# initialize params and fetch them
static_param_init_value = {}
static_param_name_list = []
for param in mnist.parameters():
static_param_name_list.append(param.name)
out = exe.run(fluid.default_startup_program(),
fetch_list=static_param_name_list)
for i in range(len(static_param_name_list)):
static_param_init_value[static_param_name_list[i]] = out[i]
for epoch in range(epoch_num):
for batch_id, data in enumerate(train_reader()):
static_x_data = np.array([
x[0].reshape(1, 28, 28) for x in data
]).astype('float32')
y_data = np.array([x[1] for x in data
]).astype('int64').reshape([128, 1])
fetch_list = [avg_loss.name]
fetch_list.extend(static_param_name_list)
out = exe.run(fluid.default_main_program(),
feed={
"pixel": static_x_data,
"label": y_data
},
fetch_list=fetch_list)
static_param_value = {}
static_out = out[0]
for i in range(1, len(out)):
static_param_value[static_param_name_list[i -
1]] = out[i]
if batch_id == 20:
break
self.assertTrue(np.allclose(dy_x_data2.all(), static_x_data.all()))
for key, value in six.iteritems(static_param_init_value):
self.assertTrue(np.allclose(value, dy_param_init_value2[key]))
self.assertTrue(np.allclose(static_out, dy_out2))
for key, value in six.iteritems(static_param_value):
self.assertTrue(np.allclose(value, dy_param_value2[key],
atol=1e-5))
cost = fluid.layers.cross_entropy(input=model, label=label)
avg_cost = fluid.layers.mean(cost)
acc = fluid.layers.accuracy(input=model, label=label)
# 获取训练和测试程序
test_program = fluid.default_main_program().clone(for_test=True)
# 定义优化方法
optimizer = fluid.optimizer.AdamOptimizer(
learning_rate=1e-3,
regularization=fluid.regularizer.L2DecayRegularizer(1e-4))
opts = optimizer.minimize(avg_cost)
# 获取自定义数据
train_reader = paddle.batch(reader=reader.train_reader('images/train.list',
crop_size, resize_size),
batch_size=32)
test_reader = paddle.batch(reader=reader.test_reader('images/test.list',
crop_size),
batch_size=32)
# 定义一个使用GPU的执行器
place = fluid.CUDAPlace(0)
# place = fluid.CPUPlace()
exe = fluid.Executor(place)
# 进行参数初始化
exe.run(fluid.default_startup_program())
# 定义输入数据维度
feeder = fluid.DataFeeder(place=place, feed_list=[image, label])
# -*- coding: utf-8 -*-
# @File : dist_train.py
# @Author: WangYe
# @Date : 2019/11/22
# @Software: PyCharm
import os
import paddle
import paddle.fluid as fluid
# train reader
BATCH_SIZE = 20
EPOCH_NUM = 30
BATCH_SIZE = 8
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.uci_housing.train(), buf_size=500),
batch_size=BATCH_SIZE)
def train():
y = fluid.layers.data(name='y', shape=[1], dtype='float32')
x = fluid.layers.data(name='x', shape=[13], dtype='float32')
y_predict = fluid.layers.fc(input=x, size=1, act=None)
loss = fluid.layers.square_error_cost(input=y_predict, label=y)
avg_loss = fluid.layers.mean(loss)
opt = fluid.optimizer.SGD(learning_rate=0.001)
opt.minimize(avg_loss)
place = fluid.CPUPlace()
feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
def check_network_convergence(self, is_sparse, build_strategy=None):
main = fluid.Program()
startup = fluid.Program()
with fluid.program_guard(main, startup):
word = fluid.layers.data(
name='word_data', shape=[1], dtype='int64', lod_level=1)
predicate = fluid.layers.data(
name='verb_data', shape=[1], dtype='int64', lod_level=1)
ctx_n2 = fluid.layers.data(
name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1)
ctx_n1 = fluid.layers.data(
name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1)
ctx_0 = fluid.layers.data(
name='ctx_0_data', shape=[1], dtype='int64', lod_level=1)
ctx_p1 = fluid.layers.data(
name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1)
ctx_p2 = fluid.layers.data(
name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1)
mark = fluid.layers.data(
name='mark_data', shape=[1], dtype='int64', lod_level=1)
feature_out = db_lstm(**locals())
target = fluid.layers.data(
name='target', shape=[1], dtype='int64', lod_level=1)
crf_cost = fluid.layers.linear_chain_crf(
input=feature_out,
label=target,
param_attr=fluid.ParamAttr(
name='crfw', learning_rate=1e-1))
avg_cost = fluid.layers.mean(crf_cost)
sgd_optimizer = fluid.optimizer.SGD(
learning_rate=fluid.layers.exponential_decay(
learning_rate=0.01,
decay_steps=100000,
decay_rate=0.5,
staircase=True))
sgd_optimizer.minimize(avg_cost)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.conll05.test(), buf_size=8192),
batch_size=16)
place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)
exe.run(startup)
pe = fluid.ParallelExecutor(
use_cuda=True,
loss_name=avg_cost.name,
build_strategy=build_strategy)
feeder = fluid.DataFeeder(
feed_list=[
word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate,
mark, target
],
place=fluid.CPUPlace())
data = train_data()
for i in xrange(10):
cur_batch = next(data)
print map(np.array,
pe.run(feed=feeder.feed(cur_batch),
fetch_list=[avg_cost.name]))[0]
plot_cost.append(train_title, step, event.metrics[0])
if event.step % 20 == 0: # 若干个batch,记录cost
test_metrics = trainer.test(
reader=test_reader, feed_order=feed_order)
if test_metrics[0] < 10:
plot_cost.append(test_title, step, test_metrics[0])
# 将参数存储，用于预测使用
if params_dirname is not None:
trainer.save_params(params_dirname)
step += 1
# 设置 BATCH_SIZE 的大小
BATCH_SIZE = 128
# 设置训练reader
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(),
buf_size=500),
batch_size=BATCH_SIZE)
#设置测试 reader
test_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.test(),
buf_size=500),
batch_size=BATCH_SIZE)
#创建训练器
trainer = Trainer(
train_func= train_func,
place= place,
optimizer_func= optimizer_func)
def train(use_cuda, is_sparse, is_parallel, save_dirname, is_local=True):
PASS_NUM = 100
EMBED_SIZE = 32
HIDDEN_SIZE = 256
N = 5
BATCH_SIZE = 32
IS_SPARSE = is_sparse
def __network__(words):
embed_first = fluid.layers.embedding(
input=words[0],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=IS_SPARSE,
param_attr='shared_w')
embed_second = fluid.layers.embedding(
input=words[1],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=IS_SPARSE,
param_attr='shared_w')
embed_third = fluid.layers.embedding(
input=words[2],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=IS_SPARSE,
param_attr='shared_w')
embed_forth = fluid.layers.embedding(
input=words[3],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=IS_SPARSE,
param_attr='shared_w')
concat_embed = fluid.layers.concat(
input=[embed_first, embed_second, embed_third, embed_forth], axis=1)
hidden1 = fluid.layers.fc(input=concat_embed,
size=HIDDEN_SIZE,
act='sigmoid')
predict_word = fluid.layers.fc(input=hidden1,
size=dict_size,
act='softmax')
cost = fluid.layers.cross_entropy(input=predict_word, label=words[4])
avg_cost = fluid.layers.mean(cost)
return avg_cost, predict_word
word_dict = paddle.dataset.imikolov.build_dict()
dict_size = len(word_dict)
first_word = fluid.layers.data(name='firstw', shape=[1], dtype='int64')
second_word = fluid.layers.data(name='secondw', shape=[1], dtype='int64')
third_word = fluid.layers.data(name='thirdw', shape=[1], dtype='int64')
forth_word = fluid.layers.data(name='forthw', shape=[1], dtype='int64')
next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
if not is_parallel:
avg_cost, predict_word = __network__(
[first_word, second_word, third_word, forth_word, next_word])
else:
places = fluid.layers.get_places()
pd = fluid.layers.ParallelDo(places)
with pd.do():
avg_cost, predict_word = __network__(
map(pd.read_input, [
first_word, second_word, third_word, forth_word, next_word
]))
pd.write_output(avg_cost)
avg_cost = fluid.layers.mean(pd())
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
sgd_optimizer.minimize(avg_cost)
train_reader = paddle.batch(
paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(
feed_list=[first_word, second_word, third_word, forth_word, next_word],
place=place)
def train_loop(main_program):
exe.run(fluid.default_startup_program())
for pass_id in range(PASS_NUM):
for data in train_reader():
avg_cost_np = exe.run(main_program,
feed=feeder.feed(data),
fetch_list=[avg_cost])
if avg_cost_np[0] < 5.0:
if save_dirname is not None:
fluid.io.save_inference_model(save_dirname, [
'firstw', 'secondw', 'thirdw', 'forthw'
], [predict_word], exe)
return
if math.isnan(float(avg_cost_np[0])):
sys.exit("got NaN loss, training failed.")
raise AssertionError("Cost is too large {0:2.2}".format(avg_cost_np[0]))
if is_local:
train_loop(fluid.default_main_program())
else:
port = os.getenv("PADDLE_INIT_PORT", "6174")
pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip...
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist) # ip:port,ip:port...
trainers = int(os.getenv("TRAINERS"))
current_endpoint = os.getenv("POD_IP") + ":" + port
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
training_role = os.getenv("TRAINING_ROLE", "TRAINER")
t = fluid.DistributeTranspiler()
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
if training_role == "PSERVER":
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint,
pserver_prog)
exe.run(pserver_startup)
exe.run(pserver_prog)
elif training_role == "TRAINER":
train_loop(t.get_trainer_program())
def run_benchmark(model, args):
if args.use_cprof:
pr = cProfile.Profile()
pr.enable()
if args.data_set == "cifar10":
class_dim = 10
if args.data_format == 'NCHW':
dshape = [3, 32, 32]
else:
dshape = [32, 32, 3]
else:
class_dim = 102
if args.data_format == 'NCHW':
dshape = [3, 224, 224]
else:
dshape = [224, 224, 3]
input = fluid.layers.data(name='data', shape=dshape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
predict = model(input, class_dim)
cost = fluid.layers.cross_entropy(input=predict, label=label)
avg_cost = fluid.layers.mean(x=cost)
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
batch_acc = fluid.layers.accuracy(input=predict,
label=label,
total=batch_size_tensor)
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
inference_program = fluid.io.get_inference_program(
target_vars=[batch_acc, batch_size_tensor])
optimizer = fluid.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
opts = optimizer.minimize(avg_cost)
fluid.memory_optimize(fluid.default_main_program())
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.cifar.train10()
if args.data_set == 'cifar10' else paddle.dataset.flowers.train(),
buf_size=5120),
batch_size=args.batch_size)
test_reader = paddle.batch(paddle.dataset.cifar.test10() if args.data_set
== 'cifar10' else paddle.dataset.flowers.test(),
batch_size=args.batch_size)
def test(exe):
test_accuracy = fluid.average.WeightedAverage()
for batch_id, data in enumerate(test_reader()):
img_data = np.array(map(lambda x: x[0].reshape(dshape),
data)).astype("float32")
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
acc, weight = exe.run(inference_program,
feed={
"data": img_data,
"label": y_data
},
fetch_list=[batch_acc, batch_size_tensor])
test_accuracy.add(value=acc, weight=weight)
return test_accuracy.eval()
place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
accuracy = fluid.average.WeightedAverage()
if args.use_fake_data:
data = train_reader().next()
image = np.array(map(lambda x: x[0].reshape(dshape),
data)).astype('float32')
label = np.array(map(lambda x: x[1], data)).astype('int64')
label = label.reshape([-1, 1])
im_num = 0
total_train_time = 0.0
total_iters = 0
train_acc_kpi = None
for kpi in tracking_kpis:
if kpi.name == '%s_%s_train_acc' % (args.data_set, args.batch_size):
train_acc_kpi = kpi
train_speed_kpi = None
for kpi in tracking_kpis:
if kpi.name == '%s_%s_train_speed' % (args.data_set, args.batch_size):
train_speed_kpi = kpi
for pass_id in range(args.pass_num):
every_pass_loss = []
accuracy.reset()
iter = 0
pass_duration = 0.0
for batch_id, data in enumerate(train_reader()):
batch_start = time.time()
if iter == args.iterations:
break
if not args.use_fake_data:
image = np.array(map(lambda x: x[0].reshape(dshape),
data)).astype('float32')
label = np.array(map(lambda x: x[1], data)).astype('int64')
label = label.reshape([-1, 1])
loss, acc, weight = exe.run(
fluid.default_main_program(),
feed={
'data': image,
'label': label
},
fetch_list=[avg_cost, batch_acc, batch_size_tensor])
accuracy.add(value=acc, weight=weight)
if iter >= args.skip_batch_num or pass_id != 0:
batch_duration = time.time() - batch_start
pass_duration += batch_duration
im_num += label.shape[0]
every_pass_loss.append(loss)
# print("Pass: %d, Iter: %d, loss: %s, acc: %s" %
# (pass_id, iter, str(loss), str(acc)))
iter += 1
total_iters += 1
total_train_time += pass_duration
pass_train_acc = accuracy.eval()
pass_test_acc = test(exe)
print(
"Pass:%d, Loss:%f, Train Accuray:%f, Test Accuray:%f, Handle Images Duration: %f\n"
% (pass_id, np.mean(every_pass_loss), pass_train_acc,
pass_test_acc, pass_duration))
if pass_id == args.pass_num - 1 and args.data_set == 'cifar10':
train_acc_kpi.add_record(np.array(pass_train_acc, dtype='float32'))
train_acc_kpi.persist()
if total_train_time > 0.0 and iter != args.skip_batch_num:
examples_per_sec = im_num / total_train_time
sec_per_batch = total_train_time / \
(iter * args.pass_num - args.skip_batch_num)
train_speed_kpi.add_record(np.array(examples_per_sec, dtype='float32'))
train_speed_kpi.persist()
print('\nTotal examples: %d, total time: %.5f' %
(im_num, total_train_time))
print('%.5f examples/sec, %.5f sec/batch \n' %
(examples_per_sec, sec_per_batch))
if args.use_cprof:
pr.disable()
s = StringIO.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(s.getvalue())
def test_raw_api(self):
prog = Program()
startup_prog = Program()
with program_guard(prog, startup_prog):
image = layers.data(name='x', shape=[784], dtype='float32')
label = layers.data(name='y', shape=[1], dtype='int64')
limit = layers.fill_constant_batch_size_like(
input=label, dtype='int64', shape=[1], value=5.0)
cond = layers.less_than(x=label, y=limit)
true_image, false_image = layers.split_lod_tensor(
input=image, mask=cond)
true_out = layers.create_tensor(dtype='float32')
true_cond = layers.ConditionalBlock([true_image])
with true_cond.block():
hidden = layers.fc(input=true_image, size=100, act='tanh')
prob = layers.fc(input=hidden, size=10, act='softmax')
layers.assign(input=prob, output=true_out)
false_out = layers.create_tensor(dtype='float32')
false_cond = layers.ConditionalBlock([false_image])
with false_cond.block():
hidden = layers.fc(input=false_image, size=200, act='tanh')
prob = layers.fc(input=hidden, size=10, act='softmax')
layers.assign(input=prob, output=false_out)
prob = layers.merge_lod_tensor(
in_true=true_out, in_false=false_out, mask=cond, x=image)
loss = layers.cross_entropy(input=prob, label=label)
avg_loss = layers.mean(loss)
optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9)
optimizer.minimize(avg_loss, startup_prog)
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=8192),
batch_size=200)
place = core.CPUPlace()
exe = Executor(place)
exe.run(startup_prog)
PASS_NUM = 100
for pass_id in range(PASS_NUM):
for data in train_reader():
x_data = np.array(map(lambda x: x[0], data)).astype("float32")
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = np.expand_dims(y_data, axis=1)
outs = exe.run(prog,
feed={'x': x_data,
'y': y_data},
fetch_list=[avg_loss])
print outs[0]
if outs[0] < 1.0:
return
self.assertFalse(True)
def test_mnist_float32(self):
seed = 90
epoch_num = 1
batch_size = 128
batch_num = 50
traced_layer = None
with fluid.dygraph.guard():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
mnist = MNIST()
sgd = SGDOptimizer(learning_rate=1e-3,
parameter_list=mnist.parameters())
batch_py_reader = fluid.io.PyReader(capacity=1)
batch_py_reader.decorate_sample_list_generator(
paddle.batch(self.reader_decorator(
paddle.dataset.mnist.train()),
batch_size=batch_size,
drop_last=True),
places=fluid.CPUPlace())
mnist.train()
dy_param_init_value = {}
helper = DyGraphProgramDescTracerTestHelper(self)
program = None
for epoch in range(epoch_num):
for batch_id, data in enumerate(batch_py_reader()):
if batch_id >= batch_num:
break
img = data[0]
dy_x_data = img.numpy()
label = data[1]
label.stop_gradient = True
if batch_id % 10 == 0:
cost, traced_layer = TracedLayer.trace(mnist,
inputs=img)
if program is not None:
self.assertTrue(program, traced_layer.program)
program = traced_layer.program
traced_layer.save_inference_model(
'./infer_imperative_mnist')
else:
cost = mnist(img)
if traced_layer is not None:
cost_static = traced_layer([img])
helper.assertEachVar(cost, cost_static)
loss = fluid.layers.cross_entropy(cost, label)
avg_loss = fluid.layers.mean(loss)
dy_out = avg_loss.numpy()
if epoch == 0 and batch_id == 0:
for param in mnist.parameters():
dy_param_init_value[param.name] = param.numpy()
avg_loss.backward()
sgd.minimize(avg_loss)
mnist.clear_gradients()
dy_param_value = {}
for param in mnist.parameters():
dy_param_value[param.name] = param.numpy()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
exe = fluid.Executor(fluid.CPUPlace(
) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0))
mnist = MNIST()
sgd = SGDOptimizer(learning_rate=1e-3)
train_reader = paddle.batch(paddle.dataset.mnist.train(),
batch_size=batch_size,
drop_last=True)
img = fluid.layers.data(name='pixel',
shape=[1, 28, 28],
dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
cost = mnist(img)
loss = fluid.layers.cross_entropy(cost, label)
avg_loss = fluid.layers.mean(loss)
sgd.minimize(avg_loss)
# initialize params and fetch them
static_param_init_value = {}
static_param_name_list = []
for param in mnist.parameters():
static_param_name_list.append(param.name)
out = exe.run(fluid.default_startup_program(),
fetch_list=static_param_name_list)
for i in range(len(static_param_name_list)):
static_param_init_value[static_param_name_list[i]] = out[i]
for epoch in range(epoch_num):
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num:
break
static_x_data = np.array([
x[0].reshape(1, 28, 28) for x in data
]).astype('float32')
y_data = np.array([x[1]
for x in data]).astype('int64').reshape(
[batch_size, 1])
fetch_list = [avg_loss.name]
fetch_list.extend(static_param_name_list)
if traced_layer is not None:
traced_layer([static_x_data])
out = exe.run(fluid.default_main_program(),
feed={
"pixel": static_x_data,
"label": y_data
},
fetch_list=fetch_list)
static_param_value = {}
static_out = out[0]
for i in range(1, len(out)):
static_param_value[static_param_name_list[i -
1]] = out[i]
self.assertTrue(np.allclose(dy_x_data.all(), static_x_data.all()))
for key, value in six.iteritems(static_param_init_value):
self.assertTrue(np.allclose(value, dy_param_init_value[key]))
self.assertTrue(np.allclose(static_out, dy_out))
for key, value in six.iteritems(static_param_value):
self.assertTrue(np.allclose(value, dy_param_value[key], atol=1e-5))
model = vgg16(image, 10)
# 获取损失函数和准确率函数
cost = fluid.layers.cross_entropy(input=model, label=label)
avg_cost = fluid.layers.mean(cost)
acc = fluid.layers.accuracy(input=model, label=label)
# 获取训练和测试程序
test_program = fluid.default_main_program().clone(for_test=True)
# 定义优化方法
optimizer = fluid.optimizer.MomentumOptimizer(learning_rate=1e-3, momentum=0.9)
opts = optimizer.minimize(avg_cost)
# 获取CIFAR数据
train_reader = paddle.batch(cifar.train10(), batch_size=32)
test_reader = paddle.batch(cifar.test10(), batch_size=32)
# 定义一个使用CPU的执行器
place = fluid.CUDAPlace(0)
# place = fluid.CPUPlace()
exe = fluid.Executor(place)
# 进行参数初始化
exe.run(fluid.default_startup_program())
# 加载之前训练过的参数模型
save_path = 'models/params_model/'
if os.path.exists(save_path):
print('使用参数模型作为预训练模型')
fluid.io.load_params(executor=exe, dirname=save_path)
def predict_infer(conf_dict, data_reader, predict_data_path, \
predict_result_path, model_path):
"""
Predict with trained models
"""
if len(predict_result_path) > 0:
result_writer = open(predict_result_path, 'w')
else:
result_writer = sys.stdout
np.set_printoptions(precision=3)
if len(model_path) == 0:
return
place = fluid.CPUPlace()
word = fluid.layers.data(name='word_data',
shape=[1],
dtype='int64',
lod_level=1)
postag = fluid.layers.data(name='token_pos',
shape=[1],
dtype='int64',
lod_level=1)
p_word = fluid.layers.data(name='p_word',
shape=[1],
dtype='int64',
lod_level=1)
feeder = fluid.DataFeeder(feed_list=[word, postag, p_word], place=place)
exe = fluid.Executor(place)
test_batch_reader = paddle.batch(
paddle.reader.buffered(data_reader.get_predict_reader\
(predict_data_path, need_input=True, need_label=False),
size=8192),
batch_size=conf_dict['batch_size'])
inference_scope = fluid.core.Scope()
text_spo_dic = {} #final triples
with fluid.scope_guard(inference_scope):
[inference_program, feed_target_names, fetch_targets] = \
fluid.io.load_inference_model(
model_path, exe, params_filename='params')
# batch
batch_id = 0
for data in test_batch_reader():
feeder_data = []
input_data = []
for item in data:
feeder_data.append(item[1:])
input_data.append(item[0])
results = exe.run(inference_program,
feed=feeder.feed(feeder_data),
fetch_list=fetch_targets,
return_numpy=False)
tag_split_idx = results[0].lod()[0]
label_tag_scores = np.array(results[0])
# sentence
print >> sys.stderr, 'batch_id=', batch_id
for sent_idx, tag_idx in enumerate(tag_split_idx[:-1]):
input_sent = input_data[sent_idx].split('\t')[0]
input_p = input_data[sent_idx].split('\t')[1]
tag_scores = label_tag_scores[tag_idx:tag_split_idx[sent_idx +
1]]
# token
tag_list = []
for token_idx, token_tags in enumerate(tag_scores):
tag = data_reader.get_label_output(token_tags)
tag_list.append(tag)
predicted_s_list, predicted_o_list = refine_predict_seq(
input_sent, tag_list)
tag_list_str = json.dumps(tag_list, ensure_ascii=False)
if len(predicted_s_list) == 0 or len(predicted_o_list) == 0:
continue
else:
text = json.loads(input_sent)["text"]
predicted_s_list = list(set(predicted_s_list))
predicted_o_list = list(set(predicted_o_list))
for predicted_s in predicted_s_list:
for predicted_o in predicted_o_list:
if text not in text_spo_dic:
text_spo_dic[text] = set()
text_spo_dic[text].add(
(predicted_s, input_p, predicted_o))
batch_id += 1
output(text_spo_dic, result_writer)
def eval(args):
train_reader = None
test_reader = None
if args.data == "mnist":
import paddle.dataset.mnist as reader
train_reader = reader.train()
val_reader = reader.test()
class_dim = 10
image_shape = "1,28,28"
elif args.data == "imagenet":
import imagenet_reader as reader
train_reader = reader.train()
val_reader = reader.val()
class_dim = 1000
image_shape = "3,224,224"
else:
raise ValueError("{} is not supported.".format(args.data))
image_shape = [int(m) for m in image_shape.split(",")]
assert args.model in model_list, "{} is not in lists: {}".format(
args.model, model_list)
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# model definition
model = models.__dict__[args.model]()
out = model.net(input=image, class_dim=class_dim)
acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1)
acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5)
val_program = fluid.default_main_program().clone(for_test=True)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
val_reader = paddle.batch(val_reader, batch_size=args.batch_size)
val_feeder = feeder = fluid.DataFeeder([image, label],
place,
program=val_program)
load_model(exe, val_program, args.model_path)
batch_id = 0
acc_top1_ns = []
acc_top5_ns = []
for data in val_reader():
start_time = time.time()
acc_top1_n, acc_top5_n = exe.run(
val_program,
feed=val_feeder.feed(data),
fetch_list=[acc_top1.name, acc_top5.name])
end_time = time.time()
if batch_id % args.log_period == 0:
_logger.info(
"Eval batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".format(
batch_id, np.mean(acc_top1_n), np.mean(acc_top5_n),
end_time - start_time))
acc_top1_ns.append(np.mean(acc_top1_n))
acc_top5_ns.append(np.mean(acc_top5_n))
batch_id += 1
_logger.info("Final eval - acc_top1: {}; acc_top5: {}".format(
np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
def do_train(args):
train_prog = fluid.default_main_program()
startup_prog = fluid.default_startup_program()
with fluid.program_guard(train_prog, startup_prog):
train_prog.random_seed = args.random_seed
startup_prog.random_seed = args.random_seed
with fluid.unique_name.guard():
# define reader
image = fluid.layers.data(
name='image', shape=[1, 28, 28], dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
reader = fluid.io.PyReader(
feed_list=[image, label], capacity=4, iterable=False)
# define the network
loss = create_net(
is_training=True, model_input=[image, label], args=args)
# define optimizer for learning
optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
optimizer.minimize(loss)
if args.do_eval_in_training:
# if you want to perform testing in training, you need to declare an extra test-program
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
test_prog.random_seed = args.random_seed
with fluid.unique_name.guard():
test_image = fluid.layers.data(
name='test_image', shape=[1, 28, 28], dtype='float32')
test_label = fluid.layers.data(
name='test_label', shape=[1], dtype='int64')
test_reader = fluid.io.PyReader(
feed_list=[test_image, test_label],
capacity=4,
iterable=False)
predict = create_net(
is_training=False, model_input=test_image, args=args)
# prepare training
## declare data generator
generator = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=500),
batch_size=args.batch_size)
reader.decorate_sample_list_generator(generator)
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
if args.init_from_checkpoint:
init_from_checkpoint(args, exe, train_prog)
compiled_train_prog = fluid.CompiledProgram(train_prog).with_data_parallel(
loss_name=loss.name)
if args.do_eval_in_training:
compiled_test_prog = fluid.CompiledProgram(test_prog)
test_generator = paddle.batch(
paddle.dataset.mnist.test(), batch_size=args.batch_size)
test_reader.decorate_sample_list_generator(test_generator)
# start training
step = 0
for epoch_step in range(args.epoch_num):
reader.start()
while True:
try:
# this is for minimizing the fetching op, saving the training speed.
if step % args.print_step == 0:
fetch_list = [loss.name]
else:
fetch_list = []
output = exe.run(compiled_train_prog, fetch_list=fetch_list)
if step % args.print_step == 0:
print("step: %d, loss: %.4f" % (step, np.sum(output[0])))
if step % args.save_step == 0 and step != 0:
if args.save_checkpoint:
save_checkpoint(args, exe, train_prog,
"step_" + str(step))
if args.save_param:
save_param(args, exe, train_prog, "step_" + str(step))
if args.do_eval_in_training:
if step != 0 and step % args.eval_step == 0:
acc = do_evaluate_in_training(
reader=test_reader,
executor=exe,
program=test_prog,
fetch_list=[predict.name])
print("evaluation acc for step %d is %.4f" %
(step, acc))
step += 1
except fluid.core.EOFException:
reader.reset()
break
if args.save_checkpoint:
save_checkpoint(args, exe, train_prog, "step_final")
if args.save_param:
save_param(args, exe, train_prog, "step_final")
def cluster_train(learning_rate,
batch_size,
num_passes,
model_save_dir='model'):
class_dim = 102
image_shape = [3, 224, 224]
image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
out = mobile_net(image, class_dim=class_dim)
cost = fluid.layers.cross_entropy(input=out, label=label)
avg_cost = fluid.layers.mean(x=cost)
optimizer = fluid.optimizer.Momentum(
learning_rate=learning_rate,
momentum=0.9,
regularization=fluid.regularizer.L2Decay(5 * 1e-5))
optimize_ops, params_grads = optimizer.minimize(avg_cost)
place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(
args.device_id)
exe = fluid.Executor(place)
standalone = int(os.getenv("STANDALONE", 0))
if standalone:
pserver_endpoints = os.getenv("PSERVERS")
trainers = int(os.getenv("TRAINERS"))
current_endpoint = os.getenv("SERVER_ENDPOINT")
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
training_role = os.getenv("TRAINING_ROLE", "TRAINER")
else:
pport = os.getenv("PADDLE_INIT_PSERVER_PORT", "6174")
tport = os.getenv("PADDLE_INIT_TRAINER_PORT", "6174")
pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip...
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, pport]))
pserver_endpoints = ",".join(eplist) # ip:port,ip:port...
trainers = int(os.getenv("TRAINERS"))
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
training_role = os.getenv("TRAINING_ROLE", "TRAINER")
if training_role == "PSERVER":
current_endpoint = os.getenv("POD_IP") + ":" + pport
else:
current_endpoint = os.getenv("POD_IP") + ":" + tport
print(
"pserver_endpoints: {0}, trainers: {1}, current_endpoint: {2}, trainer_id: {3}, training_role: {4}"
.format(pserver_endpoints, trainers, current_endpoint, trainer_id,
training_role))
t = fluid.DistributeTranspiler()
t.transpile(optimize_ops,
params_grads,
trainer_id=trainer_id,
pservers=pserver_endpoints,
trainers=trainers)
if training_role == "PSERVER":
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint, pserver_prog)
exe.run(pserver_startup)
exe.run(pserver_prog)
elif training_role == "TRAINER":
b_size_var = fluid.layers.create_tensor(dtype='int64')
b_acc_var = fluid.layers.accuracy(input=out,
label=label,
total=b_size_var)
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
inference_program = fluid.io.get_inference_program(
target_vars=[b_acc_var, b_size_var])
exe.run(fluid.default_startup_program())
train_reader = paddle.batch(paddle.dataset.flowers.train(),
batch_size=batch_size)
test_reader = paddle.batch(paddle.dataset.flowers.test(),
batch_size=batch_size)
feeder = fluid.DataFeeder(place=place, feed_list=[image, label])
train_proc = t.get_trainer_program()
train_pass_acc_evaluator = fluid.average.WeightedAverage()
test_pass_acc_evaluator = fluid.average.WeightedAverage()
# with profiler.profiler(args.device, 'total', "profilers.log"):
for pass_id in range(num_passes):
start = time.clock()
train_pass_acc_evaluator.reset()
for batch_id, data in enumerate(train_reader()):
if args.accuracy:
data = accuracy_data(trainers, trainer_id, data)
loss, acc, size = exe.run(
train_proc,
feed=feeder.feed(data),
fetch_list=[avg_cost, b_acc_var, b_size_var])
train_pass_acc_evaluator.add(value=acc, weight=size)
print("Pass {0}, batch {1}, loss {2}, acc {3}".format(
pass_id, batch_id, loss[0], acc[0]))
test_pass_acc_evaluator.reset()
for data in test_reader():
acc, size = exe.run(inference_program,
feed=feeder.feed(data),
fetch_list=[b_acc_var, b_size_var])
test_pass_acc_evaluator.add(value=acc, weight=size)
print("End pass {0}, train_acc {1}, test_acc {2}, cost {3} second".
format(pass_id, train_pass_acc_evaluator.eval(),
test_pass_acc_evaluator.eval(),
time.clock() - start))
def main():
if args.data_set == "cifar10":
classdim = 10
if args.data_format == 'NCHW':
data_shape = [3, 32, 32]
else:
data_shape = [32, 32, 3]
else:
classdim = 102
if args.data_format == 'NCHW':
data_shape = [3, 224, 224]
else:
data_shape = [224, 224, 3]
# Input data
images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
# Train program
net = vgg16_bn_drop(images)
predict = fluid.layers.fc(input=net, size=classdim, act='softmax')
cost = fluid.layers.cross_entropy(input=predict, label=label)
avg_cost = fluid.layers.mean(x=cost)
# Evaluator
batch_size = fluid.layers.create_tensor(dtype='int64')
batch_acc = fluid.layers.accuracy(input=predict,
label=label,
total=batch_size)
test_program = fluid.default_main_program().clone(for_test=True)
# inference program
inference_program = fluid.default_main_program().clone()
with fluid.program_guard(inference_program):
inference_program = fluid.io.get_inference_program(batch_acc)
# Optimization
optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
optimize_ops, params_grads = optimizer.minimize(avg_cost)
fetch_list = [avg_cost.name, batch_acc.name, batch_size.name]
# test
def test(exe11):
test_pass_acc = fluid.average.WeightedAverage()
for batch_id, data in enumerate(test_reader()):
img_data = np.array(map(lambda x: x[0].reshape(data_shape),
data)).astype("float32")
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
loss, acc, b_size = exe11.run(feed={
"pixel": img_data,
"label": y_data
},
fetch_list=fetch_list)
np_acc = np.array(acc)
np_b_size = np.array(b_size)
test_pass_acc.add(value=float(np.mean(np_acc)),
weight=float(np.mean(np_b_size)))
return test_pass_acc.eval()
def train_loop(train_exe, test_exe):
iters = 0
ts = time.time()
train_pass_acc = fluid.average.WeightedAverage()
for pass_id in range(args.num_passes):
# train
start_time = time.time()
num_samples = 0
train_pass_acc.reset()
for batch_id, data in enumerate(train_reader()):
ts = time.time()
img_data = np.array(
map(lambda x: x[0].reshape(data_shape),
data)).astype("float32")
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
loss, acc, b_size = train_exe.run(feed={
"pixel": img_data,
"label": y_data
},
fetch_list=[
avg_cost.name,
batch_acc.name,
batch_size.name
])
iters += 1
num_samples += len(data)
np_acc = np.array(acc)
np_loss = np.array(loss)
np_b_size = np.array(b_size)
train_pass_acc.add(value=float(np.mean(np_acc)),
weight=float(np.mean(np_b_size)))
print(
"Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, Speed = %.2f img/s, Period = %d secs"
% (pass_id, iters, np.mean(np_loss), np.mean(np_acc),
len(data) /
(time.time() - ts), time.time() - start_time)
) # The accuracy is the accumulation of batches, but not the current batch.
pass_elapsed = time.time() - start_time
pass_train_acc = train_pass_acc.eval()
pass_test_acc = test(test_exe)
print(
"Pass = %d, Elapsed = %d, Training performance = %f imgs/s, Train accuracy = %f, Test accuracy = %f\n"
% (pass_id, pass_elapsed, num_samples / pass_elapsed,
pass_train_acc, pass_test_acc))
# Initialize executor
use_gpu = False if args.device == 'CPU' else True
# data reader
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.cifar.train10()
if args.data_set == 'cifar10' else paddle.dataset.flowers.train(),
buf_size=5120),
batch_size=args.batch_size)
test_reader = paddle.batch(paddle.dataset.cifar.test10() if args.data_set
== 'cifar10' else paddle.dataset.flowers.test(),
batch_size=args.batch_size)
if args.local:
# Parameter initialization
place = core.CPUPlace() if not use_gpu else core.CUDAPlace(
args.device_id)
startup_exe = fluid.Executor(place)
startup_exe.run(fluid.default_startup_program())
train_exe = fluid.ParallelExecutor(use_cuda=True,
loss_name=avg_cost.name)
test_exe = fluid.ParallelExecutor(use_cuda=True,
main_program=test_program,
share_vars_from=train_exe)
# data reader
train_reader = paddle.batch(paddle.reader.shuffle(
paddle.dataset.cifar.train10()
if args.data_set == 'cifar10' else paddle.dataset.flowers.train(),
buf_size=5120),
batch_size=args.batch_size)
test_reader = paddle.batch(
paddle.dataset.cifar.test10()
if args.data_set == 'cifar10' else paddle.dataset.flowers.test(),
batch_size=args.batch_size)
train_loop(train_exe, test_exe)
else:
standalone = int(os.getenv("STANDALONE", 0))
if standalone:
pserver_endpoints = os.getenv("PSERVERS")
trainers = int(os.getenv("TRAINERS"))
current_endpoint = os.getenv("SERVER_ENDPOINT")
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
training_role = os.getenv("TRAINING_ROLE", "TRAINER")
else:
pport = os.getenv("PADDLE_INIT_PSERVER_PORT", "6174")
tport = os.getenv("PADDLE_INIT_TRAINER_PORT", "6174")
pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip...
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, pport]))
pserver_endpoints = ",".join(eplist) # ip:port,ip:port...
trainers = int(os.getenv("TRAINERS"))
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
training_role = os.getenv("TRAINING_ROLE", "TRAINER")
if training_role == "PSERVER":
current_endpoint = os.getenv("POD_IP") + ":" + pport
else:
current_endpoint = os.getenv("POD_IP") + ":" + tport
print(
"pserver_endpoints: {0}, trainers: {1}, current_endpoint: {2}, trainer_id: {3}, training_role: {4}"
.format(pserver_endpoints, trainers, current_endpoint, trainer_id,
training_role))
t = fluid.DistributeTranspiler()
t.transpile(optimize_ops,
params_grads,
trainer_id=trainer_id,
pservers=pserver_endpoints,
trainers=trainers)
if training_role == "PSERVER":
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint,
pserver_prog)
place = core.CPUPlace() if not use_gpu else core.CUDAPlace(
args.device_id)
exe = fluid.Executor(place)
exe.run(pserver_startup)
exe.run(pserver_prog)
elif training_role == "TRAINER":
# Parameter initialization
place = core.CPUPlace() if not use_gpu else core.CUDAPlace(
args.device_id)
startup_exe = fluid.Executor(place)
startup_exe.run(fluid.default_startup_program())
trainer_prog = t.get_trainer_program()
train_exe = fluid.ParallelExecutor(use_cuda=True,
main_program=trainer_prog,
loss_name=avg_cost.name)
test_exe = fluid.ParallelExecutor(use_cuda=True,
main_program=test_program,
share_vars_from=train_exe)
train_loop(train_exe, test_exe)
else:
print("environment var TRAINER_ROLE should be TRAINER os PSERVER")
def train(use_cuda, save_dirname, is_local):
x = fluid.layers.data(name='x', shape=[13], dtype='float32')
y_predict = fluid.layers.fc(input=x, size=1, act=None)
y = fluid.layers.data(name='y', shape=[1], dtype='float32')
cost = fluid.layers.square_error_cost(input=y_predict, label=y)
avg_cost = fluid.layers.mean(cost)
sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
sgd_optimizer.minimize(avg_cost)
BATCH_SIZE = 20
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.uci_housing.train(), buf_size=500),
batch_size=BATCH_SIZE)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
def train_loop(main_program):
feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
exe.run(fluid.default_startup_program())
PASS_NUM = 100
for pass_id in range(PASS_NUM):
for data in train_reader():
avg_loss_value, = exe.run(main_program,
feed=feeder.feed(data),
fetch_list=[avg_cost])
print(avg_loss_value)
if avg_loss_value[0] < 10.0:
if save_dirname is not None:
fluid.io.save_inference_model(save_dirname, ['x'],
[y_predict], exe)
return
if math.isnan(float(avg_loss_value)):
sys.exit("got NaN loss, training failed.")
raise AssertionError("Fit a line cost is too large, {0:2.2}".format(
avg_loss_value[0]))
if is_local:
train_loop(fluid.default_main_program())
else:
port = os.getenv("PADDLE_INIT_PORT", "6174")
pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip...
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist) # ip:port,ip:port...
trainers = int(os.getenv("TRAINERS"))
current_endpoint = os.getenv("POD_IP") + ":" + port
trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
training_role = os.getenv("TRAINING_ROLE", "TRAINER")
t = fluid.DistributeTranspiler()
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
if training_role == "PSERVER":
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint,
pserver_prog)
exe.run(pserver_startup)
exe.run(pserver_prog)
elif training_role == "TRAINER":
train_loop(t.get_trainer_program())
return __reader__
place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id)
with fluid.dygraph.guard(place):
strategy = fluid.dygraph.parallel.prepare_context()
simple_net = SimpleNet(hidden_size=hidden_size,
vocab_size=vocab_size,
num_steps=num_steps,
init_scale=init_scale,
is_sparse=True)
simple_net = fluid.dygraph.parallel.DataParallel(simple_net, strategy)
train_reader = paddle.batch(ptb_train_reader(),
batch_size=batch_size,
drop_last=True)
train_reader = fluid.contrib.reader.distributed_batch_reader(train_reader)
sgd = fluid.optimizer.SGD(learning_rate=1e-3,
parameter_list=simple_net.parameters())
dy_loss = None
for i, data in enumerate(train_reader()):
x_data = np.array([x[0].reshape(3) for x in data]).astype('int64')
y_data = np.array([x[1].reshape(3) for x in data]).astype('int64')
x_data = x_data.reshape((-1, num_steps, 1))
y_data = y_data.reshape((-1, 1))
x = to_variable(x_data)
y = to_variable(y_data)
batch_size = train_parameters["batch_size"]
batch_num = 10
with fluid.dygraph.guard():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
resnet = ResNet("resnet")
optimizer = optimizer_setting(train_parameters)
np.random.seed(seed)
import random
random.seed = seed
train_reader = paddle.batch(reader_decorator(
paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True)
batch_py_reader = fluid.io.DataLoader.from_generator(capacity=10)
batch_py_reader.set_sample_list_generator(train_reader,
places=fluid.CPUPlace())
dy_param_init_value = {}
for param in resnet.parameters():
dy_param_init_value[param.name] = param.numpy()
for batch_id, data in enumerate(batch_py_reader()):
if batch_id >= batch_num:
break
img = data[0]
def train(word_dict,
net_method,
use_cuda,
parallel=False,
save_dirname=None,
is_local=True):
BATCH_SIZE = 128
PASS_NUM = 5
dict_dim = len(word_dict)
class_dim = 2
data = fluid.layers.data(
name="words", shape=[1], dtype="int64", lod_level=1)
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
if not parallel:
cost, acc_out, prediction = net_method(
data, label, input_dim=dict_dim, class_dim=class_dim)
else:
places = get_places()
pd = ParallelDo(places)
with pd.do():
cost, acc, _ = net_method(
pd.read_input(data),
pd.read_input(label),
input_dim=dict_dim,
class_dim=class_dim)
pd.write_output(cost)
pd.write_output(acc)
cost, acc = pd()
cost = fluid.layers.mean(cost)
acc_out = fluid.layers.mean(acc)
prediction = None
assert save_dirname is None
adagrad = fluid.optimizer.Adagrad(learning_rate=0.002)
adagrad.minimize(cost)
train_data = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.imdb.train(word_dict), buf_size=1000),
batch_size=BATCH_SIZE)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[data, label], place=place)
def train_loop(main_program):
exe.run(fluid.default_startup_program())
for pass_id in range(PASS_NUM):
for data in train_data():
cost_val, acc_val = exe.run(main_program,
feed=feeder.feed(data),
fetch_list=[cost, acc_out])
print("cost=" + str(cost_val) + " acc=" + str(acc_val))
if cost_val < 0.4 and acc_val > 0.8:
if save_dirname is not None:
fluid.io.save_inference_model(save_dirname, ["words"],
prediction, exe)
return
if math.isnan(float(cost_val)):
sys.exit("got NaN loss, training failed.")
raise AssertionError("Cost is too large for {0}".format(
net_method.__name__))
if is_local:
train_loop(fluid.default_main_program())
else:
port = os.getenv("PADDLE_PSERVER_PORT", "6174")
pserver_ips = os.getenv("PADDLE_PSERVER_IPS") # ip,ip...
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist) # ip:port,ip:port...
trainers = int(os.getenv("PADDLE_TRAINERS"))
current_endpoint = os.getenv("POD_IP") + ":" + port
trainer_id = int(os.getenv("PADDLE_TRAINER_ID"))
training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER")
t = fluid.DistributeTranspiler()
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
if training_role == "PSERVER":
pserver_prog = t.get_pserver_program(current_endpoint)
pserver_startup = t.get_startup_program(current_endpoint,
pserver_prog)
exe.run(pserver_startup)
exe.run(pserver_prog)
elif training_role == "TRAINER":
train_loop(t.get_trainer_program())
