def train(use_cuda, train_program, params_dirname): train_reader = paddle.batch( paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE) test_reader = paddle.batch( paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() def event_handler(event): if isinstance(event, fluid.EndStepEvent): outs = trainer.test( reader=test_reader, feed_order=['firstw', 'secondw', 'thirdw', 'forthw', 'nextw']) avg_cost = outs[0] print("loss= ", avg_cost) if avg_cost < 10.0: trainer.save_params(params_dirname) trainer.stop() if math.isnan(avg_cost): sys.exit("got NaN loss, training failed.") trainer = fluid.Trainer( train_func=train_program, optimizer=fluid.optimizer.SGD(learning_rate=0.001), place=place) trainer.train( reader=train_reader, num_epochs=1, event_handler=event_handler, feed_order=['firstw', 'secondw', 'thirdw', 'forthw', 'nextw'])
def get_model(args): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) label = fluid.layers.data(name='label', shape=[1], dtype='int64') # Train program predict = cnn_model(images) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') batch_acc = fluid.layers.accuracy( input=predict, label=label, total=batch_size_tensor) # inference program inference_program = fluid.default_main_program().clone() # Optimization opt = fluid.optimizer.AdamOptimizer( learning_rate=0.001, beta1=0.9, beta2=0.999) # Reader train_reader = paddle.batch( paddle.dataset.mnist.train(), batch_size=args.batch_size) test_reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=args.batch_size) return avg_cost, inference_program, opt, train_reader, test_reader, batch_acc
def train(use_cuda, train_program, params_dirname): BATCH_SIZE = 128 train_reader = paddle.batch( paddle.reader.shuffle( cifar10_small_test_set.train10(batch_size=10), buf_size=128 * 10), batch_size=BATCH_SIZE) test_reader = paddle.batch( paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE) def event_handler(event): if isinstance(event, fluid.EndStepEvent): avg_cost, accuracy = trainer.test( reader=test_reader, feed_order=['pixel', 'label']) print('Loss {0:2.2}, Acc {1:2.2}'.format(avg_cost, accuracy)) if accuracy > 0.01: # Low threshold for speeding up CI if params_dirname is not None: trainer.save_params(params_dirname) return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() trainer = fluid.Trainer( train_func=train_program, place=place, optimizer=fluid.optimizer.Adam(learning_rate=0.001)) trainer.train( reader=train_reader, num_epochs=1, event_handler=event_handler, feed_order=['pixel', 'label'])
def get_model(args): if args.data_set == "cifar10": classdim = 10 if args.data_format == 'NCHW': data_shape = [3, 32, 32] else: data_shape = [32, 32, 3] else: classdim = 102 if args.data_format == 'NCHW': data_shape = [3, 224, 224] else: data_shape = [224, 224, 3] # Input data images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') # Train program net = vgg16_bn_drop(images) predict = fluid.layers.fc(input=net, size=classdim, act='softmax') cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') batch_acc = fluid.layers.accuracy( input=predict, label=label, total=batch_size_tensor) # inference program inference_program = fluid.default_main_program().clone() with fluid.program_guard(inference_program): inference_program = fluid.io.get_inference_program( target_vars=[batch_acc, batch_size_tensor]) # Optimization optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) # data reader train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.cifar.train10() if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), buf_size=5120), batch_size=args.batch_size) test_reader = paddle.batch( paddle.dataset.cifar.test10() if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), batch_size=args.batch_size) return avg_cost, inference_program, optimizer, train_reader, test_reader, batch_acc
def test_accuracy(executor, inference_program, feed_target_names, fetch_targets): if args.data_set == "cifar10": data_shape = [3, 32, 32] elif args.data_set == "imagenet": data_shape = [3, 224, 224] else: raise ValueError("%s dataset is not supported" % data_set) test_reader = paddle.batch( paddle.dataset.cifar.test10() if args.data_set == "cifar10" else paddle.dataset.flowers.test(), batch_size=args.inf_batch_size) test_num = 0 correct_num = 0 for test_data in test_reader(): test_image = np.array( map(lambda x: x[0].reshape(data_shape), test_data)).astype( "float32") test_label = np.array(map(lambda x: x[1], test_data)).astype("int64") test_label = test_label.reshape([-1, 1]) results = executor.run(program=inference_program, feed={feed_target_names[0]: test_image}, fetch_list=fetch_targets) prediction = np.argmax(results[0], axis=1).reshape([-1, 1]) correct_num += np.sum(prediction == test_label) test_num += test_label.size print("{0} out of {1} predictions are correct.".format(correct_num, test_num)) print("Test accuray is {0}.".format(float(correct_num) / float(test_num)))
def train(use_cuda, train_program, params_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() optimizer = fluid.optimizer.Adam(learning_rate=0.001) trainer = fluid.Trainer( train_func=train_program, place=place, optimizer=optimizer) def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) avg_cost, acc = trainer.test( reader=test_reader, feed_order=['img', 'label']) print("avg_cost: %s" % avg_cost) print("acc : %s" % acc) if acc > 0.2: # Smaller value to increase CI speed trainer.save_params(params_dirname) else: print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( event.epoch + 1, avg_cost, acc)) if math.isnan(avg_cost): sys.exit("got NaN loss, training failed.") train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=BATCH_SIZE) trainer.train( num_epochs=1, event_handler=event_handler, reader=train_reader, feed_order=['img', 'label'])
def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) avg_cost, acc = trainer.test( reader=test_reader, feed_order=['words', 'label']) print("avg_cost: %s" % avg_cost) print("acc : %s" % acc) if acc > 0.2: # Smaller value to increase CI speed trainer.save_params(params_dirname) trainer.stop() else: print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( event.epoch + 1, avg_cost, acc)) if math.isnan(avg_cost): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( event.step, event.epoch, map(np.array, event.metrics))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop()
def train(use_cuda, is_sparse, is_local=True): EPOCH_NUM = 1 if use_cuda and not fluid.core.is_compiled_with_cuda(): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) feed_order = [ 'src_word_id', 'target_language_word', 'target_language_next_word' ] def event_handler(event): if isinstance(event, fluid.EndStepEvent): print('pass_id=' + str(event.epoch) + ' batch=' + str(event.step)) if event.step == 10: trainer.stop() trainer = fluid.Trainer( train_func=partial(train_program, is_sparse), optimizer=fluid.optimizer.Adagrad( learning_rate=1e-4, regularization=fluid.regularizer.L2DecayRegularizer( regularization_coeff=0.1)), place=place) trainer.train( reader=train_reader, num_epochs=EPOCH_NUM, event_handler=event_handler, feed_order=feed_order)
def test_main(self): main = fluid.Program() startup = fluid.Program() startup.random_seed = 1 with fluid.scope_guard(fluid.core.Scope()): with fluid.program_guard(main, startup): data = fluid.layers.data( name='image', shape=[3, 224, 224], dtype='float32') label = fluid.layers.data( name='label', shape=[1], dtype='int64') out = Lenet(data, class_dim=102) loss = fluid.layers.cross_entropy(input=out, label=label) loss = fluid.layers.mean(loss) opt = fluid.optimizer.Momentum( learning_rate=0.1, momentum=0.9, regularization=fluid.regularizer.L2Decay(1e-4)) opt.minimize(loss) place = fluid.CUDAPlace(0) feeder = fluid.DataFeeder(place=place, feed_list=[data, label]) reader = feeder.decorate_reader( paddle.batch( flowers.train(), batch_size=16), multi_devices=True) exe = fluid.Executor(place) exe.run(startup) pe = fluid.ParallelExecutor( use_cuda=True, loss_name=loss.name, main_program=main) for batch_id, data in enumerate(reader()): loss_np = np.array(pe.run(feed=data, fetch_list=[loss.name])[0]) print batch_id, loss_np if batch_id == 2: break
def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( paddle.dataset.conll05.test(), batch_size=BATCH_SIZE) avg_cost_set = trainer.test( reader=test_reader, feed_order=feed_order) # get avg cost avg_cost = np.array(avg_cost_set).mean() print("avg_cost: %s" % avg_cost) if float(avg_cost) < 100.0: # Large value to increase CI speed trainer.save_params(params_dirname) else: print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, float(avg_cost))) if math.isnan(float(avg_cost)): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( event.step, event.epoch, map(np.array, event.metrics))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop()
def test_fetch_op(self): tst_reader = paddle.batch(flowers.test(use_xmap=False), batch_size=16) tst_reader_iter = tst_reader() iters = 3 train_inputs = [] for i in range(iters): train_inputs.append(tst_reader_iter.next()) self.parallel_exe(train_inputs, seed=1)
def test_ifelse(self): prog = Program() startup_prog = Program() with program_guard(prog, startup_prog): image = layers.data(name='x', shape=[784], dtype='float32') label = layers.data(name='y', shape=[1], dtype='int64') limit = layers.fill_constant_batch_size_like( input=label, dtype='int64', shape=[1], value=5.0) cond = layers.less_than(x=label, y=limit) ie = layers.IfElse(cond) with ie.true_block(): true_image = ie.input(image) hidden = layers.fc(input=true_image, size=100, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) with ie.false_block(): false_image = ie.input(image) hidden = layers.fc(input=false_image, size=200, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) prob = ie() loss = layers.cross_entropy(input=prob[0], label=label) avg_loss = layers.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=200) place = core.CPUPlace() exe = Executor(place) exe.run(kwargs['startup_program']) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) outs = exe.run(kwargs['main_program'], feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) print outs[0] if outs[0] < 1.0: return self.assertFalse(True)
def train(use_cuda, train_program, params_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() optimizer = fluid.optimizer.SGD(learning_rate=0.01) trainer = fluid.Trainer( train_func=train_program, place=place, optimizer=optimizer) feed_order = [ 'word_data', 'ctx_n2_data', 'ctx_n1_data', 'ctx_0_data', 'ctx_p1_data', 'ctx_p2_data', 'verb_data', 'mark_data', 'target' ] #embedding_param = fluid.global_scope().find_var( # EMBEDDING_NAME).get_tensor() #embedding_param.set( # load_parameter(conll05.get_embedding(), WORD_DICT_LEN, WORD_DIM), # place) def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( paddle.dataset.conll05.test(), batch_size=BATCH_SIZE) avg_cost_set = trainer.test( reader=test_reader, feed_order=feed_order) # get avg cost avg_cost = np.array(avg_cost_set).mean() print("avg_cost: %s" % avg_cost) if float(avg_cost) < 100.0: # Large value to increase CI speed trainer.save_params(params_dirname) else: print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, float(avg_cost))) if math.isnan(float(avg_cost)): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( event.step, event.epoch, map(np.array, event.metrics))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop() train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.conll05.test(), buf_size=8192), batch_size=BATCH_SIZE) trainer.train( num_epochs=1, event_handler=event_handler, reader=train_reader, feed_order=feed_order)
def setUpClass(cls): # Convert mnist to recordio file with fluid.program_guard(fluid.Program(), fluid.Program()): reader = paddle.batch(mnist.train(), batch_size=4) feeder = fluid.DataFeeder( feed_list=[ # order is image and label fluid.layers.data( name='image', shape=[784]), fluid.layers.data( name='label', shape=[1], dtype='int64'), ], place=fluid.CPUPlace()) fluid.recordio_writer.convert_reader_to_recordio_file( MNIST_RECORDIO_FILE, reader, feeder)
def iter_infer(self, input, feeding=None): from data_feeder import DataFeeder feeder = DataFeeder(self.__data_types__, feeding) batch_size = len(input) def __reader_impl__(): for each_sample in input: yield each_sample reader = paddle.batch(__reader_impl__, batch_size=batch_size) self.__gradient_machine__.start() for data_batch in reader(): yield self.__gradient_machine__.forwardTest(feeder(data_batch)) self.__gradient_machine__.finish()
def main(): rnn_out = encoder_decoder() label = layers.data( name="target_language_next_word", shape=[1], dtype='int64', lod_level=1) cost = layers.cross_entropy(input=rnn_out, label=label) avg_cost = fluid.layers.mean(cost) optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) # fluid.memory_optimize(fluid.default_main_program()) fluid.release_memory(fluid.default_main_program()) # fix the order of training data train_data = paddle.batch( paddle.dataset.wmt14.train(dict_size), batch_size=batch_size) # train_data = paddle.batch( # paddle.reader.shuffle( # paddle.dataset.wmt14.train(dict_size), buf_size=1000), # batch_size=batch_size) place = core.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) batch_id = 0 for pass_id in xrange(10): for data in train_data(): word_data = to_lodtensor(map(lambda x: x[0], data), place) trg_word = to_lodtensor(map(lambda x: x[1], data), place) trg_word_next = to_lodtensor(map(lambda x: x[2], data), place) outs = exe.run(fluid.default_main_program(), feed={ 'src_word_id': word_data, 'target_language_word': trg_word, 'target_language_next_word': trg_word_next }, fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if batch_id > 2: exit(0) if math.isnan(float(avg_cost_val)): sys.exit("got NaN loss, training failed.") batch_id += 1
def setUpClass(cls): reader = paddle.batch( wmt16.train(ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size), batch_size=transformer_model.batch_size) with fluid.recordio_writer.create_recordio_writer( WMT16_RECORDIO_FILE) as writer: for batch in reader(): for tensor in prepare_batch_input( batch, ModelHyperParams.src_pad_idx, ModelHyperParams.trg_pad_idx, ModelHyperParams.n_head): t = fluid.LoDTensor() t.set(tensor, fluid.CPUPlace()) writer.append_tensor(t) writer.complete_append_tensor()
def decode_main(use_cuda, is_sparse): if use_cuda and not fluid.core.is_compiled_with_cuda(): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() context = encoder(is_sparse) translation_ids, translation_scores = decode(context, is_sparse) exe = Executor(place) exe.run(framework.default_startup_program()) init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64') init_scores_data = np.array( [1. for _ in range(batch_size)], dtype='float32') init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_lod = [1] * batch_size init_lod = [init_lod, init_lod] init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place) init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) feed_order = ['src_word_id'] feed_list = [ framework.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) for data in train_data(): feed_dict = feeder.feed(map(lambda x: [x[0]], data)) feed_dict['init_ids'] = init_ids feed_dict['init_scores'] = init_scores result_ids, result_scores = exe.run( framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], return_numpy=False) print result_ids.lod() break
def train(use_cuda, train_program, params_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() optimizer = fluid.optimizer.Adagrad(learning_rate=0.002) word_dict = paddle.dataset.imdb.word_dict() trainer = fluid.Trainer( train_func=partial(train_program, word_dict), place=place, optimizer=optimizer) def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) avg_cost, acc = trainer.test( reader=test_reader, feed_order=['words', 'label']) print("avg_cost: %s" % avg_cost) print("acc : %s" % acc) if acc > 0.2: # Smaller value to increase CI speed trainer.save_params(params_dirname) trainer.stop() else: print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( event.epoch + 1, avg_cost, acc)) if math.isnan(avg_cost): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( event.step, event.epoch, map(np.array, event.metrics))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop() train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.imdb.train(word_dict), buf_size=25000), batch_size=BATCH_SIZE) trainer.train( num_epochs=1, event_handler=event_handler, reader=train_reader, feed_order=['words', 'label'])
def setUp(self): self.batch_size = 64 # Convert mnist to recordio file with fluid.program_guard(fluid.Program(), fluid.Program()): reader = paddle.batch(mnist.train(), batch_size=self.batch_size) feeder = fluid.DataFeeder( feed_list=[ # order is image and label fluid.layers.data( name='image', shape=[784]), fluid.layers.data( name='label', shape=[1], dtype='int64'), ], place=fluid.CPUPlace()) self.num_batch = fluid.recordio_writer.convert_reader_to_recordio_file( './mnist_0.recordio', reader, feeder) copyfile('./mnist_0.recordio', './mnist_1.recordio') copyfile('./mnist_0.recordio', './mnist_2.recordio')
def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) avg_cost, acc = trainer.test( reader=test_reader, feed_order=['img', 'label']) print("avg_cost: %s" % avg_cost) print("acc : %s" % acc) if acc > 0.2: # Smaller value to increase CI speed trainer.save_params(params_dirname) else: print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( event.epoch + 1, avg_cost, acc)) if math.isnan(avg_cost): sys.exit("got NaN loss, training failed.")
def train(use_cuda, train_program, params_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() optimizer = fluid.optimizer.SGD(learning_rate=0.2) trainer = fluid.Trainer( train_func=train_program, place=place, optimizer=optimizer) feed_order = [ 'user_id', 'gender_id', 'age_id', 'job_id', 'movie_id', 'category_id', 'movie_title', 'score' ] def event_handler(event): if isinstance(event, fluid.EndStepEvent): test_reader = paddle.batch( paddle.dataset.movielens.test(), batch_size=BATCH_SIZE) avg_cost_set = trainer.test( reader=test_reader, feed_order=feed_order) # get avg cost avg_cost = np.array(avg_cost_set).mean() print("avg_cost: %s" % avg_cost) if float(avg_cost) < 4: # Smaller value to increase CI speed trainer.save_params(params_dirname) trainer.stop() else: print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, float(avg_cost))) if math.isnan(float(avg_cost)): sys.exit("got NaN loss, training failed.") train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.movielens.train(), buf_size=8192), batch_size=BATCH_SIZE) trainer.train( num_epochs=1, event_handler=event_handler, reader=train_reader, feed_order=feed_order)
def train(use_cuda, save_dirname=None): [avg_cost, prediction] = seq_to_seq_net() optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.wmt14.train(dict_size), buf_size=1000), batch_size=batch_size) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) exe.run(framework.default_startup_program()) feed_order = ['source_sequence', 'target_sequence', 'label_sequence'] feed_list = [ framework.default_main_program().global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 for pass_id in xrange(2): for data in train_data(): outs = exe.run(framework.default_main_program(), feed=feeder.feed(data), fetch_list=[avg_cost]) avg_cost_val = np.array(outs[0]) print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if math.isnan(float(avg_cost_val[0])): sys.exit("got NaN loss, training failed.") if batch_id > 3: if save_dirname is not None: fluid.io.save_inference_model( save_dirname, ['source_sequence', 'target_sequence'], [prediction], exe) return batch_id += 1
def event_handler(event): if isinstance(event, fluid.EndStepEvent): test_reader = paddle.batch( paddle.dataset.movielens.test(), batch_size=BATCH_SIZE) avg_cost_set = trainer.test( reader=test_reader, feed_order=feed_order) # get avg cost avg_cost = np.array(avg_cost_set).mean() print("avg_cost: %s" % avg_cost) if float(avg_cost) < 4: # Smaller value to increase CI speed trainer.save_params(params_dirname) trainer.stop() else: print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, float(avg_cost))) if math.isnan(float(avg_cost)): sys.exit("got NaN loss, training failed.")
redis_port = 6379 redis_password = "" r = redis.StrictRedis(host=redis_host, port=redis_port, password=redis_password) # reader generation reader = fluid.layers.py_reader(capacity=64, shapes=[(-1, 64, 8, 8), (-1, 1)], dtypes=['float32', 'int64']) samples = r.keys() train_data = data_generater(samples, r) reader.decorate_paddle_reader( paddle.batch(paddle.reader.shuffle(train_data, buf_size=5000), batch_size=64)) conv1, label = fluid.layers.read_file(reader) # train program place = fluid.CUDAPlace(0) model = ResNet(layers=50) predicts = model.net(conv1, 10) cost = fluid.layers.cross_entropy(input=predicts, label=label) accuracy = fluid.layers.accuracy(input=predicts, label=label) loss = fluid.layers.mean(cost) optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(loss) exe = fluid.Executor(place)
def train(nn_type, use_cuda, save_dirname=None, model_filename=None, params_filename=None): if use_cuda and not fluid.core.is_compiled_with_cuda(): return startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() if args.enable_ce: train_reader = paddle.batch( paddle.dataset.mnist.train(), batch_size=BATCH_SIZE) test_reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) startup_program.random_seed = 90 main_program.random_seed = 90 else: train_reader = paddle.batch( paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), batch_size=BATCH_SIZE) test_reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) img = fluid.data(name='img', shape=[None, 1, 28, 28], dtype='float32') label = fluid.data(name='label', shape=[None, 1], dtype='int64') if nn_type == 'softmax_regression': net_conf = softmax_regression elif nn_type == 'multilayer_perceptron': net_conf = multilayer_perceptron else: net_conf = convolutional_neural_network prediction, avg_loss, acc = net_conf(img, label) test_program = main_program.clone(for_test=True) optimizer = fluid.optimizer.Adam(learning_rate=0.001) # 使用Adam优化器 optimizer.minimize(avg_loss) def train_test(train_test_program, train_test_feed, train_test_reader): acc_set = [] avg_loss_set = [] for test_data in train_test_reader(): acc_np, avg_loss_np = exe.run( program=train_test_program, feed=train_test_feed.feed(test_data), fetch_list=[acc, avg_loss]) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) # get test acc and loss acc_val_mean = numpy.array(acc_set).mean() avg_loss_val_mean = numpy.array(avg_loss_set).mean() return avg_loss_val_mean, acc_val_mean place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) exe.run(startup_program) epochs = [epoch_id for epoch_id in range(PASS_NUM)] lists = [] step = 0 for epoch_id in epochs: for step_id, data in enumerate(train_reader()): metrics = exe.run( main_program, feed=feeder.feed(data), fetch_list=[avg_loss, acc]) if step % 100 == 0: print("Pass %d, Epoch %d, Cost %f" % (step, epoch_id, metrics[0])) step += 1 # test for epoch avg_loss_val, acc_val = train_test( train_test_program=test_program, train_test_reader=test_reader, train_test_feed=feeder) print("Test with Epoch %d, avg_cost: %s, acc: %s" % (epoch_id, avg_loss_val, acc_val)) lists.append((epoch_id, avg_loss_val, acc_val)) if save_dirname is not None: fluid.io.save_inference_model( save_dirname, ["img"], [prediction], exe, model_filename=model_filename, params_filename=params_filename) if args.enable_ce: print("kpis\ttrain_cost\t%f" % metrics[0]) print("kpis\ttest_cost\t%s" % avg_loss_val) print("kpis\ttest_acc\t%s" % acc_val) # find the best pass best = sorted(lists, key=lambda list: float(list[1]))[0] print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1])) print('The classification accuracy is %.2f%%' % (float(best[2]) * 100))
def train(word_dict, net_method, use_cuda, seed, quality, save_dirname=None): BATCH_SIZE = 128 PASS_NUM = 100 dict_dim = len(word_dict) class_dim = 2 target_val_acc = quality # Seed for batch producer random.seed(seed) # Seed for weight initialization fluid.default_startup_program().random_seed = seed # Setup input features and label as data layers data = fluid.layers.data( name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") cost, acc_out, prediction = net_method( data, label, input_dim=dict_dim, class_dim=class_dim) # Initialize a test program for obtaining test accuracy and cost test_program = fluid.default_main_program().clone(for_test=True) # Setup Adam optimizer adam = fluid.optimizer.Adam(learning_rate=0.0005) #Learning rate of 5e-4 works for conv models and 2e-3 for LSTM model optimize_ops, params_grads = adam.minimize(cost) # Create reader to iterate over training set train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.imdb.train(word_dict), buf_size=25000), batch_size=BATCH_SIZE) # Setup place and executor for runtime place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[data, label], place=place) # Create reader to iterate over validation set test_reader = paddle.batch( paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) def train_loop(main_program): exe.run(fluid.default_startup_program()) for pass_id in xrange(PASS_NUM): train_loss_set = [] train_acc_set = [] # Calculate average training loss and accuracy # across all mini-batches in the training set for batch_id, data in enumerate(train_reader()): cost_val, acc_val = exe.run(main_program, feed=feeder.feed(data), fetch_list=[cost, acc_out]) train_loss_set.append(float(cost_val)) train_acc_set.append(float(acc_val)) train_loss = np.array(train_loss_set).mean() train_acc = np.array(train_acc_set).mean() * 100 # Calculate average valication loss and accuracy # across all mini-batches in the validation set acc_set = [] avg_loss_set = [] for tid, test_data in enumerate(test_reader()): avg_loss_np, acc_np = exe.run( program=test_program, feed=feeder.feed(test_data), fetch_list=[cost, acc_out]) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) acc_val = np.array(acc_set).mean() * 100 avg_loss_val = np.array(avg_loss_set).mean() print("Epoch =", pass_id, ", train-accuracy =", train_acc, ", train-loss =", train_loss, ", validation-accuracy =", acc_val, ", validation-loss =", avg_loss_val) if acc_val > target_val_acc: ## Exit the program on reaching desired accuracy value break train_loop(fluid.default_main_program())
avg_cost_clip = prog_clip.block(0).var(avg_cost.name) p_g = fluid.backward.append_backward(loss=avg_cost) p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip) with fluid.program_guard(main_program=prog_clip): fluid.clip.set_gradient_clip( fluid.clip.GradientClipByGlobalNorm(clip_norm=CLIP)) p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip) grad_list = [elem[1] for elem in p_g] grad_clip_list = [elem[1] for elem in p_g_clip] train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=BATCH_SIZE) place = fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[image, label], place=place) exe.run(fluid.default_startup_program()) count = 0 for data in train_reader(): count += 1 if count > 5: break out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list) out_clip = exe.run(prog_clip, feed=feeder.feed(data),
def train(use_cuda, save_dirname, is_local=True): scale_infer, avg_cost = model() # test program test_program = fluid.default_main_program().clone(for_test=True) sgd_optimizer = SGDOptimizer(learning_rate=0.2) sgd_optimizer.minimize(avg_cost) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.movielens.train(), buf_size=8192), batch_size=BATCH_SIZE) test_reader = paddle.batch(paddle.dataset.movielens.test(), batch_size=BATCH_SIZE) feed_order = [ 'user_id', 'gender_id', 'age_id', 'job_id', 'movie_id', 'category_id', 'movie_title', 'score' ] def train_loop(main_program): exe.run(framework.default_startup_program()) feed_list = [ main_program.global_block().var(var_name) for var_name in feed_order ] feeder = fluid.DataFeeder(feed_list, place) PASS_NUM = 100 for pass_id in range(PASS_NUM): for batch_id, data in enumerate(train_reader()): # train a mini-batch outs = exe.run(program=main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) out = np.array(outs[0]) if (batch_id + 1) % 10 == 0: avg_cost_set = [] for test_data in test_reader(): avg_cost_np = exe.run(program=test_program, feed=feeder.feed(test_data), fetch_list=[avg_cost]) avg_cost_set.append(avg_cost_np[0]) break # test only 1 segment for speeding up CI # get test avg_cost test_avg_cost = np.array(avg_cost_set).mean() if test_avg_cost < 6.0: # if avg_cost less than 6.0, we think our code is good. if save_dirname is not None: fluid.io.save_inference_model( save_dirname, [ "user_id", "gender_id", "age_id", "job_id", "movie_id", "category_id", "movie_title" ], [scale_infer], exe) return if math.isnan(float(out[0])): sys.exit("got NaN loss, training failed.") if is_local: train_loop(fluid.default_main_program()) else: port = os.getenv("PADDLE_PSERVER_PORT", "6174") pserver_ips = os.getenv("PADDLE_PSERVER_IPS") # ip,ip... eplist = [] for ip in pserver_ips.split(","): eplist.append(':'.join([ip, port])) pserver_endpoints = ",".join(eplist) # ip:port,ip:port... trainers = int(os.getenv("PADDLE_TRAINERS")) current_endpoint = os.getenv("POD_IP") + ":" + port trainer_id = int(os.getenv("PADDLE_TRAINER_ID")) training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER") t = fluid.DistributeTranspiler() t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": train_loop(t.get_trainer_program())
def test_se_resnext_float32(self): seed = 90 batch_size = train_parameters["batch_size"] batch_num = 1 epoch_num = 1 with fluid.dygraph.guard(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed se_resnext = SeResNeXt("se_resnext") optimizer = optimizer_setting(train_parameters) np.random.seed(seed) import random random.seed = seed batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( paddle.batch(self.reader_decorator( paddle.dataset.flowers.train(use_xmap=False)), batch_size=batch_size, drop_last=True), places=fluid.CPUPlace()) dy_param_init_value = {} for param in se_resnext.parameters(): dy_param_init_value[param.name] = param.numpy() for epoch_id in range(epoch_num): for batch_id, data in enumerate(batch_py_reader()): if batch_id >= batch_num and batch_num != -1: break img = data[0] label = data[1] label.stop_gradient = True label.stop_gradient = True out = se_resnext(img) loss = fluid.layers.cross_entropy(input=out, label=label) avg_loss = fluid.layers.mean(x=loss) dy_out = avg_loss.numpy() if batch_id == 0: for param in se_resnext.parameters(): if param.name not in dy_param_init_value: dy_param_init_value[param.name] = param.numpy() avg_loss.backward() #dy_grad_value = {} #for param in se_resnext.parameters(): # if param.trainable: # np_array = np.array(param._ivar._grad_ivar().value() # .get_tensor()) # dy_grad_value[param.name + core.grad_var_suffix()] = np_array optimizer.minimize(avg_loss) se_resnext.clear_gradients() dy_param_value = {} for param in se_resnext.parameters(): dy_param_value[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed exe = fluid.Executor(fluid.CPUPlace( ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) se_resnext = SeResNeXt("se_resnext") optimizer = optimizer_setting(train_parameters) np.random.seed(seed) import random random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size, drop_last=True) img = fluid.layers.data(name='pixel', shape=[3, 224, 224], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = se_resnext(img) loss = fluid.layers.cross_entropy(input=out, label=label) avg_loss = fluid.layers.mean(x=loss) optimizer.minimize(avg_loss) # initialize params and fetch them static_param_init_value = {} static_param_name_list = [] static_grad_name_list = [] for param in se_resnext.parameters(): static_param_name_list.append(param.name) for param in se_resnext.parameters(): if param.trainable: static_grad_name_list.append(param.name + core.grad_var_suffix()) out = exe.run(fluid.default_startup_program(), fetch_list=static_param_name_list) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch_id in range(epoch_num): for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num and batch_num != -1: break static_x_data = np.array([ x[0].reshape(3, 224, 224) for x in data ]).astype('float32') y_data = np.array([x[1] for x in data]).astype('int64').reshape( [batch_size, 1]) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) out = exe.run(fluid.default_main_program(), feed={ "pixel": static_x_data, "label": y_data }, fetch_list=fetch_list) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 grad_start_pos = len( static_param_name_list) + param_start_pos for i in range( param_start_pos, len(static_param_name_list) + param_start_pos): static_param_value[static_param_name_list[ i - param_start_pos]] = out[i] for i in range(grad_start_pos, len(static_grad_name_list) + grad_start_pos): static_grad_value[static_grad_name_list[ i - grad_start_pos]] = out[i] self.assertTrue(np.allclose(static_out, dy_out)) self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in six.iteritems(static_param_init_value): self.assertTrue(np.allclose(value, dy_param_init_value[key])) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) # FIXME(Yancey1989): np.array(_ivar.value().get_tensor()) leads to memory lake #self.assertEqual(len(dy_grad_value), len(static_grad_value)) #for key, value in six.iteritems(static_grad_value): # self.assertTrue(np.allclose(value, dy_grad_value[key])) # self.assertTrue(np.isfinite(value.all())) # self.assertFalse(np.isnan(value.any())) self.assertEqual(len(dy_param_value), len(static_param_value)) for key, value in six.iteritems(static_param_value): self.assertTrue(np.allclose(value, dy_param_value[key])) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any()))
def local_train(learning_rate, batch_size, num_passes, model_save_dir='model'): class_dim = 102 image_shape = [3, 224, 224] image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = mobile_net(image, class_dim=class_dim) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) optimizer = fluid.optimizer.Momentum( learning_rate=learning_rate, momentum=0.9, regularization=fluid.regularizer.L2Decay(5 * 1e-5)) # optimizer = fluid.optimizer.SGD(learning_rate=learning_rate) opts = optimizer.minimize(avg_cost) b_size_var = fluid.layers.create_tensor(dtype='int64') b_acc_var = fluid.layers.accuracy(input=out, label=label, total=b_size_var) inference_program = fluid.default_main_program().clone() with fluid.program_guard(inference_program): inference_program = fluid.io.get_inference_program( target_vars=[b_acc_var, b_size_var]) place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace( args.device_id) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) train_reader = paddle.batch(paddle.dataset.flowers.train(), batch_size=batch_size) test_reader = paddle.batch(paddle.dataset.flowers.test(), batch_size=batch_size) feeder = fluid.DataFeeder(place=place, feed_list=[image, label]) train_pass_acc_evaluator = fluid.average.WeightedAverage() test_pass_acc_evaluator = fluid.average.WeightedAverage() train_proc = fluid.default_main_program() for pass_id in range(num_passes): start = time.clock() train_pass_acc_evaluator.reset() for batch_id, data in enumerate(train_reader()): loss, acc, size = exe.run( train_proc, feed=feeder.feed(data), fetch_list=[avg_cost, b_acc_var, b_size_var]) train_pass_acc_evaluator.add(value=acc, weight=size) print("Pass {0}, batch {1}, loss {2}, acc {3}".format( pass_id, batch_id, loss[0], acc[0])) test_pass_acc_evaluator.reset() for data in test_reader(): acc, size = exe.run(inference_program, feed=feeder.feed(data), fetch_list=[b_acc_var, b_size_var]) test_pass_acc_evaluator.add(value=acc, weight=size) print("End pass {0}, train_acc {1}, test_acc {2}, cost {3} second". format(pass_id, train_pass_acc_evaluator.eval(), test_pass_acc_evaluator.eval(), time.clock() - start))
def evaluate(): place = fluid.CUDAPlace(0) if cfg.use_cuda else fluid.CPUPlace() inference_scope = fluid.Scope() test_files = [ os.path.join(cfg.evaluate_file_path, x) for x in os.listdir(cfg.evaluate_file_path) ] dataset = CriteoDataset() test_reader = paddle.batch(dataset.test(test_files), batch_size=cfg.batch_size) with fluid.dygraph.guard(place): if cfg.train_model == 'drnn': model = DRNN() elif cfg.train_model == 'dnn': model = DNN() elif cfg.train_model == 'fcdnn': model = FCDNN() model_path = os.path.join(cfg.save_path, model.name, model.name + "_epoch_" + str(cfg.test_epoch)) model_dict, optimizer_dict = fluid.dygraph.load_dygraph(model_path) model.set_dict(model_dict) logger.info("load model {} finished.".format(model_path)) model.eval() logger.info('Begin evaluate model.') run_index = 0 infer_auc = 0.0 L = [] for batch_id, data in enumerate(test_reader()): dense_feature, sparse_feature, label = zip(*data) sparse_feature = np.array(sparse_feature, dtype=np.int64) dense_feature = np.array(dense_feature, dtype=np.float32) label = np.array(label, dtype=np.int64) sparse_feature, dense_feature, label = [ to_variable(i) for i in [sparse_feature, dense_feature, label] ] avg_cost, auc_var = model(dense_feature, sparse_feature, label) run_index += 1 infer_auc += auc_var.numpy().item() L.append(avg_cost.numpy() / cfg.batch_size) if batch_id % cfg.log_interval == 0: logger.info("TEST --> batch: {} loss: {} auc: {}".format( batch_id, avg_cost.numpy() / cfg.batch_size, infer_auc / run_index)) infer_loss = np.mean(L) infer_auc = infer_auc / run_index infer_result = {} infer_result['loss'] = infer_loss infer_result['auc'] = infer_auc if not os.path.isdir(cfg.log_dir): os.makedirs(cfg.log_dir) log_path = os.path.join(cfg.log_dir, model.name + '_infer_result.log') logger.info(str(infer_result)) with open(log_path, 'w+') as f: f.write(str(infer_result)) logger.info("Done.") return infer_result
def eval(args): # parameters from arguments seg_num = args.seg_num class_dim = args.class_dim num_layers = args.num_layers batch_size = args.batch_size test_model = args.test_model if test_model == None: print('Please specify the test model ...') return image_shape = [int(m) for m in args.image_shape.split(",")] image_shape = [seg_num] + image_shape # model definition model = TSN_ResNet(layers=num_layers, seg_num=seg_num) image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = model.net(input=image, class_dim=class_dim) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) # for test inference_program = fluid.default_main_program().clone(for_test=True) if args.with_mem_opt: fluid.memory_optimize(fluid.default_main_program()) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) def is_parameter(var): if isinstance(var, Parameter): return isinstance(var, Parameter) if test_model is not None: vars = filter(is_parameter, inference_program.list_vars()) fluid.io.load_vars(exe, test_model, vars=vars) # reader test_reader = paddle.batch(reader.test(seg_num), batch_size=batch_size / 16) feeder = fluid.DataFeeder(place=place, feed_list=[image, label]) fetch_list = [avg_cost.name, acc_top1.name, acc_top5.name] # test cnt = 0 pass_id = 0 test_info = [[], [], []] for batch_id, data in enumerate(test_reader()): t1 = time.time() loss, acc1, acc5 = exe.run(inference_program, fetch_list=fetch_list, feed=feeder.feed(data)) t2 = time.time() period = t2 - t1 loss = np.mean(loss) acc1 = np.mean(acc1) acc5 = np.mean(acc5) test_info[0].append(loss * len(data)) test_info[1].append(acc1 * len(data)) test_info[2].append(acc5 * len(data)) cnt += len(data) if batch_id % 10 == 0: print( "[TEST] Pass: {0}\ttestbatch: {1}\tloss: {2}\tacc1: {3}\tacc5: {4}\ttime: {5}" .format(pass_id, batch_id, '%.6f' % loss, acc1, acc5, "%2.2f sec" % period)) sys.stdout.flush() test_loss = np.sum(test_info[0]) / cnt test_acc1 = np.sum(test_info[1]) / cnt test_acc5 = np.sum(test_info[2]) / cnt print("+ End pass: {0}, test_loss: {1}, test_acc1: {2}, test_acc5: {3}". format(pass_id, '%.3f' % test_loss, '%.3f' % test_acc1, '%.3f' % test_acc5)) sys.stdout.flush()
#导入必要的库 import paddle.fluid as fluid import paddle import numpy as np import os # 从paddle接口中获取房价数据训练集 train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.uci_housing.train(), #获取uci_housing训练数据 buf_size=500), #在buf_size的空间内进行乱序 batch_size=20) #batch_size:每个批次读入的训练数据量 # 从paddle接口中获取房价数据测试集 test_reader = paddle.batch( paddle.reader.shuffle(paddle.dataset.uci_housing.test(), buf_size=500), #获取uci_housing的测试数据 batch_size=20) #batch_size:每个批次读入的测试数据量 #定义一个简单的线性网络,网络有1层。输入层为x,输出大小为1,激活函数是relu的全连接层 #定义张量变量x,表示13维的特征值 x = fluid.layers.data(name='x', shape=[13], dtype='float32') #定义一个简单的线性网络 net = fluid.layers.fc(input=x, size=1, act=None) # 定义损失函数 y = fluid.layers.data(name='y', shape=[1], dtype='float32') #定义张量y,表示目标值 cost = fluid.layers.square_error_cost(input=net, label=y) #求一个batch的损失值 avg_cost = fluid.layers.mean(cost) #对损失值求平均值 # 定义优化方法 optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.001) opts = optimizer.minimize(avg_cost) # 创建一个使用CPU的解释器
def freeze_program(self, use_cuda, seed): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): img = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) return [img, label], loss main = fluid.Program() startup = fluid.Program() test_program = fluid.Program() import random random.seed(0) np.random.seed(0) feeds, loss = build_program(main, startup, False) build_program(test_program, startup, True) test_program = test_program.clone(for_test=True) quant_transpiler = QuantizeTranspiler() quant_transpiler.training_transpile(main) quant_transpiler.training_transpile(test_program) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) iters = 5 batch_size = 8 class_num = 10 exe.run(startup) train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=batch_size) test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=batch_size) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.program_guard(main): for _ in range(iters): data = next(train_reader()) loss_v = exe.run(program=main, feed=feeder.feed(data), fetch_list=[loss]) with fluid.program_guard(test_program): test_data = next(test_reader()) w_var = fluid.framework._get_var('conv2d_1.w_0.quantized', test_program) # Testing during training test_loss1, w_quant = exe.run(program=test_program, feed=feeder.feed(test_data), fetch_list=[loss, w_var]) # Freeze program for inference, but the weight of fc/conv is still float type. quant_transpiler.freeze_program(test_program, place) test_loss2, = exe.run(program=test_program, feed=feeder.feed(test_data), fetch_list=[loss]) self.assertAlmostEqual(test_loss1, test_loss2, delta=5e-3) w_freeze = np.array( fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) # fail: -432.0 != -433.0, this is due to the calculation precision #self.assertAlmostEqual(np.sum(w_freeze), np.sum(w_quant)) # Convert parameter to 8-bit. quant_transpiler.convert_to_int8(test_program, place) # Save the 8-bit parameter and model file. fluid.io.save_inference_model('model_8bit', ['image', 'label'], [loss], exe, test_program) # Test whether the 8-bit parameter and model file can be loaded successfully. [infer, feed, fetch] = fluid.io.load_inference_model('model_8bit', exe) # Check the loaded 8-bit weight. w_8bit = np.array(fluid.global_scope().find_var( 'conv2d_1.w_0.int8').get_tensor()) self.assertEqual(w_8bit.dtype, np.int8) self.assertEqual(np.sum(w_8bit), np.sum(w_freeze))
def do_training(self, fleet, args): """ training_from_pyreader Args: fleet (DistributedTranspiler): DistributedTranspiler inherited base class Fleet args (ArgumentParser): run args to config dist fleet. Returns: list """ if args.run_params.get("increment", False): global DATA_PATH DATA_PATH = DATA_PATH + '.increment' exe = fluid.Executor(fluid.CPUPlace()) fleet.init_worker() exe.run(fleet.startup_program) train_generator = py_reader1.CriteoDataset(1000001) file_list = [str(DATA_PATH)] * 2 train_reader = paddle.batch( train_generator.train(file_list, args.trainers, args.current_id), batch_size=4) self.pyreader.decorate_paddle_reader(train_reader) if os.getenv("PADDLE_COMPATIBILITY_CHECK", False): exec_strategy = fluid.ExecutionStrategy() exec_strategy.num_threads = int(2) build_strategy = fluid.BuildStrategy() build_strategy.async_mode = self.async_mode if args.run_params["sync_mode"] == "async": build_strategy.memory_optimize = False if args.run_params['cpu_num'] > 1: build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce else: build_strategy = self.strategy.get_build_strategy() if args.run_params["sync_mode"] == "async": build_strategy.memory_optimize = False self.strategy.set_build_strategy(build_strategy) exec_strategy = self.strategy.get_execute_strategy() compiled_prog = fluid.compiler.CompiledProgram( fleet.main_program).with_data_parallel( loss_name=self.avg_cost.name, build_strategy=build_strategy, exec_strategy=exec_strategy) # Notice: py_reader should use try & catch EOFException method to enter the dataset # reader.start() must declare in advance self.pyreader.start() train_info = [] batch_id = 0 try: while True: avg_cost = exe.run(program=compiled_prog, fetch_list=[self.avg_cost.name]) avg_cost = np.mean(avg_cost) train_info.append(avg_cost) batch_id += 1 if batch_id == args.run_params.get("total_batch_size", 5): if params["is_first_trainer"]: if params["is_pyreader_train"]: model_path = str(params["model_path"] + "/final" + "_pyreader") fleet.save_persistables( executor=fluid.Executor(fluid.CPUPlace()), dirname=model_path) elif params["is_dataset_train"]: model_path = str(params["model_path"] + '/final' + "_dataset") fleet.save_persistables( executor=fluid.Executor(fluid.CPUPlace()), dirname=model_path) else: raise ValueError( "Program must has Date feed method: is_pyreader_train / is_dataset_train" ) break except fluid.core.EOFException: self.pyreader.reset() fleet.stop_worker() return train_info
def do_compress(args): train_program = fluid.default_main_program() startup_program = fluid.default_startup_program() dataset = reader.Dataset(args) with fluid.program_guard(train_program, startup_program): train_program.random_seed = args.random_seed startup_program.random_seed = args.random_seed with fluid.unique_name.guard(): train_ret = creator.create_model(args, dataset.vocab_size, dataset.num_labels, mode='train') test_program = train_program.clone() optimizer = fluid.optimizer.Adam(learning_rate=args.base_learning_rate) # init executor if args.use_cuda: place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0'))) dev_count = fluid.core.get_cuda_device_count() else: dev_count = min(multiprocessing.cpu_count(), args.cpu_num) if (dev_count < args.cpu_num): print( "WARNING: The total CPU NUM in this machine is %d, which is less than cpu_num parameter you set. " "Change the cpu_num from %d to %d" % (dev_count, args.cpu_num, dev_count)) os.environ['CPU_NUM'] = str(dev_count) place = fluid.CPUPlace() train_reader = paddle.batch(dataset.file_reader(args.train_data), batch_size=args.batch_size) test_reader = paddle.batch(dataset.file_reader(args.test_data), batch_size=args.batch_size) exe = fluid.Executor(place) exe.run(startup_program) if args.init_checkpoint: utils.init_checkpoint(exe, args.init_checkpoint + '.pdckpt', train_program) train_feed_list = [('words', train_ret['words'].name), ("targets", train_ret["targets"].name)] train_fetch_list = [('loss', train_ret['avg_cost'].name)] test_feed_list = [('words', train_ret['words'].name), ("targets", train_ret["targets"].name)] test_fetch_list = [('f1_score', train_ret['f1_score'].name)] print(train_ret['crf_decode'].name) com_pass = Compressor(place, fluid.global_scope(), train_program=train_program, train_reader=train_reader, train_feed_list=train_feed_list, train_fetch_list=train_fetch_list, eval_program=test_program, eval_reader=test_reader, eval_feed_list=test_feed_list, eval_fetch_list=test_fetch_list, teacher_programs=[], train_optimizer=optimizer, distiller_optimizer=None) com_pass.config(args.compress_config) com_pass.run()
def main(dict_path): word_dict = load_vocab(dict_path) word_dict["<unk>"] = len(word_dict) dict_dim = len(word_dict) print("The dictionary size is : %d" % dict_dim) data, label, prediction, avg_cost = conv_net(dict_dim) sgd_optimizer = fluid.optimizer.SGD(learning_rate=conf.learning_rate) optimize_ops, params_grads = sgd_optimizer.minimize(avg_cost) batch_size_var = fluid.layers.create_tensor(dtype='int64') batch_acc_var = fluid.layers.accuracy(input=prediction, label=label, total=batch_size_var) inference_program = fluid.default_main_program().clone() with fluid.program_guard(inference_program): inference_program = fluid.io.get_inference_program( target_vars=[batch_acc_var, batch_size_var]) # The training data set. train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.imdb.train(word_dict), buf_size=51200), batch_size=conf.batch_size) # The testing data set. test_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.imdb.test(word_dict), buf_size=51200), batch_size=conf.batch_size) if conf.use_gpu: place = fluid.CUDAPlace(0) else: place = fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[data, label], place=place) train_pass_acc_evaluator = fluid.average.WeightedAverage() test_pass_acc_evaluator = fluid.average.WeightedAverage() def test(exe): test_pass_acc_evaluator.reset() for batch_id, data in enumerate(test_reader()): input_seq = to_lodtensor(map(lambda x: x[0], data), place) y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape([-1, 1]) b_acc, b_size = exe.run(inference_program, feed={ "words": input_seq, "label": y_data }, fetch_list=[batch_acc_var, batch_size_var]) test_pass_acc_evaluator.add(value=b_acc, weight=b_size) test_acc = test_pass_acc_evaluator.eval() return test_acc def train_loop(exe, train_program, trainer_id): total_time = 0. for pass_id in xrange(conf.num_passes): train_pass_acc_evaluator.reset() start_time = time.time() total_samples = 0 with profiler.profiler("CPU", 'total', profile_path='./profile_res_%d' % trainer_id) as prof: for batch_id, data in enumerate(train_reader()): batch_start = time.time() cost_val, acc_val, size_val = exe.run( train_program, feed=feeder.feed(data), fetch_list=[avg_cost, batch_acc_var, batch_size_var]) train_pass_acc_evaluator.add(value=acc_val, weight=size_val) total_samples += float(size_val) if batch_id and batch_id % conf.log_period == 0: print( "Pass id: %d, batch id: %d, cost: %f, pass_acc: %f, speed: %f, time: %f" % (pass_id, batch_id, cost_val, train_pass_acc_evaluator.eval(), float(size_val) / (time.time() - batch_start), time.time() - batch_start)) end_time = time.time() total_time += (end_time - start_time) pass_test_acc = test(exe) print("Pass id: %d, test_acc: %f, speed: %f" % (pass_id, pass_test_acc, total_samples / (end_time - start_time))) print("Total train time: %f" % (total_time)) if args.local: print("run as local mode") exe.run(fluid.default_startup_program()) train_loop(exe, fluid.default_main_program(), 0) else: pserver_ips = os.getenv( "PADDLE_INIT_PSERVERS") # all pserver endpoints eplist = [] port = os.getenv("PADDLE_INIT_PORT") for ip in pserver_ips.split(","): eplist.append(':'.join([ip, port])) pserver_endpoints = ",".join(eplist) print("pserver endpoints: ", pserver_endpoints) trainers = int(os.getenv("TRAINERS")) # total trainer count print("trainers total: ", trainers) trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID", "0")) current_endpoint = os.getenv( "POD_IP") + ":" + port # current pserver endpoint training_role = os.getenv( "TRAINING_ROLE", "TRAINER") # get the training role: trainer/pserver t = fluid.DistributeTranspiler() t.transpile(optimize_ops, params_grads, trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) print("starting server side startup") exe.run(pserver_startup) print("starting parameter server...") exe.run(pserver_prog) elif training_role == "TRAINER": trainer_prog = t.get_trainer_program() exe.run(fluid.default_startup_program()) train_loop(exe, trainer_prog, trainer_id) else: print("environment var TRAINER_ROLE should be TRAINER os PSERVER")
def train(conf_dict, data_reader, use_cuda=False): """ Training of p classification model """ label_dict_len = data_reader.get_dict_size('label_dict') # input layer word = fluid.layers.data(name='word_data', shape=[1], dtype='int64', lod_level=1) postag = fluid.layers.data(name='token_pos', shape=[1], dtype='int64', lod_level=1) arc_relation = fluid.layers.data(name="arc_relation", shape=[1], dtype='int64', lod_level=1) arc_head = fluid.layers.data(name="arc_head", shape=[1], dtype="int64", lod_level=1) # label target = fluid.layers.data(name='target', shape=[label_dict_len], dtype='float32', lod_level=0) # NN: embedding + lstm + pooling feature_out = p_model3.db_lstm(data_reader, word, postag, arc_relation, arc_head, conf_dict) # loss function for multi-label classification class_cost = fluid.layers.sigmoid_cross_entropy_with_logits(x=feature_out, label=target) avg_cost = fluid.layers.mean(class_cost) acc = fluid.layers.accuracy(input=feature_out, label=target) test_program =fluid.default_main_program().clone(for_test=True) # optimization method sgd_optimizer = fluid.optimizer.AdamOptimizer(learning_rate=2e-3, ) sgd_optimizer.minimize(avg_cost) train_batch_reader = paddle.batch(paddle.reader.shuffle(data_reader.get_train_reader(), buf_size=8192), batch_size=conf_dict['batch_size']) test_batch_reader = paddle.batch(paddle.reader.shuffle(data_reader.get_test_reader(), buf_size=8192), batch_size=conf_dict['batch_size']) def train_test(train_test_program, train_test_feed, train_test_reader): acc_set = [] avg_loss_set = [] for test_data in train_test_reader(): avg_loss_np, acc_np = exe.run( program=train_test_program, feed=train_test_feed.feed(test_data), fetch_list=[avg_cost, acc]) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) # get test acc and loss acc_val_mean = np.array(acc_set).mean() avg_loss_val_mean = np.array(avg_loss_set).mean() return avg_loss_val_mean, acc_val_mean place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() feeder = fluid.DataFeeder(feed_list=[word, postag, arc_relation, arc_head, target], place=place) exe = fluid.Executor(place) save_dirname = conf_dict['p_model_save_dir'] def train_loop(main_program, trainer_id=0): """start train""" exe.run(fluid.default_startup_program()) start_time = time.time() batch_id = 0 lists = [] best_epoch = 100000 for pass_id in six.moves.xrange(conf_dict['pass_num']): pass_start_time = time.time() cost_sum, cost_counter = 0, 0 for data in tqdm(train_batch_reader()): cost = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) cost = cost[0] cost_sum += cost cost_counter += 1 if batch_id % 10 == 0 and batch_id != 0: print("batch %d finished, second per batch: %02f" % (batch_id, (time.time() - start_time) / batch_id), file=sys.stderr) # cost expected, training over if float(cost) < 0.01: pass_avg_cost = cost_sum / cost_counter if cost_counter > 0 else 0.0 print("%d pass end, cost time: %02f, avg_cost: %f" % (pass_id, time.time() - pass_start_time, pass_avg_cost), file=sys.stderr) save_path = os.path.join(save_dirname, 'final') fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'arc_relation', 'arc_head'], [feature_out], exe, params_filename='params') return batch_id = batch_id + 1 #test for epoch avg_loss_val, acc_val = train_test(train_test_program=test_program,train_test_reader=test_batch_reader, train_test_feed=feeder) if best_epoch > avg_loss_val: best_epoch = avg_loss_val save_path = os.path.join(save_dirname, 'final') fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'arc_relation', 'arc_head'], [feature_out], exe, params_filename='params') print("Test with Epoch %d, avg_cost: %s, acc: %s" % (pass_id, avg_loss_val, acc_val)) lists.append((pass_id, avg_loss_val, acc_val)) # save the model once each pass ends pass_avg_cost = cost_sum / cost_counter if cost_counter > 0 else 0.0 print("Train with Epoch %d, cost time: %02f, avg_cost: %f" % (pass_id, time.time() - pass_start_time, pass_avg_cost), file=sys.stderr) save_path = os.path.join(save_dirname, 'pass_%04d-%f' % (pass_id, pass_avg_cost)) fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'arc_relation', 'arc_head'], [feature_out], exe, params_filename='params') # find the best pass best = sorted(lists, key=lambda list: float(list[1]))[0] print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1])) print('The classification accuracy is %.2f%%' % (float(best[2]) * 100)) return train_loop(fluid.default_main_program())
划分训练集与验证集,乱序,生成数据列表 ''' #每次生成数据列表前,首先清空train.txt和eval.txt with open(train_list_path, 'w') as f: f.seek(0) f.truncate() with open(eval_list_path, 'w') as f: f.seek(0) f.truncate() #生成数据列表 get_data_list(target_path, train_list_path, eval_list_path) ''' 构造数据提供器 ''' train_reader = paddle.batch(custom_reader(train_list_path), batch_size=batch_size, drop_last=True) eval_reader = paddle.batch(custom_reader(eval_list_path), batch_size=batch_size, drop_last=True) # # **二、模型配置** # # ![](https://ai-studio-static-online.cdn.bcebos.com/9ca0744272b0449186f766afcabadd598e24679088a4438dafede05a71b7c141) # # VGG的核心是五组卷积操作,每两组之间做Max-Pooling空间降维。同一组内采用多次连续的3X3卷积,卷积核的数目由较浅组的64增多到最深组的512,同一组内的卷积核数目是一样的。卷积之后接两层全连接层,之后是分类层。由于每组内卷积层的不同,有11、13、16、19层这几种模型,上图展示一个16层的网络结构。 # # # In[ ]:
def train(topology, train_data_dir=None, test_data_dir=None, word_dict_path=None, label_dict_path=None, model_save_dir="models", use_cuda=False, window_size=5, learning_rate=0.001, batch_size=64, num_passes=10): """ train window_net model or sentence_net model :params train_data_path: path of training data, if this parameter is not specified, Brown Corpus will be used to run this example :type train_data_path: str :params test_data_path: path of testing data, if this parameter is not specified, Brown Corpus will be used to run this example :type test_data_path: str :params word_dict_path: path of word dictionary data, if this parameter is not specified, a default dictionary file will be used to run this example :type word_dict_path: str :params label_dict_path: path of label dictionary data, if this parameter is not specified, a default dictionary file will be used to run this example :type label_dict_path: str :params use_cuda: whether use the cuda :type use_cuda: bool :params window_size: size of window width :type window_size: int :params num_pass: train pass number :type num_pass: int """ if not os.path.exists(model_save_dir): os.mkdir(model_save_dir) use_default_data = (train_data_dir is None) if use_default_data: logger.info(("No training data are provided, " "use Brown corpus to train the model.")) logger.info("downloading Brown corpus...") train_data_dir, test_data_dir, word_dict_path, label_dict_path = load_default_data( ) logger.info("please wait to build the word dictionary ...") if word_dict_path is None or not os.path.exists(word_dict_path): logger.info(("word dictionary is not given, the dictionary " "is automatically built from the training data.")) # build the word dictionary to map the original string-typed # words into integer-typed index build_dict(data_dir=train_data_dir, save_path=word_dict_path, use_col=0, cutoff_fre=1, insert_extra_words=["<UNK>"]) logger.info("the word dictionary path is %s" % word_dict_path) if not os.path.exists(label_dict_path): logger.info(("label dictionary is not given, the dictionary " "is automatically built from the training data.")) # build the label dictionary to map the original string-typed # label into integer-typed index build_dict(data_dir=train_data_dir, save_path=label_dict_path, use_col=1, cutoff_fre=10, insert_extra_words=["<UNK>"]) logger.info("the label dictionary path is %s" % label_dict_path) # get index info word_dict = load_dict(word_dict_path) lbl_dict = load_dict(label_dict_path) class_num = len(lbl_dict) logger.info("class number is : %d." % (len(lbl_dict))) # get train data reader train_reader = paddle.batch(paddle.reader.shuffle(reader.train_reader( train_data_dir, word_dict, lbl_dict, window_size), buf_size=51200), batch_size=batch_size) # get test data reader if test_data_dir is not None: # here, because training and testing data share a same format, # we still use the reader.train_reader to read the testing data. test_reader = paddle.batch(reader.train_reader(test_data_dir, word_dict, lbl_dict, window_size), batch_size=batch_size) else: test_reader = None # get size of word dictionary dict_dim = len(word_dict) + 1 logger.info("length of word dictionary is : %d." % (dict_dim)) # define the input layers data = fluid.layers.data(name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") # return the network result cost, acc, prediction = topology(data, label, dict_dim, class_num=class_num) # create optimizer sgd_optimizer = fluid.optimizer.Adam(learning_rate=learning_rate) sgd_optimizer.minimize(cost) # create trainer place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[data, label], place=place) # initialize training network exe.run(fluid.default_startup_program()) prog = fluid.default_main_program() # begin training network for pass_id in range(num_passes): ## running the train data data_size, data_count, total_acc, total_cost = 0, 0, 0.0, 0.0 for i, data_ in enumerate(train_reader()): avg_cost_np, avg_acc_np = exe.run(prog, feed=feeder.feed(data_), fetch_list=[cost, acc]) data_size = len(data_) total_acc += data_size * avg_acc_np total_cost += data_size * avg_cost_np data_count += data_size if (i + 1) % 1000 == 0: logger.info( "pass_id: %d, batch %d, avg_acc: %f, avg_cost: %f" % (pass_id, i + 1, total_acc / data_count, total_cost / data_count)) avg_cost = total_cost / data_count avg_acc = total_acc / data_count logger.info("Train result -- pass_id: %d, avg_acc: %f, avg_cost: %f" % (pass_id, avg_acc, avg_cost)) ## running the test data if test_reader is not None: data_size, data_count, total_acc, total_cost = 0, 0, 0.0, 0.0 for i, data in enumerate(test_reader()): avg_cost_np, avg_acc_np, prediction_np = exe.run( prog, feed=feeder.feed(data), fetch_list=[cost, acc, prediction]) data_size = len(data) total_acc += data_size * avg_acc_np total_cost += data_size * avg_cost_np data_count += data_size avg_cost = total_cost / data_count avg_acc = total_acc / data_count logger.info( "Test result -- pass_id: %d, avg_acc: %f, avg_cost: %f" % (pass_id, avg_acc, avg_cost)) ## save inference model epoch_model = model_save_dir + "/" + args.nn_type + "_epoch" + str( pass_id % 5) logger.info("Saving inference model at %s" % (epoch_model)) ##prediction is the topology return value ##if we use the prediction value as the infer result fluid.io.save_inference_model(epoch_model, ["words"], prediction, exe) logger.info("Training has finished.")
def test_mnist_sort_gradient_float32(self): seed = 90 epoch_num = 1 with fluid.dygraph.guard(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed backward_strategy = fluid.dygraph.BackwardStrategy() backward_strategy.sort_sum_gradient = True mnist2 = MNIST("mnist") sgd2 = SGDOptimizer(learning_rate=1e-3) train_reader2 = paddle.batch(paddle.dataset.mnist.train(), batch_size=128, drop_last=True) mnist2.train() dy_param_init_value2 = {} for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader2()): dy_x_data2 = np.array([ x[0].reshape(1, 28, 28) for x in data ]).astype('float32') y_data2 = np.array([x[1] for x in data ]).astype('int64').reshape(128, 1) img2 = to_variable(dy_x_data2) label2 = to_variable(y_data2) label2.stop_gradient = True cost2 = mnist2(img2) loss2 = fluid.layers.cross_entropy(cost2, label2) avg_loss2 = fluid.layers.mean(loss2) dy_out2 = avg_loss2.numpy() if epoch == 0 and batch_id == 0: for param in mnist2.parameters(): dy_param_init_value2[param.name] = param.numpy() avg_loss2.backward(backward_strategy) sgd2.minimize(avg_loss2) mnist2.clear_gradients() dy_param_value2 = {} for param in mnist2.parameters(): dy_param_value2[param.name] = param.numpy() if batch_id == 20: break with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed exe = fluid.Executor(fluid.CPUPlace( ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) mnist = MNIST("mnist") sgd = SGDOptimizer(learning_rate=1e-3) train_reader = paddle.batch(paddle.dataset.mnist.train(), batch_size=128, drop_last=True) img = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) avg_loss = fluid.layers.mean(loss) sgd.minimize(avg_loss) # initialize params and fetch them static_param_init_value = {} static_param_name_list = [] for param in mnist.parameters(): static_param_name_list.append(param.name) out = exe.run(fluid.default_startup_program(), fetch_list=static_param_name_list) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): static_x_data = np.array([ x[0].reshape(1, 28, 28) for x in data ]).astype('float32') y_data = np.array([x[1] for x in data ]).astype('int64').reshape([128, 1]) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) out = exe.run(fluid.default_main_program(), feed={ "pixel": static_x_data, "label": y_data }, fetch_list=fetch_list) static_param_value = {} static_out = out[0] for i in range(1, len(out)): static_param_value[static_param_name_list[i - 1]] = out[i] if batch_id == 20: break self.assertTrue(np.allclose(dy_x_data2.all(), static_x_data.all())) for key, value in six.iteritems(static_param_init_value): self.assertTrue(np.allclose(value, dy_param_init_value2[key])) self.assertTrue(np.allclose(static_out, dy_out2)) for key, value in six.iteritems(static_param_value): self.assertTrue(np.allclose(value, dy_param_value2[key], atol=1e-5))
cost = fluid.layers.cross_entropy(input=model, label=label) avg_cost = fluid.layers.mean(cost) acc = fluid.layers.accuracy(input=model, label=label) # 获取训练和测试程序 test_program = fluid.default_main_program().clone(for_test=True) # 定义优化方法 optimizer = fluid.optimizer.AdamOptimizer( learning_rate=1e-3, regularization=fluid.regularizer.L2DecayRegularizer(1e-4)) opts = optimizer.minimize(avg_cost) # 获取自定义数据 train_reader = paddle.batch(reader=reader.train_reader('images/train.list', crop_size, resize_size), batch_size=32) test_reader = paddle.batch(reader=reader.test_reader('images/test.list', crop_size), batch_size=32) # 定义一个使用GPU的执行器 place = fluid.CUDAPlace(0) # place = fluid.CPUPlace() exe = fluid.Executor(place) # 进行参数初始化 exe.run(fluid.default_startup_program()) # 定义输入数据维度 feeder = fluid.DataFeeder(place=place, feed_list=[image, label])
# -*- coding: utf-8 -*- # @File : dist_train.py # @Author: WangYe # @Date : 2019/11/22 # @Software: PyCharm import os import paddle import paddle.fluid as fluid # train reader BATCH_SIZE = 20 EPOCH_NUM = 30 BATCH_SIZE = 8 train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.uci_housing.train(), buf_size=500), batch_size=BATCH_SIZE) def train(): y = fluid.layers.data(name='y', shape=[1], dtype='float32') x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None) loss = fluid.layers.square_error_cost(input=y_predict, label=y) avg_loss = fluid.layers.mean(loss) opt = fluid.optimizer.SGD(learning_rate=0.001) opt.minimize(avg_loss) place = fluid.CPUPlace() feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
def check_network_convergence(self, is_sparse, build_strategy=None): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): word = fluid.layers.data( name='word_data', shape=[1], dtype='int64', lod_level=1) predicate = fluid.layers.data( name='verb_data', shape=[1], dtype='int64', lod_level=1) ctx_n2 = fluid.layers.data( name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1) ctx_n1 = fluid.layers.data( name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1) ctx_0 = fluid.layers.data( name='ctx_0_data', shape=[1], dtype='int64', lod_level=1) ctx_p1 = fluid.layers.data( name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1) ctx_p2 = fluid.layers.data( name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1) mark = fluid.layers.data( name='mark_data', shape=[1], dtype='int64', lod_level=1) feature_out = db_lstm(**locals()) target = fluid.layers.data( name='target', shape=[1], dtype='int64', lod_level=1) crf_cost = fluid.layers.linear_chain_crf( input=feature_out, label=target, param_attr=fluid.ParamAttr( name='crfw', learning_rate=1e-1)) avg_cost = fluid.layers.mean(crf_cost) sgd_optimizer = fluid.optimizer.SGD( learning_rate=fluid.layers.exponential_decay( learning_rate=0.01, decay_steps=100000, decay_rate=0.5, staircase=True)) sgd_optimizer.minimize(avg_cost) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.conll05.test(), buf_size=8192), batch_size=16) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) exe.run(startup) pe = fluid.ParallelExecutor( use_cuda=True, loss_name=avg_cost.name, build_strategy=build_strategy) feeder = fluid.DataFeeder( feed_list=[ word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target ], place=fluid.CPUPlace()) data = train_data() for i in xrange(10): cur_batch = next(data) print map(np.array, pe.run(feed=feeder.feed(cur_batch), fetch_list=[avg_cost.name]))[0]
plot_cost.append(train_title, step, event.metrics[0]) if event.step % 20 == 0: # 若干个batch,记录cost test_metrics = trainer.test( reader=test_reader, feed_order=feed_order) if test_metrics[0] < 10: plot_cost.append(test_title, step, test_metrics[0]) # 将参数存储,用于预测使用 if params_dirname is not None: trainer.save_params(params_dirname) step += 1 # 设置 BATCH_SIZE 的大小 BATCH_SIZE = 128 # 设置训练reader train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=BATCH_SIZE) #设置测试 reader test_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.test(), buf_size=500), batch_size=BATCH_SIZE) #创建训练器 trainer = Trainer( train_func= train_func, place= place, optimizer_func= optimizer_func)
def train(use_cuda, is_sparse, is_parallel, save_dirname, is_local=True): PASS_NUM = 100 EMBED_SIZE = 32 HIDDEN_SIZE = 256 N = 5 BATCH_SIZE = 32 IS_SPARSE = is_sparse def __network__(words): embed_first = fluid.layers.embedding( input=words[0], size=[dict_size, EMBED_SIZE], dtype='float32', is_sparse=IS_SPARSE, param_attr='shared_w') embed_second = fluid.layers.embedding( input=words[1], size=[dict_size, EMBED_SIZE], dtype='float32', is_sparse=IS_SPARSE, param_attr='shared_w') embed_third = fluid.layers.embedding( input=words[2], size=[dict_size, EMBED_SIZE], dtype='float32', is_sparse=IS_SPARSE, param_attr='shared_w') embed_forth = fluid.layers.embedding( input=words[3], size=[dict_size, EMBED_SIZE], dtype='float32', is_sparse=IS_SPARSE, param_attr='shared_w') concat_embed = fluid.layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], axis=1) hidden1 = fluid.layers.fc(input=concat_embed, size=HIDDEN_SIZE, act='sigmoid') predict_word = fluid.layers.fc(input=hidden1, size=dict_size, act='softmax') cost = fluid.layers.cross_entropy(input=predict_word, label=words[4]) avg_cost = fluid.layers.mean(cost) return avg_cost, predict_word word_dict = paddle.dataset.imikolov.build_dict() dict_size = len(word_dict) first_word = fluid.layers.data(name='firstw', shape=[1], dtype='int64') second_word = fluid.layers.data(name='secondw', shape=[1], dtype='int64') third_word = fluid.layers.data(name='thirdw', shape=[1], dtype='int64') forth_word = fluid.layers.data(name='forthw', shape=[1], dtype='int64') next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64') if not is_parallel: avg_cost, predict_word = __network__( [first_word, second_word, third_word, forth_word, next_word]) else: places = fluid.layers.get_places() pd = fluid.layers.ParallelDo(places) with pd.do(): avg_cost, predict_word = __network__( map(pd.read_input, [ first_word, second_word, third_word, forth_word, next_word ])) pd.write_output(avg_cost) avg_cost = fluid.layers.mean(pd()) sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(avg_cost) train_reader = paddle.batch( paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder( feed_list=[first_word, second_word, third_word, forth_word, next_word], place=place) def train_loop(main_program): exe.run(fluid.default_startup_program()) for pass_id in range(PASS_NUM): for data in train_reader(): avg_cost_np = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) if avg_cost_np[0] < 5.0: if save_dirname is not None: fluid.io.save_inference_model(save_dirname, [ 'firstw', 'secondw', 'thirdw', 'forthw' ], [predict_word], exe) return if math.isnan(float(avg_cost_np[0])): sys.exit("got NaN loss, training failed.") raise AssertionError("Cost is too large {0:2.2}".format(avg_cost_np[0])) if is_local: train_loop(fluid.default_main_program()) else: port = os.getenv("PADDLE_INIT_PORT", "6174") pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip... eplist = [] for ip in pserver_ips.split(","): eplist.append(':'.join([ip, port])) pserver_endpoints = ",".join(eplist) # ip:port,ip:port... trainers = int(os.getenv("TRAINERS")) current_endpoint = os.getenv("POD_IP") + ":" + port trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID")) training_role = os.getenv("TRAINING_ROLE", "TRAINER") t = fluid.DistributeTranspiler() t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": train_loop(t.get_trainer_program())
def run_benchmark(model, args): if args.use_cprof: pr = cProfile.Profile() pr.enable() if args.data_set == "cifar10": class_dim = 10 if args.data_format == 'NCHW': dshape = [3, 32, 32] else: dshape = [32, 32, 3] else: class_dim = 102 if args.data_format == 'NCHW': dshape = [3, 224, 224] else: dshape = [224, 224, 3] input = fluid.layers.data(name='data', shape=dshape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') predict = model(input, class_dim) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) batch_size_tensor = fluid.layers.create_tensor(dtype='int64') batch_acc = fluid.layers.accuracy(input=predict, label=label, total=batch_size_tensor) inference_program = fluid.default_main_program().clone() with fluid.program_guard(inference_program): inference_program = fluid.io.get_inference_program( target_vars=[batch_acc, batch_size_tensor]) optimizer = fluid.optimizer.Momentum(learning_rate=0.01, momentum=0.9) opts = optimizer.minimize(avg_cost) fluid.memory_optimize(fluid.default_main_program()) train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.cifar.train10() if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), buf_size=5120), batch_size=args.batch_size) test_reader = paddle.batch(paddle.dataset.cifar.test10() if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), batch_size=args.batch_size) def test(exe): test_accuracy = fluid.average.WeightedAverage() for batch_id, data in enumerate(test_reader()): img_data = np.array(map(lambda x: x[0].reshape(dshape), data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape([-1, 1]) acc, weight = exe.run(inference_program, feed={ "data": img_data, "label": y_data }, fetch_list=[batch_acc, batch_size_tensor]) test_accuracy.add(value=acc, weight=weight) return test_accuracy.eval() place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) accuracy = fluid.average.WeightedAverage() if args.use_fake_data: data = train_reader().next() image = np.array(map(lambda x: x[0].reshape(dshape), data)).astype('float32') label = np.array(map(lambda x: x[1], data)).astype('int64') label = label.reshape([-1, 1]) im_num = 0 total_train_time = 0.0 total_iters = 0 train_acc_kpi = None for kpi in tracking_kpis: if kpi.name == '%s_%s_train_acc' % (args.data_set, args.batch_size): train_acc_kpi = kpi train_speed_kpi = None for kpi in tracking_kpis: if kpi.name == '%s_%s_train_speed' % (args.data_set, args.batch_size): train_speed_kpi = kpi for pass_id in range(args.pass_num): every_pass_loss = [] accuracy.reset() iter = 0 pass_duration = 0.0 for batch_id, data in enumerate(train_reader()): batch_start = time.time() if iter == args.iterations: break if not args.use_fake_data: image = np.array(map(lambda x: x[0].reshape(dshape), data)).astype('float32') label = np.array(map(lambda x: x[1], data)).astype('int64') label = label.reshape([-1, 1]) loss, acc, weight = exe.run( fluid.default_main_program(), feed={ 'data': image, 'label': label }, fetch_list=[avg_cost, batch_acc, batch_size_tensor]) accuracy.add(value=acc, weight=weight) if iter >= args.skip_batch_num or pass_id != 0: batch_duration = time.time() - batch_start pass_duration += batch_duration im_num += label.shape[0] every_pass_loss.append(loss) # print("Pass: %d, Iter: %d, loss: %s, acc: %s" % # (pass_id, iter, str(loss), str(acc))) iter += 1 total_iters += 1 total_train_time += pass_duration pass_train_acc = accuracy.eval() pass_test_acc = test(exe) print( "Pass:%d, Loss:%f, Train Accuray:%f, Test Accuray:%f, Handle Images Duration: %f\n" % (pass_id, np.mean(every_pass_loss), pass_train_acc, pass_test_acc, pass_duration)) if pass_id == args.pass_num - 1 and args.data_set == 'cifar10': train_acc_kpi.add_record(np.array(pass_train_acc, dtype='float32')) train_acc_kpi.persist() if total_train_time > 0.0 and iter != args.skip_batch_num: examples_per_sec = im_num / total_train_time sec_per_batch = total_train_time / \ (iter * args.pass_num - args.skip_batch_num) train_speed_kpi.add_record(np.array(examples_per_sec, dtype='float32')) train_speed_kpi.persist() print('\nTotal examples: %d, total time: %.5f' % (im_num, total_train_time)) print('%.5f examples/sec, %.5f sec/batch \n' % (examples_per_sec, sec_per_batch)) if args.use_cprof: pr.disable() s = StringIO.StringIO() sortby = 'cumulative' ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() print(s.getvalue())
def test_raw_api(self): prog = Program() startup_prog = Program() with program_guard(prog, startup_prog): image = layers.data(name='x', shape=[784], dtype='float32') label = layers.data(name='y', shape=[1], dtype='int64') limit = layers.fill_constant_batch_size_like( input=label, dtype='int64', shape=[1], value=5.0) cond = layers.less_than(x=label, y=limit) true_image, false_image = layers.split_lod_tensor( input=image, mask=cond) true_out = layers.create_tensor(dtype='float32') true_cond = layers.ConditionalBlock([true_image]) with true_cond.block(): hidden = layers.fc(input=true_image, size=100, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=true_out) false_out = layers.create_tensor(dtype='float32') false_cond = layers.ConditionalBlock([false_image]) with false_cond.block(): hidden = layers.fc(input=false_image, size=200, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=false_out) prob = layers.merge_lod_tensor( in_true=true_out, in_false=false_out, mask=cond, x=image) loss = layers.cross_entropy(input=prob, label=label) avg_loss = layers.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=200) place = core.CPUPlace() exe = Executor(place) exe.run(startup_prog) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = np.expand_dims(y_data, axis=1) outs = exe.run(prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) print outs[0] if outs[0] < 1.0: return self.assertFalse(True)
def test_mnist_float32(self): seed = 90 epoch_num = 1 batch_size = 128 batch_num = 50 traced_layer = None with fluid.dygraph.guard(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3, parameter_list=mnist.parameters()) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( paddle.batch(self.reader_decorator( paddle.dataset.mnist.train()), batch_size=batch_size, drop_last=True), places=fluid.CPUPlace()) mnist.train() dy_param_init_value = {} helper = DyGraphProgramDescTracerTestHelper(self) program = None for epoch in range(epoch_num): for batch_id, data in enumerate(batch_py_reader()): if batch_id >= batch_num: break img = data[0] dy_x_data = img.numpy() label = data[1] label.stop_gradient = True if batch_id % 10 == 0: cost, traced_layer = TracedLayer.trace(mnist, inputs=img) if program is not None: self.assertTrue(program, traced_layer.program) program = traced_layer.program traced_layer.save_inference_model( './infer_imperative_mnist') else: cost = mnist(img) if traced_layer is not None: cost_static = traced_layer([img]) helper.assertEachVar(cost, cost_static) loss = fluid.layers.cross_entropy(cost, label) avg_loss = fluid.layers.mean(loss) dy_out = avg_loss.numpy() if epoch == 0 and batch_id == 0: for param in mnist.parameters(): dy_param_init_value[param.name] = param.numpy() avg_loss.backward() sgd.minimize(avg_loss) mnist.clear_gradients() dy_param_value = {} for param in mnist.parameters(): dy_param_value[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed exe = fluid.Executor(fluid.CPUPlace( ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) train_reader = paddle.batch(paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True) img = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) avg_loss = fluid.layers.mean(loss) sgd.minimize(avg_loss) # initialize params and fetch them static_param_init_value = {} static_param_name_list = [] for param in mnist.parameters(): static_param_name_list.append(param.name) out = exe.run(fluid.default_startup_program(), fetch_list=static_param_name_list) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break static_x_data = np.array([ x[0].reshape(1, 28, 28) for x in data ]).astype('float32') y_data = np.array([x[1] for x in data]).astype('int64').reshape( [batch_size, 1]) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) if traced_layer is not None: traced_layer([static_x_data]) out = exe.run(fluid.default_main_program(), feed={ "pixel": static_x_data, "label": y_data }, fetch_list=fetch_list) static_param_value = {} static_out = out[0] for i in range(1, len(out)): static_param_value[static_param_name_list[i - 1]] = out[i] self.assertTrue(np.allclose(dy_x_data.all(), static_x_data.all())) for key, value in six.iteritems(static_param_init_value): self.assertTrue(np.allclose(value, dy_param_init_value[key])) self.assertTrue(np.allclose(static_out, dy_out)) for key, value in six.iteritems(static_param_value): self.assertTrue(np.allclose(value, dy_param_value[key], atol=1e-5))
model = vgg16(image, 10) # 获取损失函数和准确率函数 cost = fluid.layers.cross_entropy(input=model, label=label) avg_cost = fluid.layers.mean(cost) acc = fluid.layers.accuracy(input=model, label=label) # 获取训练和测试程序 test_program = fluid.default_main_program().clone(for_test=True) # 定义优化方法 optimizer = fluid.optimizer.MomentumOptimizer(learning_rate=1e-3, momentum=0.9) opts = optimizer.minimize(avg_cost) # 获取CIFAR数据 train_reader = paddle.batch(cifar.train10(), batch_size=32) test_reader = paddle.batch(cifar.test10(), batch_size=32) # 定义一个使用CPU的执行器 place = fluid.CUDAPlace(0) # place = fluid.CPUPlace() exe = fluid.Executor(place) # 进行参数初始化 exe.run(fluid.default_startup_program()) # 加载之前训练过的参数模型 save_path = 'models/params_model/' if os.path.exists(save_path): print('使用参数模型作为预训练模型') fluid.io.load_params(executor=exe, dirname=save_path)
def predict_infer(conf_dict, data_reader, predict_data_path, \ predict_result_path, model_path): """ Predict with trained models """ if len(predict_result_path) > 0: result_writer = open(predict_result_path, 'w') else: result_writer = sys.stdout np.set_printoptions(precision=3) if len(model_path) == 0: return place = fluid.CPUPlace() word = fluid.layers.data(name='word_data', shape=[1], dtype='int64', lod_level=1) postag = fluid.layers.data(name='token_pos', shape=[1], dtype='int64', lod_level=1) p_word = fluid.layers.data(name='p_word', shape=[1], dtype='int64', lod_level=1) feeder = fluid.DataFeeder(feed_list=[word, postag, p_word], place=place) exe = fluid.Executor(place) test_batch_reader = paddle.batch( paddle.reader.buffered(data_reader.get_predict_reader\ (predict_data_path, need_input=True, need_label=False), size=8192), batch_size=conf_dict['batch_size']) inference_scope = fluid.core.Scope() text_spo_dic = {} #final triples with fluid.scope_guard(inference_scope): [inference_program, feed_target_names, fetch_targets] = \ fluid.io.load_inference_model( model_path, exe, params_filename='params') # batch batch_id = 0 for data in test_batch_reader(): feeder_data = [] input_data = [] for item in data: feeder_data.append(item[1:]) input_data.append(item[0]) results = exe.run(inference_program, feed=feeder.feed(feeder_data), fetch_list=fetch_targets, return_numpy=False) tag_split_idx = results[0].lod()[0] label_tag_scores = np.array(results[0]) # sentence print >> sys.stderr, 'batch_id=', batch_id for sent_idx, tag_idx in enumerate(tag_split_idx[:-1]): input_sent = input_data[sent_idx].split('\t')[0] input_p = input_data[sent_idx].split('\t')[1] tag_scores = label_tag_scores[tag_idx:tag_split_idx[sent_idx + 1]] # token tag_list = [] for token_idx, token_tags in enumerate(tag_scores): tag = data_reader.get_label_output(token_tags) tag_list.append(tag) predicted_s_list, predicted_o_list = refine_predict_seq( input_sent, tag_list) tag_list_str = json.dumps(tag_list, ensure_ascii=False) if len(predicted_s_list) == 0 or len(predicted_o_list) == 0: continue else: text = json.loads(input_sent)["text"] predicted_s_list = list(set(predicted_s_list)) predicted_o_list = list(set(predicted_o_list)) for predicted_s in predicted_s_list: for predicted_o in predicted_o_list: if text not in text_spo_dic: text_spo_dic[text] = set() text_spo_dic[text].add( (predicted_s, input_p, predicted_o)) batch_id += 1 output(text_spo_dic, result_writer)
def eval(args): train_reader = None test_reader = None if args.data == "mnist": import paddle.dataset.mnist as reader train_reader = reader.train() val_reader = reader.test() class_dim = 10 image_shape = "1,28,28" elif args.data == "imagenet": import imagenet_reader as reader train_reader = reader.train() val_reader = reader.val() class_dim = 1000 image_shape = "3,224,224" else: raise ValueError("{} is not supported.".format(args.data)) image_shape = [int(m) for m in image_shape.split(",")] assert args.model in model_list, "{} is not in lists: {}".format( args.model, model_list) image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') # model definition model = models.__dict__[args.model]() out = model.net(input=image, class_dim=class_dim) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) val_program = fluid.default_main_program().clone(for_test=True) place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) val_reader = paddle.batch(val_reader, batch_size=args.batch_size) val_feeder = feeder = fluid.DataFeeder([image, label], place, program=val_program) load_model(exe, val_program, args.model_path) batch_id = 0 acc_top1_ns = [] acc_top5_ns = [] for data in val_reader(): start_time = time.time() acc_top1_n, acc_top5_n = exe.run( val_program, feed=val_feeder.feed(data), fetch_list=[acc_top1.name, acc_top5.name]) end_time = time.time() if batch_id % args.log_period == 0: _logger.info( "Eval batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".format( batch_id, np.mean(acc_top1_n), np.mean(acc_top5_n), end_time - start_time)) acc_top1_ns.append(np.mean(acc_top1_n)) acc_top5_ns.append(np.mean(acc_top5_n)) batch_id += 1 _logger.info("Final eval - acc_top1: {}; acc_top5: {}".format( np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
def do_train(args): train_prog = fluid.default_main_program() startup_prog = fluid.default_startup_program() with fluid.program_guard(train_prog, startup_prog): train_prog.random_seed = args.random_seed startup_prog.random_seed = args.random_seed with fluid.unique_name.guard(): # define reader image = fluid.layers.data( name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') reader = fluid.io.PyReader( feed_list=[image, label], capacity=4, iterable=False) # define the network loss = create_net( is_training=True, model_input=[image, label], args=args) # define optimizer for learning optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) optimizer.minimize(loss) if args.do_eval_in_training: # if you want to perform testing in training, you need to declare an extra test-program test_prog = fluid.Program() with fluid.program_guard(test_prog, startup_prog): test_prog.random_seed = args.random_seed with fluid.unique_name.guard(): test_image = fluid.layers.data( name='test_image', shape=[1, 28, 28], dtype='float32') test_label = fluid.layers.data( name='test_label', shape=[1], dtype='int64') test_reader = fluid.io.PyReader( feed_list=[test_image, test_label], capacity=4, iterable=False) predict = create_net( is_training=False, model_input=test_image, args=args) # prepare training ## declare data generator generator = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=args.batch_size) reader.decorate_sample_list_generator(generator) if args.use_cuda: place = fluid.CUDAPlace(0) else: place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) if args.init_from_checkpoint: init_from_checkpoint(args, exe, train_prog) compiled_train_prog = fluid.CompiledProgram(train_prog).with_data_parallel( loss_name=loss.name) if args.do_eval_in_training: compiled_test_prog = fluid.CompiledProgram(test_prog) test_generator = paddle.batch( paddle.dataset.mnist.test(), batch_size=args.batch_size) test_reader.decorate_sample_list_generator(test_generator) # start training step = 0 for epoch_step in range(args.epoch_num): reader.start() while True: try: # this is for minimizing the fetching op, saving the training speed. if step % args.print_step == 0: fetch_list = [loss.name] else: fetch_list = [] output = exe.run(compiled_train_prog, fetch_list=fetch_list) if step % args.print_step == 0: print("step: %d, loss: %.4f" % (step, np.sum(output[0]))) if step % args.save_step == 0 and step != 0: if args.save_checkpoint: save_checkpoint(args, exe, train_prog, "step_" + str(step)) if args.save_param: save_param(args, exe, train_prog, "step_" + str(step)) if args.do_eval_in_training: if step != 0 and step % args.eval_step == 0: acc = do_evaluate_in_training( reader=test_reader, executor=exe, program=test_prog, fetch_list=[predict.name]) print("evaluation acc for step %d is %.4f" % (step, acc)) step += 1 except fluid.core.EOFException: reader.reset() break if args.save_checkpoint: save_checkpoint(args, exe, train_prog, "step_final") if args.save_param: save_param(args, exe, train_prog, "step_final")
def cluster_train(learning_rate, batch_size, num_passes, model_save_dir='model'): class_dim = 102 image_shape = [3, 224, 224] image = fluid.layers.data(name='image', shape=image_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = mobile_net(image, class_dim=class_dim) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) optimizer = fluid.optimizer.Momentum( learning_rate=learning_rate, momentum=0.9, regularization=fluid.regularizer.L2Decay(5 * 1e-5)) optimize_ops, params_grads = optimizer.minimize(avg_cost) place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace( args.device_id) exe = fluid.Executor(place) standalone = int(os.getenv("STANDALONE", 0)) if standalone: pserver_endpoints = os.getenv("PSERVERS") trainers = int(os.getenv("TRAINERS")) current_endpoint = os.getenv("SERVER_ENDPOINT") trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID")) training_role = os.getenv("TRAINING_ROLE", "TRAINER") else: pport = os.getenv("PADDLE_INIT_PSERVER_PORT", "6174") tport = os.getenv("PADDLE_INIT_TRAINER_PORT", "6174") pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip... eplist = [] for ip in pserver_ips.split(","): eplist.append(':'.join([ip, pport])) pserver_endpoints = ",".join(eplist) # ip:port,ip:port... trainers = int(os.getenv("TRAINERS")) trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID")) training_role = os.getenv("TRAINING_ROLE", "TRAINER") if training_role == "PSERVER": current_endpoint = os.getenv("POD_IP") + ":" + pport else: current_endpoint = os.getenv("POD_IP") + ":" + tport print( "pserver_endpoints: {0}, trainers: {1}, current_endpoint: {2}, trainer_id: {3}, training_role: {4}" .format(pserver_endpoints, trainers, current_endpoint, trainer_id, training_role)) t = fluid.DistributeTranspiler() t.transpile(optimize_ops, params_grads, trainer_id=trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": b_size_var = fluid.layers.create_tensor(dtype='int64') b_acc_var = fluid.layers.accuracy(input=out, label=label, total=b_size_var) inference_program = fluid.default_main_program().clone() with fluid.program_guard(inference_program): inference_program = fluid.io.get_inference_program( target_vars=[b_acc_var, b_size_var]) exe.run(fluid.default_startup_program()) train_reader = paddle.batch(paddle.dataset.flowers.train(), batch_size=batch_size) test_reader = paddle.batch(paddle.dataset.flowers.test(), batch_size=batch_size) feeder = fluid.DataFeeder(place=place, feed_list=[image, label]) train_proc = t.get_trainer_program() train_pass_acc_evaluator = fluid.average.WeightedAverage() test_pass_acc_evaluator = fluid.average.WeightedAverage() # with profiler.profiler(args.device, 'total', "profilers.log"): for pass_id in range(num_passes): start = time.clock() train_pass_acc_evaluator.reset() for batch_id, data in enumerate(train_reader()): if args.accuracy: data = accuracy_data(trainers, trainer_id, data) loss, acc, size = exe.run( train_proc, feed=feeder.feed(data), fetch_list=[avg_cost, b_acc_var, b_size_var]) train_pass_acc_evaluator.add(value=acc, weight=size) print("Pass {0}, batch {1}, loss {2}, acc {3}".format( pass_id, batch_id, loss[0], acc[0])) test_pass_acc_evaluator.reset() for data in test_reader(): acc, size = exe.run(inference_program, feed=feeder.feed(data), fetch_list=[b_acc_var, b_size_var]) test_pass_acc_evaluator.add(value=acc, weight=size) print("End pass {0}, train_acc {1}, test_acc {2}, cost {3} second". format(pass_id, train_pass_acc_evaluator.eval(), test_pass_acc_evaluator.eval(), time.clock() - start))
def main(): if args.data_set == "cifar10": classdim = 10 if args.data_format == 'NCHW': data_shape = [3, 32, 32] else: data_shape = [32, 32, 3] else: classdim = 102 if args.data_format == 'NCHW': data_shape = [3, 224, 224] else: data_shape = [224, 224, 3] # Input data images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') # Train program net = vgg16_bn_drop(images) predict = fluid.layers.fc(input=net, size=classdim, act='softmax') cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) # Evaluator batch_size = fluid.layers.create_tensor(dtype='int64') batch_acc = fluid.layers.accuracy(input=predict, label=label, total=batch_size) test_program = fluid.default_main_program().clone(for_test=True) # inference program inference_program = fluid.default_main_program().clone() with fluid.program_guard(inference_program): inference_program = fluid.io.get_inference_program(batch_acc) # Optimization optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) optimize_ops, params_grads = optimizer.minimize(avg_cost) fetch_list = [avg_cost.name, batch_acc.name, batch_size.name] # test def test(exe11): test_pass_acc = fluid.average.WeightedAverage() for batch_id, data in enumerate(test_reader()): img_data = np.array(map(lambda x: x[0].reshape(data_shape), data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape([-1, 1]) loss, acc, b_size = exe11.run(feed={ "pixel": img_data, "label": y_data }, fetch_list=fetch_list) np_acc = np.array(acc) np_b_size = np.array(b_size) test_pass_acc.add(value=float(np.mean(np_acc)), weight=float(np.mean(np_b_size))) return test_pass_acc.eval() def train_loop(train_exe, test_exe): iters = 0 ts = time.time() train_pass_acc = fluid.average.WeightedAverage() for pass_id in range(args.num_passes): # train start_time = time.time() num_samples = 0 train_pass_acc.reset() for batch_id, data in enumerate(train_reader()): ts = time.time() img_data = np.array( map(lambda x: x[0].reshape(data_shape), data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape([-1, 1]) loss, acc, b_size = train_exe.run(feed={ "pixel": img_data, "label": y_data }, fetch_list=[ avg_cost.name, batch_acc.name, batch_size.name ]) iters += 1 num_samples += len(data) np_acc = np.array(acc) np_loss = np.array(loss) np_b_size = np.array(b_size) train_pass_acc.add(value=float(np.mean(np_acc)), weight=float(np.mean(np_b_size))) print( "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, Speed = %.2f img/s, Period = %d secs" % (pass_id, iters, np.mean(np_loss), np.mean(np_acc), len(data) / (time.time() - ts), time.time() - start_time) ) # The accuracy is the accumulation of batches, but not the current batch. pass_elapsed = time.time() - start_time pass_train_acc = train_pass_acc.eval() pass_test_acc = test(test_exe) print( "Pass = %d, Elapsed = %d, Training performance = %f imgs/s, Train accuracy = %f, Test accuracy = %f\n" % (pass_id, pass_elapsed, num_samples / pass_elapsed, pass_train_acc, pass_test_acc)) # Initialize executor use_gpu = False if args.device == 'CPU' else True # data reader train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.cifar.train10() if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), buf_size=5120), batch_size=args.batch_size) test_reader = paddle.batch(paddle.dataset.cifar.test10() if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), batch_size=args.batch_size) if args.local: # Parameter initialization place = core.CPUPlace() if not use_gpu else core.CUDAPlace( args.device_id) startup_exe = fluid.Executor(place) startup_exe.run(fluid.default_startup_program()) train_exe = fluid.ParallelExecutor(use_cuda=True, loss_name=avg_cost.name) test_exe = fluid.ParallelExecutor(use_cuda=True, main_program=test_program, share_vars_from=train_exe) # data reader train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.cifar.train10() if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), buf_size=5120), batch_size=args.batch_size) test_reader = paddle.batch( paddle.dataset.cifar.test10() if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), batch_size=args.batch_size) train_loop(train_exe, test_exe) else: standalone = int(os.getenv("STANDALONE", 0)) if standalone: pserver_endpoints = os.getenv("PSERVERS") trainers = int(os.getenv("TRAINERS")) current_endpoint = os.getenv("SERVER_ENDPOINT") trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID")) training_role = os.getenv("TRAINING_ROLE", "TRAINER") else: pport = os.getenv("PADDLE_INIT_PSERVER_PORT", "6174") tport = os.getenv("PADDLE_INIT_TRAINER_PORT", "6174") pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip... eplist = [] for ip in pserver_ips.split(","): eplist.append(':'.join([ip, pport])) pserver_endpoints = ",".join(eplist) # ip:port,ip:port... trainers = int(os.getenv("TRAINERS")) trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID")) training_role = os.getenv("TRAINING_ROLE", "TRAINER") if training_role == "PSERVER": current_endpoint = os.getenv("POD_IP") + ":" + pport else: current_endpoint = os.getenv("POD_IP") + ":" + tport print( "pserver_endpoints: {0}, trainers: {1}, current_endpoint: {2}, trainer_id: {3}, training_role: {4}" .format(pserver_endpoints, trainers, current_endpoint, trainer_id, training_role)) t = fluid.DistributeTranspiler() t.transpile(optimize_ops, params_grads, trainer_id=trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) place = core.CPUPlace() if not use_gpu else core.CUDAPlace( args.device_id) exe = fluid.Executor(place) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": # Parameter initialization place = core.CPUPlace() if not use_gpu else core.CUDAPlace( args.device_id) startup_exe = fluid.Executor(place) startup_exe.run(fluid.default_startup_program()) trainer_prog = t.get_trainer_program() train_exe = fluid.ParallelExecutor(use_cuda=True, main_program=trainer_prog, loss_name=avg_cost.name) test_exe = fluid.ParallelExecutor(use_cuda=True, main_program=test_program, share_vars_from=train_exe) train_loop(train_exe, test_exe) else: print("environment var TRAINER_ROLE should be TRAINER os PSERVER")
def train(use_cuda, save_dirname, is_local): x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = fluid.layers.mean(cost) sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(avg_cost) BATCH_SIZE = 20 train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.uci_housing.train(), buf_size=500), batch_size=BATCH_SIZE) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) def train_loop(main_program): feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe.run(fluid.default_startup_program()) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): avg_loss_value, = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) print(avg_loss_value) if avg_loss_value[0] < 10.0: if save_dirname is not None: fluid.io.save_inference_model(save_dirname, ['x'], [y_predict], exe) return if math.isnan(float(avg_loss_value)): sys.exit("got NaN loss, training failed.") raise AssertionError("Fit a line cost is too large, {0:2.2}".format( avg_loss_value[0])) if is_local: train_loop(fluid.default_main_program()) else: port = os.getenv("PADDLE_INIT_PORT", "6174") pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # ip,ip... eplist = [] for ip in pserver_ips.split(","): eplist.append(':'.join([ip, port])) pserver_endpoints = ",".join(eplist) # ip:port,ip:port... trainers = int(os.getenv("TRAINERS")) current_endpoint = os.getenv("POD_IP") + ":" + port trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID")) training_role = os.getenv("TRAINING_ROLE", "TRAINER") t = fluid.DistributeTranspiler() t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": train_loop(t.get_trainer_program())
return __reader__ place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id) with fluid.dygraph.guard(place): strategy = fluid.dygraph.parallel.prepare_context() simple_net = SimpleNet(hidden_size=hidden_size, vocab_size=vocab_size, num_steps=num_steps, init_scale=init_scale, is_sparse=True) simple_net = fluid.dygraph.parallel.DataParallel(simple_net, strategy) train_reader = paddle.batch(ptb_train_reader(), batch_size=batch_size, drop_last=True) train_reader = fluid.contrib.reader.distributed_batch_reader(train_reader) sgd = fluid.optimizer.SGD(learning_rate=1e-3, parameter_list=simple_net.parameters()) dy_loss = None for i, data in enumerate(train_reader()): x_data = np.array([x[0].reshape(3) for x in data]).astype('int64') y_data = np.array([x[1].reshape(3) for x in data]).astype('int64') x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) x = to_variable(x_data) y = to_variable(y_data)
batch_size = train_parameters["batch_size"] batch_num = 10 with fluid.dygraph.guard(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed resnet = ResNet("resnet") optimizer = optimizer_setting(train_parameters) np.random.seed(seed) import random random.seed = seed train_reader = paddle.batch(reader_decorator( paddle.dataset.flowers.train(use_xmap=False)), batch_size=batch_size, drop_last=True) batch_py_reader = fluid.io.DataLoader.from_generator(capacity=10) batch_py_reader.set_sample_list_generator(train_reader, places=fluid.CPUPlace()) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() for batch_id, data in enumerate(batch_py_reader()): if batch_id >= batch_num: break img = data[0]
def train(word_dict, net_method, use_cuda, parallel=False, save_dirname=None, is_local=True): BATCH_SIZE = 128 PASS_NUM = 5 dict_dim = len(word_dict) class_dim = 2 data = fluid.layers.data( name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") if not parallel: cost, acc_out, prediction = net_method( data, label, input_dim=dict_dim, class_dim=class_dim) else: places = get_places() pd = ParallelDo(places) with pd.do(): cost, acc, _ = net_method( pd.read_input(data), pd.read_input(label), input_dim=dict_dim, class_dim=class_dim) pd.write_output(cost) pd.write_output(acc) cost, acc = pd() cost = fluid.layers.mean(cost) acc_out = fluid.layers.mean(acc) prediction = None assert save_dirname is None adagrad = fluid.optimizer.Adagrad(learning_rate=0.002) adagrad.minimize(cost) train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.imdb.train(word_dict), buf_size=1000), batch_size=BATCH_SIZE) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[data, label], place=place) def train_loop(main_program): exe.run(fluid.default_startup_program()) for pass_id in range(PASS_NUM): for data in train_data(): cost_val, acc_val = exe.run(main_program, feed=feeder.feed(data), fetch_list=[cost, acc_out]) print("cost=" + str(cost_val) + " acc=" + str(acc_val)) if cost_val < 0.4 and acc_val > 0.8: if save_dirname is not None: fluid.io.save_inference_model(save_dirname, ["words"], prediction, exe) return if math.isnan(float(cost_val)): sys.exit("got NaN loss, training failed.") raise AssertionError("Cost is too large for {0}".format( net_method.__name__)) if is_local: train_loop(fluid.default_main_program()) else: port = os.getenv("PADDLE_PSERVER_PORT", "6174") pserver_ips = os.getenv("PADDLE_PSERVER_IPS") # ip,ip... eplist = [] for ip in pserver_ips.split(","): eplist.append(':'.join([ip, port])) pserver_endpoints = ",".join(eplist) # ip:port,ip:port... trainers = int(os.getenv("PADDLE_TRAINERS")) current_endpoint = os.getenv("POD_IP") + ":" + port trainer_id = int(os.getenv("PADDLE_TRAINER_ID")) training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER") t = fluid.DistributeTranspiler() t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": train_loop(t.get_trainer_program())