def infer(topology, data_dir, model_path, word_dict_path, label_dict_path, batch_size): def _infer_a_batch(inferer, test_batch, ids_2_word, ids_2_label): probs = inferer.infer(input=test_batch, field=["value"]) assert len(probs) == len(test_batch) for word_ids, prob in zip(test_batch, probs): word_text = " ".join([ids_2_word[id] for id in word_ids[0]]) print("%s\t%s\t%s" % (ids_2_label[prob.argmax()], " ".join(["{:0.4f}".format(p) for p in prob]), word_text)) logger.info("begin to predict...") use_default_data = (data_dir is None) if use_default_data: word_dict = paddle.dataset.imdb.word_dict() word_reverse_dict = dict((value, key) for key, value in word_dict.iteritems()) label_reverse_dict = {0: "positive", 1: "negative"} test_reader = paddle.dataset.imdb.test(word_dict) else: assert os.path.exists( word_dict_path), "the word dictionary file does not exist" assert os.path.exists( label_dict_path), "the label dictionary file does not exist" word_dict = load_dict(word_dict_path) word_reverse_dict = load_reverse_dict(word_dict_path) label_reverse_dict = load_reverse_dict(label_dict_path) test_reader = reader.test_reader(data_dir, word_dict)() dict_dim = len(word_dict) class_num = len(label_reverse_dict) prob_layer = topology(dict_dim, class_num, is_infer=True) # initialize PaddlePaddle paddle.init(use_gpu=False, trainer_count=1) # load the trained models parameters = paddle.parameters.Parameters.from_tar( gzip.open(model_path, "r")) inferer = paddle.inference.Inference( output_layer=prob_layer, parameters=parameters) test_batch = [] for idx, item in enumerate(test_reader): test_batch.append([item[0]]) if len(test_batch) == batch_size: _infer_a_batch(inferer, test_batch, word_reverse_dict, label_reverse_dict) test_batch = [] if len(test_batch): _infer_a_batch(inferer, test_batch, word_reverse_dict, label_reverse_dict) test_batch = []
def infer(args): id2word_dict = reader.load_dict(args.word_dict_path) word2id_dict = reader.load_reverse_dict(args.word_dict_path) id2label_dict = reader.load_dict(args.label_dict_path) label2id_dict = reader.load_reverse_dict(args.label_dict_path) q2b_dict = reader.load_dict(args.word_rep_dict_path) test_data = paddle.batch(reader.test_reader(args.test_data_dir, word2id_dict, label2id_dict, q2b_dict), batch_size=args.batch_size) place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.core.Scope() with fluid.scope_guard(inference_scope): [inference_program, feed_target_names, fetch_targets] = fluid.io.load_inference_model(args.model_path, exe) for data in test_data(): full_out_str = "" word_idx = to_lodtensor([x[0] for x in data], place) word_list = [x[1] for x in data] (crf_decode, ) = exe.run(inference_program, feed={"word": word_idx}, fetch_list=fetch_targets, return_numpy=False) lod_info = (crf_decode.lod())[0] np_data = np.array(crf_decode) assert len(data) == len(lod_info) - 1 for sen_index in xrange(len(data)): assert len( data[sen_index][0]) == lod_info[sen_index + 1] - lod_info[sen_index] word_index = 0 outstr = "" cur_full_word = "" cur_full_tag = "" words = word_list[sen_index] for tag_index in xrange(lod_info[sen_index], lod_info[sen_index + 1]): cur_word = words[word_index] cur_tag = id2label_dict[str(np_data[tag_index][0])] if cur_tag.endswith("-B") or cur_tag.endswith("O"): if len(cur_full_word) != 0: outstr += cur_full_word.encode( 'utf8') + "/" + cur_full_tag.encode( 'utf8') + " " cur_full_word = cur_word cur_full_tag = get_real_tag(cur_tag) else: cur_full_word += cur_word word_index += 1 outstr += cur_full_word.encode( 'utf8') + "/" + cur_full_tag.encode('utf8') + " " outstr = outstr.strip() full_out_str += outstr + "\n" print full_out_str.strip()
def main(args): input, output, data_size = construct_sample(args) places = fluid.cuda_places() if args.use_cuda else fluid.cpu_places() exe = Executor(places[0]) [inference_program, feed_target_names, fetch_targets] = (fluid.io.load_inference_model(dirname="./model", executor=exe)) feat, lod = test_reader() result = exe.run(inference_program, feed={ feed_target_names[0]: feat, feed_target_names[1]: lod }, fetch_list=fetch_targets) print(result[0].shape) output_final_result(result[0])
def test(model_path, vocabs_path, test_data_path, batch_size, beam_size, save_file, use_gpu): assert os.path.exists(model_path), "The given model does not exist." assert os.path.exists( test_data_path), "The given test data does not exist." with gzip.open(model_path, "r") as f: parameters = paddle.parameters.Parameters.from_tar(f) id_to_text = {} assert os.path.exists( vocabs_path), "The given word dictionary path does not exist." with open(vocabs_path, "r") as f: for i, line in enumerate(f): id_to_text[i] = line.strip().split("\t")[0] paddle.init(use_gpu=use_gpu, trainer_count=1) beam_gen = encoder_decoder_network(word_count=len(id_to_text), emb_dim=512, encoder_depth=3, encoder_hidden_dim=512, decoder_depth=3, decoder_hidden_dim=512, bos_id=0, eos_id=1, max_length=50, beam_size=beam_size, is_generating=True) inferer = paddle.inference.Inference(output_layer=beam_gen, parameters=parameters) test_batch = [] with open(save_file, "w") as fout: for idx, item in enumerate( reader.test_reader(test_data_path, vocabs_path)()): test_batch.append([item]) if len(test_batch) == batch_size: infer_a_batch(inferer, test_batch, beam_size, id_to_text, fout) test_batch = [] if len(test_batch): infer_a_batch(inferer, test_batch, beam_size, id_to_text, fout) test_batch = [] end = time.clock() print('time:{}'.format(end - start))
cost = fluid.layers.square_error_cost(input=model, label=label) avg_cost = fluid.layers.mean(cost) # 获取训练和测试程序 test_program = fluid.default_main_program().clone(for_test=True) # 定义优化方法 optimizer = fluid.optimizer.AdamOptimizer(learning_rate=0.001) opts = optimizer.minimize(avg_cost) # 获取自定义数据 train_reader = paddle.batch(reader=reader.train_reader(train_list, crop_size, resize_size), batch_size=32) test_reader = paddle.batch(reader=reader.test_reader(test_list, crop_size), batch_size=32) # 定义执行器 place = fluid.CUDAPlace(0) # place = fluid.CPUPlace() exe = fluid.Executor(place) # 进行参数初始化 exe.run(fluid.default_startup_program()) # 定义输入数据维度 feeder = fluid.DataFeeder(place=place, feed_list=[image, label]) # 训练 all_test_cost = [] for pass_id in range(100): # 进行训练
cost = fluid.layers.cross_entropy(input=model, label=label) avg_cost = fluid.layers.mean(cost) acc = fluid.layers.accuracy(input=model, label=label) # 获取训练和测试程序 test_program = fluid.default_main_program().clone(for_test=True) # 定义优化方法 optimizer = fluid.optimizer.AdamOptimizer(learning_rate=1e-3, regularization=fluid.regularizer.L2Decay( regularization_coeff=0.001)) opts = optimizer.minimize(avg_cost) # 获取自定义数据 train_reader = reader.train_reader('dataset/train', batch_size=32) test_reader = reader.test_reader('dataset/test', batch_size=32) # 定义一个使用GPU的执行器 place = fluid.CUDAPlace(0) exe = fluid.Executor(place) # 进行参数初始化 exe.run(fluid.default_startup_program()) # 加载初始化模型 if init_model: fluid.load(program=fluid.default_main_program(), model_path=init_model, executor=exe, var_list=fluid.io.get_program_parameter(fluid.default_main_program())) print("Init model from: %s." % init_model)
acc = fluid.layers.accuracy(input=model, label=label) # 获取训练和测试程序 test_program = fluid.default_main_program().clone(for_test=True) # 定义优化方法 l2 = fluid.regularizer.L2DecayRegularizer(1e-4) optimizer = fluid.optimizer.AdamOptimizer(learning_rate=1e-3, regularization=l2) opts = optimizer.minimize(avg_cost) # 获取自定义数据 train_reader = paddle.batch(reader=reader.train_reader('images/train.list', crop_size, resize_size), batch_size=32) test_reader = paddle.batch(reader=reader.test_reader('images/test.list', crop_size), batch_size=32) # 定义一个使用GPU的执行器 place = fluid.CUDAPlace(0) # place = fluid.CPUPlace() exe = fluid.Executor(place) # 进行参数初始化 exe.run(fluid.default_startup_program()) # 定义输入数据维度 feeder = fluid.DataFeeder(place=place, feed_list=[image, label]) # 训练100次 for pass_id in range(100): # 进行训练
def main(): # parse the argument parser = argparse.ArgumentParser() parser.add_argument('-m', '--model', help='The model for image classification', choices=[ 'alexnet', 'vgg13', 'vgg16', 'vgg19', 'resnet', 'googlenet', 'inception-resnet-v2', 'inception_v4', 'xception' ]) parser.add_argument( '-r', '--retrain_file', type=str, default='', help="The model file to retrain, none is for train from scratch") args = parser.parse_args() # PaddlePaddle init paddle.init(use_gpu=True, trainer_count=1) image = paddle.layer.data(name="image", type=paddle.data_type.dense_vector(DATA_DIM)) lbl = paddle.layer.data(name="label", type=paddle.data_type.integer_value(CLASS_DIM)) extra_layers = None learning_rate = 0.0001 if args.model == 'alexnet': out = alexnet.alexnet(image, class_dim=CLASS_DIM) elif args.model == 'vgg13': out = vgg.vgg13(image, class_dim=CLASS_DIM) elif args.model == 'vgg16': out = vgg.vgg16(image, class_dim=CLASS_DIM) elif args.model == 'vgg19': out = vgg.vgg19(image, class_dim=CLASS_DIM) elif args.model == 'resnet': conv, pool, out = resnet.resnet_imagenet(image, class_dim=CLASS_DIM) learning_rate = 0.1 elif args.model == 'googlenet': out, out1, out2 = googlenet.googlenet(image, class_dim=CLASS_DIM) loss1 = paddle.layer.cross_entropy_cost(input=out1, label=lbl, coeff=0.3) paddle.evaluator.classification_error(input=out1, label=lbl) loss2 = paddle.layer.cross_entropy_cost(input=out2, label=lbl, coeff=0.3) paddle.evaluator.classification_error(input=out2, label=lbl) extra_layers = [loss1, loss2] elif args.model == 'inception-resnet-v2': assert DATA_DIM == 3 * 331 * 331 or DATA_DIM == 3 * 299 * 299 out = inception_resnet_v2.inception_resnet_v2(image, class_dim=CLASS_DIM, dropout_rate=0.5, data_dim=DATA_DIM) elif args.model == 'inception_v4': conv, pool, out = inception_v4.inception_v4(image, class_dim=CLASS_DIM) elif args.model == 'xception': out = xception.xception(image, class_dim=CLASS_DIM) cost = paddle.layer.classification_cost(input=out, label=lbl) # Create parameters parameters = paddle.parameters.create(cost) for k, v in parameters.__param_conf__.items(): print(" config key {0}\t\t\tval{1}".format(k, v)) print("-" * 50) #print(parameters.__param_conf__[0]) if args.retrain_file is not None and '' != args.retrain_file: print("restore parameters from {0}".format(args.retrain_file)) exclude_params = [ param for param in parameters.names() if param.startswith('___fc_layer_0__') ] parameters.init_from_tar(gzip.open(args.retrain_file), exclude_params) # Create optimizer optimizer = paddle.optimizer.Momentum( momentum=0.9, regularization=paddle.optimizer.L2Regularization(rate=0.0005 * BATCH_SIZE), learning_rate=learning_rate / BATCH_SIZE, learning_rate_decay_a=0.1, learning_rate_decay_b=128000 * 35, learning_rate_schedule="discexp", ) train_reader = paddle.batch( paddle.reader.shuffle( # flowers.train(), # To use other data, replace the above line with: reader.train_reader('valid_train0.lst'), buf_size=2048), batch_size=BATCH_SIZE) test_reader = paddle.batch( # flowers.valid(), # To use other data, replace the above line with: reader.test_reader('valid_val.lst'), batch_size=BATCH_SIZE) # Create trainer trainer = paddle.trainer.SGD(cost=cost, parameters=parameters, update_equation=optimizer, extra_layers=extra_layers) # End batch and end pass event handler def event_handler(event): global step global start if isinstance(event, paddle.event.EndIteration): if event.batch_id % 10 == 0: print "\nPass %d, Batch %d, Cost %f, %s, %s" % ( event.pass_id, event.batch_id, event.cost, event.metrics, time.time() - start) start = time.time() loss_scalar.add_record(step, event.cost) acc_scalar.add_record( step, 1 - event.metrics['classification_error_evaluator']) start = time.time() step += 1 if event.batch_id % 100 == 0: with gzip.open('params_pass_%d.tar.gz' % event.pass_id, 'w') as f: trainer.save_parameter_to_tar(f) if isinstance(event, paddle.event.EndPass): with gzip.open('params_pass_%d.tar.gz' % event.pass_id, 'w') as f: trainer.save_parameter_to_tar(f) result = trainer.test(reader=test_reader) print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics) trainer.train(reader=train_reader, num_passes=200, event_handler=event_handler)
cost = paddle.layer.classification_cost(input=out, label=lbl) # Create parameters # parameters = paddle.parameters.create(cost) with gzip.open('/book/working/models/params_pass_47.tar.gz', 'r') as f: parameters = paddle.parameters.Parameters.from_tar(f) # Read training data train_reader = paddle.batch( paddle.reader.shuffle( reader.train_reader('/book/working/data/train.list', buffered_size=1024), buf_size=20000), batch_size=BATCH_SIZE) # Read testing data test_reader = paddle.batch( reader.test_reader('/book/working/data/val.list', buffered_size=1024), batch_size=BATCH_SIZE) # End batch and end pass event handler def event_handler(event): # Report result of batch. if isinstance(event, paddle.event.EndIteration): print "\nPass %d, Batch %d, Cost %f, %s" % ( event.pass_id, event.batch_id, event.cost, event.metrics) # Report result of pass. if isinstance(event, paddle.event.EndPass): if (event.pass_id + 1) % 4 == 0: # Save parameters with gzip.open('/book/working/models/params_pass_%d.tar.gz' % event.pass_id, 'w') as f: parameters.to_tar(f) # Report validation accuracy
# 获取训练和测试程序 test_program = fluid.default_main_program().clone(for_test=True) # 定义优化方法 l2 = fluid.regularizer.L2DecayRegularizer(1e-4) optimizer = fluid.optimizer.AdamOptimizer(learning_rate=1e-3, regularization=l2) opts = optimizer.minimize(avg_cost) # 获取自定义数据 train_reader = paddle.batch( reader=reader.train_reader('images/train.list', crop_size, resize_size), batch_size=32) test_reader = paddle.batch( reader=reader.test_reader('images/test.list', crop_size), batch_size=32) # 定义一个使用GPU的执行器 place = fluid.CUDAPlace(0) # place = fluid.CPUPlace() exe = fluid.Executor(place) # 进行参数初始化 exe.run(fluid.default_startup_program()) # 定义输入数据维度 feeder = fluid.DataFeeder(place=place, feed_list=[image, label]) # 训练100次 for pass_id in range(100): # 进行训练 for batch_id, data in enumerate(train_reader()):