예제 #1
0
def do_save_inference_model(args):

    ernie_config = ErnieConfig(args.ernie_config_path)
    ernie_config.print_config()

    if args.use_cuda:
        dev_count = fluid.core.get_cuda_device_count()
        place = fluid.CUDAPlace(0)
    else:
        dev_count = int(os.environ.get('CPU_NUM', 1))
        place = fluid.CPUPlace()

    exe = fluid.Executor(place)

    test_prog = fluid.Program()
    startup_prog = fluid.Program()

    with fluid.program_guard(test_prog, startup_prog):
        with fluid.unique_name.guard():
            infer_pyreader, ernie_inputs, labels = ernie_pyreader(
                args, pyreader_name="infer_reader")

            if args.use_paddle_hub:
                embeddings = ernie_encoder_with_paddle_hub(
                    ernie_inputs, args.max_seq_len)
            else:
                embeddings = ernie_encoder(ernie_inputs,
                                           ernie_config=ernie_config)

            probs = create_model(args,
                                 embeddings,
                                 labels=labels,
                                 is_prediction=True)
    test_prog = test_prog.clone(for_test=True)
    exe.run(startup_prog)

    assert (args.init_checkpoint)

    if args.init_checkpoint:
        utils.init_checkpoint(exe, args.init_checkpoint, test_prog)

    fluid.io.save_inference_model(args.inference_model_dir,
                                  feeded_var_names=[
                                      ernie_inputs["src_ids"].name,
                                      ernie_inputs["sent_ids"].name,
                                      ernie_inputs["pos_ids"].name,
                                      ernie_inputs["input_mask"].name,
                                      ernie_inputs["seq_lens"].name
                                  ],
                                  target_vars=[probs],
                                  executor=exe,
                                  main_program=test_prog,
                                  model_filename="model.pdmodel",
                                  params_filename="params.pdparams")

    print("save inference model at %s" % (args.inference_model_dir))
예제 #2
0
def test_inference_model(args):
    ernie_config = ErnieConfig(args.ernie_config_path)
    ernie_config.print_config()

    if args.use_cuda:
        dev_count = fluid.core.get_cuda_device_count()
        place = fluid.CUDAPlace(0)
    else:
        dev_count = int(os.environ.get('CPU_NUM', 1))
        place = fluid.CPUPlace()

    exe = fluid.Executor(place)

    reader = task_reader.ClassifyReader(vocab_path=args.vocab_path,
                                        label_map_config=args.label_map_config,
                                        max_seq_len=args.max_seq_len,
                                        do_lower_case=args.do_lower_case,
                                        random_seed=args.random_seed)

    test_prog = fluid.Program()
    startup_prog = fluid.Program()

    with fluid.program_guard(test_prog, startup_prog):
        with fluid.unique_name.guard():
            infer_pyreader, ernie_inputs, labels = ernie_pyreader(
                args, pyreader_name="infer_pyreader")

            embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config)
            probs = create_model(args,
                                 embeddings,
                                 labels=labels,
                                 is_prediction=True)

    test_prog = test_prog.clone(for_test=True)
    exe.run(startup_prog)

    assert (args.inference_model_dir)
    infer_data_generator = reader.data_generator(input_file=args.test_set,
                                                 batch_size=args.batch_size /
                                                 dev_count,
                                                 phase="infer",
                                                 epoch=1,
                                                 shuffle=False)

    infer_program, feed_names, fetch_targets = fluid.io.load_inference_model(
        dirname=args.inference_model_dir,
        executor=exe,
        model_filename="model.pdmodel",
        params_filename="params.pdparams")

    infer_pyreader.set_batch_generator(infer_data_generator)
    inference(exe, test_prog, infer_pyreader, [probs.name], "infer")
def main(args):
    """
    Main Function
    """
    args = parser.parse_args()
    ernie_config = ErnieConfig(args.ernie_config_path)
    ernie_config.print_config()

    if args.use_cuda:
        place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
        dev_count = fluid.core.get_cuda_device_count()
    else:
        place = fluid.CPUPlace()
        dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
    exe = fluid.Executor(place)

    reader = task_reader.SequenceLabelReader(
        vocab_path=args.vocab_path,
        label_map_config=args.label_map_config,
        max_seq_len=args.max_seq_len,
        do_lower_case=args.do_lower_case,
        in_tokens=False,
        random_seed=args.random_seed)

    if not (args.do_train or args.do_test or args.do_infer):
        raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
                         "least one of them must be True.")

    startup_prog = fluid.Program()
    if args.random_seed is not None:
        startup_prog.random_seed = args.random_seed

    if args.do_train:
        num_train_examples = reader.get_num_examples(args.train_set)
        max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
        print("Device count: %d" % dev_count)
        print("Num train examples: %d" % num_train_examples)
        print("Max train steps: %d" % max_train_steps)

        train_program = fluid.Program()

        with fluid.program_guard(train_program, startup_prog):
            with fluid.unique_name.guard():
                # create ernie_pyreader
                train_pyreader, ernie_inputs, words, labels = ernie_pyreader(
                    args, pyreader_name='train_reader')
                train_pyreader.decorate_tensor_provider(
                    reader.data_generator(args.train_set,
                                          args.batch_size,
                                          args.epoch,
                                          shuffle=True,
                                          phase="train"))
                # get ernie_embeddings
                embeddings = ernie_encoder(ernie_inputs,
                                           ernie_config=ernie_config)
                # user defined model based on ernie embeddings
                train_ret = create_model(args,
                                         embeddings,
                                         labels=labels,
                                         is_prediction=False)

                optimizer = fluid.optimizer.Adam(learning_rate=args.lr)
                fluid.clip.set_gradient_clip(
                    clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0))
                optimizer.minimize(train_ret["loss"])

        lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
            program=train_program, batch_size=args.batch_size)
        print("Theoretical memory usage in training: %.3f - %.3f %s" %
              (lower_mem, upper_mem, unit))

    if args.do_test:
        test_program = fluid.Program()
        with fluid.program_guard(test_program, startup_prog):
            with fluid.unique_name.guard():
                # create ernie_pyreader
                test_pyreader, ernie_inputs, words, labels = ernie_pyreader(
                    args, pyreader_name='test_reader')
                test_pyreader.decorate_tensor_provider(
                    reader.data_generator(args.test_set,
                                          args.batch_size,
                                          phase='test',
                                          epoch=1,
                                          shuffle=False))
                # get ernie_embeddings
                embeddings = ernie_encoder(ernie_inputs,
                                           ernie_config=ernie_config)
                # user defined model based on ernie embeddings
                test_ret = create_model(args,
                                        embeddings,
                                        labels=labels,
                                        is_prediction=False)

        test_program = test_program.clone(for_test=True)

    if args.do_infer:
        infer_program = fluid.Program()
        with fluid.program_guard(infer_program, startup_prog):
            with fluid.unique_name.guard():
                # create ernie_pyreader
                infer_pyreader, ernie_inputs, words, labels = ernie_pyreader(
                    args, pyreader_name='infer_reader')
                infer_pyreader.decorate_tensor_provider(
                    reader.data_generator(args.infer_set,
                                          args.batch_size,
                                          phase='infer',
                                          epoch=1,
                                          shuffle=False))
                # get ernie_embeddings
                embeddings = ernie_encoder(ernie_inputs,
                                           ernie_config=ernie_config)
                # user defined model based on ernie embeddings
                infer_ret = create_model(args,
                                         embeddings,
                                         labels=labels,
                                         is_prediction=True)
                infer_ret["words"] = words

        infer_program = infer_program.clone(for_test=True)

    exe.run(startup_prog)

    # load checkpoints
    if args.do_train:
        if args.init_checkpoint and args.init_pretraining_params:
            print(
                "WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
                "both are set! Only arg 'init_checkpoint' is made valid.")
        if args.init_checkpoint:
            utils.init_checkpoint(exe, args.init_checkpoint, startup_prog)
        elif args.init_pretraining_params:
            utils.init_pretraining_params(exe, args.init_pretraining_params,
                                          startup_prog)
    elif args.do_test or args.do_infer:
        if not args.init_checkpoint:
            raise ValueError(
                "args 'init_checkpoint' should be set if only doing test or infer!"
            )
        utils.init_checkpoint(exe, args.init_checkpoint, startup_prog)

    if args.do_train:
        train_pyreader.start()
        steps = 0
        total_cost, total_acc, total_num_seqs = [], [], []
        while True:
            try:
                steps += 1
                if steps % args.skip_steps == 0:
                    fetch_list = [
                        train_ret["loss"],
                        train_ret["num_infer_chunks"],
                        train_ret["num_label_chunks"],
                        train_ret["num_correct_chunks"],
                    ]
                else:
                    fetch_list = []

                start_time = time.time()
                outputs = exe.run(program=train_program, fetch_list=fetch_list)
                end_time = time.time()
                if steps % args.skip_steps == 0:
                    loss, nums_infer, nums_label, nums_correct = outputs
                    train_ret["chunk_evaluator"].reset()
                    train_ret["chunk_evaluator"].update(
                        nums_infer, nums_label, nums_correct)
                    precision, recall, f1_score = train_ret[
                        "chunk_evaluator"].eval()
                    print(
                        "[train] batch_id = %d, loss = %.5f, P: %.5f, R: %.5f, F1: %.5f, elapsed time %.5f, "
                        "pyreader queue_size: %d " %
                        (steps, loss, precision, recall, f1_score,
                         end_time - start_time, train_pyreader.queue.size()))

                if steps % args.save_steps == 0:
                    save_path = os.path.join(args.checkpoints,
                                             "step_" + str(steps))
                    print("\tsaving model as %s" % (save_path))
                    fluid.io.save_persistables(exe, save_path, train_program)

                if steps % args.validation_steps == 0:
                    # evaluate test set
                    if args.do_test:
                        evaluate(exe, test_program, test_pyreader, test_ret)

            except fluid.core.EOFException:
                save_path = os.path.join(args.checkpoints,
                                         "step_" + str(steps))
                fluid.io.save_persistables(exe, save_path, train_program)
                train_pyreader.reset()
                break

    # final eval on test set
    if args.do_test:
        evaluate(exe, test_program, test_pyreader, test_ret)

    if args.do_infer:
        # create dict
        id2word_dict = dict([(str(word_id), word)
                             for word, word_id in reader.vocab.items()])
        id2label_dict = dict([(str(label_id), label)
                              for label, label_id in reader.label_map.items()])
        Dataset = namedtuple("Dataset", ["id2word_dict", "id2label_dict"])
        dataset = Dataset(id2word_dict, id2label_dict)

        infer_pyreader.start()
        while True:
            try:
                (words, crf_decode) = exe.run(
                    infer_program,
                    fetch_list=[infer_ret["words"], infer_ret["crf_decode"]],
                    return_numpy=False)
                # User should notice that words had been clipped if long than args.max_seq_len
                results = utils.parse_result(words, crf_decode, dataset)
                for result in results:
                    print(result)
            except fluid.core.EOFException:
                infer_pyreader.reset()
                break
예제 #4
0
def main(args):
    """
    Main Function
    """
    ernie_config = ErnieConfig(args.ernie_config_path)
    ernie_config.print_config()

    if args.use_cuda:
        place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
        dev_count = fluid.core.get_cuda_device_count()
    else:
        place = fluid.CPUPlace()
        dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
    exe = fluid.Executor(place)

    reader = task_reader.ClassifyReader(vocab_path=args.vocab_path,
                                        label_map_config=args.label_map_config,
                                        max_seq_len=args.max_seq_len,
                                        do_lower_case=args.do_lower_case,
                                        random_seed=args.random_seed)

    if not (args.do_train or args.do_val or args.do_infer):
        raise ValueError("For args `do_train`, `do_val` and `do_infer`, at "
                         "least one of them must be True.")

    startup_prog = fluid.Program()
    if args.random_seed is not None:
        startup_prog.random_seed = args.random_seed

    if args.do_train:
        train_data_generator = reader.data_generator(
            input_file=args.train_set,
            batch_size=args.batch_size,
            epoch=args.epoch,
            shuffle=True,
            phase="train")

        num_train_examples = reader.get_num_examples(args.train_set)

        max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count

        print("Device count: %d" % dev_count)
        print("Num train examples: %d" % num_train_examples)
        print("Max train steps: %d" % max_train_steps)

        train_program = fluid.Program()

        with fluid.program_guard(train_program, startup_prog):
            with fluid.unique_name.guard():
                # create ernie_pyreader
                train_pyreader, ernie_inputs, labels = ernie_pyreader(
                    args, pyreader_name='train_pyreader')

                # get ernie_embeddings
                if args.use_paddle_hub:
                    embeddings = ernie_encoder_with_paddle_hub(
                        ernie_inputs, args.max_seq_len)
                else:
                    embeddings = ernie_encoder(ernie_inputs,
                                               ernie_config=ernie_config)

                # user defined model based on ernie embeddings
                loss, accuracy, num_seqs = create_model(args,
                                                        embeddings,
                                                        labels=labels,
                                                        is_prediction=False)

                optimizer = fluid.optimizer.Adam(learning_rate=args.lr)
                optimizer.minimize(loss)

        if args.verbose:
            lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
                program=train_program, batch_size=args.batch_size)
            print("Theoretical memory usage in training: %.3f - %.3f %s" %
                  (lower_mem, upper_mem, unit))

    if args.do_val:
        test_data_generator = reader.data_generator(input_file=args.dev_set,
                                                    batch_size=args.batch_size,
                                                    phase='dev',
                                                    epoch=1,
                                                    shuffle=False)
        test_prog = fluid.Program()
        with fluid.program_guard(test_prog, startup_prog):
            with fluid.unique_name.guard():
                # create ernie_pyreader
                test_pyreader, ernie_inputs, labels = ernie_pyreader(
                    args, pyreader_name='eval_reader')

                # get ernie_embeddings
                if args.use_paddle_hub:
                    embeddings = ernie_encoder_with_paddle_hub(
                        ernie_inputs, args.max_seq_len)
                else:
                    embeddings = ernie_encoder(ernie_inputs,
                                               ernie_config=ernie_config)

                # user defined model based on ernie embeddings
                loss, accuracy, num_seqs = create_model(args,
                                                        embeddings,
                                                        labels=labels,
                                                        is_prediction=False)

        test_prog = test_prog.clone(for_test=True)

    if args.do_infer:
        infer_data_generator = reader.data_generator(
            input_file=args.test_set,
            batch_size=args.batch_size,
            phase='infer',
            epoch=1,
            shuffle=False)
        infer_prog = fluid.Program()
        with fluid.program_guard(infer_prog, startup_prog):
            with fluid.unique_name.guard():
                infer_pyreader, ernie_inputs, labels = ernie_pyreader(
                    args, pyreader_name="infer_pyreader")

                # get ernie_embeddings
                if args.use_paddle_hub:
                    embeddings = ernie_encoder_with_paddle_hub(
                        ernie_inputs, args.max_seq_len)
                else:
                    embeddings = ernie_encoder(ernie_inputs,
                                               ernie_config=ernie_config)

                probs = create_model(args,
                                     embeddings,
                                     labels=labels,
                                     is_prediction=True)

        infer_prog = infer_prog.clone(for_test=True)

    exe.run(startup_prog)

    if args.do_train:
        if args.init_checkpoint:
            init_checkpoint(exe,
                            args.init_checkpoint,
                            main_program=train_program)
    elif args.do_val:
        if not args.init_checkpoint:
            raise ValueError("args 'init_checkpoint' should be set if"
                             "only doing validation or testing!")
        init_checkpoint(exe, args.init_checkpoint, main_program=test_prog)
    elif args.do_infer:
        if not args.init_checkpoint:
            raise ValueError("args 'init_checkpoint' should be set if"
                             "only doing validation or testing!")
        init_checkpoint(exe, args.init_checkpoint, main_program=infer_prog)

    if args.do_train:
        train_exe = exe
        train_pyreader.set_batch_generator(train_data_generator)
    else:
        train_exe = None
    if args.do_val:
        test_exe = exe
        test_pyreader.set_batch_generator(test_data_generator)
    if args.do_infer:
        test_exe = exe
        infer_pyreader.set_batch_generator(infer_data_generator)

    if args.do_train:
        train_pyreader.start()
        steps = 0
        total_cost, total_acc, total_num_seqs = [], [], []
        time_begin = time.time()
        while True:
            try:
                steps += 1
                if steps % args.skip_steps == 0:
                    fetch_list = [loss.name, accuracy.name, num_seqs.name]
                else:
                    fetch_list = []

                outputs = train_exe.run(program=train_program,
                                        fetch_list=fetch_list,
                                        return_numpy=False)
                if steps % args.skip_steps == 0:
                    np_loss, np_acc, np_num_seqs = outputs
                    np_loss = np.array(np_loss)
                    np_acc = np.array(np_acc)
                    np_num_seqs = np.array(np_num_seqs)
                    total_cost.extend(np_loss * np_num_seqs)
                    total_acc.extend(np_acc * np_num_seqs)
                    total_num_seqs.extend(np_num_seqs)

                    if args.verbose:
                        verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
                        )
                        print(verbose)

                    time_end = time.time()
                    used_time = time_end - time_begin
                    print("step: %d, ave loss: %f, "
                          "ave acc: %f, speed: %f steps/s" %
                          (steps, np.sum(total_cost) / np.sum(total_num_seqs),
                           np.sum(total_acc) / np.sum(total_num_seqs),
                           args.skip_steps / used_time))
                    total_cost, total_acc, total_num_seqs = [], [], []
                    time_begin = time.time()

                if steps % args.save_steps == 0:
                    save_path = os.path.join(args.checkpoints,
                                             "step_" + str(steps),
                                             "checkpoint")
                    fluid.save(train_program, save_path)

                if steps % args.validation_steps == 0:
                    # evaluate dev set
                    if args.do_val:
                        evaluate(exe, test_prog, test_pyreader,
                                 [loss.name, accuracy.name, num_seqs.name],
                                 "dev")

            except fluid.core.EOFException:
                save_path = os.path.join(args.checkpoints,
                                         "step_" + str(steps), "checkpoint")
                fluid.save(train_program, save_path)
                train_pyreader.reset()
                break

    # final eval on dev set
    if args.do_val:
        print("Final validation result:")
        evaluate(exe, test_prog, test_pyreader,
                 [loss.name, accuracy.name, num_seqs.name], "dev")

    # final eval on test set
    if args.do_infer:
        print("Final test result:")
        infer(exe, infer_prog, infer_pyreader, [probs.name], "infer")
예제 #5
0
def create_ernie_model(args, ernie_config):
    """
    Create Model for LAC based on ERNIE encoder
    """
    # ERNIE's input data

    src_ids = fluid.data(name='src_ids',
                         shape=[-1, args.max_seq_len, 1],
                         dtype='int64')
    sent_ids = fluid.data(name='sent_ids',
                          shape=[-1, args.max_seq_len, 1],
                          dtype='int64')
    pos_ids = fluid.data(name='pos_ids',
                         shape=[-1, args.max_seq_len, 1],
                         dtype='int64')
    input_mask = fluid.data(name='input_mask',
                            shape=[-1, args.max_seq_len, 1],
                            dtype='float32')

    padded_labels = fluid.data(name='padded_labels',
                               shape=[-1, args.max_seq_len, 1],
                               dtype='int64')

    seq_lens = fluid.data(name='seq_lens',
                          shape=[-1],
                          dtype='int64',
                          lod_level=0)

    squeeze_labels = fluid.layers.squeeze(padded_labels, axes=[-1])

    # ernie_pyreader
    ernie_inputs = {
        "src_ids": src_ids,
        "sent_ids": sent_ids,
        "pos_ids": pos_ids,
        "input_mask": input_mask,
        "seq_lens": seq_lens
    }
    embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config)

    padded_token_embeddings = embeddings["padded_token_embeddings"]

    emission = fluid.layers.fc(
        size=args.num_labels,
        input=padded_token_embeddings,
        param_attr=fluid.ParamAttr(
            initializer=fluid.initializer.Uniform(low=-args.init_bound,
                                                  high=args.init_bound),
            regularizer=fluid.regularizer.L2DecayRegularizer(
                regularization_coeff=1e-4)),
        num_flatten_dims=2)

    crf_cost = fluid.layers.linear_chain_crf(
        input=emission,
        label=padded_labels,
        param_attr=fluid.ParamAttr(name='crfw',
                                   learning_rate=args.crf_learning_rate),
        length=seq_lens)

    avg_cost = fluid.layers.mean(x=crf_cost)
    crf_decode = fluid.layers.crf_decoding(
        input=emission,
        param_attr=fluid.ParamAttr(name='crfw'),
        length=seq_lens)

    (precision, recall, f1_score, num_infer_chunks, num_label_chunks,
     num_correct_chunks) = fluid.layers.chunk_eval(
         input=crf_decode,
         label=squeeze_labels,
         chunk_scheme="IOB",
         num_chunk_types=int(math.ceil((args.num_labels - 1) / 2.0)),
         seq_length=seq_lens)
    chunk_evaluator = fluid.metrics.ChunkEvaluator()
    chunk_evaluator.reset()

    ret = {
        "feed_list":
        [src_ids, sent_ids, pos_ids, input_mask, padded_labels, seq_lens],
        "words":
        src_ids,
        "pos_ids":
        pos_ids,
        "sent_ids":
        sent_ids,
        "input_mask":
        input_mask,
        "labels":
        padded_labels,
        "seq_lens":
        seq_lens,
        "avg_cost":
        avg_cost,
        "crf_decode":
        crf_decode,
        "precision":
        precision,
        "recall":
        recall,
        "f1_score":
        f1_score,
        "chunk_evaluator":
        chunk_evaluator,
        "num_infer_chunks":
        num_infer_chunks,
        "num_label_chunks":
        num_label_chunks,
        "num_correct_chunks":
        num_correct_chunks,
        "emission":
        emission,
        "alpha":
        None
    }

    return ret