Python process_vocabulary示例

示例#1

def override_parameters(params, args):
    if args.parameters:
        params.parse(args.parameters)

    params.vocabulary = {
        "source": vocabulary.load_vocabulary(args.vocabulary[0]),
        "target": vocabulary.load_vocabulary(args.vocabulary[1])
    }
    params.vocabulary["source"] = vocabulary.process_vocabulary(
        params.vocabulary["source"], params)
    params.vocabulary["target"] = vocabulary.process_vocabulary(
        params.vocabulary["target"], params)

    control_symbols = [params.pad, params.bos, params.eos, params.unk]

    params.mapping = {
        "source":
        vocabulary.get_control_mapping(params.vocabulary["source"],
                                       control_symbols),
        "target":
        vocabulary.get_control_mapping(params.vocabulary["target"],
                                       control_symbols)
    }

    return params

示例#2

文件： translator_zeng.py 项目： ljwmusclenlper/NER-of-Experimental-Facilities

def override_parameters(params, args):
    params.input = args.input
    params.glove_emb_path = args.glove_emb_path
    params.bert_emb_path = args.bert_emb_path
    if args.parameters:
        params.parse(args.parameters)
    # args.vocabulary = args.vocabulary.split()
    params.vocabulary = {
        "source": vocabulary.load_vocabulary(args.vocabulary[0]),
        "target": vocabulary.load_vocabulary(args.vocabulary[1]),
        "char": vocabulary.load_vocabulary(args.vocabulary[2])
    }
    params.vocabulary["source"] = vocabulary.process_vocabulary(
        params.vocabulary["source"], params)
    params.vocabulary["target"] = vocabulary.process_vocabulary(
        params.vocabulary["target"], params)
    params.vocabulary["char"] = vocabulary.process_vocabulary(
        params.vocabulary["char"], params)

    control_symbols = [params.pad, params.bos, params.eos, params.unk]

    params.mapping = {
        "source":
        vocabulary.get_control_mapping(params.vocabulary["source"],
                                       control_symbols),
        "target":
        vocabulary.get_control_mapping(params.vocabulary["target"],
                                       control_symbols),
        "char":
        vocabulary.get_control_mapping(params.vocabulary["char"],
                                       control_symbols)
    }

    return params

示例#3

def override_parameters(params, args):
    params.model = args.model
    params.input = args.input or params.input
    params.output = args.output or params.output
    params.record = args.record or params.record
    params.vocab = args.vocabulary or params.vocab
    params.validation = args.validation or params.validation
    params.references = args.references or params.references
    params.parse(args.parameters)

    params.vocabulary = {
        "source": vocabulary.load_vocabulary(params.vocab[0]),
        "target": vocabulary.load_vocabulary(params.vocab[1])
    }
    params.vocabulary["source"] = vocabulary.process_vocabulary(
        params.vocabulary["source"], params)
    params.vocabulary["target"] = vocabulary.process_vocabulary(
        params.vocabulary["target"], params)

    control_symbols = [params.pad, params.bos, params.eos, params.unk]

    params.mapping = {
        "source":
        vocabulary.get_control_mapping(params.vocabulary["source"],
                                       control_symbols),
        "target":
        vocabulary.get_control_mapping(params.vocabulary["target"],
                                       control_symbols)
    }

    return params

示例#4

def override_parameters(params, args):
    if args.parameters:
        params.parse(args.parameters)

    params.vocabulary = {
        "source": vocabulary.load_vocabulary(args.vocabulary[0]),
        "target": vocabulary.load_vocabulary(args.vocabulary[1])
    }
    params.vocabulary["source"] = vocabulary.process_vocabulary(
        params.vocabulary["source"], params)
    params.vocabulary["target"] = vocabulary.process_vocabulary(
        params.vocabulary["target"], params)

    control_symbols = [params.pad, params.bos, params.eos, params.unk]

    params.mapping = {
        "source":
        vocabulary.get_control_mapping(params.vocabulary["source"],
                                       control_symbols),
        "target":
        vocabulary.get_control_mapping(params.vocabulary["target"],
                                       control_symbols)
    }

    params.add_hparam("constraints", args.constraints)
    params.add_hparam("pos_extend", args.pos_extend)
    params.add_hparam("weight_threshold", args.weight_threshold)
    params.add_hparam("encdec_att_layer", args.encdec_att_layer)
    params.add_hparam("heads", args.heads)
    params.add_hparam("head_model", args.head_model)
    params.add_hparam("hgbs", args.hgbs)
    params.add_hparam("decode_hyp_num", args.decode_hyp_num)
    return params

示例#5

文件： trainer.py 项目： ljwmusclenlper/NER-of-Experimental-Facilities

def override_parameters(params, args):
    params.model = args.model
    params.input = args.input or params.input
    params.glove_emb_path = args.glove_emb_path
    params.bert_emb_path = args.bert_emb_path
    params.output = args.output or params.output
    params.record = args.record or params.record
    params.vocab = args.vocabulary.split() or params.vocab
    params.validation = args.validation or params.validation
    params.references = args.references or params.references
    params.parse(args.parameters)

    params.vocabulary = {
        "source": vocabulary.load_vocabulary(params.vocab[0]),  #9092个中国字
        "target": vocabulary.load_vocabulary(
            params.vocab[1]
        ),  #['E-PRODUCT', 'O', 'B-PRODUCT', 'S-PRODUCT', 'I-PRODUCT']
        "char": vocabulary.load_vocabulary(params.vocab[2])  #char跟source一样
    }
    params.vocabulary["source"] = vocabulary.process_vocabulary(
        params.vocabulary["source"], params)
    params.vocabulary["target"] = vocabulary.process_vocabulary(
        params.vocabulary["target"], params)
    params.vocabulary["char"] = vocabulary.process_vocabulary(
        params.vocabulary["char"], params)

    control_symbols = [params.pad, params.bos, params.eos,
                       params.unk]  #pad,bos,eos,unk
    #记录字典中#pad,bos,eos,unk的位置
    params.mapping = {
        "source":
        vocabulary.get_control_mapping(  #{'<unk>': 3890}source3890的位置有个unk
            params.vocabulary["source"], control_symbols),
        "target":
        vocabulary.get_control_mapping(  #{}
            params.vocabulary["target"], control_symbols),
        "char":
        vocabulary.get_control_mapping(  #{'<unk>': 3890}source3890的位置有个unk
            params.vocabulary["char"], control_symbols)
    }

    return params

示例#6

def override_parameters(params, args):
    if args.parameters:
        params.parse(args.parameters)

    params.vocabulary = {
        "source": vocabulary.load_vocabulary(args.vocabulary[0]),
        "target": vocabulary.load_vocabulary(args.vocabulary[1])
    }
    params.vocabulary["source"] = vocabulary.process_vocabulary(
        params.vocabulary["source"], params)
    params.vocabulary["target"] = vocabulary.process_vocabulary(
        params.vocabulary["target"], params)

    control_symbols = [
        params.pad, params.bos, params.eos, params.unk, '!', '\"', '&', '\\',
        ',', '(', ')', '*', ',', '-', '.', '...', '/', ':', ';', '?', '{', '|',
        '}', '~', '、', '。', '—', '……', '‘', '’ ', '“', '”', '〈', '〉', '《', '》',
        '！', '（', '）', '，', '．', '：', '；'
    ]
    params.mapping = {
        "source":
        vocabulary.get_control_mapping(params.vocabulary["source"],
                                       control_symbols),
        "target":
        vocabulary.get_control_mapping(params.vocabulary["target"],
                                       control_symbols)
    }

    params.add_hparam("constraints", args.constraints)
    params.add_hparam("pos_extend", args.pos_extend)
    params.add_hparam("weight_threshold", args.weight_threshold)
    params.add_hparam("encdec_att_layer", args.encdec_att_layer)
    params.add_hparam("heads", args.heads)
    params.add_hparam("head_model", args.head_model)
    params.add_hparam("hgbs", args.hgbs)
    params.add_hparam("decode_hyp_num", args.decode_hyp_num)
    return params

示例#7

文件： predicter.py 项目： cxncu001/PSSAttention-1

def override_parameters(params, args):
    if args.parameters:
        params.parse(args.parameters)

    print(args.vocabulary)

    params.vocabulary = vocabulary.load_vocabulary(args.vocabulary)

    params.vocabulary = vocabulary.process_vocabulary(params.vocabulary,
                                                      params)

    control_symbols = [params.pad, params.bos, params.eos, params.unk]

    params.mapping = vocabulary.get_control_mapping(params.vocabulary,
                                                    control_symbols)

    return params

示例#8

def override_parameters(params, args):
    params.model = args.model
    params.input = args.input or params.input
    params.output = args.output or params.output
    params.vocab = args.vocabulary or params.vocab
    params.pretrained_embedding = args.pretrained_embedding or params.pretrained_embedding
    params.validation = args.validation or params.validation
    params.parse(args.parameters)

    params.vocabulary = vocabulary.load_vocabulary(params.vocab)

    params.vocabulary = vocabulary.process_vocabulary(params.vocabulary,
                                                      params)

    control_symbols = [params.pad, params.bos, params.eos, params.unk]
    params.mapping = vocabulary.get_control_mapping(params.vocabulary,
                                                    control_symbols)

    return params