def override_parameters(params, args):
if args.parameters:
params.parse(args.parameters)
params.vocabulary = {
"source": vocabulary.load_vocabulary(args.vocabulary[0]),
"target": vocabulary.load_vocabulary(args.vocabulary[1])
}
params.vocabulary["source"] = vocabulary.process_vocabulary(
params.vocabulary["source"], params)
params.vocabulary["target"] = vocabulary.process_vocabulary(
params.vocabulary["target"], params)
control_symbols = [params.pad, params.bos, params.eos, params.unk]
params.mapping = {
"source":
vocabulary.get_control_mapping(params.vocabulary["source"],
control_symbols),
"target":
vocabulary.get_control_mapping(params.vocabulary["target"],
control_symbols)
}
return params
def override_parameters(params, args):
params.input = args.input
params.glove_emb_path = args.glove_emb_path
params.bert_emb_path = args.bert_emb_path
if args.parameters:
params.parse(args.parameters)
# args.vocabulary = args.vocabulary.split()
params.vocabulary = {
"source": vocabulary.load_vocabulary(args.vocabulary[0]),
"target": vocabulary.load_vocabulary(args.vocabulary[1]),
"char": vocabulary.load_vocabulary(args.vocabulary[2])
}
params.vocabulary["source"] = vocabulary.process_vocabulary(
params.vocabulary["source"], params)
params.vocabulary["target"] = vocabulary.process_vocabulary(
params.vocabulary["target"], params)
params.vocabulary["char"] = vocabulary.process_vocabulary(
params.vocabulary["char"], params)
control_symbols = [params.pad, params.bos, params.eos, params.unk]
params.mapping = {
"source":
vocabulary.get_control_mapping(params.vocabulary["source"],
control_symbols),
"target":
vocabulary.get_control_mapping(params.vocabulary["target"],
control_symbols),
"char":
vocabulary.get_control_mapping(params.vocabulary["char"],
control_symbols)
}
return params
def override_parameters(params, args):
params.model = args.model
params.input = args.input or params.input
params.output = args.output or params.output
params.record = args.record or params.record
params.vocab = args.vocabulary or params.vocab
params.validation = args.validation or params.validation
params.references = args.references or params.references
params.parse(args.parameters)
params.vocabulary = {
"source": vocabulary.load_vocabulary(params.vocab[0]),
"target": vocabulary.load_vocabulary(params.vocab[1])
}
params.vocabulary["source"] = vocabulary.process_vocabulary(
params.vocabulary["source"], params)
params.vocabulary["target"] = vocabulary.process_vocabulary(
params.vocabulary["target"], params)
control_symbols = [params.pad, params.bos, params.eos, params.unk]
params.mapping = {
"source":
vocabulary.get_control_mapping(params.vocabulary["source"],
control_symbols),
"target":
vocabulary.get_control_mapping(params.vocabulary["target"],
control_symbols)
}
return params
def override_parameters(params, args):
if args.parameters:
params.parse(args.parameters)
params.vocabulary = {
"source": vocabulary.load_vocabulary(args.vocabulary[0]),
"target": vocabulary.load_vocabulary(args.vocabulary[1])
}
params.vocabulary["source"] = vocabulary.process_vocabulary(
params.vocabulary["source"], params)
params.vocabulary["target"] = vocabulary.process_vocabulary(
params.vocabulary["target"], params)
control_symbols = [params.pad, params.bos, params.eos, params.unk]
params.mapping = {
"source":
vocabulary.get_control_mapping(params.vocabulary["source"],
control_symbols),
"target":
vocabulary.get_control_mapping(params.vocabulary["target"],
control_symbols)
}
params.add_hparam("constraints", args.constraints)
params.add_hparam("pos_extend", args.pos_extend)
params.add_hparam("weight_threshold", args.weight_threshold)
params.add_hparam("encdec_att_layer", args.encdec_att_layer)
params.add_hparam("heads", args.heads)
params.add_hparam("head_model", args.head_model)
params.add_hparam("hgbs", args.hgbs)
params.add_hparam("decode_hyp_num", args.decode_hyp_num)
return params
def override_parameters(params, args):
params.model = args.model
params.input = args.input or params.input
params.glove_emb_path = args.glove_emb_path
params.bert_emb_path = args.bert_emb_path
params.output = args.output or params.output
params.record = args.record or params.record
params.vocab = args.vocabulary.split() or params.vocab
params.validation = args.validation or params.validation
params.references = args.references or params.references
params.parse(args.parameters)
params.vocabulary = {
"source": vocabulary.load_vocabulary(params.vocab[0]), #9092个中国字
"target": vocabulary.load_vocabulary(
params.vocab[1]
), #['E-PRODUCT', 'O', 'B-PRODUCT', 'S-PRODUCT', 'I-PRODUCT']
"char": vocabulary.load_vocabulary(params.vocab[2]) #char跟source一样
}
params.vocabulary["source"] = vocabulary.process_vocabulary(
params.vocabulary["source"], params)
params.vocabulary["target"] = vocabulary.process_vocabulary(
params.vocabulary["target"], params)
params.vocabulary["char"] = vocabulary.process_vocabulary(
params.vocabulary["char"], params)
control_symbols = [params.pad, params.bos, params.eos,
params.unk] #pad,bos,eos,unk
#记录字典中#pad,bos,eos,unk的位置
params.mapping = {
"source":
vocabulary.get_control_mapping( #{'<unk>': 3890}source3890的位置有个unk
params.vocabulary["source"], control_symbols),
"target":
vocabulary.get_control_mapping( #{}
params.vocabulary["target"], control_symbols),
"char":
vocabulary.get_control_mapping( #{'<unk>': 3890}source3890的位置有个unk
params.vocabulary["char"], control_symbols)
}
return params
def override_parameters(params, args):
if args.parameters:
params.parse(args.parameters)
params.vocabulary = {
"source": vocabulary.load_vocabulary(args.vocabulary[0]),
"target": vocabulary.load_vocabulary(args.vocabulary[1])
}
params.vocabulary["source"] = vocabulary.process_vocabulary(
params.vocabulary["source"], params)
params.vocabulary["target"] = vocabulary.process_vocabulary(
params.vocabulary["target"], params)
control_symbols = [
params.pad, params.bos, params.eos, params.unk, '!', '\"', '&', '\\',
',', '(', ')', '*', ',', '-', '.', '...', '/', ':', ';', '?', '{', '|',
'}', '~', '、', '。', '—', '……', '‘', '’ ', '“', '”', '〈', '〉', '《', '》',
'！', '（', '）', '，', '．', '：', '；'
]
params.mapping = {
"source":
vocabulary.get_control_mapping(params.vocabulary["source"],
control_symbols),
"target":
vocabulary.get_control_mapping(params.vocabulary["target"],
control_symbols)
}
params.add_hparam("constraints", args.constraints)
params.add_hparam("pos_extend", args.pos_extend)
params.add_hparam("weight_threshold", args.weight_threshold)
params.add_hparam("encdec_att_layer", args.encdec_att_layer)
params.add_hparam("heads", args.heads)
params.add_hparam("head_model", args.head_model)
params.add_hparam("hgbs", args.hgbs)
params.add_hparam("decode_hyp_num", args.decode_hyp_num)
return params
def override_parameters(params, args):
if args.parameters:
params.parse(args.parameters)
print(args.vocabulary)
params.vocabulary = vocabulary.load_vocabulary(args.vocabulary)
params.vocabulary = vocabulary.process_vocabulary(params.vocabulary,
params)
control_symbols = [params.pad, params.bos, params.eos, params.unk]
params.mapping = vocabulary.get_control_mapping(params.vocabulary,
control_symbols)
return params
def override_parameters(params, args):
params.model = args.model
params.input = args.input or params.input
params.output = args.output or params.output
params.vocab = args.vocabulary or params.vocab
params.pretrained_embedding = args.pretrained_embedding or params.pretrained_embedding
params.validation = args.validation or params.validation
params.parse(args.parameters)
params.vocabulary = vocabulary.load_vocabulary(params.vocab)
params.vocabulary = vocabulary.process_vocabulary(params.vocabulary,
params)
control_symbols = [params.pad, params.bos, params.eos, params.unk]
params.mapping = vocabulary.get_control_mapping(params.vocabulary,
control_symbols)
return params
