def build_slot_decoder(opt, config, emission_scorer, emb_log):
transition_scorer = None
if opt.decoder == 'sms':
decoder = SequenceLabeler()
elif opt.decoder == 'rule':
decoder = RuleSequenceLabeler(config['id2label']['slot'])
elif opt.decoder == 'crf':
# logger.info('We only support back-off trans training now!')
# Notice: only train back-off now
trans_normalizer = build_scale_controller(name=opt.trans_normalizer)
trans_scaler = build_scale_controller(
name=opt.trans_scaler, kwargs=make_scaler_args(opt.trans_scaler, trans_normalizer, opt.trans_scale_r))
if opt.transition == 'learn':
transition_scorer = FewShotTransitionScorer(
num_tags=config['num_tags']['slot'], normalizer=trans_normalizer, scaler=trans_scaler,
r=opt.trans_r, backoff_init=opt.backoff_init)
elif opt.transition == 'learn_with_label':
label_trans_normalizer = build_scale_controller(name=opt.label_trans_normalizer)
label_trans_scaler = build_scale_controller(name=opt.label_trans_scaler, kwargs=make_scaler_args(
opt.label_trans_scaler, label_trans_normalizer, opt.label_trans_scale_r))
transition_scorer = FewShotTransitionScorerFromLabel(
num_tags=config['num_tags']['slot'], normalizer=trans_normalizer, scaler=trans_scaler,
r=opt.trans_r, backoff_init=opt.backoff_init, label_scaler=label_trans_scaler)
else:
raise ValueError('Wrong choice of transition.')
if opt.add_transition_rules and 'id2label' in config: # 0 is [PAD] label id, here remove it.
non_pad_id2label = copy.deepcopy(config['id2label']['slot']).__delitem__(0)
print('before - non_pad_id2label: {}'.format(non_pad_id2label))
for k, v in non_pad_id2label.items():
# TODO: maybe error? `k - 1` rather than `v - 1`
non_pad_id2label[k] = v - 1 # we 0 as [PAD] label id, here remove it.
print('after - non_pad_id2label: {}'.format(non_pad_id2label))
constraints = allowed_transitions(constraint_type='BIO', labels=non_pad_id2label)
else:
constraints = None
decoder = ConditionalRandomField(
num_tags=transition_scorer.num_tags, constraints=constraints) # accurate tags
else:
raise TypeError('wrong component type')
seq_laber = SchemaFewShotSeqLabeler if opt.use_schema else FewShotSeqLabeler
slot_model = seq_laber(opt=opt,
emission_scorer=emission_scorer,
decoder=decoder,
transition_scorer=transition_scorer,
config=config,
emb_log=emb_log)
return slot_model
def make_model(opt, config):
""" Customize and build the few-shot learning model from components """
''' Build context_embedder '''
if opt.context_emb == 'bert':
context_embedder = BertSchemaContextEmbedder(opt=opt) if opt.use_schema else BertContextEmbedder(opt=opt)
elif opt.context_emb == 'sep_bert':
context_embedder = BertSchemaSeparateContextEmbedder(opt=opt) if opt.use_schema else \
BertSeparateContextEmbedder(opt=opt)
elif opt.context_emb == 'electra':
context_embedder = ElectraSchemaContextEmbedder(opt=opt) if opt.use_schema else ElectraContextEmbedder(opt=opt)
elif opt.context_emb == 'elmo':
raise NotImplementedError
elif opt.context_emb == 'glove':
context_embedder = NormalContextEmbedder(opt=opt, num_token=len(opt.word2id))
context_embedder.load_embedding()
elif opt.context_emb == 'raw':
context_embedder = NormalContextEmbedder(opt=opt, num_token=len(opt.word2id))
else:
raise TypeError('wrong component type')
''' Create log file to record testing data '''
if opt.emb_log:
emb_log = open(os.path.join(opt.output_dir, 'emb.log'), 'w')
if 'id2label' in config:
emb_log.write('id2label\t' + '\t'.join([str(k) + ':' + str(v) for k, v in config['id2label'].items()]) + '\n')
else:
emb_log = None
'''Build emission scorer and similarity scorer '''
# build scaler
ems_normalizer = build_scale_controller(name=opt.emission_normalizer)
ems_scaler = build_scale_controller(
name=opt.emission_scaler, kwargs=make_scaler_args(opt.emission_scaler, ems_normalizer, opt.ems_scale_r))
if opt.similarity == 'dot':
sim_func = reps_dot
elif opt.similarity == 'cosine':
sim_func = reps_cosine_sim
elif opt.similarity == 'l2':
sim_func = reps_l2_sim
else:
raise TypeError('wrong component type')
if opt.emission == 'mnet':
similarity_scorer = MatchingSimilarityScorer(sim_func=sim_func, emb_log=emb_log)
emission_scorer = MNetEmissionScorer(similarity_scorer, ems_scaler, opt.div_by_tag_num)
elif opt.emission == 'proto':
similarity_scorer = PrototypeSimilarityScorer(sim_func=sim_func, emb_log=emb_log)
emission_scorer = PrototypeEmissionScorer(similarity_scorer, ems_scaler)
elif opt.emission == 'proto_with_label':
similarity_scorer = ProtoWithLabelSimilarityScorer(sim_func=sim_func, scaler=opt.ple_scale_r, emb_log=emb_log)
emission_scorer = ProtoWithLabelEmissionScorer(similarity_scorer, ems_scaler)
elif opt.emission == 'tapnet':
# set num of anchors:
# (1) if provided in config, use it (usually in load model case.)
# (2) *3 is used to ensure enough anchors ( > num_tags of unseen domains )
num_anchors = config['num_anchors'] if 'num_anchors' in config else config['num_tags'] * 3
config['num_anchors'] = num_anchors
anchor_dim = 256 if opt.context_emb == 'electra' else 768
similarity_scorer = TapNetSimilarityScorer(
sim_func=sim_func, num_anchors=num_anchors, mlp_out_dim=opt.tap_mlp_out_dim,
random_init=opt.tap_random_init, random_init_r=opt.tap_random_init_r,
mlp=opt.tap_mlp, emb_log=emb_log, tap_proto=opt.tap_proto, tap_proto_r=opt.tap_proto_r,
anchor_dim=anchor_dim)
emission_scorer = TapNetEmissionScorer(similarity_scorer, ems_scaler)
else:
raise TypeError('wrong component type')
''' Build decoder '''
if opt.task == 'sl': # for sequence labeling
if opt.decoder == 'sms':
transition_scorer = None
decoder = SequenceLabeler()
elif opt.decoder == 'rule':
transition_scorer = None
decoder = RuleSequenceLabeler(config['id2label'])
elif opt.decoder == 'crf':
# Notice: only train back-off now
trans_normalizer = build_scale_controller(name=opt.trans_normalizer)
trans_scaler = build_scale_controller(
name=opt.trans_scaler, kwargs=make_scaler_args(opt.trans_scaler, trans_normalizer, opt.trans_scale_r))
if opt.transition == 'learn':
transition_scorer = FewShotTransitionScorer(
num_tags=config['num_tags'], normalizer=trans_normalizer, scaler=trans_scaler,
r=opt.trans_r, backoff_init=opt.backoff_init)
elif opt.transition == 'learn_with_label':
label_trans_normalizer = build_scale_controller(name=opt.label_trans_normalizer)
label_trans_scaler = build_scale_controller(name=opt.label_trans_scaler, kwargs=make_scaler_args(
opt.label_trans_scaler, label_trans_normalizer, opt.label_trans_scale_r))
transition_scorer = FewShotTransitionScorerFromLabel(
num_tags=config['num_tags'], normalizer=trans_normalizer, scaler=trans_scaler,
r=opt.trans_r, backoff_init=opt.backoff_init, label_scaler=label_trans_scaler)
else:
raise ValueError('Wrong choice of transition.')
if opt.add_transition_rules and 'id2label' in config: # 0 is [PAD] label id, here remove it.
non_pad_id2label = copy.deepcopy(config['id2label']).__delitem__(0)
for k, v in non_pad_id2label.items():
non_pad_id2label[k] = v - 1 # we 0 as [PAD] label id, here remove it.
constraints = allowed_transitions(constraint_type='BIO', labels=non_pad_id2label)
else:
constraints = None
decoder = ConditionalRandomField(
num_tags=transition_scorer.num_tags, constraints=constraints) # accurate tags
else:
raise TypeError('wrong component type')
elif opt.task == 'mlc': # for multi-label text classification task
grad_threshold = True if opt.threshold_type == 'learn' else False
if opt.decoder == 'mlc':
decoder = MultiLabelTextClassifier(opt.threshold, grad_threshold)
elif opt.decoder == 'eamlc':
decoder = EAMultiLabelTextClassifier(opt.threshold, grad_threshold)
elif opt.decoder == 'msmlc':
decoder = MetaStatsMultiLabelTextClassifier(opt.threshold, grad_threshold, meta_rate=opt.meta_rate,
ab_ea=opt.ab_ea)
elif opt.decoder == 'krnmsmlc':
map_dict = {
"feature_map": opt.feature_map,
"feature_num": opt.feature_num,
"feature_map_dim": opt.feature_map_dim,
"feature_map_act": opt.feature_map_act,
"feature_map_layer_num": opt.feature_map_layer_num,
}
decoder = KRNMetaStatsMultiLabelTextClassifier(opt.threshold, grad_threshold, meta_rate=opt.meta_rate,
ab_ea=opt.ab_ea, kernel=opt.kernel, bandwidth=opt.bandwidth,
use_gold=opt.use_gold, learnable=opt.kernel_learnable,
map_dict=map_dict)
else:
raise TypeError('wrong component type')
elif opt.task == 'sc': # for single-label text classification task
decoder = SingleLabelTextClassifier()
else:
raise TypeError('wrong task type')
''' Build the whole model '''
if opt.task == 'sl':
seq_labeler = SchemaFewShotSeqLabeler if opt.use_schema else FewShotSeqLabeler
model = seq_labeler(
opt=opt,
context_embedder=context_embedder,
emission_scorer=emission_scorer,
decoder=decoder,
transition_scorer=transition_scorer,
config=config,
emb_log=emb_log
)
elif opt.task in ['sc', 'mlc']:
text_classifier = SchemaFewShotTextClassifier if opt.use_schema else FewShotTextClassifier
model = text_classifier(
opt=opt,
context_embedder=context_embedder,
emission_scorer=emission_scorer,
decoder=decoder,
config=config,
emb_log=emb_log
)
else:
raise TypeError('wrong task type')
return model
def make_model(opt, config):
""" Customize and build the few-shot learning model from components """
''' Create log file to record testing data '''
emb_log = open(os.path.join(opt.output_dir, 'emb.log'), 'w') if opt.emb_log else None
context_embedder = build_context_embedder(opt)
'''Build emission scorer and similarity scorer '''
ems_normalizer = build_scale_controller(name=opt.emission_normalizer)
ems_scaler = build_scale_controller(
name=opt.emission_scaler, kwargs=make_scaler_args(opt.emission_scaler, ems_normalizer, opt.ems_scale_r))
if opt.similarity == 'dot':
sim_func = reps_dot
elif opt.similarity == 'cosine':
sim_func = reps_cosine_sim
elif opt.similarity == 'l2':
sim_func = reps_l2_sim
elif opt.similarity == 'relation':
emb_dim = 768 if opt.context_emb in ['bert', 'sep_bert'] else 256
pair_emb_dim = 2 * emb_dim
sim_func = RelationNetSim(opt=opt, input_size=pair_emb_dim, hidden_size=opt.relation_hidden_size)
else:
raise TypeError('wrong component type')
intent_emission_scorer = build_emission_scorer(opt, config, ems_scaler, sim_func, emb_log, 'intent')
slot_emission_scorer = build_emission_scorer(opt, config, ems_scaler, sim_func, emb_log, 'slot')
''' Build decoder '''
intent_decoder = None
slot_decoder = None
if opt.task in ['slot_filling', 'slu']:
slot_decoder = build_slot_decoder(opt, config, slot_emission_scorer, emb_log)
if opt.task in ['intent', 'slu']:
intent_decoder = build_intent_decoder(opt, config, intent_emission_scorer, emb_log)
''' Build the whole model '''
opt.id2label = config['id2label']
if opt.slu_model_type == 'simple':
few_shot_slu = SchemaFewShotSLU if opt.use_schema else FewShotSLU
elif opt.slu_model_type == 'split_metric':
few_shot_slu = SplitMetricFewShotSLU
elif opt.slu_model_type == 'emission_merge_intent':
few_shot_slu = EmissionMergeIntentFewShotSLU
elif opt.slu_model_type == 'emission_merge_slot':
few_shot_slu = EmissionMergeSlotFewShotSLU
elif opt.slu_model_type == 'emission_merge_iteration':
few_shot_slu = EmissionMergeIterationFewShotSLU
else:
raise NotImplementedError
model = few_shot_slu(
opt=opt,
context_embedder=context_embedder,
slot_decoder=slot_decoder,
intent_decoder=intent_decoder,
config=config,
emb_log=emb_log
)
return model
def make_model(opt, config):
""" Customize and build the few-shot learning model from components """
''' Build context_embedder '''
if opt.context_emb == 'bert':
context_embedder = BertSchemaContextEmbedder(
opt=opt) if opt.use_schema else BertContextEmbedder(opt=opt)
elif opt.context_emb == 'sep_bert':
context_embedder = BertSchemaSeparateContextEmbedder(opt=opt) if opt.use_schema else \
BertSeparateContextEmbedder(opt=opt)
elif opt.context_emb == 'electra':
context_embedder = ElectraSchemaContextEmbedder(
opt=opt) if opt.use_schema else ElectraContextEmbedder(opt=opt)
elif opt.context_emb == 'elmo':
raise NotImplementedError
elif opt.context_emb == 'glove':
context_embedder = NormalContextEmbedder(opt=opt,
num_token=len(opt.word2id))
context_embedder.load_embedding()
elif opt.context_emb == 'raw':
context_embedder = NormalContextEmbedder(opt=opt,
num_token=len(opt.word2id))
else:
raise TypeError('wrong component type')
''' Create log file to record testing data '''
if opt.emb_log:
emb_log = open(os.path.join(opt.output_dir, 'emb.log'), 'w')
if 'id2label_map' in config:
emb_log.write('id2label_map\t' + '\t'.join([
str(k) + ':' + str(v)
for k, v in config['id2label_map'].items()
]) + '\n')
else:
emb_log = None
'''Build emission scorer and similarity scorer '''
# build scaler
ems_normalizer = build_scale_controller(name=opt.emission_normalizer)
ems_scaler = build_scale_controller(name=opt.emission_scaler,
kwargs=make_scaler_args(
opt.emission_scaler,
ems_normalizer, opt.ems_scale_r))
if opt.similarity == 'dot':
sim_func = reps_dot
elif opt.similarity == 'cosine':
sim_func = reps_cosine_sim
elif opt.similarity == 'l2':
sim_func = reps_l2_sim
else:
raise TypeError('wrong component type')
assert len(opt.emission) == len(
opt.task), "the emission list should match with task list"
emission_scorer_map = {}
for task, emission in zip(opt.task, opt.emission):
if emission == 'mnet':
similarity_scorer = MatchingSimilarityScorer(sim_func=sim_func,
emb_log=emb_log)
emission_scorer = MNetEmissionScorer(similarity_scorer, ems_scaler,
opt.div_by_tag_num)
elif emission == 'proto':
similarity_scorer = PrototypeSimilarityScorer(sim_func=sim_func,
emb_log=emb_log)
emission_scorer = PrototypeEmissionScorer(similarity_scorer,
ems_scaler)
elif emission == 'proto_with_label':
similarity_scorer = ProtoWithLabelSimilarityScorer(
sim_func=sim_func, scaler=opt.ple_scale_r, emb_log=emb_log)
emission_scorer = ProtoWithLabelEmissionScorer(
similarity_scorer, ems_scaler)
elif emission == 'tapnet':
# set num of anchors:
# (1) if provided in config, use it (usually in load model case.)
# (2) *3 is used to ensure enough anchors ( > num_tags of unseen domains )
print('config: {}'.format(config))
num_anchors = config[
'num_anchors'] if 'num_anchors' in config else config[
'num_tags'] * 3
config['num_anchors'] = num_anchors
anchor_dim = 256 if opt.context_emb == 'electra' else 768
similarity_scorer = TapNetSimilarityScorer(
sim_func=sim_func,
num_anchors=num_anchors,
mlp_out_dim=opt.tap_mlp_out_dim,
random_init=opt.tap_random_init,
random_init_r=opt.tap_random_init_r,
mlp=opt.tap_mlp,
emb_log=emb_log,
tap_proto=opt.tap_proto,
tap_proto_r=opt.tap_proto_r,
anchor_dim=anchor_dim)
emission_scorer = TapNetEmissionScorer(similarity_scorer,
ems_scaler)
else:
raise TypeError('wrong component type')
emission_scorer_map[task] = emission_scorer
''' Build decoder '''
model_map = {}
transition_scorer = None
if 'sl' in opt.task: # for sequence labeling
if opt.decoder == 'sms':
decoder = SequenceLabeler()
elif opt.decoder == 'rule':
decoder = RuleSequenceLabeler(config['id2label'])
elif opt.decoder == 'crf':
# logger.info('We only support back-off trans training now!')
# Notice: only train back-off now
trans_normalizer = build_scale_controller(
name=opt.trans_normalizer)
trans_scaler = build_scale_controller(
name=opt.trans_scaler,
kwargs=make_scaler_args(opt.trans_scaler, trans_normalizer,
opt.trans_scale_r))
if opt.transition == 'learn':
transition_scorer = FewShotTransitionScorer(
num_tags=config['num_tags'],
normalizer=trans_normalizer,
scaler=trans_scaler,
r=opt.trans_r,
backoff_init=opt.backoff_init)
elif opt.transition == 'learn_with_label':
label_trans_normalizer = build_scale_controller(
name=opt.label_trans_normalizer)
label_trans_scaler = build_scale_controller(
name=opt.label_trans_scaler,
kwargs=make_scaler_args(opt.label_trans_scaler,
label_trans_normalizer,
opt.label_trans_scale_r))
transition_scorer = FewShotTransitionScorerFromLabel(
num_tags=config['num_tags'],
normalizer=trans_normalizer,
scaler=trans_scaler,
r=opt.trans_r,
backoff_init=opt.backoff_init,
label_scaler=label_trans_scaler)
else:
raise ValueError('Wrong choice of transition.')
if opt.add_transition_rules and 'id2label' in config: # 0 is [PAD] label id, here remove it.
non_pad_id2label = copy.deepcopy(
config['id2label']).__delitem__(0)
for k, v in non_pad_id2label.items():
non_pad_id2label[
k] = v - 1 # we 0 as [PAD] label id, here remove it.
constraints = allowed_transitions(constraint_type='BIO',
labels=non_pad_id2label)
else:
constraints = None
decoder = ConditionalRandomField(
num_tags=transition_scorer.num_tags,
constraints=constraints) # accurate tags
else:
raise TypeError('wrong component type')
seq_laber = SchemaFewShotSeqLabeler if opt.use_schema else FewShotSeqLabeler
model_map['sl'] = seq_laber(opt=opt,
context_embedder=context_embedder,
emission_scorer=emission_scorer_map['sl'],
decoder=decoder,
transition_scorer=transition_scorer,
config=config,
emb_log=emb_log)
if 'sc' in opt.task: # for single-label text classification task
decoder = SingleLabelTextClassifier()
text_classifier = SchemaFewShotTextClassifier if opt.use_schema else FewShotTextClassifier
model_map['sc'] = text_classifier(
opt=opt,
context_embedder=context_embedder,
emission_scorer=emission_scorer_map['sc'],
decoder=decoder,
config=config,
emb_log=emb_log)
''' Build the whole model '''
few_shot_learner = SchemaFewShotLearner if opt.use_schema else FewShotLearner
model = few_shot_learner(opt=opt,
context_embedder=context_embedder,
model_map=model_map,
config=config,
emb_log=emb_log)
return model