tokenizer = tokenizer_class.from_pretrained(opt.pretrained_model_name)
if opt.fix_pretrained_model:
pretrained_model = pretrained_model_class.from_pretrained(
opt.pretrained_model_name, output_hidden_states=True)
else:
pretrained_model = pretrained_model_class.from_pretrained(
opt.pretrained_model_name)
print(pretrained_model.config)
opt.emb_size = None
if opt.task_st == 'slot_tagger':
model_tag = slot_tagger.LSTMTagger(
opt.emb_size,
opt.hidden_size,
None,
len(tag_to_idx),
bidirectional=opt.bidirectional,
num_layers=opt.num_layers,
dropout=opt.dropout,
device=opt.device,
pretrained_model=pretrained_model,
pretrained_model_type=opt.pretrained_model_type,
fix_pretrained_model=opt.fix_pretrained_model)
elif opt.task_st == 'slot_tagger_with_focus':
model_tag = slot_tagger_with_focus.LSTMTagger_focus(
opt.emb_size,
opt.tag_emb_size,
opt.hidden_size,
None,
len(tag_to_idx),
bidirectional=opt.bidirectional,
num_layers=opt.num_layers,
dropout=opt.dropout,
keep_order=opt.testing,
raw_word=True)
rec_test_feats, rec_test_slot_tags = data_reader.read_seqtag_data_with_unali_act(
rec_test_data_dir,
word_to_idx,
slot_tag_to_idx,
keep_order=opt.testing,
raw_word=True)
if opt.task == 'slot_tagger':
import models.slot_tagger as slot_tagger
model_tag = slot_tagger.LSTMTagger(opt.emb_size,
opt.hidden_size,
len(word_to_idx),
len(slot_tag_to_idx),
word_to_idx['<pad>'],
bidirectional=opt.bidirectional,
num_layers=opt.num_layers,
dropout=opt.dropout,
device=opt.device)
elif opt.task == 'slot_tagger_with_focus':
import models.enc_dec.slot_tagger_with_focus as slot_tagger_with_focus
model_tag = slot_tagger_with_focus.LSTMTagger_focus(
opt.emb_size,
opt.hidden_size,
len(word_to_idx),
len(slot_tag_to_idx),
word_to_idx['<pad>'],
slot_tag_to_idx['<pad>'],
bidirectional=opt.bidirectional,
num_layers=opt.num_layers,
test_data_dir,
tag_to_idx,
class_to_idx,
multiClass=opt.multiClass,
keep_order=opt.testing,
lowercase=opt.word_lowercase)
model_bert = BertModel.from_pretrained(opt.bert_model_name)
model_elmo = Elmo(opt.elmo_json, opt.elmo_weight, 1, dropout=0)
opt.emb_size = None
if opt.task_st == 'slot_tagger':
model_tag = slot_tagger.LSTMTagger(opt.emb_size,
opt.hidden_size,
None,
len(tag_to_idx),
bidirectional=opt.bidirectional,
num_layers=opt.num_layers,
dropout=opt.dropout,
device=opt.device,
bert_model=model_bert,
elmo_model=model_elmo)
elif opt.task_st == 'slot_tagger_with_focus':
model_tag = slot_tagger_with_focus.LSTMTagger_focus(
opt.emb_size,
opt.tag_emb_size,
opt.hidden_size,
None,
len(tag_to_idx),
bidirectional=opt.bidirectional,
num_layers=opt.num_layers,
dropout=opt.dropout,
device=opt.device,
if not opt.testing:
train_feats, train_tags, train_class = data_reader.read_seqtag_data_with_class(train_data_dir, word_to_idx, tag_to_idx, class_to_idx, multiClass=opt.multiClass, lowercase=opt.word_lowercase)
valid_feats, valid_tags, valid_class = data_reader.read_seqtag_data_with_class(valid_data_dir, word_to_idx, tag_to_idx, class_to_idx, multiClass=opt.multiClass, keep_order=opt.testing, lowercase=opt.word_lowercase)
test_feats, test_tags, test_class = data_reader.read_seqtag_data_with_class(test_data_dir, word_to_idx, tag_to_idx, class_to_idx, multiClass=opt.multiClass, keep_order=opt.testing, lowercase=opt.word_lowercase)
else:
valid_feats, valid_tags, valid_class = data_reader.read_seqtag_data_with_class(valid_data_dir, word_to_idx, tag_to_idx, class_to_idx, multiClass=opt.multiClass, keep_order=opt.testing, lowercase=opt.word_lowercase)
test_feats, test_tags, test_class = data_reader.read_seqtag_data_with_class(test_data_dir, word_to_idx, tag_to_idx, class_to_idx, multiClass=opt.multiClass, keep_order=opt.testing, lowercase=opt.word_lowercase)
if opt.word_digit_features:
feature_extractor = word_features.word_digit_features_extractor(device=opt.device)
extFeats_dim = feature_extractor.get_feature_dim()
else:
extFeats_dim = None
if opt.task_st == 'slot_tagger':
model_tag = slot_tagger.LSTMTagger(opt.emb_size, opt.hidden_size, len(word_to_idx), len(tag_to_idx), bidirectional=opt.bidirectional, num_layers=opt.num_layers, dropout=opt.dropout, device=opt.device, extFeats_dim=extFeats_dim)
elif opt.task_st == 'slot_tagger_with_focus':
model_tag = slot_tagger_with_focus.LSTMTagger_focus(opt.emb_size, opt.tag_emb_size, opt.hidden_size, len(word_to_idx), len(tag_to_idx), bidirectional=opt.bidirectional, num_layers=opt.num_layers, dropout=opt.dropout, device=opt.device, extFeats_dim=extFeats_dim)
elif opt.task_st == 'slot_tagger_with_crf':
model_tag = slot_tagger_with_crf.LSTMTagger_CRF(opt.emb_size, opt.hidden_size, len(word_to_idx), len(tag_to_idx), bidirectional=opt.bidirectional, num_layers=opt.num_layers, dropout=opt.dropout, device=opt.device, extFeats_dim=extFeats_dim)
else:
exit()
if opt.task_sc == '2tails':
model_class = snt_classifier.sntClassifier_2tails(opt.hidden_size, len(class_to_idx), bidirectional=opt.bidirectional, num_layers=opt.num_layers, dropout=opt.dropout, device=opt.device, multi_class=opt.multiClass)
encoder_info_filter = lambda info: info[0]
elif opt.task_sc == 'maxPooling':
model_class = snt_classifier.sntClassifier_hiddenPooling(opt.hidden_size, len(class_to_idx), bidirectional=opt.bidirectional, num_layers=opt.num_layers, dropout=opt.dropout, device=opt.device, multi_class=opt.multiClass, pooling='max')
encoder_info_filter = lambda info: (info[1], info[2])
elif opt.task_sc == 'hiddenCNN':
model_class = snt_classifier.sntClassifier_hiddenCNN(opt.hidden_size, len(class_to_idx), bidirectional=opt.bidirectional, num_layers=opt.num_layers, dropout=opt.dropout, device=opt.device, multi_class=opt.multiClass)
