def __init__(self, opt, bert_config=None):
super(SANBertNetwork, self).__init__()
self.dropout_list = nn.ModuleList()
self.bert_config = BertConfig.from_dict(opt)
self.bert = BertModel(self.bert_config)
if opt.get('dump_feature', False):
self.opt = opt
return
if opt['update_bert_opt'] > 0:
for p in self.bert.parameters():
p.requires_grad = False
mem_size = self.bert_config.hidden_size
self.decoder_opt = opt['answer_opt']
self.scoring_list = nn.ModuleList()
labels = [int(ls) for ls in opt['label_size'].split(',')]
task_dropout_p = opt['tasks_dropout_p']
self.bert_pooler = None
for task, lab in enumerate(labels):
decoder_opt = self.decoder_opt[task]
dropout = DropoutWrapper(task_dropout_p[task], opt['vb_dropout'])
self.dropout_list.append(dropout)
if decoder_opt == 1:
out_proj = SANClassifier(mem_size, mem_size, lab, opt, prefix='answer', dropout=dropout)
self.scoring_list.append(out_proj)
else:
out_proj = nn.Linear(self.bert_config.hidden_size, lab)
self.scoring_list.append(out_proj)
self.opt = opt
self._my_init()
self.set_embed(opt)
def __init__(self, model_name, cache_dir, task_list):
super(MultiTaskModel, self).__init__()
cache = os.path.join(cache_dir, model_name)
self.transformer = XLNetModel.from_pretrained(model_name,
cache_dir=cache)
self.transformer_config = self.transformer.config
self.dropout = DropoutWrapper(self.transformer_config.dropout)
self.decoderID = {} #模型内部的task_id与decoder_id的映射
# self.decoder = {}
self.decoder_list = nn.ModuleList()
for innerid, task in enumerate(task_list):
if task[1] == TaskType["classification"]: # task[1] = tasktype
classifier = Classification(self.transformer_config)
# classifier = Classification(self.transformer_config)
print("use simple classification")
self.decoder_list.append(classifier)
elif task[1] == TaskType["SANclassification"]:
classifier = SANClassifier(self.transformer_config.hidden_size,
self.transformer_config.hidden_size,
label_size=1,
dropout=self.dropout)
print("use SANClassifier")
self.decoder_list.append(classifier)
else:
pass
self.decoderID[task[0]] = innerid
def __init__(self, opt, bert_config=None):
super(SANBertNetwork, self).__init__()
self.dropout_list = nn.ModuleList()
self.encoder_type = opt['encoder_type']
if opt['encoder_type'] == EncoderModelType.ROBERTA:
from fairseq.models.roberta import RobertaModel
self.bert = RobertaModel.from_pretrained(opt['init_checkpoint'])
hidden_size = self.bert.args.encoder_embed_dim
self.pooler = LinearPooler(hidden_size)
else:
self.bert_config = BertConfig.from_dict(opt)
self.bert = BertModel(self.bert_config)
hidden_size = self.bert_config.hidden_size
if opt.get('dump_feature', False):
self.opt = opt
return
if opt['update_bert_opt'] > 0:
for p in self.bert.parameters():
p.requires_grad = False
self.decoder_opt = opt['answer_opt']
self.task_types = opt["task_types"]
self.scoring_list = nn.ModuleList()
labels = [int(ls) for ls in opt['label_size'].split(',')]
task_dropout_p = opt['tasks_dropout_p']
for task, lab in enumerate(labels):
decoder_opt = self.decoder_opt[task]
task_type = self.task_types[task]
dropout = DropoutWrapper(task_dropout_p[task], opt['vb_dropout'])
self.dropout_list.append(dropout)
if task_type == TaskType.Span:
assert decoder_opt != 1
out_proj = nn.Linear(hidden_size, 2)
elif task_type == TaskType.SeqenceLabeling:
out_proj = nn.Linear(hidden_size, lab)
elif task_type == TaskType.MaskLM:
if opt['encoder_type'] == EncoderModelType.ROBERTA:
# TODO: xiaodl
out_proj = MaskLmHeader(
self.bert.embeddings.word_embeddings.weight)
else:
out_proj = MaskLmHeader(
self.bert.embeddings.word_embeddings.weight)
else:
if decoder_opt == 1:
out_proj = SANClassifier(hidden_size,
hidden_size,
lab,
opt,
prefix='answer',
dropout=dropout)
else:
out_proj = nn.Linear(hidden_size, lab)
self.scoring_list.append(out_proj)
self.opt = opt
self._my_init()
def train_build_task_layer(decoder_opt, hidden_size, lab, opt, prefix,
dropout):
if decoder_opt == 1:
out_proj = SANClassifier(hidden_size,
hidden_size,
lab,
opt,
prefix,
dropout=dropout)
else:
out_proj = nn.Linear(hidden_size, lab)
return out_proj
def __init__(self, opt, bert_config=None, initial_from_local=False):
super(SANBertNetwork, self).__init__()
self.dropout_list = nn.ModuleList()
if opt['encoder_type'] not in EncoderModelType._value2member_map_:
raise ValueError("encoder_type is out of pre-defined types")
self.encoder_type = opt['encoder_type']
self.preloaded_config = None
literal_encoder_type = EncoderModelType(self.encoder_type).name.lower()
config_class, model_class, tokenizer_class = MODEL_CLASSES[
literal_encoder_type]
self.preloaded_config = config_class.from_dict(
opt) # load config from opt
self.preloaded_config.output_hidden_states = True # return all hidden states
self.bert = model_class(self.preloaded_config)
hidden_size = self.bert.config.hidden_size
if opt.get('dump_feature', False):
self.opt = opt
return
if opt['update_bert_opt'] > 0:
for p in self.bert.parameters():
p.requires_grad = False
task_def_list = opt['task_def_list']
self.task_def_list = task_def_list
self.decoder_opt = []
self.task_types = []
for task_id, task_def in enumerate(task_def_list):
self.decoder_opt.append(
generate_decoder_opt(task_def.enable_san, opt['answer_opt']))
self.task_types.append(task_def.task_type)
# create output header
self.scoring_list = nn.ModuleList()
self.dropout_list = nn.ModuleList()
for task_id in range(len(task_def_list)):
task_def: TaskDef = task_def_list[task_id]
lab = task_def.n_class
decoder_opt = self.decoder_opt[task_id]
task_type = self.task_types[task_id]
task_dropout_p = opt[
'dropout_p'] if task_def.dropout_p is None else task_def.dropout_p
dropout = DropoutWrapper(task_dropout_p, opt['vb_dropout'])
self.dropout_list.append(dropout)
task_obj = tasks.get_task_obj(task_def)
if task_obj is not None:
out_proj = task_obj.train_build_task_layer(decoder_opt,
hidden_size,
lab,
opt,
prefix='answer',
dropout=dropout)
elif task_type == TaskType.Span:
assert decoder_opt != 1
out_proj = nn.Linear(hidden_size, 2)
elif task_type == TaskType.SeqenceLabeling:
out_proj = nn.Linear(hidden_size, lab)
elif task_type == TaskType.MaskLM:
if opt['encoder_type'] == EncoderModelType.ROBERTA:
# TODO: xiaodl
out_proj = MaskLmHeader(
self.bert.embeddings.word_embeddings.weight)
else:
out_proj = MaskLmHeader(
self.bert.embeddings.word_embeddings.weight)
else:
if decoder_opt == 1:
out_proj = SANClassifier(hidden_size,
hidden_size,
lab,
opt,
prefix='answer',
dropout=dropout)
else:
out_proj = nn.Linear(hidden_size, lab)
self.scoring_list.append(out_proj)
self.opt = opt
self._my_init()
# if not loading from local, loading model weights from pre-trained model, after initialization
if not initial_from_local:
config_class, model_class, tokenizer_class = MODEL_CLASSES[
literal_encoder_type]
self.bert = model_class.from_pretrained(
opt['init_checkpoint'], config=self.preloaded_config)
def __init__(self, opt, bert_config=None, initial_from_local=False):
super(SANBertNetwork, self).__init__()
self.dropout_list = nn.ModuleList()
if opt["encoder_type"] not in EncoderModelType._value2member_map_:
raise ValueError("encoder_type is out of pre-defined types")
self.encoder_type = opt["encoder_type"]
self.preloaded_config = None
literal_encoder_type = EncoderModelType(self.encoder_type).name.lower()
config_class, model_class, _ = MODEL_CLASSES[literal_encoder_type]
if not initial_from_local:
# self.bert = model_class.from_pretrained(opt['init_checkpoint'], config=self.preloaded_config)
self.bert = model_class.from_pretrained(
opt["init_checkpoint"], cache_dir=opt["transformer_cache"])
else:
self.preloaded_config = config_class.from_dict(
opt) # load config from opt
self.preloaded_config.output_hidden_states = (
True # return all hidden states
)
self.bert = model_class(self.preloaded_config)
hidden_size = self.bert.config.hidden_size
if opt.get("dump_feature", False):
self.config = opt
return
if opt["update_bert_opt"] > 0:
for p in self.bert.parameters():
p.requires_grad = False
task_def_list = opt["task_def_list"]
self.task_def_list = task_def_list
self.decoder_opt = []
self.task_types = []
for task_id, task_def in enumerate(task_def_list):
self.decoder_opt.append(
generate_decoder_opt(task_def.enable_san, opt["answer_opt"]))
self.task_types.append(task_def.task_type)
# create output header
self.scoring_list = nn.ModuleList()
self.dropout_list = nn.ModuleList()
for task_id in range(len(task_def_list)):
task_def: TaskDef = task_def_list[task_id]
lab = task_def.n_class
decoder_opt = self.decoder_opt[task_id]
task_type = self.task_types[task_id]
task_dropout_p = (opt["dropout_p"] if task_def.dropout_p is None
else task_def.dropout_p)
dropout = DropoutWrapper(task_dropout_p, opt["vb_dropout"])
self.dropout_list.append(dropout)
task_obj = tasks.get_task_obj(task_def)
if task_obj is not None:
# Move this to task_obj
self.pooler = Pooler(hidden_size,
dropout_p=opt["dropout_p"],
actf=opt["pooler_actf"])
out_proj = task_obj.train_build_task_layer(decoder_opt,
hidden_size,
lab,
opt,
prefix="answer",
dropout=dropout)
elif task_type == TaskType.Span:
assert decoder_opt != 1
out_proj = nn.Linear(hidden_size, 2)
elif task_type == TaskType.SpanYN:
assert decoder_opt != 1
out_proj = nn.Linear(hidden_size, 2)
elif task_type == TaskType.SeqenceLabeling:
out_proj = nn.Linear(hidden_size, lab)
# elif task_type == TaskType.MaskLM:
# if opt["encoder_type"] == EncoderModelType.ROBERTA:
# # TODO: xiaodl
# out_proj = MaskLmHeader(self.bert.embeddings.word_embeddings.weight)
# else:
# out_proj = MaskLmHeader(self.bert.embeddings.word_embeddings.weight)
elif task_type == TaskType.SeqenceGeneration:
# use orginal header
out_proj = None
elif task_type == TaskType.ClozeChoice:
self.pooler = Pooler(hidden_size,
dropout_p=opt["dropout_p"],
actf=opt["pooler_actf"])
out_proj = nn.Linear(hidden_size, lab)
else:
if decoder_opt == 1:
out_proj = SANClassifier(
hidden_size,
hidden_size,
lab,
opt,
prefix="answer",
dropout=dropout,
)
else:
out_proj = nn.Linear(hidden_size, lab)
self.scoring_list.append(out_proj)
self.config = opt