def __init__(self,
vocab_size,
embedding_size,
char_vocab_size,
char_embedding_size,
num_filter,
ngram_filter_size,
num_classes,
bert_weight_path=False):
super().__init__()
self.char_embedding = nn.Embedding(char_vocab_size,
char_embedding_size)
init.uniform_(self.char_embedding.weight, -0.1, 0.1)
if bert_weight_path:
self.bert = PretrainedBertEmbedder(bert_weight_path)
else:
self.embedding = nn.Embedding(vocab_size,
embedding_dim=embedding_size)
init.uniform_(self.embedding.weight, -0.1, 0.1)
self.bert = None
self.cnn_encoder = CnnEncoder(char_embedding_size,
num_filters=num_filter,
ngram_filter_sizes=ngram_filter_size)
self.char_encoder = TokenCharactersEncoder(self.char_embedding,
self.cnn_encoder)
if bert_weight_path:
embedding_size = 768
self.linear_layer = nn.Linear(embedding_size + num_filter, num_classes)
init.xavier_normal_(self.linear_layer.weight)
def __init__(self, args, num_authors: int, out_sz: int,
vocab: Vocabulary):
super().__init__(vocab)
# init word embedding
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
self.word_embeddings = BasicTextFieldEmbedder({"tokens": bert_embedder},
# we'll be ignoring masks so we'll need to set this to True
allow_unmatched_keys=True)
self.encoder = BertSentencePooler(vocab, self.word_embeddings.get_output_dim())
self.num_authors = num_authors
# skills dim
self.num_sk, self.sk_dim = 20, 768
self.author_embeddings = nn.Parameter(torch.randn(num_authors, self.sk_dim), requires_grad=True) # (m, d)
self.attention = nn.Parameter(torch.randn(self.word_embeddings.get_output_dim(), self.sk_dim), requires_grad=True)
# nn.Linear(self.word_embeddings.get_output_dim(), self.sk_dim)
self.tanh = nn.Tanh()
self.softmax = nn.Softmax(dim=2)
self.sigmoid = nn.Sigmoid()
self.projection = nn.Linear(self.encoder.get_output_dim(), out_sz)
# self.loss = nn.CrossEntropyLoss()
# loss related
# self.cohere_loss = CoherenceLoss(self.encoder.get_output_dim(), out_sz)
self.triplet_loss = TripletLoss(self.encoder.get_output_dim(), out_sz)
def __init__(self, num_authors: int, out_sz: int, vocab: Vocabulary):
super().__init__(vocab)
# init word embedding
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
self.word_embeddings = BasicTextFieldEmbedder(
{"tokens": bert_embedder},
# we'll be ignoring masks so we'll need to set this to True
allow_unmatched_keys=True)
self.encoder = BertSentencePooler(
vocab, self.word_embeddings.get_output_dim())
self.num_authors = num_authors
# skills dim
self.num_sk, self.sk_dim, self.time_dim = 20, 768, 32
self.author_embeddings = nn.Parameter(torch.randn(
num_authors, self.num_sk, self.sk_dim),
requires_grad=True) # (m, k, d)
self.multihead_att = TempCtxAttention(h=8, d_model=self.sk_dim)
self.attention = nn.Parameter(torch.randn(
self.word_embeddings.get_output_dim(), self.sk_dim),
requires_grad=True)
# nn.Linear(self.word_embeddings.get_output_dim(), self.sk_dim)
self.cohere_loss = CoherenceLoss(self.encoder.get_output_dim(), out_sz)
def __init__(self, num_authors: int, out_sz: int, vocab: Vocabulary,
date_span: Any):
super().__init__(vocab)
# init word embedding
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
self.date_span = date_span
self.word_embeddings = BasicTextFieldEmbedder(
{"tokens": bert_embedder},
# we'll be ignoring masks so we'll need to set this to True
allow_unmatched_keys=True)
self.encoder = BertSentencePooler(
vocab, self.word_embeddings.get_output_dim())
self.num_authors = num_authors
# skills dim
# self.num_sk, self.sk_dim, self.time_dim = 20, 768, 32
self.num_sk, self.sk_dim, self.time_dim = 20, 768, 768
# self.author_dim = self.sk_dim + self.time_dim
self.author_dim = self.sk_dim
self.author_embeddings = nn.Parameter(torch.randn(
num_authors, self.author_dim),
requires_grad=True) # (m, d)
# self.ctx_attention = MultiHeadCtxAttention(h=8, d_model=self.sk_dim + self.time_dim)
self.temp_ctx_attention_ns = TempCtxAttentionNS(
h=8,
d_model=self.author_dim,
d_query=self.sk_dim,
d_time=self.time_dim)
# temporal context
self.time_encoder = TimeEncoder(self.time_dim,
dropout=0.1,
span=1,
date_range=date_span)
# layer_norm
self.ctx_layer_norm = LayerNorm(self.author_dim)
# loss related
# self.cohere_loss = CoherenceLoss(self.encoder.get_output_dim(), out_sz)
self.triplet_loss = TripletLoss(self.encoder.get_output_dim(), out_sz)
self.htemp_loss = HTempLoss(self.encoder.get_output_dim(), out_sz)
self.rank_loss = MarginRankLoss(self.encoder.get_output_dim(), out_sz)
self.coherence_func = CoherenceInnerProd()
def create_module(self):
self.embedding = PretrainedBertEmbedder(
pretrained_model=self.pretrained_model, )
return BasicTextFieldEmbedder({self.key: self.embedding},
embedder_to_indexer_map={
self.key: [
self.key,
"{}-offsets".format(self.key),
"{}-type-ids".format(self.key)
]
},
allow_unmatched_keys=True)
def bert_embeddings(pretrained_model: Path, training: bool = False,
top_layer_only: bool = True
) -> BasicTextFieldEmbedder:
"Pre-trained embeddings using BERT"
bert = PretrainedBertEmbedder(
requires_grad=training,
pretrained_model=pretrained_model,
top_layer_only=top_layer_only
)
word_embeddings = BasicTextFieldEmbedder(
token_embedders={'tokens': bert},
embedder_to_indexer_map={'tokens': ['tokens', 'tokens-offsets']},
allow_unmatched_keys=True)
return word_embeddings
def get_pretrained_bert(model_name: str = 'bert-base-uncased',
cache_dir: str = PATH_ALLENNLP_CACHE,
top_layer_only=True,
requires_grad=True,
**kwargs):
model_path = path.join(cache_dir, 'bert', f'{model_name}.tar.gz')
msgex.assert_path_exist(
path_str=model_path,
arg_name='model_path',
extra_msg=f"the specified BERT model '{model_name}' is not found")
model = PretrainedBertEmbedder(pretrained_model=model_path,
top_layer_only=top_layer_only,
requires_grad=requires_grad,
**kwargs)
return model, model.output_dim
def __init__(self, args, out_sz: int,
vocab: Vocabulary):
super().__init__(vocab)
# init word embedding
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
self.word_embeddings = BasicTextFieldEmbedder({"tokens": bert_embedder},
# we'll be ignoring masks so we'll need to set this to True
allow_unmatched_keys=True)
self.encoder = BertSentencePooler(vocab, self.word_embeddings.get_output_dim())
self.projection = nn.Linear(self.encoder.get_output_dim(), out_sz)
self.loss = nn.CrossEntropyLoss()
def __init__(
self,
vocab: Vocabulary,
dropout: float,
pool: Text = "cls",
model_name_or_path: Text = "bert-base-uncased",
label_namespace: str = "page_labels",
):
bert = PretrainedBertEmbedder(model_name_or_path, requires_grad=True)
super().__init__(
vocab=vocab,
dropout=dropout,
label_namespace=label_namespace,
hidden_dim=bert.get_output_dim(),
)
self._model_name_or_path = model_name_or_path
self._bert = bert
self._pool = pool
def __init__(self, num_authors: int, out_sz: int, vocab: Vocabulary,
date_span: Any, num_shift: int, span: int):
super().__init__(vocab)
# init word embedding
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
self.date_span = date_span
self.word_embeddings = BasicTextFieldEmbedder(
{"tokens": bert_embedder},
# we'll be ignoring masks so we'll need to set this to True
allow_unmatched_keys=True)
self.encoder = BertSentencePooler(
vocab, self.word_embeddings.get_output_dim())
self.num_authors = num_authors
# skills dim
self.num_sk, self.sk_dim = 20, 768
self.author_embeddings = nn.Parameter(torch.randn(
num_authors, self.num_sk, self.sk_dim),
requires_grad=True) # (m, k, d)
self.ctx_attention = TempCtxAttention(h=8, d_model=self.sk_dim)
# layer_norm
self.ctx_layer_norm = nn.LayerNorm(self.sk_dim)
self.shift_temp_att = ShiftTempAttention(self.num_authors, self.sk_dim,
date_span, num_shift, span)
# self.cohere_loss = CoherenceLoss(self.encoder.get_output_dim(), out_sz)
self.triplet_loss = TripletLoss(self.encoder.get_output_dim(), out_sz)
self.temp_loss = TemporalLoss(self.encoder.get_output_dim(), out_sz)
self.rank_loss = MarginRankLoss(self.encoder.get_output_dim(), out_sz)
self.weight_temp = 0.3
def __init__(self,
num_authors: int,
out_sz: int,
vocab: Vocabulary,
date_span: Any,
num_shift: int,
spans: List,
encoder: Any,
max_vocab_size: int,
ignore_time: bool,
ns_mode: bool = False,
num_sk: int = 20):
super().__init__(vocab)
self.date_span = date_span
self.num_authors = num_authors
# skills dim
self.num_sk, self.sk_dim = num_sk, 768
self.ignore_time = ignore_time
self.ns_mode = ns_mode
if self.ns_mode:
self.author_embeddings = nn.Parameter(torch.randn(
num_authors, self.sk_dim),
requires_grad=True) # (m, d)
else:
self.author_embeddings = nn.Parameter(
torch.randn(num_authors, self.num_sk, self.sk_dim),
requires_grad=True) # (m, k, d)
self.encode_type = encoder
if self.encode_type == "bert":
# init word embedding
bert_embedder = PretrainedBertEmbedder(
pretrained_model="bert-base-uncased",
top_layer_only=True, # conserve memory
)
self.word_embeddings = BasicTextFieldEmbedder(
{"tokens": bert_embedder},
# we'll be ignoring masks so we'll need to set this to True
allow_unmatched_keys=True)
self.encoder = BertSentencePooler(
vocab, self.word_embeddings.get_output_dim())
else:
# prepare embeddings
token_embedding = Embedding(num_embeddings=max_vocab_size + 2,
embedding_dim=300,
padding_index=0)
self.word_embeddings: TextFieldEmbedder = BasicTextFieldEmbedder(
{"tokens": token_embedding})
self.encoder: Seq2VecEncoder = PytorchSeq2VecWrapper(
nn.LSTM(self.word_embeddings.get_output_dim(),
hidden_size=int(self.sk_dim / 2),
bidirectional=True,
batch_first=True))
self.ctx_attention = TempCtxAttention(h=8, d_model=self.sk_dim)
self.ctx_layer_norm = nn.LayerNorm(self.sk_dim) # layer_norm
# shifted temporal attentions
self.spans = spans
self.span_temp_atts = nn.ModuleList()
for span in self.spans:
self.span_temp_atts.append(
ShiftTempAttention(self.num_authors, self.sk_dim, date_span,
num_shift, span, self.ignore_time))
self.span_projection = nn.Linear(len(spans), 1)
self.num_shift = num_shift
# temporal encoder: used only for adding temporal information into token embedding
self.time_encoder = TimeEncoder(self.sk_dim,
dropout=0.1,
span=spans[0],
date_range=date_span)
# loss
# self.cohere_loss = CoherenceLoss(self.encoder.get_output_dim(), out_sz)
# self.triplet_loss = TripletLoss(self.encoder.get_output_dim(), out_sz)
self.temp_loss = TemporalLoss(self.encoder.get_output_dim(), out_sz)
self.rank_loss = MarginRankLoss(self.encoder.get_output_dim(), out_sz)
self.weight_temp = 0.3
self.visual_id = 0
def __init__(
self,
vocab: Vocabulary,
task_type: str,
model_type: str,
random_init_bert:
bool, # set True to shuffle the BERT encoder and get random init
initializer: InitializerApplicator = InitializerApplicator()
) -> None:
super().__init__(vocab)
assert task_type in ["unary", "binary"] # unary or binary edges
assert model_type in ["clf", "reg"] # classification or regression
self.task_type = task_type
self.model_type = model_type
mix_params = None
if self.task_type == "binary": # for binary tasks train two separate mixes
self.bert_embedder = PretrainedBertEmbedderSplitMix(
BERT_MODEL_NAME,
requires_grad=False,
top_layer_only=False,
scalar_mix_parameters=mix_params)
else: # for unary task train a single mix
self.bert_embedder = PretrainedBertEmbedder(
BERT_MODEL_NAME,
requires_grad=False,
top_layer_only=False,
scalar_mix_parameters=mix_params)
if random_init_bert:
self.bert_embedder.bert_model.apply(init_weights)
self.vocab = vocab
self.num_classes = self.vocab.get_vocab_size("labels")
self.num_classes = self.num_classes if self.num_classes > 0 else 1
self.span_projection_dim = self.bert_embedder.output_dim
# represent each span by its first wordpiece token
self.span_extractor = EndpointSpanExtractor(self.span_projection_dim,
combination="x")
if self.task_type == "binary":
clf_input_dim = self.span_projection_dim * 2
else:
clf_input_dim = self.span_projection_dim
self.classifier = Linear(
clf_input_dim,
self.num_classes) # just a linear tag projection layer
if self.model_type == "clf":
self.loss = torch.nn.CrossEntropyLoss(
) # cross-entropy for classification
else:
self.loss = torch.nn.SmoothL1Loss() # smooth L1 for regresison
self.m_acc = CategoricalAccuracy()
self.m_fmicro = FBetaMeasure(average="micro")
self.mse = MeanSquaredError()
initializer(self)
def main():
parser = argparse.ArgumentParser(description='BM25 Pipeline reader')
parser.add_argument(
'--k',
type=int,
default=1,
help=
'number of evidence paragraphs to pick from the classifier (default: 1)'
)
parser.add_argument('--probs',
type=str,
default=None,
help='Pickled sentence probs file (default: None)')
args = parser.parse_args()
with torch.no_grad():
bert_token_indexer = {
'bert': PretrainedBertIndexer('scibert/vocab.txt', max_pieces=512)
}
pipeline_train = pickle.load(open('data/train_instances.p', 'rb'))
pipeline_val = pickle.load(open('data/val_instances.p', 'rb'))
pipeline_test = pickle.load(open('data/test_instances.p', 'rb'))
pipeline_reader = PipelineDatasetReader(bert_token_indexer)
p_train = pipeline_reader.read(pipeline_train)
p_val = pipeline_reader.read(pipeline_val)
p_test = pipeline_reader.read(pipeline_test)
p_vocab = Vocabulary.from_instances(p_train + p_val + p_test)
bert_token_embedding = PretrainedBertEmbedder('scibert/weights.tar.gz',
requires_grad=False)
word_embeddings = BasicTextFieldEmbedder(
{"bert": bert_token_embedding}, {"bert": ['bert']},
allow_unmatched_keys=True)
predictor = Oracle(word_embeddings=word_embeddings, vocab=p_vocab)
cuda_device = 0
if torch.cuda.is_available():
predictor = predictor.cuda()
else:
cuda_device = -1
predictor.load_state_dict(
torch.load('model_checkpoints/f_oracle_full/best.th'))
logger.info('Predictor model loaded successfully')
predictor.eval()
iterator = BasicIterator(batch_size=256)
iterator.index_with(p_vocab)
top_k_sentences = []
prob_counter = 0
for i in range(len(pipeline_test)):
sentences = [
' '.join(pipeline_test[i]['sentence_span'][k][0] +
pipeline_test[i]['sentence_span'][k + 1][0] +
pipeline_test[i]['sentence_span'][k + 2][0]).lower().
split()
for k in range(len(pipeline_test[i]['sentence_span']) - 2)
]
bm25 = BM25Okapi(sentences)
prompt = pipeline_test[i]['I'] + pipeline_test[i]['C'] + pipeline_test[i]['O'] + \
['no', 'significant', 'difference']
doc_scores = np.array(bm25.get_scores(prompt))
probs = list(doc_scores)
prob_counter += len(sentences)
sorted_sentences = sorted(zip(sentences, probs),
key=lambda x: x[1],
reverse=True)
top_k = [' '.join(s[0]) for s in sorted_sentences[:args.k]]
top_k_sentences.append({
'I': pipeline_test[i]['I'],
'C': pipeline_test[i]['C'],
'O': pipeline_test[i]['O'],
'y_label': pipeline_test[i]['y'][0][0],
'evidence': ' '.join(top_k)
})
logger.info('Obtained the top sentences from the bm25 classifier')
predictor_reader = EIDatasetReader(bert_token_indexer)
predictor_test = predictor_reader.read(top_k_sentences)
test_metrics = evaluate(predictor,
predictor_test,
iterator,
cuda_device=cuda_device,
batch_weight_key="")
print('Test Data statistics:')
for key, value in test_metrics.items():
print(str(key) + ': ' + str(value))
def main():
parser = create_parser()
args = parser.parse_args()
torch.manual_seed(args.seed)
model_id = create_model_id(args)
if not path.exists(args.out_dir):
print("# Create directory: {}".format(args.out_dir))
os.mkdir(args.out_dir)
# log file
out_dir = path.join(args.out_dir, "out-" + model_id)
print("# Create output directory: {}".format(out_dir))
os.mkdir(out_dir)
log = StandardLogger(path.join(out_dir, "log-" + model_id + ".txt"))
log.write(args=args)
write_args_log(args, path.join(out_dir, "args.json"))
# dataset reader
token_indexers = {
"tokens": SingleIdTokenIndexer(),
"elmo": ELMoTokenCharactersIndexer(),
"bert": PretrainedBertIndexer(BERT_MODEL, use_starting_offsets=True),
"xlnet": PretrainedTransformerIndexer(XLNET_MODEL, do_lowercase=False)
}
reader = SrlDatasetReader(token_indexers)
# dataset
train_dataset = reader.read_with_ratio(args.train, args.data_ratio)
validation_dataset = reader.read_with_ratio(args.dev, 100)
pseudo_dataset = reader.read_with_ratio(
args.pseudo, args.data_ratio) if args.pseudo else []
all_dataset = train_dataset + validation_dataset + pseudo_dataset
if args.test:
test_dataset = reader.read_with_ratio(args.test, 100)
all_dataset += test_dataset
vocab = Vocabulary.from_instances(all_dataset)
# embedding
input_size = args.binary_dim * 2 if args.multi_predicate else args.binary_dim
if args.glove:
token_embedding = Embedding(
num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=GLOVE_DIM,
trainable=True,
pretrained_file=GLOVE)
input_size += GLOVE_DIM
else:
token_embedding = Embedding(
num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=args.embed_dim,
trainable=True)
input_size += args.embed_dim
token_embedders = {"tokens": token_embedding}
if args.elmo:
elmo_embedding = ElmoTokenEmbedder(options_file=ELMO_OPT,
weight_file=ELMO_WEIGHT)
token_embedders["elmo"] = elmo_embedding
input_size += ELMO_DIM
if args.bert:
bert_embedding = PretrainedBertEmbedder(BERT_MODEL)
token_embedders["bert"] = bert_embedding
input_size += BERT_DIM
if args.xlnet:
xlnet_embedding = PretrainedTransformerEmbedder(XLNET_MODEL)
token_embedders["xlnet"] = xlnet_embedding
input_size += XLNET_DIM
word_embeddings = BasicTextFieldEmbedder(token_embedders=token_embedders,
allow_unmatched_keys=True,
embedder_to_indexer_map={
"bert":
["bert", "bert-offsets"],
"elmo": ["elmo"],
"tokens": ["tokens"],
"xlnet": ["xlnet"]
})
# encoder
if args.highway:
lstm = PytorchSeq2SeqWrapper(
StackedAlternatingLstm(input_size=input_size,
hidden_size=args.hidden_dim,
num_layers=args.n_layers,
recurrent_dropout_probability=args.dropout))
else:
pytorch_lstm = torch.nn.LSTM(input_size=input_size,
hidden_size=args.hidden_dim,
num_layers=int(args.n_layers / 2),
batch_first=True,
dropout=args.dropout,
bidirectional=True)
# initialize
for name, param in pytorch_lstm.named_parameters():
if 'weight_ih' in name:
torch.nn.init.xavier_uniform_(param.data)
elif 'weight_hh' in name:
# Wii, Wif, Wic, Wio
for n in range(4):
torch.nn.init.orthogonal_(
param.data[args.hidden_dim * n:args.hidden_dim *
(n + 1)])
elif 'bias' in name:
param.data.fill_(0)
lstm = PytorchSeq2SeqWrapper(pytorch_lstm)
# model
hidden_dim = args.hidden_dim if args.highway else args.hidden_dim * 2 # pytorch.nn.LSTMはconcatされるので2倍
model = SemanticRoleLabelerWithAttention(
vocab=vocab,
text_field_embedder=word_embeddings,
encoder=lstm,
binary_feature_dim=args.binary_dim,
embedding_dropout=args.embed_dropout,
attention_dropout=0.0,
use_attention=args.attention,
use_multi_predicate=args.multi_predicate,
hidden_dim=hidden_dim)
if args.model:
print("# Load model parameter: {}".format(args.model))
with open(args.model, 'rb') as f:
state_dict = torch.load(f, map_location='cpu')
model.load_state_dict(state_dict)
if torch.cuda.is_available():
cuda_device = 0
model = model.cuda(cuda_device)
else:
cuda_device = -1
# optimizer
if args.optimizer == "Adam":
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
elif args.optimizer == "SGD":
optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate)
elif args.optimizer == "Adadelta":
optimizer = torch.optim.Adadelta(model.parameters(), rho=0.95)
else:
raise ValueError("unsupported value: '{}'".format(args.optimizer))
# iterator
# iterator = BucketIterator(batch_size=args.batch, sorting_keys=[("tokens", "num_tokens")])
iterator = BasicIterator(batch_size=args.batch)
iterator.index_with(vocab)
if not args.test_only:
# Train
print("# Train Method: {}".format(args.train_method))
print("# Start Train", flush=True)
if args.train_method == "concat":
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_dataset + pseudo_dataset,
validation_dataset=validation_dataset,
validation_metric="+f1-measure-overall",
patience=args.early_stopping,
num_epochs=args.max_epoch,
num_serialized_models_to_keep=5,
grad_clipping=args.grad_clipping,
serialization_dir=out_dir,
cuda_device=cuda_device)
trainer.train()
elif args.train_method == "pre-train":
pre_train_out_dir = path.join(out_dir + "pre-train")
fine_tune_out_dir = path.join(out_dir + "fine-tune")
os.mkdir(pre_train_out_dir)
os.mkdir(fine_tune_out_dir)
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=pseudo_dataset,
validation_dataset=validation_dataset,
validation_metric="+f1-measure-overall",
patience=args.early_stopping,
num_epochs=args.max_epoch,
num_serialized_models_to_keep=3,
grad_clipping=args.grad_clipping,
serialization_dir=pre_train_out_dir,
cuda_device=cuda_device)
trainer.train()
if args.optimizer == "Adam":
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate)
elif args.optimizer == "SGD":
optimizer = torch.optim.SGD(model.parameters(),
lr=args.learning_rate)
elif args.optimizer == "Adadelta":
optimizer = torch.optim.Adadelta(model.parameters(), rho=0.95)
else:
raise ValueError("unsupported value: '{}'".format(
args.optimizer))
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
validation_metric="+f1-measure-overall",
patience=args.early_stopping,
num_epochs=args.max_epoch,
num_serialized_models_to_keep=3,
grad_clipping=args.grad_clipping,
serialization_dir=fine_tune_out_dir,
cuda_device=cuda_device)
trainer.train()
else:
raise ValueError("Unsupported Value '{}'".format(
args.train_method))
# Test
if args.test:
print("# Test")
result = evaluate(model=model,
instances=test_dataset,
data_iterator=iterator,
cuda_device=cuda_device,
batch_weight_key="")
with open(path.join(out_dir, "test.score"), 'w') as fo:
json.dump(result, fo)
log.write_endtime()
def main():
parser = argparse.ArgumentParser(description='Evidence sentence classifier')
parser.add_argument('--k', type=int, default=1,
help='number of evidence paragraphs to pick from the classifier (default: 1)')
parser.add_argument('--probs', type=str, default=None,
help='Pickled sentence probs file (default: None)')
args = parser.parse_args()
with torch.no_grad():
bert_token_indexer = {'bert': PretrainedBertIndexer('scibert/vocab.txt', max_pieces=512)}
pipeline_train = pickle.load(open('data/train_instances.p', 'rb'))
pipeline_val = pickle.load(open('data/val_instances.p', 'rb'))
pipeline_test = pickle.load(open('data/test_instances.p', 'rb'))
pipeline_reader = PipelineDatasetReader(bert_token_indexer)
p_train = pipeline_reader.read(pipeline_train)
p_val = pipeline_reader.read(pipeline_val)
p_test = pipeline_reader.read(pipeline_test)
p_vocab = Vocabulary.from_instances(p_train + p_val + p_test)
bert_token_embedding = PretrainedBertEmbedder(
'scibert/weights.tar.gz', requires_grad=False
)
word_embeddings = BasicTextFieldEmbedder(
{"bert": bert_token_embedding},
{"bert": ['bert']},
allow_unmatched_keys=True
)
ev_classifier = Classifier(word_embeddings=word_embeddings,
vocab=p_vocab,
loss='bce',
hinge_margin=0)
predictor = Oracle(word_embeddings=word_embeddings,
vocab=p_vocab)
cuda_device = 0
if torch.cuda.is_available():
ev_classifier = ev_classifier.cuda()
predictor = predictor.cuda()
else:
cuda_device = -1
ev_classifier.load_state_dict(torch.load('model_checkpoints/f_evidence_sentence_classifier_para/best.th'))
predictor.load_state_dict(torch.load('model_checkpoints/f_oracle_full/best.th'))
logger.info('Classifier and Predictor models loaded successfully')
ev_classifier.eval()
predictor.eval()
iterator = BasicIterator(batch_size=256)
iterator.index_with(p_vocab)
if args.probs is None:
iterator_obj = iterator(p_test, num_epochs=1, shuffle=False)
generator_tqdm = Tqdm.tqdm(iterator_obj, total=iterator.get_num_batches(p_test))
output_probs = []
for batch in generator_tqdm:
batch = nn_util.move_to_device(batch, cuda_device)
probs = ev_classifier.predict_evidence_probs(**batch)
probs = probs.cpu().numpy()
output_probs.append(probs)
output_probs = [i for item in output_probs for i in item]
logger.info('Obtained all sentence evidence probabilities - total {}'.format(len(output_probs)))
pickle.dump(output_probs, open('sentence_ev_probs.p', 'wb'))
else:
output_probs = pickle.load(open(args.probs, 'rb'))
top_k_sentences = []
prob_counter = 0
for i in range(len(pipeline_test)):
sentences = [' '.join(pipeline_test[i]['sentence_span'][k][0] + pipeline_test[i]['sentence_span'][k + 1][0]
+ pipeline_test[i]['sentence_span'][k + 2][0])
for k in range(len(pipeline_test[i]['sentence_span']) - 2)]
probs = list(output_probs[prob_counter: prob_counter + len(sentences)])
prob_counter += len(sentences)
sorted_sentences = sorted(zip(sentences, probs), key=lambda x: x[1], reverse=True)
top_k = [s[0] for s in sorted_sentences[:args.k]]
top_k_sentences.append({'I': pipeline_test[i]['I'],
'C': pipeline_test[i]['C'],
'O': pipeline_test[i]['O'],
'y_label': pipeline_test[i]['y'][0][0],
'evidence': ' '.join(top_k)})
logger.info('Obtained the top sentences from the evidence classifier')
predictor_reader = EIDatasetReader(bert_token_indexer)
predictor_test = predictor_reader.read(top_k_sentences)
test_metrics = evaluate(predictor, predictor_test, iterator,
cuda_device=cuda_device,
batch_weight_key="")
print('Test Data statistics:')
for key, value in test_metrics.items():
print(str(key) + ': ' + str(value))
def main():
parser = argparse.ArgumentParser(description='Evidence Inference experiments')
parser.add_argument('--cuda_device', type=int, default=0,
help='GPU number (default: 0)')
parser.add_argument('--epochs', type=int, default=2,
help='upper epoch limit (default: 2)')
parser.add_argument('--patience', type=int, default=1,
help='trainer patience (default: 1)')
parser.add_argument('--batch_size', type=int, default=32,
help='batch size (default: 32)')
parser.add_argument('--dropout', type=float, default=0.2,
help='dropout for the model (default: 0.2)')
parser.add_argument('--model_name', type=str, default='baseline',
help='model name (default: baseline)')
parser.add_argument('--tunable', action='store_true',
help='tune the underlying embedding model (default: False)')
args = parser.parse_args()
annotations = pd.read_csv('data/data/annotations_merged.csv')
prompts = pd.read_csv('data/data/prompts_merged.csv')
feature_dictionary = {}
prompts_dictionary = {}
for index, row in prompts.iterrows():
prompts_dictionary[row['PromptID']] = [row['Outcome'], row['Intervention'], row['Comparator']]
for index, row in annotations.iterrows():
if row['PMCID'] not in feature_dictionary:
feature_dictionary[row['PMCID']] = []
feature_dictionary[row['PMCID']].append([row['Annotations'], row['Label']]
+ prompts_dictionary[row['PromptID']])
train = []
valid = []
test = []
with open('data/splits/train_article_ids.txt') as train_file:
for line in train_file:
train.append(int(line.strip()))
with open('data/splits/validation_article_ids.txt') as valid_file:
for line in valid_file:
valid.append(int(line.strip()))
with open('data/splits/test_article_ids.txt') as test_file:
for line in test_file:
test.append(int(line.strip()))
bert_token_indexer = {'bert': PretrainedBertIndexer('scibert/vocab.txt', max_pieces=512)}
reader = EIDatasetReader(bert_token_indexer, feature_dictionary)
train_data = reader.read(train)
valid_data = reader.read(valid)
test_data = reader.read(test)
vocab = Vocabulary.from_instances(train_data + valid_data + test_data)
bert_token_embedding = PretrainedBertEmbedder(
'scibert/weights.tar.gz', requires_grad=args.tunable
)
word_embeddings = BasicTextFieldEmbedder(
{"bert": bert_token_embedding},
{"bert": ['bert']},
allow_unmatched_keys=True
)
model = Baseline(word_embeddings, vocab)
cuda_device = args.cuda_device
if torch.cuda.is_available():
model = model.cuda(cuda_device)
else:
cuda_device = -1
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
iterator = BucketIterator(batch_size=args.batch_size,
sorting_keys=[('intervention', 'num_tokens')],
padding_noise=0.1)
iterator.index_with(vocab)
serialization_dir = 'model_checkpoints/' + args.model_name
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_data,
validation_dataset=test_data,
patience=args.patience,
validation_metric='+accuracy',
num_epochs=args.epochs,
cuda_device=cuda_device,
serialization_dir=serialization_dir)
result = trainer.train()
for key in result:
print(str(key) + ': ' + str(result[key]))
test_metrics = evaluate(trainer.model, test_data, iterator,
cuda_device=cuda_device,
batch_weight_key="")
print('Test Data statistics:')
for key, value in test_metrics.items():
print(str(key) + ': ' + str(value))
def main():
parser = argparse.ArgumentParser(description='Evidence oracle QA')
parser.add_argument('--epochs', type=int, default=5,
help='upper epoch limit (default: 5)')
parser.add_argument('--patience', type=int, default=1,
help='trainer patience (default: 1)')
parser.add_argument('--batch_size', type=int, default=32,
help='batch size (default: 32)')
parser.add_argument('--model_name', type=str, default='sentence_oracle_bert',
help='model name (default: sentence_oracle_bert)')
parser.add_argument('--tunable', action='store_true',
help='tune the underlying embedding model (default: False)')
parser.add_argument('--ev_type', type=str, default='sentence',
help='how to train the oracle - sentence or full (evidence) (default: sentence)')
args = parser.parse_args()
if args.ev_type == 'sentence':
train = pickle.load(open('data/oracle_train.p', 'rb'))
valid = pickle.load(open('data/oracle_val.p', 'rb'))
test = pickle.load(open('data/oracle_test.p', 'rb'))
elif args.ev_type == 'full':
train = pickle.load(open('data/oracle_full_train.p', 'rb'))
valid = pickle.load(open('data/oracle_full_val.p', 'rb'))
test = pickle.load(open('data/oracle_full_test.p', 'rb'))
else:
print('ev_type should be either sentence or full')
return
bert_token_indexer = {'bert': PretrainedBertIndexer('scibert/vocab.txt', max_pieces=512)}
pipeline_train = pickle.load(open('data/train_instances.p', 'rb'))
pipeline_val = pickle.load(open('data/val_instances.p', 'rb'))
pipeline_test = pickle.load(open('data/test_instances.p', 'rb'))
pipeline_reader = PipelineDatasetReader(bert_token_indexer)
p_train = pipeline_reader.read(pipeline_train)
p_val = pipeline_reader.read(pipeline_val)
p_test = pipeline_reader.read(pipeline_test)
p_vocab = Vocabulary.from_instances(p_train + p_val + p_test)
reader = EIDatasetReader(bert_token_indexer)
train_data = reader.read(train)
valid_data = reader.read(valid)
test_data = reader.read(test)
bert_token_embedding = PretrainedBertEmbedder(
'scibert/weights.tar.gz', requires_grad=args.tunable
)
word_embeddings = BasicTextFieldEmbedder(
{"bert": bert_token_embedding},
{"bert": ['bert']},
allow_unmatched_keys=True
)
model = Oracle(word_embeddings, p_vocab)
cuda_device = list(range(torch.cuda.device_count()))
if torch.cuda.is_available():
model = model.cuda()
else:
cuda_device = -1
t_total = len(train_data) // args.epochs
optimizer = BertAdam(model.parameters(), lr=1e-5, warmup=0.05, t_total=t_total)
iterator = BucketIterator(batch_size=args.batch_size,
sorting_keys=[('comb_prompt_ev', 'num_tokens')],
padding_noise=0.1)
iterator.index_with(p_vocab)
serialization_dir = 'model_checkpoints/' + args.model_name
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_data,
validation_dataset=valid_data,
patience=args.patience,
validation_metric='+accuracy',
num_epochs=args.epochs,
cuda_device=cuda_device,
serialization_dir=serialization_dir)
result = trainer.train()
for key in result:
print(str(key) + ': ' + str(result[key]))
if cuda_device != -1:
cuda_device = 0
test_metrics = evaluate(trainer.model, test_data, iterator,
cuda_device=cuda_device,
batch_weight_key="")
print('Test Data statistics:')
for key, value in test_metrics.items():
print(str(key) + ': ' + str(value))
iterator = BucketIterator(
batch_size=batch_size,
# This is for testing. To see how big of batch size the GPU can handle.
biggest_batch_first=True,
sorting_keys=[("tokens", "num_tokens")],
)
iterator.index_with(vocab)
linguistic_features_embedding = Embedding(
num_embeddings=max_vocab_size + 2,
embedding_dim=linguistic_features_embedding_dim,
# padding_index=0 I do not understand what is does
)
bert_embedder = PretrainedBertEmbedder(
pretrained_model=bert_mode,
top_layer_only=False,
requires_grad=bert_finetuning,
)
word_embedder = BasicTextFieldEmbedder(
{
"bert": bert_embedder,
"deps": linguistic_features_embedding,
"ner": linguistic_features_embedding,
"pos": linguistic_features_embedding,
"lang": linguistic_features_embedding,
}, {
"bert": {
"input_ids": "bert",
"offsets": "bert-offsets"
},
"deps": {
def get_model(pretrained_file: str, WORD_EMB_DIM: int, vocab: Vocabulary,
num_tags: int):
"""
This creates a new model and returns it along with some other variables.
:param pretrained_file:
:param WORD_EMB_DIM:
:param vocab:
:param num_tags:
:return:
"""
CNN_EMB_DIM = 128
CHAR_EMB_DIM = 16
weight = _read_pretrained_embeddings_file(pretrained_file, WORD_EMB_DIM,
vocab, "tokens")
token_embedding = Embedding(num_embeddings=weight.shape[0],
embedding_dim=weight.shape[1],
weight=weight,
vocab_namespace="tokens")
char_embedding = Embedding(
num_embeddings=vocab.get_vocab_size("token_characters"),
embedding_dim=CHAR_EMB_DIM,
vocab_namespace="token_characters")
char_encoder = CnnEncoder(
embedding_dim=CHAR_EMB_DIM,
num_filters=CNN_EMB_DIM,
ngram_filter_sizes=[3],
conv_layer_activation=Activation.by_name("relu")())
token_characters_embedding = TokenCharactersEncoder(
embedding=char_embedding, encoder=char_encoder)
if USING_BERT:
print("USING BERT EMBEDDINGS")
bert_emb = PretrainedBertEmbedder("bert-base-multilingual-cased")
tfe = BasicTextFieldEmbedder(
{
"bert": bert_emb,
"token_characters": token_characters_embedding
},
embedder_to_indexer_map={
"bert": ["bert", "bert-offsets"],
"token_characters": ["token_characters"]
},
allow_unmatched_keys=True)
EMBEDDING_DIM = CNN_EMB_DIM + 768
else:
EMBEDDING_DIM = CNN_EMB_DIM + WORD_EMB_DIM
tfe = BasicTextFieldEmbedder({
"tokens":
token_embedding,
"token_characters":
token_characters_embedding
})
HIDDEN_DIM = 256
encoder = PytorchSeq2SeqWrapper(
torch.nn.LSTM(EMBEDDING_DIM,
HIDDEN_DIM,
batch_first=True,
bidirectional=True,
dropout=0.5,
num_layers=2))
model = MarginalCrfTagger(vocab,
tfe,
encoder,
num_tags,
include_start_end_transitions=False,
calculate_span_f1=True,
dropout=0.5,
label_encoding="BIOUL",
constrain_crf_decoding=True)
optimizer = optim.Adam(model.parameters(), lr=0.001)
if torch.cuda.is_available():
print("Using GPU")
cuda_device = 0
model = model.cuda(cuda_device)
else:
cuda_device = -1
return model, optimizer, cuda_device
dev_dataset = reader.read('data/stanfordSentimentTreebank/trees/dev.txt')
# You can optionally specify the minimum count of tokens/labels.
# `min_count={'tokens':3}` here means that any tokens that appear less than three times
# will be ignored and not included in the vocabulary.
vocab = Vocabulary.from_instances(train_dataset + dev_dataset,
min_count={'tokens': 3})
# token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
# embedding_dim=EMBEDDING_DIM)
# BasicTextFieldEmbedder takes a dict - we need an embedding just for tokens,
# not for labels, which are used as-is as the "answer" of the sentence classification
# word_embeddings = BasicTextFieldEmbedder({"tokens": token_embedding})
bert_embedder = PretrainedBertEmbedder(pretrained_model='bert-base-uncased',
top_layer_only=True)
word_embeddings: TextFieldEmbedder = BasicTextFieldEmbedder(
{"tokens": bert_embedder}, allow_unmatched_keys=True)
# Seq2VecEncoder is a neural network abstraction that takes a sequence of something
# (usually a sequence of embedded word vectors), processes it, and returns a single
# vector. Oftentimes this is an RNN-based architecture (e.g., LSTM or GRU), but
# AllenNLP also supports CNNs and other simple architectures (for example,
# just averaging over the input vectors).
# encoder = PytorchSeq2VecWrapper(
# torch.nn.LSTM(EMBEDDING_DIM, HIDDEN_DIM, batch_first=True))
# HW
encoder = TransformerSeq2VecEncoder(EMBEDDING_DIM,
HIDDEN_DIM,
def __init__(
self,
vocab: Vocabulary,
attention: Attention,
beam_size: int,
max_decoding_steps: int,
target_embedding_dim: int = 30,
copy_token: str = "@COPY@",
source_namespace: str = "bert",
target_namespace: str = "target_tokens",
tensor_based_metric: Metric = None,
token_based_metric: Metric = None,
initializer: InitializerApplicator = InitializerApplicator(),
) -> None:
super().__init__(vocab)
self._source_namespace = source_namespace
self._target_namespace = target_namespace
self._src_start_index = self.vocab.get_token_index(
START_SYMBOL, self._source_namespace)
self._src_end_index = self.vocab.get_token_index(
END_SYMBOL, self._source_namespace)
self._start_index = self.vocab.get_token_index(START_SYMBOL,
self._target_namespace)
self._end_index = self.vocab.get_token_index(END_SYMBOL,
self._target_namespace)
self._oov_index = self.vocab.get_token_index(self.vocab._oov_token,
self._target_namespace)
self._pad_index = self.vocab.get_token_index(self.vocab._padding_token,
self._target_namespace)
self._copy_index = self.vocab.add_token_to_namespace(
copy_token, self._target_namespace)
self._tensor_based_metric = tensor_based_metric or BLEU(
exclude_indices={
self._pad_index, self._end_index, self._start_index
})
self._token_based_metric = token_based_metric
self._target_vocab_size = self.vocab.get_vocab_size(
self._target_namespace)
# Encoding modules.
bert_token_embedding = PretrainedBertEmbedder('bert-base-uncased',
requires_grad=True)
self._source_embedder = bert_token_embedding
self._encoder = PassThroughEncoder(
input_dim=self._source_embedder.get_output_dim())
# Decoder output dim needs to be the same as the encoder output dim since we initialize the
# hidden state of the decoder with the final hidden state of the encoder.
# We arbitrarily set the decoder's input dimension to be the same as the output dimension.
self.encoder_output_dim = self._encoder.get_output_dim()
self.decoder_output_dim = self.encoder_output_dim
self.decoder_input_dim = self.decoder_output_dim
target_vocab_size = self.vocab.get_vocab_size(self._target_namespace)
# The decoder input will be a function of the embedding of the previous predicted token,
# an attended encoder hidden state called the "attentive read", and another
# weighted sum of the encoder hidden state called the "selective read".
# While the weights for the attentive read are calculated by an `Attention` module,
# the weights for the selective read are simply the predicted probabilities
# corresponding to each token in the source sentence that matches the target
# token from the previous timestep.
self._target_embedder = Embedding(target_vocab_size,
target_embedding_dim)
self._attention = attention
self._input_projection_layer = Linear(
target_embedding_dim + self.encoder_output_dim * 2,
self.decoder_input_dim)
# We then run the projected decoder input through an LSTM cell to produce
# the next hidden state.
self._decoder_cell = LSTMCell(self.decoder_input_dim,
self.decoder_output_dim)
# We create a "generation" score for each token in the target vocab
# with a linear projection of the decoder hidden state.
self._output_generation_layer = Linear(self.decoder_output_dim,
target_vocab_size)
# We create a "copying" score for each source token by applying a non-linearity
# (tanh) to a linear projection of the encoded hidden state for that token,
# and then taking the dot product of the result with the decoder hidden state.
self._output_copying_layer = Linear(self.encoder_output_dim,
self.decoder_output_dim)
# At prediction time, we'll use a beam search to find the best target sequence.
self._beam_search = BeamSearch(self._end_index,
max_steps=max_decoding_steps,
beam_size=beam_size)
initializer(self)
def main():
parser = argparse.ArgumentParser(
description='Evidence sentence classifier')
parser.add_argument('--epochs',
type=int,
default=5,
help='upper epoch limit (default: 5)')
parser.add_argument('--patience',
type=int,
default=1,
help='trainer patience (default: 1)')
parser.add_argument('--batch_size',
type=int,
default=8,
help='batch size (default: 8)')
parser.add_argument(
'--loss',
type=str,
default='hinge',
help=
'loss function to train the model - choose bce or hinge (default: hinge)'
)
parser.add_argument(
'--hinge_margin',
type=float,
default=0.5,
help='the margin for the hinge loss, if used (default: 0.5)')
parser.add_argument('--model_name',
type=str,
default='ev_classifier_bert',
help='model name (default: ev_classifier_bert)')
parser.add_argument(
'--tunable',
action='store_true',
help='tune the underlying embedding model (default: False)')
args = parser.parse_args()
if args.loss not in ['bce', 'hinge']:
print('Loss must be bce or hinge')
return
bert_token_indexer = {
'bert': PretrainedBertIndexer('scibert/vocab.txt', max_pieces=512)
}
pipeline_train = pickle.load(open('data/train_instances.p', 'rb'))
pipeline_val = pickle.load(open('data/val_instances.p', 'rb'))
pipeline_test = pickle.load(open('data/test_instances.p', 'rb'))
pipeline_reader = PipelineDatasetReader(bert_token_indexer)
p_train = pipeline_reader.read(pipeline_train)
p_val = pipeline_reader.read(pipeline_val)
p_test = pipeline_reader.read(pipeline_test)
p_vocab = Vocabulary.from_instances(p_train + p_val + p_test)
classifier_train = pickle.load(open('data/classifier_train.p', 'rb'))
classifier_val = pickle.load(open('data/classifier_val.p', 'rb'))
reader = EvidenceDatasetReader(bert_token_indexer)
train_data = reader.read(classifier_train)
valid_data = reader.read(classifier_val)
bert_token_embedding = PretrainedBertEmbedder('scibert/weights.tar.gz',
requires_grad=args.tunable)
word_embeddings = BasicTextFieldEmbedder({"bert": bert_token_embedding},
{"bert": ['bert']},
allow_unmatched_keys=True)
model = Classifier(word_embeddings=word_embeddings,
vocab=p_vocab,
loss=args.loss,
hinge_margin=args.hinge_margin)
cuda_device = list(range(torch.cuda.device_count()))
if torch.cuda.is_available():
model = model.cuda()
else:
cuda_device = -1
t_total = len(train_data) // args.epochs
optimizer = BertAdam(model.parameters(),
lr=2e-5,
warmup=0.1,
t_total=t_total)
iterator = BucketIterator(batch_size=args.batch_size,
sorting_keys=[('comb_evidence', 'num_tokens')],
padding_noise=0.1,
biggest_batch_first=True)
iterator.index_with(p_vocab)
serialization_dir = 'model_checkpoints/' + args.model_name
trainer = Trainer(
model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_data,
validation_dataset=valid_data,
patience=args.patience,
validation_metric='+accuracy',
num_epochs=args.epochs,
cuda_device=cuda_device,
# learning_rate_scheduler=scheduler,
serialization_dir=serialization_dir)
result = trainer.train()
for key in result:
print(str(key) + ': ' + str(result[key]))