def __init__(self,
weight,
vocab_size,
embedding_dim,
rnn_size_in=(1024 + 300, 1024 + 300),
rnn_size_out=(300, 300),
max_l=150,
max_span_l=50,
mlp_d=300,
num_of_class=3,
drop_r=0.5,
activation_type='relu',
use_extra_lex_feature=True):
super(Model, self).__init__()
self.glove_embd_layer = Embedding(vocab_size,
embedding_dim,
weight=weight,
padding_index=0)
options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
num_of_elmo = 1
self.max_l = max_l
self.elmo_embd_layer = Elmo(options_file,
weight_file,
num_of_elmo,
dropout=0)
self.esim_layer = ESIM_SENT_WISE(rnn_size_in, rnn_size_out, max_l,
max_span_l, mlp_d, num_of_class,
drop_r, activation_type)
self.use_extra_lex_feature = use_extra_lex_feature
def __init__(self, device, elmo_options, elmo_weights, elmo_size=None):
super().__init__()
self.device = device
self.elmo_options = elmo_options
self.elmo_weights = elmo_weights
self.elmo_size = elmo_size
self.elmo = Elmo(self.elmo_options, self.elmo_weights, 2, dropout=0)
def __init__(self,
weight,
vocab_size,
embedding_dim,
rnn_size_in=(1024 + 300, ),
rnn_size_out=(1024, ),
max_l=600,
mlp_d=1024,
num_of_class=3,
drop_r=0.5,
activation_type='relu'):
super(Model, self).__init__()
self.glove_embd_layer = Embedding(vocab_size,
embedding_dim,
weight=weight,
padding_index=0)
options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
num_of_elmo = 1
self.max_l = max_l
self.elmo_embd_layer = Elmo(options_file,
weight_file,
num_of_elmo,
dropout=0)
self.esim_layer = FastMaxout(rnn_size_in, rnn_size_out, max_l, mlp_d,
num_of_class, drop_r, activation_type)
def write(filename, filename2, rows):
options_file = 'biomed_elmo_options.json'
weight_file = 'biomed_elmo_weights.hdf5'
elmo = Elmo(options_file, weight_file, 1, dropout=0)
elmo = elmo.to("cuda")
df = pd.read_csv(filename, nrows=rows)
x = df["TEXT"].values
y = df[[
'No Finding', 'Enlarged Cardiomediastinum', 'Cardiomegaly',
'Airspace Opacity', 'Lung Lesion', 'Edema', 'Consolidation',
'Pneumonia', 'Atelectasis', 'Pneumothorax', 'Pleural Effusion',
'Pleural Other', 'Fracture', 'Support Devices'
]].values
for i in range(rows):
print(str(i) + " of " + str(rows))
text = x[i]
token_list = []
for word in tokenizer(text):
token_list.append(word)
for n in range(MAXLEN - len(token_list)):
token_list.append('PAD')
token_list = np.array([token_list])
character_ids = batch_to_ids(token_list)
character_ids = character_ids.to("cuda")
word_emb = elmo(character_ids)['elmo_representations'][0]
character_ids.to("cpu")
word_emb = word_emb.data.cpu().numpy()
label = y[i]
batch = np.array([word_emb, label])
final_filename = "./Data/" + filename2 + "_instance_" + str(i)
np.save(final_filename, batch)
def get_elmo(options_file: str,
weight_file: str,
num_output_representations: int,
requires_grad: bool = False,
do_layer_norm: bool = False,
dropout: float = 0.5,
vocab_to_cache: List[str] = None,
keep_sentence_boundaries: bool = False,
):
from allennlp.modules import Elmo
key = (options_file, weight_file)
old_elmo = global_elmo_cache.get(key)
if old_elmo:
# noinspection PyProtectedMember
module = old_elmo._elmo_lstm
options_file = None
weight_file = None
else:
module = None
ret = Elmo(options_file=options_file,
weight_file=weight_file,
num_output_representations=num_output_representations,
requires_grad=requires_grad,
do_layer_norm=do_layer_norm,
dropout=dropout,
vocab_to_cache=vocab_to_cache,
keep_sentence_boundaries=keep_sentence_boundaries,
module=module)
if not old_elmo:
global_elmo_cache[key] = ret
return ret
def __init__(self, config):
super(SeqPrototypicalNetwork, self).__init__()
self.base_path = config['base_path']
self.early_stopping = config['early_stopping']
self.lr = config.get('meta_lr', 1e-3)
self.weight_decay = config.get('meta_weight_decay', 0.0)
if 'seq' in config['learner_model']:
self.learner = RNNSequenceModel(config['learner_params'])
elif 'mlp' in config['learner_model']:
self.learner = MLPModel(config['learner_params'])
elif 'bert' in config['learner_model']:
self.learner = BERTSequenceModel(config['learner_params'])
self.num_outputs = config['learner_params']['num_outputs']
self.vectors = config.get('vectors', 'glove')
if self.vectors == 'elmo':
self.elmo = Elmo(
options_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
elif self.vectors == 'bert':
self.bert_tokenizer = BertTokenizer.from_pretrained(
'bert-base-cased')
self.loss_fn = {}
for task in config['learner_params']['num_outputs']:
self.loss_fn[task] = nn.CrossEntropyLoss(ignore_index=-1)
if config.get('trained_learner', False):
self.learner.load_state_dict(
torch.load(
os.path.join(self.base_path, 'saved_models',
config['trained_learner'])))
logger.info('Loaded trained learner model {}'.format(
config['trained_learner']))
self.device = torch.device(config.get('device', 'cpu'))
self.to(self.device)
if self.vectors == 'elmo':
self.elmo.to(self.device)
self.initialize_optimizer_scheduler()
def __init__(self, config):
super(SeqMetaModel, self).__init__()
self.base_path = config['base_path']
self.learner_lr = config.get('learner_lr', 1e-3)
self.output_lr = config.get('output_lr', 0.1)
if 'seq' in config['learner_model']:
self.learner = RNNSequenceModel(config['learner_params'])
elif 'mlp' in config['learner_model']:
self.learner = MLPModel(config['learner_params'])
elif 'bert' in config['learner_model']:
self.learner = BERTSequenceModel(config['learner_params'])
self.proto_maml = config.get('proto_maml', False)
self.fomaml = config.get('fomaml', False)
self.vectors = config.get('vectors', 'glove')
if self.vectors == 'elmo':
self.elmo = Elmo(options_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
elif self.vectors == 'bert':
self.bert_tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
self.learner_loss = {}
for task in config['learner_params']['num_outputs']:
self.learner_loss[task] = nn.CrossEntropyLoss(ignore_index=-1)
self.output_layer_weight = None
self.output_layer_bias = None
if config.get('trained_learner', False):
self.learner.load_state_dict(torch.load(
os.path.join(self.base_path, 'saved_models', config['trained_learner'])
))
logger.info('Loaded trained learner model {}'.format(config['trained_learner']))
self.device = torch.device(config.get('device', 'cpu'))
self.to(self.device)
if self.proto_maml:
logger.info('Initialization of output layer weights as per prototypical networks turned on')
params = [p for p in self.learner.parameters() if p.requires_grad]
self.learner_optimizer = optim.SGD(params, lr=self.learner_lr)
def __init__(self,
options_file: str,
weight_file: str,
do_layer_norm: bool = False,
dropout: float = 0.5,
requires_grad: bool = False,
projection_dim: int = None,
num_output_representations: int = 1) -> None:
super(ElmoTokenEmbedderWrapper, self).__init__()
# other arguments can be passed in when needed
self._elmo = Elmo(options_file=options_file,
weight_file=weight_file,
num_output_representations=num_output_representations,
dropout=dropout)
def __init__(self, config):
super(SeqBaselineModel, self).__init__()
self.base_path = config['base_path']
self.early_stopping = config['early_stopping']
self.learner_lr = config.get('learner_lr', 1e-3)
self.weight_decay = config.get('meta_weight_decay', 0.0)
if 'seq' in config['learner_model']:
self.learner = RNNSequenceModel(config['learner_params'])
elif 'mlp' in config['learner_model']:
self.learner = MLPModel(config['learner_params'])
self.vectors = config.get('vectors', 'glove')
if self.vectors == 'elmo':
self.elmo = Elmo(
options_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
self.learner_loss = {}
for task in config['learner_params']['num_outputs']:
self.learner_loss[task] = nn.CrossEntropyLoss(ignore_index=-1)
self.output_layer = None
if config.get('trained_baseline', None):
self.learner.load_state_dict(
torch.load(
os.path.join(self.base, 'saved_models',
config['trained_baseline'])))
logger.info('Loaded trained baseline model {}'.format(
config['trained_baseline']))
self.device = torch.device(config.get('device', 'cpu'))
self.to(self.device)
def __init__(self, model='original_5b', bos_eos=(True, True), n_out=0, dropout=0.5, requires_grad=False):
super().__init__()
from allennlp.modules import Elmo
self.elmo = Elmo(options_file=self.OPTION[model],
weight_file=self.WEIGHT[model],
num_output_representations=1,
dropout=dropout,
requires_grad=requires_grad,
keep_sentence_boundaries=True)
self.model = model
self.bos_eos = bos_eos
self.hidden_size = self.elmo.get_output_dim()
self.n_out = n_out or self.hidden_size
self.dropout = dropout
self.requires_grad = requires_grad
self.projection = nn.Linear(self.hidden_size, self.n_out, False) if self.hidden_size != n_out else nn.Identity()
def __init__(self, config):
self.vectors = config.get('vectors', 'elmo')
self.device = torch.device(config.get('device', 'cpu'))
if self.vectors == 'elmo':
self.elmo = Elmo(
options_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
weight_file=
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
num_output_representations=1,
dropout=0,
requires_grad=False)
self.elmo.to(self.device)
elif self.vectors == 'glove':
self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
elif self.vectors == 'bert':
self.bert_tokenizer = BertTokenizer.from_pretrained(
'bert-base-cased')
self.bert = BertModel.from_pretrained('bert-base-cased')
self.bert.to(self.device)
logger.info('Nearest neighbor classifier instantiated')
def __init__(self, config, pre_trained_embeds=None):
super(Document_Classifier, self).__init__()
self.lstm_dim = config['lstm_dim']
self.model_name = config['model_name']
self.embed_name = config['embed_dim']
self.fc_dim = config['fc_dim']
self.num_classes = config['n_classes']
self.embed_dim = config['embed_dim'] if config[
'embed_dim'] == 300 else 2 * config['embed_dim']
self.batch_size = config['batch_size']
self.num_kernels = config["kernel_num"]
self.kernel_sizes = [int(k) for k in config["kernel_sizes"].split(',')]
self.mode = 'single' if not config['parallel_computing'] else 'multi'
# Choose the right embedding method based on embed_dim given
if config['embed_dim'] == 300:
self.vocab_size = config['vocab_size']
self.embedding = nn.Embedding(self.vocab_size, self.embed_dim)
self.embedding.weight.data.copy_(pre_trained_embeds)
self.embedding.requires_grad = False
elif config['embed_dim'] == 128:
# Small
self.options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json"
self.weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5"
self.elmo = Elmo(options_file=self.options_file,
weight_file=self.weight_file,
num_output_representations=1,
requires_grad=False)
elif config['embed_dim'] == 256:
# Medium
self.options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_options.json"
self.weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5"
self.elmo = Elmo(options_file=self.options_file,
weight_file=self.weight_file,
num_output_representations=1,
requires_grad=False)
elif config['embed_dim'] == 512:
# Highest
self.options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json"
self.weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5"
self.elmo = Elmo(options_file=self.options_file,
weight_file=self.weight_file,
num_output_representations=1,
requires_grad=False)
if self.model_name == 'bilstm':
self.encoder = BiLSTM(config)
self.fc_inp_dim = 2 * config['lstm_dim']
elif self.model_name == 'bilstm_pool':
self.encoder = BiLSTM(config, max_pool=True)
self.fc_inp_dim = 2 * config['lstm_dim']
elif self.model_name == 'bilstm_reg':
self.encoder = BiLSTM_reg(config)
self.fc_inp_dim = config['lstm_dim']
elif self.model_name == 'han':
self.encoder = HAN(config)
self.fc_inp_dim = 2 * config['sent_lstm_dim']
elif self.model_name == 'cnn':
self.encoder = Kim_CNN(config)
self.fc_inp_dim = self.num_kernels * len(self.kernel_sizes)
if config['embed_name'] in ['dbert', 'roberta']:
MODEL_CLASSES = {
"dbert": (DistilBertConfig, DistilBertPreTrainedModel,
DistilBertTokenizerFast),
"roberta": (RobertaConfig, RobertaModel, RobertaTokenizerFast),
}
config_class, _, _ = MODEL_CLASSES[config['embed_name']]
self.encoder = Transformer_model(config, config_class)
self.fc_inp_dim = 768 if 'base' in config['model_name'] else 1024
self.classifier = nn.Sequential(
nn.Dropout(config["dropout"]),
nn.Linear(self.fc_inp_dim, self.fc_dim), nn.ReLU(),
nn.Linear(self.fc_dim, self.num_classes))