def __init__(self, weight, vocab_size, embedding_dim, rnn_size_in=(1024 + 300, 1024 + 300), rnn_size_out=(300, 300), max_l=150, max_span_l=50, mlp_d=300, num_of_class=3, drop_r=0.5, activation_type='relu', use_extra_lex_feature=True): super(Model, self).__init__() self.glove_embd_layer = Embedding(vocab_size, embedding_dim, weight=weight, padding_index=0) options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" num_of_elmo = 1 self.max_l = max_l self.elmo_embd_layer = Elmo(options_file, weight_file, num_of_elmo, dropout=0) self.esim_layer = ESIM_SENT_WISE(rnn_size_in, rnn_size_out, max_l, max_span_l, mlp_d, num_of_class, drop_r, activation_type) self.use_extra_lex_feature = use_extra_lex_feature
def __init__(self, device, elmo_options, elmo_weights, elmo_size=None): super().__init__() self.device = device self.elmo_options = elmo_options self.elmo_weights = elmo_weights self.elmo_size = elmo_size self.elmo = Elmo(self.elmo_options, self.elmo_weights, 2, dropout=0)
def __init__(self, weight, vocab_size, embedding_dim, rnn_size_in=(1024 + 300, ), rnn_size_out=(1024, ), max_l=600, mlp_d=1024, num_of_class=3, drop_r=0.5, activation_type='relu'): super(Model, self).__init__() self.glove_embd_layer = Embedding(vocab_size, embedding_dim, weight=weight, padding_index=0) options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" num_of_elmo = 1 self.max_l = max_l self.elmo_embd_layer = Elmo(options_file, weight_file, num_of_elmo, dropout=0) self.esim_layer = FastMaxout(rnn_size_in, rnn_size_out, max_l, mlp_d, num_of_class, drop_r, activation_type)
def write(filename, filename2, rows): options_file = 'biomed_elmo_options.json' weight_file = 'biomed_elmo_weights.hdf5' elmo = Elmo(options_file, weight_file, 1, dropout=0) elmo = elmo.to("cuda") df = pd.read_csv(filename, nrows=rows) x = df["TEXT"].values y = df[[ 'No Finding', 'Enlarged Cardiomediastinum', 'Cardiomegaly', 'Airspace Opacity', 'Lung Lesion', 'Edema', 'Consolidation', 'Pneumonia', 'Atelectasis', 'Pneumothorax', 'Pleural Effusion', 'Pleural Other', 'Fracture', 'Support Devices' ]].values for i in range(rows): print(str(i) + " of " + str(rows)) text = x[i] token_list = [] for word in tokenizer(text): token_list.append(word) for n in range(MAXLEN - len(token_list)): token_list.append('PAD') token_list = np.array([token_list]) character_ids = batch_to_ids(token_list) character_ids = character_ids.to("cuda") word_emb = elmo(character_ids)['elmo_representations'][0] character_ids.to("cpu") word_emb = word_emb.data.cpu().numpy() label = y[i] batch = np.array([word_emb, label]) final_filename = "./Data/" + filename2 + "_instance_" + str(i) np.save(final_filename, batch)
def get_elmo(options_file: str, weight_file: str, num_output_representations: int, requires_grad: bool = False, do_layer_norm: bool = False, dropout: float = 0.5, vocab_to_cache: List[str] = None, keep_sentence_boundaries: bool = False, ): from allennlp.modules import Elmo key = (options_file, weight_file) old_elmo = global_elmo_cache.get(key) if old_elmo: # noinspection PyProtectedMember module = old_elmo._elmo_lstm options_file = None weight_file = None else: module = None ret = Elmo(options_file=options_file, weight_file=weight_file, num_output_representations=num_output_representations, requires_grad=requires_grad, do_layer_norm=do_layer_norm, dropout=dropout, vocab_to_cache=vocab_to_cache, keep_sentence_boundaries=keep_sentence_boundaries, module=module) if not old_elmo: global_elmo_cache[key] = ret return ret
def __init__(self, config): super(SeqPrototypicalNetwork, self).__init__() self.base_path = config['base_path'] self.early_stopping = config['early_stopping'] self.lr = config.get('meta_lr', 1e-3) self.weight_decay = config.get('meta_weight_decay', 0.0) if 'seq' in config['learner_model']: self.learner = RNNSequenceModel(config['learner_params']) elif 'mlp' in config['learner_model']: self.learner = MLPModel(config['learner_params']) elif 'bert' in config['learner_model']: self.learner = BERTSequenceModel(config['learner_params']) self.num_outputs = config['learner_params']['num_outputs'] self.vectors = config.get('vectors', 'glove') if self.vectors == 'elmo': self.elmo = Elmo( options_file= "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json", weight_file= "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5", num_output_representations=1, dropout=0, requires_grad=False) elif self.vectors == 'glove': self.glove = torchtext.vocab.GloVe(name='840B', dim=300) elif self.vectors == 'bert': self.bert_tokenizer = BertTokenizer.from_pretrained( 'bert-base-cased') self.loss_fn = {} for task in config['learner_params']['num_outputs']: self.loss_fn[task] = nn.CrossEntropyLoss(ignore_index=-1) if config.get('trained_learner', False): self.learner.load_state_dict( torch.load( os.path.join(self.base_path, 'saved_models', config['trained_learner']))) logger.info('Loaded trained learner model {}'.format( config['trained_learner'])) self.device = torch.device(config.get('device', 'cpu')) self.to(self.device) if self.vectors == 'elmo': self.elmo.to(self.device) self.initialize_optimizer_scheduler()
def __init__(self, config): super(SeqMetaModel, self).__init__() self.base_path = config['base_path'] self.learner_lr = config.get('learner_lr', 1e-3) self.output_lr = config.get('output_lr', 0.1) if 'seq' in config['learner_model']: self.learner = RNNSequenceModel(config['learner_params']) elif 'mlp' in config['learner_model']: self.learner = MLPModel(config['learner_params']) elif 'bert' in config['learner_model']: self.learner = BERTSequenceModel(config['learner_params']) self.proto_maml = config.get('proto_maml', False) self.fomaml = config.get('fomaml', False) self.vectors = config.get('vectors', 'glove') if self.vectors == 'elmo': self.elmo = Elmo(options_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json", weight_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5", num_output_representations=1, dropout=0, requires_grad=False) elif self.vectors == 'glove': self.glove = torchtext.vocab.GloVe(name='840B', dim=300) elif self.vectors == 'bert': self.bert_tokenizer = BertTokenizer.from_pretrained('bert-base-cased') self.learner_loss = {} for task in config['learner_params']['num_outputs']: self.learner_loss[task] = nn.CrossEntropyLoss(ignore_index=-1) self.output_layer_weight = None self.output_layer_bias = None if config.get('trained_learner', False): self.learner.load_state_dict(torch.load( os.path.join(self.base_path, 'saved_models', config['trained_learner']) )) logger.info('Loaded trained learner model {}'.format(config['trained_learner'])) self.device = torch.device(config.get('device', 'cpu')) self.to(self.device) if self.proto_maml: logger.info('Initialization of output layer weights as per prototypical networks turned on') params = [p for p in self.learner.parameters() if p.requires_grad] self.learner_optimizer = optim.SGD(params, lr=self.learner_lr)
def __init__(self, options_file: str, weight_file: str, do_layer_norm: bool = False, dropout: float = 0.5, requires_grad: bool = False, projection_dim: int = None, num_output_representations: int = 1) -> None: super(ElmoTokenEmbedderWrapper, self).__init__() # other arguments can be passed in when needed self._elmo = Elmo(options_file=options_file, weight_file=weight_file, num_output_representations=num_output_representations, dropout=dropout)
def __init__(self, config): super(SeqBaselineModel, self).__init__() self.base_path = config['base_path'] self.early_stopping = config['early_stopping'] self.learner_lr = config.get('learner_lr', 1e-3) self.weight_decay = config.get('meta_weight_decay', 0.0) if 'seq' in config['learner_model']: self.learner = RNNSequenceModel(config['learner_params']) elif 'mlp' in config['learner_model']: self.learner = MLPModel(config['learner_params']) self.vectors = config.get('vectors', 'glove') if self.vectors == 'elmo': self.elmo = Elmo( options_file= "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json", weight_file= "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5", num_output_representations=1, dropout=0, requires_grad=False) elif self.vectors == 'glove': self.glove = torchtext.vocab.GloVe(name='840B', dim=300) self.learner_loss = {} for task in config['learner_params']['num_outputs']: self.learner_loss[task] = nn.CrossEntropyLoss(ignore_index=-1) self.output_layer = None if config.get('trained_baseline', None): self.learner.load_state_dict( torch.load( os.path.join(self.base, 'saved_models', config['trained_baseline']))) logger.info('Loaded trained baseline model {}'.format( config['trained_baseline'])) self.device = torch.device(config.get('device', 'cpu')) self.to(self.device)
def __init__(self, model='original_5b', bos_eos=(True, True), n_out=0, dropout=0.5, requires_grad=False): super().__init__() from allennlp.modules import Elmo self.elmo = Elmo(options_file=self.OPTION[model], weight_file=self.WEIGHT[model], num_output_representations=1, dropout=dropout, requires_grad=requires_grad, keep_sentence_boundaries=True) self.model = model self.bos_eos = bos_eos self.hidden_size = self.elmo.get_output_dim() self.n_out = n_out or self.hidden_size self.dropout = dropout self.requires_grad = requires_grad self.projection = nn.Linear(self.hidden_size, self.n_out, False) if self.hidden_size != n_out else nn.Identity()
def __init__(self, config): self.vectors = config.get('vectors', 'elmo') self.device = torch.device(config.get('device', 'cpu')) if self.vectors == 'elmo': self.elmo = Elmo( options_file= "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json", weight_file= "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5", num_output_representations=1, dropout=0, requires_grad=False) self.elmo.to(self.device) elif self.vectors == 'glove': self.glove = torchtext.vocab.GloVe(name='840B', dim=300) elif self.vectors == 'bert': self.bert_tokenizer = BertTokenizer.from_pretrained( 'bert-base-cased') self.bert = BertModel.from_pretrained('bert-base-cased') self.bert.to(self.device) logger.info('Nearest neighbor classifier instantiated')
def __init__(self, config, pre_trained_embeds=None): super(Document_Classifier, self).__init__() self.lstm_dim = config['lstm_dim'] self.model_name = config['model_name'] self.embed_name = config['embed_dim'] self.fc_dim = config['fc_dim'] self.num_classes = config['n_classes'] self.embed_dim = config['embed_dim'] if config[ 'embed_dim'] == 300 else 2 * config['embed_dim'] self.batch_size = config['batch_size'] self.num_kernels = config["kernel_num"] self.kernel_sizes = [int(k) for k in config["kernel_sizes"].split(',')] self.mode = 'single' if not config['parallel_computing'] else 'multi' # Choose the right embedding method based on embed_dim given if config['embed_dim'] == 300: self.vocab_size = config['vocab_size'] self.embedding = nn.Embedding(self.vocab_size, self.embed_dim) self.embedding.weight.data.copy_(pre_trained_embeds) self.embedding.requires_grad = False elif config['embed_dim'] == 128: # Small self.options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json" self.weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5" self.elmo = Elmo(options_file=self.options_file, weight_file=self.weight_file, num_output_representations=1, requires_grad=False) elif config['embed_dim'] == 256: # Medium self.options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_options.json" self.weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5" self.elmo = Elmo(options_file=self.options_file, weight_file=self.weight_file, num_output_representations=1, requires_grad=False) elif config['embed_dim'] == 512: # Highest self.options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json" self.weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5" self.elmo = Elmo(options_file=self.options_file, weight_file=self.weight_file, num_output_representations=1, requires_grad=False) if self.model_name == 'bilstm': self.encoder = BiLSTM(config) self.fc_inp_dim = 2 * config['lstm_dim'] elif self.model_name == 'bilstm_pool': self.encoder = BiLSTM(config, max_pool=True) self.fc_inp_dim = 2 * config['lstm_dim'] elif self.model_name == 'bilstm_reg': self.encoder = BiLSTM_reg(config) self.fc_inp_dim = config['lstm_dim'] elif self.model_name == 'han': self.encoder = HAN(config) self.fc_inp_dim = 2 * config['sent_lstm_dim'] elif self.model_name == 'cnn': self.encoder = Kim_CNN(config) self.fc_inp_dim = self.num_kernels * len(self.kernel_sizes) if config['embed_name'] in ['dbert', 'roberta']: MODEL_CLASSES = { "dbert": (DistilBertConfig, DistilBertPreTrainedModel, DistilBertTokenizerFast), "roberta": (RobertaConfig, RobertaModel, RobertaTokenizerFast), } config_class, _, _ = MODEL_CLASSES[config['embed_name']] self.encoder = Transformer_model(config, config_class) self.fc_inp_dim = 768 if 'base' in config['model_name'] else 1024 self.classifier = nn.Sequential( nn.Dropout(config["dropout"]), nn.Linear(self.fc_inp_dim, self.fc_dim), nn.ReLU(), nn.Linear(self.fc_dim, self.num_classes))