def __init__(self, model_name, num_labels: int, clf_dropout=0.2): super().__init__() self.transformer = GPT2Model.from_pretrained(model_name) self.dropout = nn.Dropout(clf_dropout) self.linear = nn.Linear(self.transformer.config.n_embd * 2, num_labels) nn.init.normal_(self.linear.weight, std=0.02) nn.init.normal_(self.linear.bias, 0)
def extract_gpt2_hidden_activations( text_path, save_activs_to): #, mode="full_model", focus_layers=[]): # read in text samples to pass through single layer of gpt2 model text_inputs = [] with open(text_path, "rb") as infile: text_inputs = pickle.load(infile) # num_inputs = len(text_inputs) # Load pre-trained model tokenizer (vocabulary) tokenizer = GPT2Tokenizer.from_pretrained('gpt2') # get the hidden activations - assumes a gpu is available layer_activs = [] for text in text_inputs: # tokenize text indexed_tokens = tokenizer.encode(text) tokens_tensor = torch.tensor([indexed_tokens]).to('cuda') # set up model model = GPT2Model.from_pretrained('gpt2') model.eval() model.to('cuda') # grab the hidden activations and save them to layer_actives with torch.no_grad(): hidden, _ = model(tokens_tensor) layer_activs.append(hidden.cpu().numpy().squeeze()) # clear gpu memory in preparation for next text sample torch.cuda.empty_cache() # save layer dimensions with open(save_activs_to, "wb") as outfile: pickle.dump(layer_activs, outfile) pass
def download_model(name): if not name in MODELS: raise Exception(str(name) + ' not a model in the list') if not exists(PATH): print("# ", str(PATH), "not found, creating dir.") mkdir(PATH) print('# Downloading model: ' + str(name)) name_path = MODEL_PATH_DICT[name] if name == 'word2vec': if not exists(join(PATH, name_path)): wget.download( 'https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz' ) shutil.move(name_path, join(PATH, name_path)) print('# Downloaded word2vec') else: print('# Already downloaded') if name == 'glove': if not exists(join(PATH, name_path)): wget.download( 'http://nlp.stanford.edu/data/wordvecs/glove.840B.300d.zip') zip = zipfile.ZipFile('./glove.840B.300d.zip') zip.extractall() _ = glove2word2vec('./glove.840B.300d.txt', join(PATH, name_path)) print('# Downloaded glove') else: print('# Already downloaded') if name == 'dict2vec': if not exists(join(PATH, name_path)): wget.download( 'https://dict2vec.s3.amazonaws.com/dict2vec300.tar.bz2') tar = tarfile.open("dict2vec300.tar.bz2") tar.extractall() tar.close() shutil.move(name_path, join(PATH, name_path)) print('# Downloaded dict2vec') else: print('# Already downloaded') if name == 'conceptnet': if not exists(join(PATH, name_path)): wget.download( 'https://conceptnet.s3.amazonaws.com/downloads/2019/numberbatch/numberbatch-en-19.08.txt.gz' ) shutil.move(name_path, join(PATH, name_path)) print('# Downloaded Conceptnet Numberbatch') else: print('# Already downloaded') if name == 'bert' or name == 'bert-context': _ = BertTokenizer.from_pretrained('bert-large-uncased') _ = BertModel.from_pretrained( 'bert-large-uncased').embeddings.word_embeddings.weight.data.numpy( ) print('# Downloaded bert') if name == 'gpt2' or name == 'gpt2-context': _ = GPT2Tokenizer.from_pretrained('gpt2') _ = GPT2LMHeadModel.from_pretrained('gpt2') _ = GPT2Model.from_pretrained('gpt2') print('# Downloaded gpt-2')
def construct_encoder(self): model = GPT2Model.from_pretrained(self.model_name) model.cuda() model = torch.nn.DataParallel(model) model.eval() tokenizer = GPT2Tokenizer.from_pretrained(self.model_name) print("Model and tokenzier are constructed!") return model, tokenizer
def __init__(self, pretrain_path, dropout=0.1): super(Gpt2Model, self).__init__() self.bert = GPT2Model.from_pretrained(pretrain_path) self.dropout = nn.Dropout(dropout) self.aux_head = nn.Sequential( OrderedDict([ ('dropout', nn.Dropout(dropout)), ('clf', nn.Linear(self.bert.config.n_embd, 6)), ])) self.main_head = nn.Sequential( OrderedDict([('dropout', nn.Dropout(dropout)), ('clf', nn.Linear(self.bert.config.n_embd, 1))]))
def from_opt(cls, opt): if 'pretrained' not in opt: return cls(opt['n_vocab'], d_word_vec=opt['d_word_vec'], d_model=opt['d_model'], len_max_seq=opt['len_max_seq'], n_layer=opt['n_layer'], d_inner=opt['d_inner'], n_head=opt['n_head'], slf_attn=opt['slf_attn'], d_k=opt['d_k'], d_v=opt['d_v'], feat_vocab=opt['feat_vocab'], d_feat_vec=opt['d_feat_vec'], layer_attn=opt['layer_attn'], slf_attn_mask=opt['mask_slf_attn'], dropout=opt['dropout'], attn_dropout=opt['attn_dropout']) elif opt['pretrained'].count('bert'): pretrained = BertModel.from_pretrained(opt['pretrained']) return cls(opt['n_vocab'], pretrained=pretrained, layer_attn=opt['layer_attn'], model_name='bert') elif opt['pretrained'].count('gpt2'): pretrained = GPT2Model.from_pretrained(opt['pretrained']) return cls(opt['n_vocab'], pretrained=pretrained, model_name='gpt2') else: raise ValueError("Other pretrained models haven't been supported yet")
def __init__(self, cuda_device=-1): super(GPT2Embedder, self).__init__() self.cuda_device = 'cpu' if cuda_device == -1 else f'cuda:{cuda_device}' # Load pre-trained model tokenizer (vocabulary) self.enc = GPT2Tokenizer.from_pretrained('gpt2') # Load pre-trained model (weights) self.model = GPT2Model.from_pretrained('gpt2') self.model.to(self.cuda_device) self.model.eval( ) # we only use the evaluation mode of the pretrained model self._bos_id = self.enc.encoder['<|endoftext|>'] self._bos_past = None
def fetch_objects(): bert = BertModel.from_pretrained( 'bert-base-uncased').embeddings.position_embeddings.weight.data gpt = OpenAIGPTModel.from_pretrained( 'openai-gpt').positions_embed.weight.data gpt2 = GPT2Model.from_pretrained('gpt2').wpe.weight.data bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') gpt_tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') gpt2_tokenizer = GPT2Tokenizer.from_pretrained('gpt2') return { 'bert': bert, 'gpt': gpt, 'gpt2': gpt2 }, { 'bert': bert_tokenizer, 'gpt': gpt_tokenizer, 'gpt2': gpt2_tokenizer }
def extract_gpt2_hidden_word_representations( word, save_activs_to): #, mode="full_model", focus_layers=[]): # text_inputs = [] # with open(text_path, "rb") as infile: # text_inputs = pickle.load(infile) # num_inputs = len(text_inputs) # Load pre-trained model tokenizer (vocabulary) tokenizer = GPT2Tokenizer.from_pretrained('gpt2') # get the hidden activations for word -- assumes gpu is available word_vec = None # initialize word vector object # tokenize word indexed_tokens = tokenizer.encode(word) num_tokens = len(indexed_tokens) tokens_tensor = torch.tensor([indexed_tokens]).to('cuda') # set up model model = GPT2Model.from_pretrained('gpt2') model.eval() model.to('cuda') # get word_vec with torch.no_grad(): # get token-wise activations hidden, _ = model(tokens_tensor) hidden_np = hidden.cpu().numpy() # identify hidden layer dimension that represents different tokens # seq_dim = hidden_np.shape.index(num_tokens) seq_dim = 1 # we know that the dimension corresponding to tokens is the 2nd dimension, indexed by 1 # sum the hidden layer element-wise along the token dimension to get word vector representation word_vec = np.sum(hidden_np, axis=seq_dim).squeeze() # clear gpu memory torch.cuda.empty_cache() # save word vector with open(save_activs_to, "wb") as outfile: pickle.dump(word_vec, outfile) pass
def transform(self, X): # Load pre-trained model tokenizer (vocabulary) tokenizer = GPT2Tokenizer.from_pretrained('gpt2') # Load pre-trained model (weights) model = GPT2Model.from_pretrained('gpt2', cache_dir='tmp/gpt2/') model.eval() output = [] for idx, row in tqdm(X.iterrows(), total=len(X)): # Encode some inputs indexed_tokens_1 = tokenizer.encode(row.text) # If you have a GPU, put everything on cuda # Convert inputs to PyTorch tensors tokens_tensor_1 = torch.tensor([indexed_tokens_1]) tokens_tensor_1 = tokens_tensor_1.to('cuda') model.to('cuda') # Predict hidden states features for each layer with torch.no_grad(): hidden_states_1, past = model(tokens_tensor_1) tokens = [ tokenizer.decoder[token].replace('Ġ', '') for token in indexed_tokens_1 ] output.append([tokens, hidden_states_1.cpu()[0]]) output = pd.DataFrame(output, columns=['tokens', 'layer_-1']) res = [] for idx, row in X.iterrows(): res.append(self.get_sample_props(output.loc[idx], **row)[1:]) res = pd.DataFrame(res, columns=[ 'tokens', 'pronoun_offset_token', 'a_offset_token', 'b_offset_token', 'a_span', 'b_span', 'pronoun_token', 'a_tokens', 'b_tokens', 'bert', 'cls' ]) cols = set(X.columns).difference(res.columns) return {'X': pd.concat([X[cols], res], axis=1)}
def load_model_fromlist(name): if not name in MODELS: raise Exception(str(name) + ' not a model in the list') print('# Loading model: ' + str(name)) name_path = MODEL_PATH_DICT[name] if name == 'word2vec': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format(join( PATH, name_path), binary=True)) if name == 'glove': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format( join(PATH, name_path))) if name == 'dict2vec': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format( join(PATH, name_path), binary=False, unicode_errors="ignore")) if name == 'conceptnet': if not exists(join(PATH, name_path)): download_model(name) return (gensim.models.KeyedVectors.load_word2vec_format( join(PATH, name_path))) if name == 'bert': tokenizer = BertTokenizer.from_pretrained('bert-large-uncased') model = BertModel.from_pretrained( 'bert-large-uncased').embeddings.word_embeddings.weight.data.numpy( ) return ([model, tokenizer]) if name == 'bert-context': tokenizer = BertTokenizer.from_pretrained('bert-large-uncased') model = BertModel.from_pretrained('bert-large-uncased', output_hidden_states=True) return ([model, tokenizer]) if name == 'gpt2': tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained( 'gpt2').transformer.wte.weight.data.numpy() return ([model, tokenizer]) if name == 'gpt2-context': tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2Model.from_pretrained('gpt2', output_hidden_states=True) return ([model, tokenizer])
def __init__(self, model_name, add_dense=True, trainable=False): super().__init__() self.model_name = model_name self.add_dense = add_dense self.trainable = trainable if self.model_name == 'GPT': self.encoder = OpenAIGPTModel.from_pretrained('openai-gpt') elif self.model_name == 'GPT-2': self.encoder = GPT2Model.from_pretrained('gpt2') else: raise NotImplementedError(f'{self.model_name} -- No such model') if not self.trainable: for p in self.encoder.parameters(): p.requires_grad = False if self.add_dense: self.dense = nn.Linear(in_features=768, out_features=128)
def Get_GPT2_Representation(self, examples): for i, example in enumerate(examples): # example.gpt2_mat = np.zeros((pb.fgt_maxlength,768)) # continue if (self.gpt2_tokenizer == None): self.gpt2_tokenizer = GPT2Tokenizer.from_pretrained('gpt2') text = example.fgt_channels[0] indexed_tokens = self.gpt2_tokenizer.encode(text) tokens_tensor = torch.tensor([indexed_tokens]) if (self.gpt2 == None): self.gpt2 = GPT2Model.from_pretrained('gpt2') self.gpt2.eval() with torch.no_grad(): hidden_states, past = self.gpt2(tokens_tensor) # (1, 5, 768) shape = np.array(hidden_states).shape representation, sum = [], 0 a, b = shape[1], shape[2] representation = np.zeros((a, b)) for layer in hidden_states: for words in layer.numpy(): representation += words sum += 1 if (sum > 0): representation = representation * 1.0 / sum representation = list(representation) while (len(representation) < pb.fgt_maxlength): representation.append(np.zeros(b)) example.gpt2_mat = representation[0:pb.fgt_maxlength] print("{:.2%}".format(i * 1.0 / len(examples)))
def __init__(self, n_layers, in_size, out_size, embed_size, dropout=0.5, initialEmbW=None, rnn_type='lstm', attention=None, q_size=-1, embedding_init=None, weights_init=None, elmo_init=False, elmo_num_outputs=1, finetune_elmo=False, bert_init=False, bert_model=None, finetune_bert=False, add_word_emb=True): """Initialize encoder with structure parameters Args: n_layers (int): Number of layers. in_size (int): Dimensionality of input vectors. out_size (int) : Dimensionality of hidden vectors to be output. embed_size (int): Dimensionality of word embedding. dropout (float): Dropout ratio. """ # TODO conv_out_size = 512 super(LSTMEncoder, self).__init__() self.embed = nn.Embedding(in_size, embed_size) if embedding_init is not None: self.embed.weight.data.copy_(torch.from_numpy(embedding_init)) elif weights_init is not None: self.embed.weight.data.copy_( torch.from_numpy(weights_init['embed'])) self.elmo_init = elmo_init self.bert_init = bert_init self.bert_model = bert_model self.add_word_emb = add_word_emb if elmo_init: options_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" self.elmo = Elmo(options_file, weight_file, elmo_num_outputs, requires_grad=finetune_elmo) elmo_layer = [ nn.Linear(elmo_num_outputs * 1024, out_size), nn.ReLU() ] self.elmo_layer = nn.Sequential(*elmo_layer) elif bert_init: if 'bert' in bert_model: self.bert = BertModel.from_pretrained(bert_model) elif 'openai-gpt' in bert_model: self.bert = OpenAIGPTModel.from_pretrained(bert_model) elif 'gpt2' in bert_model: self.bert = GPT2Model.from_pretrained(bert_model) elif 'transfo-xl' in bert_model: self.bert = TransfoXLModel.from_pretrained(bert_model) self.finetune_bert = finetune_bert if not finetune_bert: for param in self.bert.parameters(): param.requires_grad = False if bert_model in ['bert-base-uncased', 'openai-gpt', 'gpt2']: bert_in = 768 elif bert_model in [ 'bert-large-uncased', 'gpt2-medium', 'transfo-xl-wt103' ]: bert_in = 1024 bert_layer = [nn.Linear(bert_in, out_size), nn.ReLU()] self.bert_layer = nn.Sequential(*bert_layer) if rnn_type == 'lstm': self.lstm = nn.LSTM(embed_size, out_size, n_layers, batch_first=True, dropout=dropout) elif rnn_type == 'gru': self.lstm = nn.GRU(embed_size, out_size, n_layers, batch_first=True, dropout=dropout) self.attention = attention if attention == 'conv' or attention == 'conv_sum': conv_in_size = out_size self.conv1 = nn.Conv1d(in_channels=conv_in_size, out_channels=conv_out_size, kernel_size=1, padding=0) self.conv2 = nn.Conv1d(in_channels=conv_out_size, out_channels=2, kernel_size=1, padding=0) if weights_init is not None: self.conv1.weight.data.copy_( torch.from_numpy(weights_init['conv1'])) self.conv2.weight.data.copy_( torch.from_numpy(weights_init['conv2'])) elif attention == 'c_conv_sum': hidden_size = 512 conv_hidden_size = 256 layers = [ weight_norm(nn.Linear(out_size, hidden_size), dim=None), nn.ReLU() ] self.c_fa = nn.Sequential(*layers) layers = [ weight_norm(nn.Linear(q_size, hidden_size), dim=None), nn.ReLU() ] self.q_fa = nn.Sequential(*layers) layers = [ nn.Conv2d(in_channels=hidden_size, out_channels=conv_hidden_size, kernel_size=1), nn.ReLU(), nn.Conv2d(in_channels=conv_hidden_size, out_channels=1, kernel_size=1) ] self.cq_att = nn.Sequential(*layers) if weights_init is not None: self.c_fa[0].weight.data.copy_( torch.from_numpy(weights_init['c_fa'])) self.q_fa[0].weight.data.copy_( torch.from_numpy(weights_init['q_fa'])) self.cq_att[0].weight.data.copy_( torch.from_numpy(weights_init['cq_att_conv1'])) self.cq_att[2].weight.data.copy_( torch.from_numpy(weights_init['cq_att_conv2']))
def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_pretrained_bert_test/" for model_name in list(PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = GPT2Model.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) self.assertIsNotNone(model)
def _init_model_tokenizer(self): from pytorch_pretrained_bert import GPT2Model, GPT2Tokenizer self._tokenizer = GPT2Tokenizer.from_pretrained(self.model_dir) self._model = GPT2Model.from_pretrained(self.model_dir) self._model.eval()
def __init__(self, n_layers, in_size, out_size, embed_size, in_size_hier, hidden_size, proj_size, dropout=0.5, initialEmbW=None, independent=False, rnn_type='lstm', classifier='baseline', states_att=False, state_size=-1, embedding_init=None, weights_init=None, elmo_init=False, elmo_num_outputs=1, finetune_elmo=False, bert_init=False, bert_model=None, finetune_bert=False, add_word_emb=True, pretrained_all=True): """Initialize encoder with structure parameters Args: n_layers (int): Number of layers. in_size (int): Dimensionality of input vectors. out_size (int): Dimensionality of output vectors. embed_size (int): Dimensionality of word embedding. hidden_size (int) : Dimensionality of hidden vectors. proj_size (int) : Dimensionality of projection before softmax. dropout (float): Dropout ratio. """ #TODO att_size = 128 self.rnn_type = rnn_type self.classifier = classifier super(HLSTMDecoder, self).__init__() self.embed = nn.Embedding(in_size, embed_size) if embedding_init is not None: self.embed.weight.data.copy_(torch.from_numpy(embedding_init)) elif weights_init is not None: self.embed.weight.data.copy_( torch.from_numpy(weights_init['embed'])) if rnn_type == 'lstm': self.lstm = nn.LSTM(embed_size + in_size_hier, hidden_size, n_layers, batch_first=True, dropout=dropout) elif rnn_type == 'gru': self.lstm = nn.GRU(embed_size + in_size_hier, hidden_size, n_layers, batch_first=True, dropout=dropout) if weights_init is not None: lstm_wt = weights_init['lstm'] for k, v in lstm_wt.items(): self.lstm.__getattr__(k).data.copy_(torch.from_numpy(v)) self.elmo_init = elmo_init self.bert_init = bert_init self.pretrained_all = pretrained_all self.bert_model = bert_model self.add_word_emb = add_word_emb if False: #if pretrained_all and elmo_init: options_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" self.elmo = Elmo(options_file, weight_file, elmo_num_outputs, requires_grad=finetune_elmo) elmo_layer = [ nn.Linear(elmo_num_outputs * 1024, out_size), nn.ReLU() ] self.elmo_layer = nn.Sequential(*elmo_layer) elif False: #elif pretrained_all and bert_init: if 'bert' in bert_model: self.bert = BertModel.from_pretrained(bert_model) elif 'openai-gpt' in bert_model: self.bert = OpenAIGPTModel.from_pretrained(bert_model) elif 'gpt2' in bert_model: self.bert = GPT2Model.from_pretrained(bert_model) elif 'transfo-xl' in bert_model: self.bert = TransfoXLModel.from_pretrained(bert_model) self.finetune_bert = finetune_bert if not finetune_bert: for param in self.bert.parameters(): param.requires_grad = False if bert_model in ['bert-base-uncased', 'openai-gpt', 'gpt2']: bert_in = 768 elif bert_model in [ 'bert-large-uncased', 'gpt2-medium', 'transfo-xl-wt103' ]: bert_in = 1024 bert_layer = [nn.Linear(bert_in, out_size), nn.ReLU()] self.bert_layer = nn.Sequential(*bert_layer) self.n_layers = n_layers self.dropout = dropout self.independent = independent self.states_att = states_att if states_att: self.ecW = nn.Linear(state_size, att_size) self.ysW = nn.Linear(hidden_size, att_size) hidden_size += state_size if classifier == 'baseline': layers = [ nn.Linear(hidden_size, proj_size), nn.Linear(proj_size, out_size) ] self.y_classifier = nn.Sequential(*layers) elif classifier == 'weighted_norm': layers = [ weight_norm(nn.Linear(hidden_size, proj_size), dim=None), nn.ReLU(), weight_norm(nn.Linear(proj_size, out_size), dim=None) ] self.y_classifier = nn.Sequential(*layers) elif classifier == 'logit': layers = [ weight_norm(nn.Linear(hidden_size, proj_size), dim=None), nn.ReLU(), nn.Linear(proj_size, out_size) ] self.classifier_txt = nn.Sequential(*layers) layers = [ weight_norm(nn.Linear(hidden_size, 2048), dim=None), nn.ReLU(), nn.Linear(2048, out_size) ] self.classifier_ft = nn.Sequential(*layers) if weights_init is not None: self.classifier_txt[0].weight.data.copy_( torch.from_numpy(weights_init['classifier_txt'])) self.classifier_ft[0].weight.data.copy_( torch.from_numpy(weights_init['classifier_ft']))
print(tokenizer.decode(indexed_tokens_1)) # Who was Jim Henson ? print(tokenizer.decode([8727, 373, 474, 320])) # who was jim print(tokenizer.decode([30963, 1559, 5633])) # henson ? print(tokenizer.decode([30963])) # hen print(tokenizer.decode([508, 8727, 373])) # whowho was indexed_tokens_1 = tokenizer.convert_tokens_to_ids(text_1) # AttributeError: 'GPT2Tokenizer' object has no attribute 'convert_tokens_to_ids' indexed_tokens_2 = tokenizer.encode(text_2); print(indexed_tokens_2) # [18050, 367, 19069, 373, 257, 13595, 14471, 263] ## Convert inputs to PyTorch tensors tokens_tensor_1 = torch.tensor([indexed_tokens_1]); print(tokens_tensor_1) tokens_tensor_2 = torch.tensor([indexed_tokens_2]) ################################################################## ## GPT2Model ## Load pre-trained model (weights) model = GPT2Model.from_pretrained('/Users/coder352/datasets/WordVec/pytorch_pretrained_bert/gpt2/') model.eval() # INFO:pytorch_pretrained_bert.modeling_gpt2:Model config { # "initializer_range": 0.02, # "layer_norm_epsilon": 1e-05, # "n_ctx": 1024, # "n_embd": 768, # "n_head": 12, # "n_layer": 12, # "n_positions": 1024, # "vocab_size": 50257 # } ## Predict hidden states features for each layer with torch.no_grad(): hidden_states_1, past = model(tokens_tensor_1)
def main(): """Main training program.""" print('Evaluate GPT2 model') # Disable CuDNN. torch.backends.cudnn.enabled = False # Timer. timers = Timers() # Arguments. args = get_args() # Pytorch distributed. initialize_distributed(args) # Random seeds for reproducability. set_random_seed(args.seed) # Data stuff. eval_data = get_eval_data(args) # Model, optimizer, and learning rate. if args.eval_hf: from pytorch_pretrained_bert import GPT2LMHeadModel from pytorch_pretrained_bert import GPT2Model as HFGPT2Model if args.num_layers == 24: model_path = args.load #model_path = '/home/universal-lm-data.cosmos549/repos/gpt2_mp/models/345M' hfmodel = HFGPT2Model.from_pretrained(model_path, cache_dir='gpt2_weights', from_tf=True).cuda() model = GPT2LMHeadModel(hfmodel.config) model.transformer.load_state_dict(hfmodel.state_dict()) model.cuda() else: model = GPT2LMHeadModel.from_pretrained('gpt2', cache_dir='gpt2_weights').cuda() else: if args.load_openai: from utils import move_weights model_path = args.load args.load = None model = setup_model(args) from pytorch_pretrained_bert import GPT2LMHeadModel from pytorch_pretrained_bert import GPT2Model as HFGPT2Model model_path = 'gpt2' from_tf = False print('loading openai weights') model.cpu() if args.num_layers == 24: #model_path = '/home/universal-lm-data.cosmos549/repos/gpt2_mp/models/345M' hfmodel = HFGPT2Model.from_pretrained(model_path, cache_dir='gpt2_weights', from_tf=True) gpt2model = GPT2LMHeadModel(hfmodel.config) gpt2model.transformer.load_state_dict(hfmodel.state_dict()) gpt2model else: gpt2model = GPT2LMHeadModel.from_pretrained('gpt2', cache_dir='gpt2_weights') model2fill = model while isinstance(model2fill, (DDP, FP16_Module)): model2fill = model2fill.module move_weights(model2fill, gpt2model) model.cuda() else: model = setup_model(args) # Run on test data. prefix = "wiki" #os.path.basename(args.valid_data) evaluate_and_print_results(prefix, eval_data, model, args, timers)
def __init__(self, n_wlayers, n_slayers, in_size, out_size, embed_size, hidden_size, dropout=0.5, ignore_label=None, initialEmbW=None, independent=False, rnn_type='lstm', embedding_init=None, weights_init=None, elmo_init=False, elmo_num_outputs=1, finetune_elmo=False, bert_init=False, bert_model=None, finetune_bert=False, add_word_emb=True, pretrained_all=True, concat_his=False): """Initialize encoder with structure parameters Args: n_layers (int): Number of layers. in_size (int): Dimensionality of input vectors. out_size (int) : Dimensionality of hidden vectors to be output. embed_size (int): Dimensionality of word embedding. dropout (float): Dropout ratio. """ super(HLSTMEncoder, self).__init__() self.embed = nn.Embedding(in_size, embed_size) if embedding_init is not None: self.embed.weight.data.copy_(torch.from_numpy(embedding_init)) elif weights_init is not None: self.embed.weight.data.copy_( torch.from_numpy(weights_init['embed'])) if rnn_type == 'lstm': self.wlstm = nn.LSTM(embed_size, hidden_size, n_wlayers, batch_first=True, dropout=dropout) self.slstm = nn.LSTM(hidden_size, out_size, n_slayers, batch_first=True, dropout=dropout) elif rnn_type == 'gru': self.wlstm = nn.GRU(embed_size, hidden_size, n_wlayers, batch_first=True, dropout=dropout) self.slstm = nn.GRU(hidden_size, out_size, n_slayers, batch_first=True, dropout=dropout) self.elmo_init = elmo_init self.bert_init = bert_init self.pretrained_all = pretrained_all self.concat_his = concat_his self.bert_model = bert_model self.add_word_emb = add_word_emb if pretrained_all and elmo_init: options_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" self.elmo = Elmo(options_file, weight_file, elmo_num_outputs, requires_grad=finetune_elmo) elmo_layer = [ nn.Linear(elmo_num_outputs * 1024, out_size), nn.ReLU() ] self.elmo_layer = nn.Sequential(*elmo_layer) elif pretrained_all and bert_init: if 'bert' in bert_model: self.bert = BertModel.from_pretrained(bert_model) elif 'openai-gpt' in bert_model: self.bert = OpenAIGPTModel.from_pretrained(bert_model) elif 'gpt2' in bert_model: self.bert = GPT2Model.from_pretrained(bert_model) elif 'transfo-xl' in bert_model: self.bert = TransfoXLModel.from_pretrained(bert_model) self.finetune_bert = finetune_bert if not finetune_bert: for param in self.bert.parameters(): param.requires_grad = False if bert_model in ['bert-base-uncased', 'openai-gpt', 'gpt2']: bert_in = 768 elif bert_model in [ 'bert-large-uncased', 'gpt2-medium', 'transfo-xl-wt103' ]: bert_in = 1024 bert_layer = [nn.Linear(bert_in, out_size), nn.ReLU()] self.bert_layer = nn.Sequential(*bert_layer) self.independent = independent self.rnn_type = rnn_type