def batch_predict_loader(config): vocab = load_vocab(config.vocab) label_dic = load_vocab(config.label_file) test_data = read_corpus(config.test_file, max_length=config.max_length, label_dic=label_dic, vocab=vocab) test_ids = torch.LongTensor([temp.input_id for temp in test_data]).to(config.device) test_masks = torch.LongTensor([temp.input_mask for temp in test_data]).to(config.device) test_tags = torch.LongTensor([temp.label_id for temp in test_data]).to(config.device) batch_predict_dataset = TensorDataset(test_ids, test_masks) batch_predict_load = DataLoader(batch_predict_dataset, shuffle=True, batch_size=config.batch_size) test_dataset = TensorDataset(test_ids, test_masks, test_tags) test_loader = DataLoader(test_dataset, shuffle=True, batch_size=config.batch_size) return batch_predict_load, test_loader
def data_load(config): vocab = load_vocab(config.vocab) label_dic = load_vocab(config.label_file) train_data = read_corpus(config.train_file, max_length=config.max_length, label_dic=label_dic, vocab=vocab) dev_data = read_corpus(config.dev_file, max_length=config.max_length, label_dic=label_dic, vocab=vocab) test_data = read_corpus(config.test_file, max_length=config.max_length, label_dic=label_dic, vocab=vocab) train_ids = torch.LongTensor([temp.input_id for temp in train_data]).to(config.device) train_masks = torch.LongTensor([temp.input_mask for temp in train_data]).to(config.device) train_tags = torch.LongTensor([temp.label_id for temp in train_data]).to(config.device) train_dataset = TensorDataset(train_ids, train_masks, train_tags) train_loader = DataLoader(train_dataset, shuffle=True, batch_size=config.batch_size) dev_ids = torch.LongTensor([temp.input_id for temp in dev_data]).to(config.device) dev_masks = torch.LongTensor([temp.input_mask for temp in dev_data]).to(config.device) dev_tags = torch.LongTensor([temp.label_id for temp in dev_data]).to(config.device) dev_dataset = TensorDataset(dev_ids, dev_masks, dev_tags) dev_loader = DataLoader(dev_dataset, shuffle=True, batch_size=config.batch_size) test_ids = torch.LongTensor([temp.input_id for temp in test_data]).to(config.device) test_masks = torch.LongTensor([temp.input_mask for temp in test_data]).to(config.device) test_tags = torch.LongTensor([temp.label_id for temp in test_data]).to(config.device) test_dataset = TensorDataset(test_ids, test_masks, test_tags) test_loader = DataLoader(test_dataset, shuffle=True, batch_size=config.batch_size) return train_loader, dev_loader, test_loader
def __init__(self, config, embedding_dim, hidden_dim, rnn_layers, dropout_ratio, dropout1): super(BERT_LSTM_CRF, self).__init__() self.device = config.device self.num_tags = len(load_vocab(config.label_file)) self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim self.encoder = BertEmbedder.create() self.lstm = nn.LSTM(self.embedding_dim, self.hidden_dim, num_layers=rnn_layers, bidirectional=True, dropout=dropout_ratio, batch_first=True) self.rnn_layers = rnn_layers self.dropout1 = nn.Dropout(p=dropout1) self.linear = nn.Linear(hidden_dim * 2, self.num_tags) self.crf = CRF(num_tags=self.num_tags, device=self.device) self.init_weights()
def __init__(self, config, embed_dim, dropout1): super(BERT_CRF, self).__init__() self.device = config.device self.num_tags = len(load_vocab(config.label_file)) self.embed_dim = embed_dim self.encoder = BertEmbedder.create() self.dropout1 = nn.Dropout(p=dropout1) self.linear = nn.Linear(self.embed_dim, self.num_tags) self.crf = CRF(num_tags=self.num_tags, device=self.device)
def __init__(self, config, embedding_dim, dropout1): super(BERT_ATTN_CRF, self).__init__() self.device = config.device self.num_tags = len(load_vocab(config.label_file)) self.encoder = BertEmbedder.create() self.attention = Attn(config) self.dropout = nn.Dropout(dropout1) config.update(attention_dim=embedding_dim) self.linear = nn.Linear(embedding_dim, self.num_tags) self.crf = CRF(num_tags=self.num_tags, device=self.device)