Python TextDataSet示例

示例#1

import torch
from torch import nn,optim
from torch.autograd import Variable
from torch.utils.data import DataLoader
import data_preprocess
import os

os.environ['CUDA_VISIBLE_DEVICES']='0'
use_cuda=torch.cuda.is_available()


# 将数据划分为训练集和测试集
X_train, X_test, Y_train, Y_test, word_to_inx, inx_to_word = data_preprocess.tensorFromData()
trainDataSet = data_preprocess.TextDataSet(X_train, Y_train)
testDataSet = data_preprocess.TextDataSet(X_test, Y_test)
trainDataLoader = DataLoader(trainDataSet, batch_size=16, shuffle=True)
testDataLoader = DataLoader(testDataSet, batch_size=16, shuffle=False)

# 获取字典
# word_to_inx, inx_to_word = data_preprocess.get_dic()
word_to_inx, inx_to_word = word_to_inx, inx_to_word
len_dic = len(word_to_inx)

# 定义超参数
MAXLEN = 64
input_dim = MAXLEN
emb_dim = 128
num_epoches = 20
batch_size = 16
learning_rate = 0.001

示例#2

文件： predict.py 项目： lichaomessi/NER-Sequence-labeling--Textcnn-bilstm-crf-pytorch

import torch
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torch import optim, nn
import data_preprocess
import os

torch.manual_seed(1)

os.environ['CUDA_VISIBLE_DEVICES'] = '0'
use_cuda = torch.cuda.is_available()

word2index, index2word, tag2index, index2tag = data_preprocess.get_dic()
test_x_cut, test_y_cut, test_mask, test_x_len, test_x_cut_word, test_x_fenge = data_preprocess.getTest_xy(
    './data/test_data')
testDataSet = data_preprocess.TextDataSet(test_x_cut, test_y_cut, test_mask)

testDataLoader = DataLoader(testDataSet, batch_size=16, shuffle=False)

MAXLEN = 100
vcab_size = len(word2index)
emb_dim = 128
hidden_dim = 256
num_epoches = 20
batch_size = 16


class BILSTM_CRF(nn.Module):
    def __init__(self, vcab_size, tag2index, emb_dim, hidden_dim, batch_size):
        super(BILSTM_CRF, self).__init__()
        self.vcab_size = vcab_size