Python get_dic Examples

Example #1

File: CNN_BiLSTM_Concat_model.py Project: MoherX/text-classification-pytorch

from torch.utils.data import DataLoader
import data_preprocess
import os

os.environ['CUDA_VISIBLE_DEVICES'] = '0'
use_cuda = torch.cuda.is_available()

# 将数据划分为训练集和测试集
X_train, X_test, Y_train, Y_test = data_preprocess.tensorFromData()
trainDataSet = data_preprocess.TextDataSet(X_train, Y_train)
testDataSet = data_preprocess.TextDataSet(X_test, Y_test)
trainDataLoader = DataLoader(trainDataSet, batch_size=16, shuffle=True)
testDataLoader = DataLoader(testDataSet, batch_size=16, shuffle=False)

# 获取字典
word_to_inx, inx_to_word = data_preprocess.get_dic()
len_dic = len(word_to_inx)

# 定义超参数
MAXLEN = 64
input_dim = MAXLEN
emb_dim = 128
num_epoches = 20
batch_size = 16


# 定义模型
class CNN_BiLSTM_Concat_model(nn.Module):
    def __init__(self, len_dic, emb_dim, input_dim):
        super(CNN_BiLSTM_Concat_model, self).__init__()
        self.embed = nn.Embedding(len_dic, emb_dim)  # b,64,128

Example #2

File: predict.py Project: lichaomessi/NER-Sequence-labeling--Textcnn-bilstm-crf-pytorch

import numpy as np
import torch
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torch import optim, nn
import data_preprocess
import os

torch.manual_seed(1)

os.environ['CUDA_VISIBLE_DEVICES'] = '0'
use_cuda = torch.cuda.is_available()

word2index, index2word, tag2index, index2tag = data_preprocess.get_dic()
test_x_cut, test_y_cut, test_mask, test_x_len, test_x_cut_word, test_x_fenge = data_preprocess.getTest_xy(
    './data/test_data')
testDataSet = data_preprocess.TextDataSet(test_x_cut, test_y_cut, test_mask)

testDataLoader = DataLoader(testDataSet, batch_size=16, shuffle=False)

MAXLEN = 100
vcab_size = len(word2index)
emb_dim = 128
hidden_dim = 256
num_epoches = 20
batch_size = 16


class BILSTM_CRF(nn.Module):
    def __init__(self, vcab_size, tag2index, emb_dim, hidden_dim, batch_size):
        super(BILSTM_CRF, self).__init__()