batch_size=args.batch_size,
cuda=use_cuda)
validation_data = DataLoader(data['valid']['src'],
data['valid']['label'],
args.max_len,
batch_size=args.batch_size,
shuffle=False,
cuda=use_cuda)
# ##############################################################################
# Build model
# ##############################################################################
import model
rnn = model.LSTM_Text(args)
if use_cuda:
rnn = rnn.cuda()
optimizer = torch.optim.Adam(rnn.parameters(), lr=args.lr, weight_decay=0.001)
criterion = torch.nn.CrossEntropyLoss()
# ##############################################################################
# Training
# ##############################################################################
import time
from tqdm import tqdm
train_loss = []
valid_loss = []
accuracy = []
X, Y, corpus = util.read_data_omg(data)
# data set - dev
input_file = "../omg_ValidationTranscripts.csv"
input_file2 = "../omg_ValidationVideos.csv"
data = util.process_transcripts(input_file, input_file2)
# prepare data
Xv, Yv, corpus = util.read_data_omg(data)
# prepare embedding
emb_file = "../glove/glove.6B.100d.txt"
w2v = util.load_embeddings_from_glove(emb_file, corpus.word2index)
# model
emb_dim = 100
lstm_model = model.LSTM_Text(emb_dim, 128, 32).cuda()
def sent_to_tensor(x):
tensor = torch.zeros(len(x), 1, emb_dim)
for index, tok_id in enumerate(x):
tensor[index][0] = torch.from_numpy(w2v[tok_id])
return tensor
def eval(X, Y, model):
model.cpu()
y_pred = [[],[]]
for i in range(len(X)):
x = X[i]
if len(x) == 0:
y_pred[0].append(0.)
y_pred[1].append(0.)
def __init__(self, master):
''' GUI布局 '''
frame = Frame(master)
frame.pack()
self.frame = frame
self.text = ''
self.isPreprocess = 0
# 全局
self.frame.bind("<Button-3>", self.delete)
# 字体
ft = tkFont.Font(family='Fixdsys', size=13, weight=tkFont.BOLD)
ft_big = tkFont.Font(family='Fixdsys', size=21, weight=tkFont.BOLD)
ft_middle = tkFont.Font(family='Fixdsys', size=18, weight=tkFont.BOLD)
# 文本框 <请输入简历链接：>
self.label1 = Label(frame, text='请输入简历链接：', font=ft, pady=5)
#self.label1.pack(side=TOP, anchor='w')
self.label1.grid(row=0, column=0)
# 输入框 (简历链接)
self.url = StringVar()
self.urlEntry = Entry(frame, textvariable=self.url, font=ft, width=45)
self.urlEntry.bind("<Return>", self.input_url)
self.urlEntry.bind("<Button-3>", self.delete)
#self.urlEntry.pack(side=TOP, anchor='w')
self.urlEntry.grid(row=1, column=0)
# 文本框 <或输入简历文本：>
self.label2 = Label(frame, text='或输入简历文本：', font=ft, pady=5)
#self.label2.pack(side=TOP, anchor='w')
self.label2.grid(row=2, column=0)
# 输入框 (简历文本)
self.resumeText = Text(frame, width=50, height=30, font=ft)
self.resumeText.bind("<Button-3>", self.delete)
self.resumeText.bind("<Return>", self.input_text)
#self.entryText.pack(side=TOP, anchor='w')
self.resumeText.grid(row=3, column=0)
# 按钮 [开始预测]
self.predictButton = Button(frame, text="开始预测", fg="red", command=self.predict, font=ft_big)
#self.next.pack(side=RIGHT)
self.predictButton.grid(row=3, column=1)
# 文本框 < --> >
self.label5 = Label(frame, text="-> ", fg="blue", font=ft_big)
#self.next.pack(side=RIGHT)
self.label5.grid(row=3, column=2)
# 文本框 (预测结果）
self.result = StringVar()
self.label4 = Label(frame, textvariable=self.result, bg='yellow', font=ft_middle)
self.result.set(' ')
#self.label4.pack(side=RIGHT)
self.label4.grid(row=3, column=3)
# 文本 (状态显示)
self.condition = StringVar()
self.label6 = Label(frame, textvariable=self.condition, fg="green", font=ft_big)
self.condition.set('空闲中.')
#self.next.pack(side=RIGHT)
self.label6.grid(row=2, column=1)
''' 模型载入 '''
excel = './data/标注数据.xlsx'
text_path = './data/text/processed/'
args.snapshot = './snapshot/TextCNN_Word2Vec/best_steps_4100.pt'
#new_model = word2vec.Word2Vec.load(args.word2vec_model)
print("\nLoading data...")
tokenize = lambda x: re.split(' ', x)
text_field = data.Field(tokenize=tokenize, lower=True, stop_words=['\r', '\n', '\t', '\xa0', ' ', ''])
label_field = data.Field(sequential=False)
train_iter, dev_iter = mydatasets.resume(excel, text_path, text_field, label_field,
args.batch_size, device=-1, repeat=False,
use_wv=True,
wv_model=args.word2vec_model,
)
args.embed_num = len(text_field.vocab) # 9499
args.class_num = len(label_field.vocab) - 1 # 16, -1是为了除去<unk>
args.cuda = (not args.no_cuda) and torch.cuda.is_available(); del args.no_cuda
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
#args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
if args.model == 'CNN':
print('Using CNN model')
cnn = model.CNN_Text(args, text_field)
elif args.model == 'LSTM':
print('Using LSTM model')
cnn = model.LSTM_Text(args, text_field)
if args.snapshot is not None:
print('\nLoading model from {}...'.format(args.snapshot))
cnn.load_state_dict(torch.load(args.snapshot))
if args.cuda:
#torch.cuda.set_device(args.device)
print('Using CUDA')
cnn = cnn.cuda()
else:
print('Using CPU')
self.cnn = cnn
self.text_field = text_field
self.label_field = label_field
# Update args and print
args.embed_num = len(text_field.vocab)
args.class_num = len(label_field.vocab) - 1
args.vectors = text_field.vocab.vectors
args.cuda = (not args.no_cuda) and torch.cuda.is_available(); del args.no_cuda
args.save_dir = os.path.join(args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
print("\nParameters:")
for attr, value in sorted(args.__dict__.items()):
print("\t{}={}".format(attr.upper(), value))
args.TF = text_field
# Model creation and load
if args.snapshot is None:
net = model.LSTM_Text(args)
else :
print('\nLoading model from [%s]...' % args.snapshot)
try:
net = model.LSTM_Text(args)
net.load_state_dict(torch.load(args.snapshot))
except :
print("Sorry, This snapshot doesn't exist."); exit()
if args.cuda:
net = net.cuda()
# Call the main function (train.py)
if args.predict is not None:
label = train.predict(args.predict, net, text_field, label_field, args.cuda, args)
print('\n[Text] {}\n[Label] {}\n'.format(args.predict, label))
args.class_num = len(label_field.vocab) - 1 # 16, -1是为了除去<unk>
print('embed num:', args.embed_num, '\nclass num:', args.class_num)
args.cuda = (not args.no_cuda) and torch.cuda.is_available()
del args.no_cuda
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
args.save_dir = os.path.join(
args.save_dir,
datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
if args.model == 'CNN':
print('Using CNN model')
cnn = model.CNN_Text(args, text_field)
elif args.model == 'LSTM':
print('Using LSTM model')
cnn = model.LSTM_Text(args, text_field)
if args.snapshot is not None:
print('\nLoading model from {}...'.format(args.snapshot))
cnn.load_state_dict(torch.load(args.snapshot))
if args.cuda:
#torch.cuda.set_device(args.device)
print('Using CUDA')
cnn = cnn.cuda()
else:
print('Using CPU')
#------ 预测 ------#
if args.predict or args.predict_url:
args.snapshot = './snapshot/TextCNN_Word2Vec/best_steps_4100.pt'