dropout_p = 0.5
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
train_iter, test_iter, dictionary = preprocess(batch_size=batch_size)
model = TextCNNMulti(len(dictionary.vocab),
embedding_dim,
hidden_size,
num_layers,
dropout_p=dropout_p)
#def init_weights(m):
# for name, param in m.named_parameters():
# nn.init.uniform_(param.data, -0.08, 0.08)
#model.apply(init_weights)
app = Trainer(
App(model=model,
criterion=nn.NLLLoss(ignore_index=dictionary.vocab.stoi['<pad>'])))
@app.on("train")
def lm_train(e):
e.model.zero_grad()
src = e.batch.text
batch_size = src.size(1)
target = src[1:, :].contiguous()
pad = torch.tensor(
[dictionary.vocab.stoi['<pad>'] for _ in range(batch_size)])
pad = pad.view(1, batch_size).to(target.device)
target = torch.cat([target, pad])
y_predict, _ = model(src)
loss = 0.0
#pdb.set_trace()
for i in range(target.size(0)):
batch_size = 128
train_set = datasets.MNIST("data",
train=True,
download=True,
transform=transforms.ToTensor())
test_set = datasets.MNIST("data",
train=False,
download=True,
transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(dataset=train_set,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(test_set)
app = Trainer(App(model=LeNet5(), criterion=nn.CrossEntropyLoss()))
app.extend(Checkpoint())
@app.on("train")
def mnist_train(e):
e.model.zero_grad()
inputs, labels = e.batch
y_predict = e.model(inputs)
loss = e.criterion(y_predict, labels)
loss.backward()
e.optimizer.step()
e.a.loss = loss.item()
@app.on("iter_completed")
def logging(e):
_train, _test, SRC, TRG = preprocess_couplet()
train_iter = BucketIterator(_train, batch_size=batch_size, train=True,
repeat=False, shuffle=True,
#sort_within_batch=True,
#sort_key=lambda x: (len(x.src), len(x.trg)), repeat=False,
device=device)
test_iter = BucketIterator(_test, batch_size=1, train=False, repeat=False,
sort_within_batch=False, sort_key=lambda x: (len(x.src), len(x.trg)), device=device)
#import pdb; pdb.set_trace()
#model = Seq2Seq(SRC, TRG, src_vocab_size=len(SRC.vocab), trg_vocab_size=len(TRG.vocab), embedding_dim=256, hidden_size=512, dropout_p=0.5, attention=True)
model_t = Transformer(len(SRC.vocab), len(TRG.vocab), d_model=d_model, n_encoder_layers=6, n_decoder_layers=6, dropout_p=0.1)
criterion = nn.CrossEntropyLoss(ignore_index=TRG.vocab.stoi['<pad>'], reduction="sum")
app = Trainer(App(model=model_t,
criterion=criterion))
app.extend(Checkpoint())
@app.on("train")
def nmt_train(e):
e.model.zero_grad()
(src, lengths_src), (targets, lengths_trg) = e.batch.src, e.batch.trg
# (seq, batch)
#import pdb; pdb.set_trace()
#targets, gold = targets[:-1,:], targets[1:,:]
#lengths_trg -= 1
y_predict = e.model(src, targets, lengths_src, lengths_trg)
#if e.current_iter % 50 == 0:
out_classes=9,
attention=attention)
#model = model.to(device)
#def init_weights(m):
# for name, param in m.named_parameters():
# nn.init.uniform_(param.data, -0.08, 0.08)
#model.apply(init_weights)
## model = model.half()
#criterion = nn.NLLLoss(ignore_index = 1)
#criterion = criterion.to(device)
## criterion = criterion.half()
#optimizer = optim.Adam(model.parameters(), lr=lr, eps=adam_eps)
#pdb.set_trace()
app = Trainer(
App(model=model,
criterion=nn.NLLLoss(ignore_index=SEN.vocab.stoi["<pad>"])))
app.extend(Checkpoint())
def itos(s):
s.transpose(0, 1)
return [" ".join([SEN.vocab.itos[ix] for ix in sent]) for sent in s]
#@train(model, criterion, optimizer, train_iter, epochs, device=device, immediate=False, checkpoint=True, verbose=False)
@app.on("train")
def ner_train(e):
e.model.zero_grad()
#pdb.set_trace()
sent, label = e.batch.sen, e.batch.tag
y_predict = e.model(sent)
label = label[0]
#pdb.set_trace()
if __name__ == "__main__":
vocab, dataset, word_to_ix = prepare()
vocab_size = len(vocab)
embedding_dim = 100
context_size = 4
batch_size = 1
epochs = 100
train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False)
test_loader = torch.utils.data.DataLoader(dataset, batch_size=1)
model = CBOW(vocab_size, embedding_dim, context_size)
app = Trainer(App(model=model,
criterion=nn.NLLLoss()))
app.extend(Checkpoint())
@app.on("train")
def cbow_train(e):
e.model.zero_grad()
inputs, labels = e.batch
#import pdb; pdb.set_trace()
context_ids = [[word_to_ix[w] for w in context] for context in inputs]
context_ids = torch.tensor(context_ids)
labels = [word_to_ix[l] for l in labels]
labels = torch.tensor(labels)
y_predicts = e.model(context_ids)
loss = e.criterion(y_predicts, labels)
loss.backward()
e.optimizer.step()