def main(dataset_csv: ("Dataset CSV", 'option', 'd')):
classifier = RNNClassifier(1000, 256, 3, 0.2, 2)
df = pd.read_csv(dataset_csv)
dataset = VideoDataset(df)
dataloader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=10,
shuffle=False,
)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
for i, (images, labels) in enumerate(mn_dataset_loader):
images = Variable(images)
labels = Variable(labels)
# Clear gradients
optimizer.zero_grad()
# Forward pass
outputs = model(images)
# Calculate loss
loss = criterion(outputs, labels)
# Backward pass
loss.backward()
# Update weights
optimizer.step()
break
videos = Variable(torch.randn(2, 3, 3, 224, 224))
outputs = classifier(videos)
out_dim = len(answers.vocab)
cells = 2
# TODO - remove bidirectional RNN for simpleRNN
birnn = True
lr = 0.01
epochs = 10
if birnn:
cells *= 2
dropout = 0.5
fc1_dim = 50
fc2_dim = 3
hidden_size = 1000
embed_dim = 300
config = ConfigGen(vocab_dim, out_dim, cells, birnn, dropout, fc1_dim, fc2_dim,
embed_dim, hidden_size)
model = RNNClassifier(config)
model.embed.weight.data = inputs.vocab.vectors
# TODO - convert to cuda if required
criterion = nn.CrossEntropyLoss()
opt = optim.Adam(model.parameters(), lr=lr)
iterations = 0
start = time.time()
best_dev_acc = -1
train_iter.repeat = False
model.train()
for epoch in range(epochs):
train_iter.init_epoch()
n_correct, n_total = 0, 0
dataset_test = nlp.data.TSVDataset("data/test_data.txt",
field_indices=[1, 2],
num_discard_samples=1)
tokenizer = get_tokenizer()
tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)
if model_mode.lower() == "bert":
data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)
data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)
model = BERTClassifier(bertmodel, dr_rate=0.5).to(device)
elif model_mode.lower() == "rnn":
data_train = RNNDataset(dataset_train, 0, 1, tokenizer, max_len, True,
False)
data_test = RNNDataset(dataset_test, 0, 1, tokenizer, max_len, True, False)
model = RNNClassifier(bertmodel, dr_rate=0.5).to(device)
train_dataloader = torch.utils.data.DataLoader(data_train,
batch_size=batch_size,
num_workers=5)
test_dataloader = torch.utils.data.DataLoader(data_test,
batch_size=batch_size,
num_workers=5)
# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
V = 1000 # vocabulary size
emb_size = 5 # embedding size
bptt = 4 # sequence length
batch_size = 2 # batch size
# choose dropouts
dropoute = 0.2 # dropout to the embedding layer
dropouti = 0.2 # dropout to the inputs of the RNN
dropouto = 0.3 # dropout to the outputs of the RNN
dropoutw = 0.4 # dropout to the recurrent layers of the RNN
# dummy input tensor of shape (batch_size, seq_len)
words = np.random.random_integers(low=0,
high=V - 1,
size=(batch_size, bptt))
words = torch.LongTensor(words)
model = RNNClassifier(ntokens=V,
nclasses=3,
emb_size=emb_size,
dropoute=dropoute,
rnn_size=[6, 7, 8],
rnn_layers=3,
rnn_dropouti=dropouti,
rnn_dropouto=dropouto,
rnn_dropoutw=dropoutw)
print(model)
# forward pass
model(words)
args.hidden_dim,
args.embedding_dim,
len(sst_train.vocab),
padding_index,
args.dropout,
) if not args.use_lstm else LSTM(
args.hidden_dim,
args.embedding_dim,
len(sst_train.vocab),
padding_index,
args.dropout,
))
# classifier wrapper
model = RNNClassifier(
args.hidden_dim,
len(SSTClassificationDataset.labels_to_string),
rnn,
args.dropout,
)
# get training things set up
data_size = len(sst_train)
batch_size = args.batch_size
starts = list(range(0, data_size, batch_size))
optimizer = torch.optim.Adam(rnn.parameters(),
weight_decay=args.l2,
lr=args.lr)
best_loss = float("inf")
best_model = None
for epoch in range(args.num_epochs):
running_loss = 0.0
out_dim = len(answers.vocab)
cells = 2
# TODO - remove bidirectional RNN for simpleRNN
birnn = True
lr = 0.01
epochs = 10
if birnn:
cells *= 2
dropout = 0.5
fc1_dim = 50
fc2_dim = 3
hidden_size = 1000
embed_dim = 300
config = ConfigGen(vocab_dim, out_dim, cells, birnn, dropout, fc1_dim, fc2_dim,
embed_dim, hidden_size)
model = RNNClassifier(config)
model.embed.weight.data = inputs.vocab.vectors
# TODO - convert to cuda if required
criterion = nn.CrossEntropyLoss()
def init_weights(m):
if type(m) == nn.Linear:
torch.nn.init.xavier_uniform_(m.weight)
m.bias.data.fill_(0.01)
model.apply(init_weights) # xavier init: trying to avoid vanishing gradient
opt = optim.Adam(model.parameters(), lr=lr)
def test(epoch):
correct = 0
with torch.no_grad():
for i, (names, countries) in enumerate(test_loader, 1):
inputs, seq_lengths, target = util.make_tensors(names, countries)
output = model(inputs, seq_lengths)
pred= output.max(dim=1, keepdim = True)[1]
correct += pred.eq(target.view_as(pred)).sum().item()
print('Test: acc {:.4f}'.format(correct/len(test_loader.dataset)))
if __name__ == '__main__':
train_dataset = NameDataset(train=True)
test_dataset = NameDataset(train=False)
train_loader = DataLoader(train_dataset, batch_size=Config.batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=Config.batch_size, shuffle=False)
model = RNNClassifier(Config.n_chars, Config.hidden_size, train_dataset.get_country_num(),Config.n_layer)
print(model)
if Config.use_gpu:
device = torch.device("cuda:0")
model = model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr = Config.lr)
for epoch in range(1, Config.n_epochs+1):
train(epoch)
test(epoch)