def predict():
device = "cuda:0" if torch.cuda.is_available() else "cpu"
df = pd.read_csv(config.TEST_PATH, header=None)
dataset = PicDataset(df.loc[:, 1:])
preds = np.zeros((len(dataset), 256))
for i in range(5):
temp = np.zeros((len(dataset), 256))
model = DNN()
model.load_state_dict(torch.load(f'./models/model_{i}.bin'))
model.to(device)
model.eval()
for j in range(len(dataset)):
x, _ = dataset[j]
x = x.to(device)
y = model(x)
temp[j, :] = y.detach().cpu().numpy()
preds += temp
preds /= 5
df = pd.DataFrame(np.concatenate([np.arange(1, 921).reshape(-1, 1), preds], axis=1), columns=np.arange(257))
df[0] = df[0].astype('int')
df.to_csv('./predictions.csv', index=False, header=False)
def train(args, config, io):
train_loader, validation_loader = get_loader(args, config)
device = torch.device("cuda" if args.cuda else "cpu")
# print(len(train_loader), len(validation_loader))
#Try to load models
model = DNN(args).to(device)
"""if device == torch.device("cuda"):
model = nn.DataParallel(model)"""
if args.model_path != "":
model.load_state_dict(torch.load(args.model_path))
# for para in list(model.parameters())[:-5]:
# para.requires_grad=False
# print(model)
if args.use_sgd:
# print("Use SGD")
opt = optim.SGD(model.parameters(),
lr=args.lr * 100,
momentum=args.momentum,
weight_decay=1e-4)
else:
# print("Use Adam")
opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
"""opt = optim.Adam([
{'params': list(model.parameters())[:-1], 'lr':args.lr/50, 'weight_decay': 1e-4},
{'params': list(model.parameters())[-1], 'lr':args.lr, 'weight_decay': 1e-4}
])
"""
scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr)
criterion = nn.MSELoss()
best_test_loss = 9999999.
for epoch in range(args.epochs):
startTime = time.time()
####################
# Train
####################
train_loss = 0.0
train_dis = 0.0
count = 0.0
model.train()
for data, label in train_loader:
data, label = data.to(device), label.to(device)
data = drop(jitter(data, device), device)
# data = jitter(data, device, delta=0.05)
batch_size = data.shape[0]
logits = model(data)
loss = criterion(logits, label)
opt.zero_grad()
loss.backward()
opt.step()
dis = distance(logits, label)
count += batch_size
train_loss += loss.item() * batch_size
train_dis += dis.item() * batch_size
scheduler.step()
outstr = 'Train %d, loss: %.6f, distance: %.6f' % (
epoch, train_loss * 1.0 / count, train_dis * 1.0 / count)
io.cprint(outstr)
####################
# Evaluation
####################
test_loss = 0.0
test_dis = 0.0
count = 0.0
model.eval()
with torch.no_grad():
for data, label in validation_loader:
data, label = data.to(device), label.to(device)
batch_size = data.shape[0]
logits = model(data)
loss = criterion(logits, label)
dis = distance(logits, label)
count += batch_size
test_loss += loss.item() * batch_size
test_dis += dis.item() * batch_size
outstr = 'Test %d, loss: %.6f, distance: %.6f' % (
epoch, test_loss * 1.0 / count, test_dis * 1.0 / count)
io.cprint(outstr)
if test_loss <= best_test_loss:
best_test_loss = test_loss
torch.save(model.state_dict(),
'checkpoints/%s/models/model.t7' % args.exp_name)
torch.save(model, (config.root + config.model_path))
io.cprint('Time: %.3f sec' % (time.time() - startTime))
def train():
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
data_dict, topic_dict = dh.load_data(
) # data_dict, [group2topic, mem2topic]
train_data, train_label, dev_data, dev_label, test_data, test_label = dh.data_split(
data_dict, topic_dict)
train_dataset = dh.Dataset(train_data, train_label)
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
dev_dataset = dh.Dataset(dev_data, dev_label)
dev_loader = DataLoader(dev_dataset, batch_size=128, shuffle=True)
lambda1 = lambda epoch: (
epoch / args.warm_up_step
) if epoch < args.warm_up_step else 0.5 * (math.cos(
(epoch - args.warm_up_step) /
(args.n_epoch * len(train_dataset) - args.warm_up_step) * math.pi) + 1)
model = DNN(args).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.init_lr)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer,
len(train_loader) * args.n_epoch)
global_step = 0
best_f1 = 0.
loss_deq = collections.deque([], args.report_step)
for epoch in range(args.n_epoch):
for batch in tqdm(train_loader):
optimizer.zero_grad()
inputs = batch['input'].to(device)
group_topic = batch['group_topic'].to(device)
mem_topic = batch['mem_topic'].to(device)
labels = batch['label'].to(device)
output = model(inputs, mem_topic, group_topic, label=labels)
loss = output[0]
loss.backward()
loss_deq.append(loss.item())
optimizer.step()
scheduler.step()
global_step += 1
if global_step % args.report_step == 0:
logger.info('loss: {}, lr: {}, epoch: {}'.format(
np.average(loss_deq).item(),
optimizer.param_groups[0]['lr'],
global_step / len(train_dataset)))
if global_step % args.eval_step == 0:
model.eval()
eval_result = evaluation(model,
data_loader=dev_loader,
device=device)
logger.info(eval_result)
if eval_result['f1'] > best_f1:
torch.save(model,
'./model/{}/torch.pt'.format(args.task_name))
best_f1 = eval_result['f1']
model.train()
loss.backward()
optimizer.step()
if (idx + 1) % 100 == 0:
print(
'Epoch: [{}/{}], Step: [{}/{}], Loss: {:.4f}, Acc: {:.4f}'.
format(epoch + 1, args.epochs, idx + 1, len(train_loader),
loss.item(), 100 * correct / total))
# 保存模型参数
torch.save(
model.state_dict(),
os.path.join(
'./log', '{}_{}_{}.ckpt'.format(args.model, args.dataset,
args.epochs)))
model.eval()
with torch.no_grad():
correct = 0
total = 0
for idx, (x, y) in enumerate(test_loader):
x, y = x.to(device), y.to(device)
y_pred = model(x)
_, y_pred = torch.max(y_pred.data, 1)
total += y.size(0)
correct += (y_pred == y).sum().item()
# print(result)
if idx % 100 == 0:
get_grid(x.cpu().numpy(), args, args.epochs, idx)
print(y_pred.data.cpu().numpy(), y.data.cpu().numpy)
print('Test Acc: {:.4f}%, Model: {}, Epochs: {}'.format(
correct / total * 100, args.model, args.epochs))
def train(args, config, io):
train_loader, validation_loader, unlabelled_loader = get_loader(
args, config)
device = torch.device("cuda" if args.cuda else "cpu")
#Try to load models
model = DNN(args).to(device)
ema_model = DNN(args).to(device)
for param in ema_model.parameters():
param.detach_()
if device == torch.device("cuda"):
model = nn.DataParallel(model)
ema_model = nn.DataParallel(ema_model)
if args.model_path != "":
model.load_state_dict(torch.load(args.model_path))
ema_model.load_state_dict(torch.load(args.model_path))
if args.use_sgd:
print("Use SGD")
opt = optim.SGD(model.parameters(),
lr=args.lr * 100,
momentum=args.momentum,
weight_decay=1e-4)
else:
print("Use Adam")
opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr)
criterion = nn.MSELoss()
consistency_criterion = nn.MSELoss()
best_test_loss = 9999999.
global_step = 0
for epoch in range(args.epochs):
startTime = time.time()
####################
# Train
####################
train_loss = 0.0
count = 0.0
model.train()
ema_model.train()
i = -1
for (data, label), (u, _) in zip(cycle(train_loader),
unlabelled_loader):
i = i + 1
if data.shape[0] != u.shape[0]:
bt_size = np.minimum(data.shape[0], u.shape[0])
data = data[0:bt_size]
label = label[0:bt_size]
u = u[0:bt_size]
data, label, u = data.to(device), label.to(device), u.to(device)
batch_size = data.shape[0]
logits = model(data)
class_loss = criterion(logits, label)
u_student = jitter(u, device)
u_teacher = jitter(u, device)
logits_unlabeled = model(u_student)
ema_logits_unlabeled = ema_model(u_teacher)
ema_logits_unlabeled = Variable(ema_logits_unlabeled.detach().data,
requires_grad=False)
consistency_loss = consistency_criterion(logits_unlabeled,
ema_logits_unlabeled)
if epoch < args.consistency_rampup_starts:
consistency_weight = 0.0
else:
consistency_weight = get_current_consistency_weight(
args, args.final_consistency, epoch, i,
len(unlabelled_loader))
consistency_loss = consistency_weight * consistency_loss
loss = class_loss + consistency_loss
opt.zero_grad()
loss.backward()
opt.step()
global_step += 1
# print(global_step)
update_ema_variables(model, ema_model, args.ema_decay, global_step)
count += batch_size
train_loss += loss.item() * batch_size
scheduler.step()
outstr = 'Train %d, loss: %.6f' % (epoch, train_loss * 1.0 / count)
io.cprint(outstr)
####################
# Evaluation
####################
test_loss = 0.0
count = 0.0
model.eval()
ema_model.eval()
for data, label in validation_loader:
data, label = data.to(device), label.to(device)
batch_size = data.shape[0]
logits = ema_model(data)
loss = criterion(logits, label)
count += batch_size
test_loss += loss.item() * batch_size
outstr = 'Test %d, loss: %.6f' % (epoch, test_loss * 1.0 / count)
io.cprint(outstr)
if test_loss <= best_test_loss:
best_test_loss = test_loss
torch.save(ema_model.state_dict(),
'checkpoints/%s/models/model.t7' % args.exp_name)
torch.save(ema_model, (config.root + config.model_path))
io.cprint('Time: %.3f sec' % (time.time() - startTime))
