def train(model,batch,EPOCH,device,data,dataset_idx,optimizer,foward_name):
input_id,y_true = data
train_idx,val_idx,test_idx = dataset_idx
model.train()
criterion = nn.CrossEntropyLoss()
for n_epoch in range(EPOCH):
random.shuffle(train_idx)
n=0
optimizer.zero_grad()
loss_epoch=list()
for n_kousin in range(len(train_idx)//batch+1):
batch_input_idx=train_idx[n_kousin*batch:min((n_kousin+1)*batch,len(train_idx))]
train_input=list()
train_y=list()
for idx in batch_input_idx:
train_input.append(input_id[idx])
train_y.append(y_true[idx])
batch_input_idx_tensor=torch.tensor(train_input).to(device)
train_y_tensor=torch.tensor(train_y).to(device)
output=foward(model,foward_name,batch_input_idx_tensor)
loss=criterion(output,train_y_tensor)
loss_epoch.append(loss.item())
loss.backward()
optimizer.step()
optimizer.zero_grad()
print("epoch {} : {}".format(n_epoch,np.array(loss_epoch).mean()))
if n_epoch%10==0:
eval(model,val_idx,input_id,y_true,device,foward_name)
model.train()
def train_model(dataloader, model, criterion, optimizer, device, num_epochs,
dataset_size):
model.to(device)
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
for phase in ['train', 'test']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
for inputs, labels in tqdm(dataloaders[phase]):
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
_, pred = torch.max(outputs, 1)
loss = criterion(outputs, labels)
if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(pred == labels.data)
epoch_loss = running_loss / dataset_size[phase]
epoch_acc = running_corrects.double() / dataset_size[phase]
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
epoch_acc))
if phase == 'test' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
torch.save(
best_model_wts,
osp.join(Config['root_path'], Config['checkpoint_path'],
'model.pth'))
print('Model saved at: {}'.format(
osp.join(Config['root_path'], Config['checkpoint_path'],
'model.pth')))
time_elapsed = time.time() - since
print('Time taken to complete training: {:0f}m {:0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best acc: {:.4f}'.format(best_acc))
def train(args, model, train_dataset, epoch):
with torch.enable_grad():
# Turn on training mode which enables dropout.
model.train()
total_loss = 0
start_time = time.time()
hidden = model.init_hidden(args.batch_size)
for batch, i in enumerate(range(0, train_dataset.size(0) - 1, args.bptt)):
inputSeq, targetSeq = get_batch(args,train_dataset, i)
# inputSeq: [ seq_len * batch_size * feature_size ]
# targetSeq: [ seq_len * batch_size * feature_size ]
# Starting each batch, we detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
hidden = model.repackage_hidden(hidden)
hidden_ = model.repackage_hidden(hidden)
optimizer.zero_grad()
'''Loss1: Free running loss'''
outVal = inputSeq[0].unsqueeze(0)
outVals=[]
hids1 = []
for i in range(inputSeq.size(0)):
outVal, hidden_, hid = model.forward(outVal, hidden_,return_hiddens=True)
outVals.append(outVal)
hids1.append(hid)
outSeq1 = torch.cat(outVals,dim=0)
hids1 = torch.cat(hids1,dim=0)
loss1 = criterion(outSeq1.contiguous().view(args.batch_size,-1), targetSeq.contiguous().view(args.batch_size,-1))
'''Loss2: Teacher forcing loss'''
outSeq2, hidden, hids2 = model.forward(inputSeq, hidden, return_hiddens=True)
loss2 = criterion(outSeq2.contiguous().view(args.batch_size, -1), targetSeq.contiguous().view(args.batch_size, -1))
'''Loss3: Simplified Professor forcing loss'''
loss3 = criterion(hids1.contiguous().view(args.batch_size,-1), hids2.contiguous().view(args.batch_size,-1).detach())
'''Total loss = Loss1+Loss2+Loss3'''
loss = loss1+loss2+loss3
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
total_loss += loss.item()
if batch % args.log_interval == 0 and batch > 0:
cur_loss = total_loss / args.log_interval
elapsed = time.time() - start_time
print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.4f} | '
'loss {:5.5f} '.format(
epoch, batch, len(train_dataset) // args.bptt,
elapsed * 1000 / args.log_interval, cur_loss))
total_loss = 0
start_time = time.time()
def train(args, model, train_dataset,epoch):
with torch.enable_grad():
# Turn on training mode which enables dropout.
model.train()
total_loss = 0
start_time = time.time()
hidden = model.init_hidden(args.batch_size)
for batch, i in enumerate(range(0, train_dataset.size(0) - 1, args.bptt)):
inputSeq, targetSeq = get_batch(args,train_dataset, i)
# inputSeq: [ seq_len * batch_size * feature_size ]
# targetSeq: [ seq_len * batch_size * feature_size ]
# Starting each batch, we detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
hidden = model.repackage_hidden(hidden)
hidden_ = model.repackage_hidden(hidden)
optimizer.zero_grad()
'''Loss1: Free running loss'''
outVal = inputSeq[0].unsqueeze(0)
outVals=[]
hids1 = []
for i in range(inputSeq.size(0)):
outVal, hidden_, hid = model.forward(outVal, hidden_,return_hiddens=True)
outVals.append(outVal)
hids1.append(hid)
outSeq1 = torch.cat(outVals,dim=0)
hids1 = torch.cat(hids1,dim=0)
loss1 = criterion(outSeq1.view(args.batch_size,-1), targetSeq.view(args.batch_size,-1))
'''Loss2: Teacher forcing loss'''
outSeq2, hidden, hids2 = model.forward(inputSeq, hidden, return_hiddens=True)
loss2 = criterion(outSeq2.view(args.batch_size, -1), targetSeq.view(args.batch_size, -1))
'''Loss3: Simplified Professor forcing loss'''
loss3 = criterion(hids1.view(args.batch_size,-1), hids2.view(args.batch_size,-1).detach())
'''Total loss = Loss1+Loss2+Loss3'''
loss = loss1+loss2+loss3
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
total_loss += loss.item()
if batch % args.log_interval == 0 and batch > 0:
cur_loss = total_loss / args.log_interval
elapsed = time.time() - start_time
print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.4f} | '
'loss {:5.2f} '.format(
epoch, batch, len(train_dataset) // args.bptt,
elapsed * 1000 / args.log_interval, cur_loss))
total_loss = 0
start_time = time.time()
def test_learn_online(self):
e = threading.Event()
try:
urlemg = 'tcp://192.168.0.20:5555'
emgp = online.publisher.EmgPublisher(urlemg, abort=e)
emgs = online.sources.FileSource(emgp, 4000, 'emg_data', abort=e)
emgsub = online.subscriber.EmgSubscriber(urlemg, abort=e)
emgiter = online.subscriber.array_iterator(ArrayMessage, emgsub)
urlkin = 'tcp://192.168.0.20:5556'
kinp = online.publisher.KinPublisher(urlkin, abort=e)
kins = online.sources.FileSource(kinp, 500, 'kin_data', abort=e)
kinsub = online.subscriber.KinSubscriber(urlkin, abort=e)
kiniter = online.subscriber.array_iterator(ArrayMessage, kinsub)
#sigmoid = lambda X: 1 / (1 + np.exp(X))
identity = lambda X: X
model = online.regression.LinReg(
dim_in=ArrayMessage.duration * kins.samplingrate * 5,
dim_out=ArrayMessage.duration * kins.samplingrate * 3,
dim_basis=ArrayMessage.duration * kins.samplingrate * 5,
basis_fcts=identity
)
print 'Calculated shapes: dim_in={}, dim_out={}, dim_basis={}'.format(
ArrayMessage.duration * kins.samplingrate * 5,
ArrayMessage.duration * kins.samplingrate * 3,
ArrayMessage.duration * kins.samplingrate * 5
)
print 'start threads'
emgp.start()
emgs.start()
emgsub.start()
kinp.start()
kins.start()
kinsub.start()
count = 0
while count < 1000:
Z = kiniter.next().data[:, [2,7,9]]
X = emgiter.next().data
X_ = X.reshape(Z.shape[0], -1, X.shape[1])
X = np.mean(X_, axis=1)
Z = Z.flatten().reshape(1, -1)
X = X.flatten().reshape(1, -1)
model.train(X,Z)
if count % 100 == 0:
print '{}\t\t{}'.format(count, model.loss(X, Z))
count += 1
e.set()
except Exception as ex:
e.set()
raise ex
e.set()
def train(args, model, train_dataset):
# Turn on training mode which enables dropout.
model.train()
total_loss = 0
start_time = time.time()
hidden = model.init_hidden(args.batch_size)
for batch, i in enumerate(range(0, train_dataset.size(0) - 1, args.bptt)):
inputSeq, targetSeq = get_batch(train_dataset, i)
# inputSeq: [ seq_len * batch_size * feature_size ]
# targetSeq: [ seq_len * batch_size * feature_size ]
# Starting each batch, we detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
hidden = model.repackage_hidden(hidden)
optimizer.zero_grad()
USE_TEACHER_FORCING = random.random() < args.teacher_forcing_ratio
if USE_TEACHER_FORCING:
outSeq, hidden = model.forward(inputSeq, hidden)
else:
outVal = inputSeq[0].unsqueeze(0)
outVals = []
for i in range(inputSeq.size(0)):
outVal, hidden = model.forward(outVal, hidden)
outVals.append(outVal)
outSeq = torch.cat(outVals, dim=0)
#print('outSeq:',outSeq.size())
#print('targetSeq:', targetSeq.size())
loss = criterion(outSeq.view(args.batch_size, -1),
targetSeq.view(args.batch_size, -1))
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
optimizer.step()
# for p in model2_for_timeDiff.parameters():
# p.data.add_(-lr, p.grad.data)
total_loss += loss.data
if batch % args.log_interval == 0 and batch > 0:
cur_loss = total_loss[0] / args.log_interval
elapsed = time.time() - start_time
print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.4f} | '
'loss {:5.2f} '.format(epoch, batch,
len(train_dataset) // args.bptt,
elapsed * 1000 / args.log_interval,
cur_loss))
total_loss = 0
start_time = time.time()
def test_learn_online(self):
e = threading.Event()
try:
urlemg = 'tcp://192.168.0.20:5555'
emgp = online.publisher.EmgPublisher(urlemg, abort=e)
emgs = online.sources.FileSource(emgp, 4000, 'emg_data', abort=e)
emgsub = online.subscriber.EmgSubscriber(urlemg, abort=e)
emgiter = online.subscriber.array_iterator(ArrayMessage, emgsub)
urlkin = 'tcp://192.168.0.20:5556'
kinp = online.publisher.KinPublisher(urlkin, abort=e)
kins = online.sources.FileSource(kinp, 500, 'kin_data', abort=e)
kinsub = online.subscriber.KinSubscriber(urlkin, abort=e)
kiniter = online.subscriber.array_iterator(ArrayMessage, kinsub)
#sigmoid = lambda X: 1 / (1 + np.exp(X))
identity = lambda X: X
model = online.regression.LinReg(
dim_in=ArrayMessage.duration * kins.samplingrate * 5,
dim_out=ArrayMessage.duration * kins.samplingrate * 3,
dim_basis=ArrayMessage.duration * kins.samplingrate * 5,
basis_fcts=identity)
print 'Calculated shapes: dim_in={}, dim_out={}, dim_basis={}'.format(
ArrayMessage.duration * kins.samplingrate * 5,
ArrayMessage.duration * kins.samplingrate * 3,
ArrayMessage.duration * kins.samplingrate * 5)
print 'start threads'
emgp.start()
emgs.start()
emgsub.start()
kinp.start()
kins.start()
kinsub.start()
count = 0
while count < 1000:
Z = kiniter.next().data[:, [2, 7, 9]]
X = emgiter.next().data
X_ = X.reshape(Z.shape[0], -1, X.shape[1])
X = np.mean(X_, axis=1)
Z = Z.flatten().reshape(1, -1)
X = X.flatten().reshape(1, -1)
model.train(X, Z)
if count % 100 == 0:
print '{}\t\t{}'.format(count, model.loss(X, Z))
count += 1
e.set()
except Exception as ex:
e.set()
raise ex
e.set()
def train():
# Turn on training mode which enables dropout.
model.train()
total_loss = 0.
start_time = time.time()
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(args.batch_size)
for batch, i in enumerate(range(0, train_data.size(0) - 1, args.bptt)):
data, targets = get_batch(train_data, i)
# Starting each batch, we detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
hidden = repackage_hidden(hidden)
optimizer.zero_grad()
output, hidden = model(data, hidden)
loss = criterion(output.view(-1, ntokens), targets)
loss.backward()
optimizer.step()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
# for p in model.parameters():
# p.data.add_(-lr, p.grad.data)
total_loss += loss.item()
if batch % args.log_interval == 0 and batch > 0:
cur_loss = total_loss / args.log_interval
elapsed = time.time() - start_time
try:
print(
'| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | '
'loss {:5.2f} | ppl {:8.2f}'.format(
epoch, batch,
len(train_data) // args.bptt, lr,
elapsed * 1000 / args.log_interval, cur_loss,
math.exp(cur_loss)))
except OverflowError as err:
print(
'| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | '
'loss {:5.2f} | ppl {:8.2f}'.format(
epoch, batch,
len(train_data) // args.bptt, lr,
elapsed * 1000 / args.log_interval, cur_loss,
math.exp(cur_loss)))
total_loss = 0
start_time = time.time()
data.to("cpu")
targets.to("cpu")
def train(epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item()))
def _fit(self, model, X, Z, batchsize=10, valeval=50, alpha=0.01):
iter = 0
best_loss = 1000000
best_W = None
while iter + batchsize < X.shape[0]:
model.train(X[iter:iter+batchsize], Z[iter:iter+batchsize],alpha)
if iter/batchsize % 50 == 0:
loss = model.loss(X[iter:iter+batchsize], Z[iter:iter+batchsize])
if loss < best_loss:
best_loss = loss
best_W = model._W
print 'best loss: {} -- current loss: {}'.format(best_loss, loss)
iter += batchsize
model._W = best_W
def train(epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
if use_cuda:
data, target = data.cuda(), target.cuda()
optimizer.zero_grad()
output = model(data)
criterion = torch.nn.CrossEntropyLoss(reduction='mean')
loss = criterion(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data.item()))
def run_expreminent(look_forward, hidden_size, batch_size, epochs, dropout,
dataset):
x, y = get_sentiments_prices(dataset['twitter_sentiments'],
dataset["reddit_sentiments"],
dataset["coin_price"], look_forward)
for i in range(x.shape[1]):
x[:, i] = normalize_array(x[:, i])
# split into train and test sets
train_x, test_x = split(x)
train_y, test_y = split(y)
train_x = np.reshape(train_x,
(train_x.shape[0], look_forward, train_x.shape[1]))
test_x = np.reshape(test_x,
(test_x.shape[0], look_forward, test_x.shape[1]))
model = create_model(hidden_size=hidden_size,
look_forward=look_forward,
dropout=dropout)
model = train(model,
train_x,
train_y,
batch_size=batch_size,
epochs=epochs)
y_pred = test(model, test_x)
score = evaluate(test_y, y_pred)
print('Test Score: %.2f RMSE' % score)
return score
def train(model, iterator, optimizer, criterion, clip):
model.train()
epoch_loss = 0
len_iterator = 0
for i, batch in enumerate(iterator):
src = batch.src
trg = batch.trg
optimizer.zero_grad()
output = model(src, trg)
loss = criterion(output, trg)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
optimizer.step()
epoch_loss += loss.item()
len_iterator += 1
return epoch_loss / len_iterator
def train(model):
model = model.train()
optimizer.zero_grad()
losses = []
accurate_preds = 0
all_targets = []
all_predictions = []
for d in train_loader:
inputs = d['input_ids'].to(device)
masks = d['attention_mask'].to(device)
all_targets.extend(list(d['targets'].squeeze().numpy()))
targets = d['targets'].to(device)
outputs = model(
input_ids=inputs,
attention_mask=masks
)
_, preds = torch.max(outputs, dim=1)
loss = criterion(outputs, targets)
all_predictions.extend(list(preds.cpu().squeeze().numpy()))
accurate_preds += torch.sum(preds == targets)
losses.append(loss.item())
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
optimizer.step()
scheduler.step()
return accurate_preds / train_len, np.mean(losses), all_targets, all_predictions
def _fit(self, model, X, Z, batchsize=10, valeval=50, alpha=0.01):
iter = 0
best_loss = 1000000
best_W = None
while iter + batchsize < X.shape[0]:
model.train(X[iter:iter + batchsize], Z[iter:iter + batchsize],
alpha)
if iter / batchsize % 50 == 0:
loss = model.loss(X[iter:iter + batchsize],
Z[iter:iter + batchsize])
if loss < best_loss:
best_loss = loss
best_W = model._W
print 'best loss: {} -- current loss: {}'.format(
best_loss, loss)
iter += batchsize
model._W = best_W
def train_model(dataloader, model, optimizer, device, num_epochs,
dataset_size):
model.to(device)
for epoch in range(num_epochs):
print('-' * 15)
print('Epoch {}/{}'.format(epoch + 1, num_epochs))
for phase in ['train', 'val']: #train and validate every epoch
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
for i in tqdm(range(len(dataloader[phase].dataset[0]))):
inputs = dataloader[phase].dataset[0][i]
#print(inputs.shape)
labels = dataloader[phase].dataset[1][i]
#print(labels.shape)
inputs = inputs.unsqueeze(0)
labels = labels.unsqueeze(0)
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
loss = criterion(outputs, labels)
if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
epoch_loss = running_loss / dataset_size[phase]
print('{} Loss: {:.4f} '.format(phase, epoch_loss))
# save the model
#saved_model = copy.deepcopy(model.state_dict())
with open(osp.join(Config['path'], "my_model.pth"), "wb") as output_file:
torch.save(model.state_dict(), output_file)
def train(epoch):
print('#' * 15)
print('Epoch {}, Latent Size {}'.format(epoch, model.latent_size))
print('#' * 15)
model.train()
for index, (x, _) in enumerate(loader):
x = x.mean(dim=1, keepdim=True).to(device)
optimizer.zero_grad()
x_generated, mu, logvar = model(x)
loss = get_loss(x_generated, x, mu, logvar)
loss.backward()
optimizer.step()
if index % 100 == 0:
print('Loss at iteration {0}: {1:.4f}'.format(index, loss.item()))
if epoch == 4:
filename = 'epoch{}_ls{}.pkl'.format(epoch, model.latent_size)
torch.save(model.state_dict(), os.path.join(weights_dir, filename))
if epoch < 4:
scheduler.step()
def train():
model.train() # Turn on the train mode
total_loss = 0.0
start_time = time.time()
batches = np.random.permutation(
range(0,
train_data.size(0) - seq_len, seq_len))
for batch_counter, i in enumerate(batches):
data, targets = get_batch(train_data, i)
optimizer.zero_grad()
src_mask = model.generate_square_subsequent_mask(
data.size(0)).to(device)
output = model(
data,
src_mask=src_mask,
)
loss = criterion(output, targets)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
optimizer.step()
total_loss += loss.item()
log_interval = 100
if batch_counter % log_interval == 0 and batch_counter > 0:
cur_loss = total_loss / log_interval
elapsed = time.time() - start_time
print("| epoch {:3d} | {:5d}/{:5d} batches | {:5.6f}"
"| ms/batch {:5.4f} | "
"loss {:5.4f}".format(
epoch,
batch_counter,
len(train_data) // seq_len,
scheduler.get_last_lr()[0],
elapsed * 1000 / log_interval,
cur_loss,
))
total_loss = 0
start_time = time.time()
def train(epoch):
model.train()
utils.adjust_learning_rate(optimizer, epoch, args.step_size, args.lr,
args.gamma)
print('epoch =', epoch, 'lr = ', optimizer.param_groups[0]['lr'])
for iteration, (lr_tensor, hr_tensor) in enumerate(training_data_loader,
1):
if args.cuda:
lr_tensor = lr_tensor.to(device) # ranges from [0, 1]
hr_tensor = hr_tensor.to(device) # ranges from [0, 1]
optimizer.zero_grad()
sr_tensor = model(lr_tensor)
loss_l1 = l1_criterion(sr_tensor, hr_tensor)
loss_sr = loss_l1
loss_sr.backward()
optimizer.step()
# if iteration % 1000 == 0:
print("===> Epoch[{}]({}/{}): Loss_l1: {:.5f}".format(
epoch, iteration, len(training_data_loader), loss_l1.item()))
def train(epoch):
global niter
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
niter += 1
if use_cuda:
data, target = data.cuda(), target.cuda()
optimizer.zero_grad()
output = model(data)
criterion = torch.nn.CrossEntropyLoss(reduction='elementwise_mean')
loss = criterion(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data.item()))
writer.add_scalar('data/train_loss', loss.data.item(), niter)
writer.add_scalar('data/learning_rate',
optimizer.state_dict()['param_groups'][0]['lr'],
niter)
def train(train_loader, model, optimizer, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
model.train()
end = time.time()
for i, (im, gt) in (enumerate(train_loader)):
adjust_learning_rate(optimizer, epoch)
# measure data loading time
data_time.update(time.time() - end)
optimizer.zero_grad()
loss, scores, boxes = model((im, gt))
loss.backward()
optimizer.step()
losses.update(loss.item(), im.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
#'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
#'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'
.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses,
#top1=top1, top5=top5
))
break;
def train_model(dataloader, model, criterion, optimizer, device, num_epochs, dataset_size):
model.to(device)
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
acc_list = []
loss_list = []
test_acc_list= []
test_loss_list = []
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
for phase in ['train', 'test']:
if phase=='train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
for input1, input2, labels in tqdm(dataloaders[phase], position=0, leave=True):
input1 = input1.to(device)
input2 = input2.to(device)
labels = labels.to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
outputs = model(input1, input2)
outputs = torch.reshape(outputs, (outputs.shape[0],))
outputs = outputs.type(torch.DoubleTensor)
labels = labels.type(torch.DoubleTensor)
pred = []
for i in outputs:
if i>0.5:
pred.append(0)
else:
pred.append(1)
pred = torch.FloatTensor(pred)
loss = criterion(outputs,labels)
if phase=='train':
loss.backward()
optimizer.step()
running_loss += loss.item() * input1.size(0)
running_corrects += torch.sum(pred==labels.data)
epoch_loss = running_loss / dataset_size[phase]
epoch_acc = running_corrects.double() / dataset_size[phase]
if phase=='train':
acc_list.append(epoch_acc)
loss_list.append(epoch_loss)
elif phase=='test':
test_acc_list.append(epoch_acc)
test_loss_list.append(epoch_loss)
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
if phase=='test' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
torch.save(best_model_wts, osp.join(Config['root_path'], Config['checkpoint_path'], 'model.pth'))
print('Model saved at: {}'.format(osp.join(Config['root_path'], Config['checkpoint_path'], 'model.pth')))
time_elapsed = time.time() - since
print('Time taken to complete training: {:0f}m {:0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best acc: {:.4f}'.format(best_acc))
np.savetxt('acc_list.txt',acc_list)
np.savetxt('test_acc_list.txt',test_acc_list)
np.savetxt('loss_list.txt',loss_list)
np.savetxt('test_loss_list.txt',test_loss_list)
def train_hypergcn(args):
if args.dataset_name in ['citeseer', 'cora']:
if args.do_svd:
data_path = 'data/citeseer.pt' if args.dataset_name == 'citeseer' else 'data/cora_author_10cls300.pt'
else:
data_path = 'data/citeseer6cls3703.pt' if args.dataset_name == 'citeseer' else 'data/cora_author_10cls1000.pt'
args = hypergraph.gen_data_cora(args,
data_path=data_path,
flip_edge_node=False)
elif args.dataset_name in ['dblp', 'pubmed']:
data_path = 'data/pubmed_data.pt' if args.dataset_name == 'pubmed' else 'data/dblp_data.pt'
args = hypergraph.gen_data_dblp(args, data_path=data_path)
else:
raise Exception('dataset {} not supported'.format(args.dataset_name))
#data_path = 'data/citeseer.pt' if args.dataset_name == 'citeseer' else 'data/cora_author.pt'
#args = hypergraph.gen_data_cora(args, data_path=data_path)
#dict_keys(['hypergraph', 'features', 'labels', 'n'])
if args.predict_edge:
hyp_struct = create_hypergcn_struct(args)
n_edge = len(args.edge_X)
n_node = len(args.all_labels)
labels = torch.cat([args.all_labels, args.edge_classes], 0)
X = torch.cat([args.v, args.edge_X], 0)
#dict_keys(['hypergraph', 'features', 'labels', 'n'])
#pdb.set_trace()
datadict = {
'hypergraph': hyp_struct,
'features': args.v,
'labels': labels,
'n': len(args.v) + n_edge
}
n_labels = max(
1, math.ceil(n_edge * utils.get_label_percent(args.dataset_name)))
train_idx = torch.from_numpy(
np.random.choice(n_edge, size=(n_labels, ),
replace=False)).to(torch.int64)
all_idx = torch.LongTensor(list(range(n_edge)))
all_idx[train_idx] = -1
test_idx = all_idx[all_idx > -1]
train_idx += n_node
test_idx += n_node #len(args.all_labels)
hg_args = hgcn_config.parse()
else:
hyp_struct = create_hypergcn_struct(args)
#dict_keys(['hypergraph', 'features', 'labels', 'n'])
#pdb.set_trace()
datadict = {
'hypergraph': hyp_struct,
'features': args.v,
'labels': args.all_labels,
'n': len(args.v)
}
train_idx = args.label_idx
all_idx = torch.LongTensor(list(range(len(args.v))))
all_idx[train_idx] = -1
test_idx = all_idx[all_idx > -1]
hg_args = hgcn_config.parse()
'''
HyperGCN = model.initialise(dataset, args)
# train and test HyperGCN
HyperGCN = model.train(HyperGCN, dataset, train, args)
'''
hg_args.n_cls = int(args.all_labels.max() + 1)
hg_args.fast = args.fast
save_data = False #True
if save_data:
full_datadict = dict(datadict)
full_datadict.update({
'train_idx': train_idx,
'test_idx': test_idx,
'n_cls': hg_args.n_cls
})
torch.save(full_datadict,
'../hypergcn/data/{}_torch.pt'.format(args.dataset_name))
pdb.set_trace()
hg = hgcn_model.initialise(datadict, hg_args)
time0 = time.time()
hg = hgcn_model.train(hg, datadict, train_idx, hg_args)
dur = time.time() - time0
hg_acc = hgcn_model.test(hg, datadict, test_idx, hg_args)
print('hg acc ', hg_acc)
#pdb.set_trace()
return hg_acc, dur
song_folder = "songs_splited"
# train and test
model = model(X_train,
Y_train,
S,
Tx,
Ty,
lr=0.005,
n_a=128,
n_s=64,
jump_step=jump_step,
epoch=1000,
sec=sec,
optimizer="Adam")
if sys.argv[1] == "-train":
model.train(songs)
elif sys.argv[1] == "-test":
count = 0
for s in songs_test:
print("song: ", s)
if sys.argv[2] == "-sf":
song_folder = sys.argv[3]
X_predict, duration = song_preprocessing.graph_spectrogram(
"../" + song_folder + "/" + s)
if sys.argv[4] == "-f":
folder = sys.argv[5]
p_s = model.predict(np.transpose(X_predict), songs, folder)
from model import model
from dataset.mnist import load_mnist
if __name__ == '__main__':
model = model()
trainset, testset = load_mnist(normalize=True, one_hot_label=True)
model.train(trainset, testset)
## add more data: velocity , omega, past position
if __name__ == '__main__':
train_inputs, train_outputs = read_input()
m = m()
m.load_state_dict(torch.load('model_1.pt'))
optimizer = optim.Adam(m.parameters(), lr=0.005)
minibatch_size = 3
num_minibatches = len(train_inputs) // minibatch_size
for epoch in (range(30)):
# Training
print("Training")
# Put the model in training mode
m.train()
start_train = time.time()
for group in tqdm(range(num_minibatches)):
total_loss = None
optimizer.zero_grad()
for i in range(group * minibatch_size,
(group + 1) * minibatch_size):
input_seq = train_inputs[i]
gold_seq = torch.tensor(train_outputs[i])
prediction = m(input_seq)
loss = m.compute_Loss(prediction, gold_seq)
# On the first gradient update
if total_loss is None:
total_loss = loss
else:
def train():
"""Train datasets for a number of steps."""
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# Get images and labels for model.
images, labels = model.distorted_inputs()
# Build a Graph that computes the logits predictions from the
# inference model.
logits = model.inference(images)
# Calculate loss.
loss = model.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = model.train(loss, global_step)
# Create a saver.
saver = tf.train.Saver(tf.global_variables())
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Build an initialization operation to run below.
init = tf.global_variables_initializer()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement
)) # log_device_placement=True,该参数表示程序会将运行每一个操作的设备输出到屏幕
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph_def=sess.graph_def)
for step in range(FLAGS.max_steps):
start_time = time.time()
_, loss_value = sess.run([train_op, loss])
duration = time.time() - start_time
assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
if step % 10 == 0:
num_examples_per_step = FLAGS.batch_size
examples_per_sec = num_examples_per_step / duration
sec_per_batch = float(duration)
format_str = (
'%s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
)
print(format_str % (datetime.now(), step, loss_value,
examples_per_sec, sec_per_batch))
if step % 100 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
# Save the model checkpoint periodically.
if step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
flags = tf.app.flags
flags.DEFINE_integer("epoch", 25, "Epoch to train [25]")
flags.DEFINE_float("learning_rate", 0.0002,
"Learning rate of for adam [0.0002]")
flags.DEFINE_float("beta1", 0.5, "Momentum term of adam [0.5]")
flags.DEFINE_integer("load_size", 250, "The size of images to be loaded [250]")
flags.DEFINE_integer("fine_size", 256, "The fine size of images [256]")
flags.DEFINE_integer("batch_size", 64, "The size of batch images [64]")
flags.DEFINE_integer("c_dim", 3, "The channal size of the images [3]")
flags.DEFINE_integer("num_shots", 3, "The number of exposure shots [3]")
flags.DEFINE_string("dataset", "dataset/tf_records", "Dataset directory.")
flags.DEFINE_string("checkpoint_dir", "checkpoint",
"Directory name to save the checkpoints [checkpoint]")
flags.DEFINE_string("sample_dir", "samples",
"Directory name to save the image samples [samples]")
flags.DEFINE_string("log_dir", "logs",
"Directory name to save the logs [logs]")
flags.DEFINE_integer("save_freq", 0, "Save frequency [0]")
FLAGS = flags.FLAGS
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.69
with tf.Session(config=config) as sess:
model = model(sess, config=FLAGS, train=True)
model.train(FLAGS)
def train_model(model,
device,
train_data_loader,
valid_data_loader,
criterion,
optimizer,
scheduler,
num_epochs=5):
"""
training
Parameters
--------------
model : DogClassificationModel
Network model to be trained.
device : device
cuda or cpu
train_data_loader : dataloader
dataloader for training
valid_data_loader : dataloader
dataloader for validation
criterion :
Loss function.
optimizer :
Optimizer.
scheduler :
Learning rate scheduler.
num_epochs : int
The number of epochs.
Returns
--------------
model : DogClassificationModel
Trained model.
"""
since = time.time()
model = model.to(device)
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
for epoch in range(num_epochs):
bar = tqdm(total=len(train_data_loader))
bar.set_description("Epoch: {}/{}".format(epoch + 1, num_epochs))
"""
Training Phase
"""
model.train()
running_loss = 0.0
running_corrects = 0
for j, (inputs, labels) in enumerate(train_data_loader):
optimizer.zero_grad()
tmp_loss_item = 0.0
# training
with torch.set_grad_enabled(True):
outputs = model(inputs.to(device))
torch.cuda.empty_cache()
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels.to(device))
# backward + optimize only if in training phase
loss.backward()
optimizer.step()
tmp_loss_item = loss.item()
# statistics
running_loss += tmp_loss_item * inputs.size(0)
running_corrects += torch.sum(preds.to('cpu') == labels.data)
# progress bar
bar.update(1)
tmp_loss = float(running_loss /
(j + 1)) / 32 # 32: mini-batch size
tmp_acc = float(running_corrects // (j + 1)) / 32
bar.set_postfix(OrderedDict(loss=tmp_loss, acc=tmp_acc))
# update learning rate scheduler
scheduler.step()
dataset_size = len(train_data_loader.dataset)
epoch_loss = running_loss / dataset_size
epoch_acc = running_corrects.double() / dataset_size
"""
Validation Phase
"""
model.eval() # Set model to validation mode
val_running_loss = 0.0
val_running_corrects = 0
# Iterate over data.
for inputs, labels in valid_data_loader:
val_inputs = inputs.to(device)
val_labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.no_grad():
val_outputs = model(val_inputs)
_, preds = torch.max(val_outputs, 1)
loss = criterion(val_outputs, val_labels)
# statistics
val_running_loss += loss.item() * val_inputs.size(0)
val_running_corrects += torch.sum(preds == val_labels.data)
dataset_size = len(valid_data_loader.dataset)
val_epoch_loss = val_running_loss / dataset_size
val_epoch_acc = val_running_corrects.double() / dataset_size
print('VALIDATION Loss: {:.4f} Acc: {:.4f}'.format(
val_epoch_loss, val_epoch_acc))
print("Elapsed time: {} [sec]".format(time.time() - since))
# deep copy the model
if val_epoch_acc > best_acc:
best_acc = val_epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
model.load_state_dict(best_model_wts)
return model
init_seeds(0)
model = SVHN_Model1()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), 0.001)
best_loss = 1000.0
use_cuda = True
if use_cuda:
model = model.cuda()
for epoch in range(17):
start = time.time()
print('start', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start)))
train_loss = train(train_loader, model, criterion, optimizer, epoch)
val_loss = validate(val_loader, model, criterion)
val_label = [''.join(map(str, x)) for x in val_loader.dataset.img_label]
val_predict_label = predict(val_loader, model, 1)
val_predict_label = np.vstack([
val_predict_label[:, :11].argmax(1),
val_predict_label[:, 11:22].argmax(1),
val_predict_label[:, 22:33].argmax(1),
val_predict_label[:, 33:44].argmax(1),
val_predict_label[:, 44:55].argmax(1),
]).T
val_label_pred = []
for x in val_predict_label:
val_label_pred.append(''.join(map(str, x[x != 10])))
val_char_acc = np.mean(np.array(val_label_pred) == np.array(val_label))
eval_dataloader = DataLoader(eval_dataset, batch_size=BATCH_SIZE, shuffle=True)
train_dataloader = DataLoader(train_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
#使用交叉熵为 loss，使用 SGD 优化方法
criterion = NN.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
#载入上次训练
model.load_state_dict(torch.load("./chkpoint_res.bin"))
#开始训练
for epoch in range(0, 100):
model.train()
with tqdm(train_dataloader, unit="batch") as tepoch: #进度条
correct = 0
batch = 0
for data, target in tepoch:
batch += 1
tepoch.set_description(f"Epoch {epoch}")
data, target = data.cuda(), target.cuda() #数据载入 GPU
optimizer.zero_grad() #梯度归零
output = model(data) #前向计算
loss = criterion(output, target) #计算 loss
loss.backward() #反向传播
optimizer.step() #优化器梯度下降
predictions = output.argmax(dim=1, keepdim=True).squeeze() #预测
train_filename = data_folder + 'dataset_' + keyword + '_'
test_filename = data_folder + 'dataset_' + keyword + '_'
dropout = [1, 1, 1, 1]
wd = 0.0
lr = 0.001
epochs = 10
num_of_threads = 16
min_after_dequeue = 10000
capacity = min_after_dequeue + (num_of_threads + 1) * batch_size
# Train
train(data_folder,
train_filename,
test_filename,
train_data_count,
file_count,
weights,
dropout,
wd,
img_size,
max_char,
class_count,
batch_size=batch_size,
learning_rate=lr,
epochs=epochs,
restore=False,
var_lr=[None, None])
# In[ ]:
from data import data
dataset, train, test = data.load(args)
print("length of train is", len(train))
# # initialise HyperGCN
# In[ ]:
from model import model
HyperGCN = model.initialise(dataset, args)
# # train and test HyperGCN
# In[ ]:
HyperGCN = model.train(HyperGCN, dataset, train, args)
acc = model.test(HyperGCN, dataset, test, args)
print(float(acc))
# # store result
# In[ ]:
# # 'r': [run all cells](https://stackoverflow.com/questions/33143753/jupyter-ipython-notebooks-shortcut-for-run-all)
# In[ ]:
