def train(model,batch,EPOCH,device,data,dataset_idx,optimizer,foward_name): input_id,y_true = data train_idx,val_idx,test_idx = dataset_idx model.train() criterion = nn.CrossEntropyLoss() for n_epoch in range(EPOCH): random.shuffle(train_idx) n=0 optimizer.zero_grad() loss_epoch=list() for n_kousin in range(len(train_idx)//batch+1): batch_input_idx=train_idx[n_kousin*batch:min((n_kousin+1)*batch,len(train_idx))] train_input=list() train_y=list() for idx in batch_input_idx: train_input.append(input_id[idx]) train_y.append(y_true[idx]) batch_input_idx_tensor=torch.tensor(train_input).to(device) train_y_tensor=torch.tensor(train_y).to(device) output=foward(model,foward_name,batch_input_idx_tensor) loss=criterion(output,train_y_tensor) loss_epoch.append(loss.item()) loss.backward() optimizer.step() optimizer.zero_grad() print("epoch {} : {}".format(n_epoch,np.array(loss_epoch).mean())) if n_epoch%10==0: eval(model,val_idx,input_id,y_true,device,foward_name) model.train()
def train_model(dataloader, model, criterion, optimizer, device, num_epochs, dataset_size): model.to(device) since = time.time() best_model_wts = copy.deepcopy(model.state_dict()) best_acc = 0.0 for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) for phase in ['train', 'test']: if phase == 'train': model.train() else: model.eval() running_loss = 0.0 running_corrects = 0 for inputs, labels in tqdm(dataloaders[phase]): inputs = inputs.to(device) labels = labels.to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): outputs = model(inputs) _, pred = torch.max(outputs, 1) loss = criterion(outputs, labels) if phase == 'train': loss.backward() optimizer.step() running_loss += loss.item() * inputs.size(0) running_corrects += torch.sum(pred == labels.data) epoch_loss = running_loss / dataset_size[phase] epoch_acc = running_corrects.double() / dataset_size[phase] print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc)) if phase == 'test' and epoch_acc > best_acc: best_acc = epoch_acc best_model_wts = copy.deepcopy(model.state_dict()) torch.save( best_model_wts, osp.join(Config['root_path'], Config['checkpoint_path'], 'model.pth')) print('Model saved at: {}'.format( osp.join(Config['root_path'], Config['checkpoint_path'], 'model.pth'))) time_elapsed = time.time() - since print('Time taken to complete training: {:0f}m {:0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best acc: {:.4f}'.format(best_acc))
def train(args, model, train_dataset, epoch): with torch.enable_grad(): # Turn on training mode which enables dropout. model.train() total_loss = 0 start_time = time.time() hidden = model.init_hidden(args.batch_size) for batch, i in enumerate(range(0, train_dataset.size(0) - 1, args.bptt)): inputSeq, targetSeq = get_batch(args,train_dataset, i) # inputSeq: [ seq_len * batch_size * feature_size ] # targetSeq: [ seq_len * batch_size * feature_size ] # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. hidden = model.repackage_hidden(hidden) hidden_ = model.repackage_hidden(hidden) optimizer.zero_grad() '''Loss1: Free running loss''' outVal = inputSeq[0].unsqueeze(0) outVals=[] hids1 = [] for i in range(inputSeq.size(0)): outVal, hidden_, hid = model.forward(outVal, hidden_,return_hiddens=True) outVals.append(outVal) hids1.append(hid) outSeq1 = torch.cat(outVals,dim=0) hids1 = torch.cat(hids1,dim=0) loss1 = criterion(outSeq1.contiguous().view(args.batch_size,-1), targetSeq.contiguous().view(args.batch_size,-1)) '''Loss2: Teacher forcing loss''' outSeq2, hidden, hids2 = model.forward(inputSeq, hidden, return_hiddens=True) loss2 = criterion(outSeq2.contiguous().view(args.batch_size, -1), targetSeq.contiguous().view(args.batch_size, -1)) '''Loss3: Simplified Professor forcing loss''' loss3 = criterion(hids1.contiguous().view(args.batch_size,-1), hids2.contiguous().view(args.batch_size,-1).detach()) '''Total loss = Loss1+Loss2+Loss3''' loss = loss1+loss2+loss3 loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip) optimizer.step() total_loss += loss.item() if batch % args.log_interval == 0 and batch > 0: cur_loss = total_loss / args.log_interval elapsed = time.time() - start_time print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.4f} | ' 'loss {:5.5f} '.format( epoch, batch, len(train_dataset) // args.bptt, elapsed * 1000 / args.log_interval, cur_loss)) total_loss = 0 start_time = time.time()
def train(args, model, train_dataset,epoch): with torch.enable_grad(): # Turn on training mode which enables dropout. model.train() total_loss = 0 start_time = time.time() hidden = model.init_hidden(args.batch_size) for batch, i in enumerate(range(0, train_dataset.size(0) - 1, args.bptt)): inputSeq, targetSeq = get_batch(args,train_dataset, i) # inputSeq: [ seq_len * batch_size * feature_size ] # targetSeq: [ seq_len * batch_size * feature_size ] # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. hidden = model.repackage_hidden(hidden) hidden_ = model.repackage_hidden(hidden) optimizer.zero_grad() '''Loss1: Free running loss''' outVal = inputSeq[0].unsqueeze(0) outVals=[] hids1 = [] for i in range(inputSeq.size(0)): outVal, hidden_, hid = model.forward(outVal, hidden_,return_hiddens=True) outVals.append(outVal) hids1.append(hid) outSeq1 = torch.cat(outVals,dim=0) hids1 = torch.cat(hids1,dim=0) loss1 = criterion(outSeq1.view(args.batch_size,-1), targetSeq.view(args.batch_size,-1)) '''Loss2: Teacher forcing loss''' outSeq2, hidden, hids2 = model.forward(inputSeq, hidden, return_hiddens=True) loss2 = criterion(outSeq2.view(args.batch_size, -1), targetSeq.view(args.batch_size, -1)) '''Loss3: Simplified Professor forcing loss''' loss3 = criterion(hids1.view(args.batch_size,-1), hids2.view(args.batch_size,-1).detach()) '''Total loss = Loss1+Loss2+Loss3''' loss = loss1+loss2+loss3 loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip) optimizer.step() total_loss += loss.item() if batch % args.log_interval == 0 and batch > 0: cur_loss = total_loss / args.log_interval elapsed = time.time() - start_time print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.4f} | ' 'loss {:5.2f} '.format( epoch, batch, len(train_dataset) // args.bptt, elapsed * 1000 / args.log_interval, cur_loss)) total_loss = 0 start_time = time.time()
def test_learn_online(self): e = threading.Event() try: urlemg = 'tcp://192.168.0.20:5555' emgp = online.publisher.EmgPublisher(urlemg, abort=e) emgs = online.sources.FileSource(emgp, 4000, 'emg_data', abort=e) emgsub = online.subscriber.EmgSubscriber(urlemg, abort=e) emgiter = online.subscriber.array_iterator(ArrayMessage, emgsub) urlkin = 'tcp://192.168.0.20:5556' kinp = online.publisher.KinPublisher(urlkin, abort=e) kins = online.sources.FileSource(kinp, 500, 'kin_data', abort=e) kinsub = online.subscriber.KinSubscriber(urlkin, abort=e) kiniter = online.subscriber.array_iterator(ArrayMessage, kinsub) #sigmoid = lambda X: 1 / (1 + np.exp(X)) identity = lambda X: X model = online.regression.LinReg( dim_in=ArrayMessage.duration * kins.samplingrate * 5, dim_out=ArrayMessage.duration * kins.samplingrate * 3, dim_basis=ArrayMessage.duration * kins.samplingrate * 5, basis_fcts=identity ) print 'Calculated shapes: dim_in={}, dim_out={}, dim_basis={}'.format( ArrayMessage.duration * kins.samplingrate * 5, ArrayMessage.duration * kins.samplingrate * 3, ArrayMessage.duration * kins.samplingrate * 5 ) print 'start threads' emgp.start() emgs.start() emgsub.start() kinp.start() kins.start() kinsub.start() count = 0 while count < 1000: Z = kiniter.next().data[:, [2,7,9]] X = emgiter.next().data X_ = X.reshape(Z.shape[0], -1, X.shape[1]) X = np.mean(X_, axis=1) Z = Z.flatten().reshape(1, -1) X = X.flatten().reshape(1, -1) model.train(X,Z) if count % 100 == 0: print '{}\t\t{}'.format(count, model.loss(X, Z)) count += 1 e.set() except Exception as ex: e.set() raise ex e.set()
def train(args, model, train_dataset): # Turn on training mode which enables dropout. model.train() total_loss = 0 start_time = time.time() hidden = model.init_hidden(args.batch_size) for batch, i in enumerate(range(0, train_dataset.size(0) - 1, args.bptt)): inputSeq, targetSeq = get_batch(train_dataset, i) # inputSeq: [ seq_len * batch_size * feature_size ] # targetSeq: [ seq_len * batch_size * feature_size ] # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. hidden = model.repackage_hidden(hidden) optimizer.zero_grad() USE_TEACHER_FORCING = random.random() < args.teacher_forcing_ratio if USE_TEACHER_FORCING: outSeq, hidden = model.forward(inputSeq, hidden) else: outVal = inputSeq[0].unsqueeze(0) outVals = [] for i in range(inputSeq.size(0)): outVal, hidden = model.forward(outVal, hidden) outVals.append(outVal) outSeq = torch.cat(outVals, dim=0) #print('outSeq:',outSeq.size()) #print('targetSeq:', targetSeq.size()) loss = criterion(outSeq.view(args.batch_size, -1), targetSeq.view(args.batch_size, -1)) loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) optimizer.step() # for p in model2_for_timeDiff.parameters(): # p.data.add_(-lr, p.grad.data) total_loss += loss.data if batch % args.log_interval == 0 and batch > 0: cur_loss = total_loss[0] / args.log_interval elapsed = time.time() - start_time print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.4f} | ' 'loss {:5.2f} '.format(epoch, batch, len(train_dataset) // args.bptt, elapsed * 1000 / args.log_interval, cur_loss)) total_loss = 0 start_time = time.time()
def test_learn_online(self): e = threading.Event() try: urlemg = 'tcp://192.168.0.20:5555' emgp = online.publisher.EmgPublisher(urlemg, abort=e) emgs = online.sources.FileSource(emgp, 4000, 'emg_data', abort=e) emgsub = online.subscriber.EmgSubscriber(urlemg, abort=e) emgiter = online.subscriber.array_iterator(ArrayMessage, emgsub) urlkin = 'tcp://192.168.0.20:5556' kinp = online.publisher.KinPublisher(urlkin, abort=e) kins = online.sources.FileSource(kinp, 500, 'kin_data', abort=e) kinsub = online.subscriber.KinSubscriber(urlkin, abort=e) kiniter = online.subscriber.array_iterator(ArrayMessage, kinsub) #sigmoid = lambda X: 1 / (1 + np.exp(X)) identity = lambda X: X model = online.regression.LinReg( dim_in=ArrayMessage.duration * kins.samplingrate * 5, dim_out=ArrayMessage.duration * kins.samplingrate * 3, dim_basis=ArrayMessage.duration * kins.samplingrate * 5, basis_fcts=identity) print 'Calculated shapes: dim_in={}, dim_out={}, dim_basis={}'.format( ArrayMessage.duration * kins.samplingrate * 5, ArrayMessage.duration * kins.samplingrate * 3, ArrayMessage.duration * kins.samplingrate * 5) print 'start threads' emgp.start() emgs.start() emgsub.start() kinp.start() kins.start() kinsub.start() count = 0 while count < 1000: Z = kiniter.next().data[:, [2, 7, 9]] X = emgiter.next().data X_ = X.reshape(Z.shape[0], -1, X.shape[1]) X = np.mean(X_, axis=1) Z = Z.flatten().reshape(1, -1) X = X.flatten().reshape(1, -1) model.train(X, Z) if count % 100 == 0: print '{}\t\t{}'.format(count, model.loss(X, Z)) count += 1 e.set() except Exception as ex: e.set() raise ex e.set()
def train(): # Turn on training mode which enables dropout. model.train() total_loss = 0. start_time = time.time() ntokens = len(corpus.dictionary) hidden = model.init_hidden(args.batch_size) for batch, i in enumerate(range(0, train_data.size(0) - 1, args.bptt)): data, targets = get_batch(train_data, i) # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. hidden = repackage_hidden(hidden) optimizer.zero_grad() output, hidden = model(data, hidden) loss = criterion(output.view(-1, ntokens), targets) loss.backward() optimizer.step() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip) # for p in model.parameters(): # p.data.add_(-lr, p.grad.data) total_loss += loss.item() if batch % args.log_interval == 0 and batch > 0: cur_loss = total_loss / args.log_interval elapsed = time.time() - start_time try: print( '| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | ' 'loss {:5.2f} | ppl {:8.2f}'.format( epoch, batch, len(train_data) // args.bptt, lr, elapsed * 1000 / args.log_interval, cur_loss, math.exp(cur_loss))) except OverflowError as err: print( '| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | ' 'loss {:5.2f} | ppl {:8.2f}'.format( epoch, batch, len(train_data) // args.bptt, lr, elapsed * 1000 / args.log_interval, cur_loss, math.exp(cur_loss))) total_loss = 0 start_time = time.time() data.to("cpu") targets.to("cpu")
def train(epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): data, target = data.to(device), target.to(device) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item()))
def _fit(self, model, X, Z, batchsize=10, valeval=50, alpha=0.01): iter = 0 best_loss = 1000000 best_W = None while iter + batchsize < X.shape[0]: model.train(X[iter:iter+batchsize], Z[iter:iter+batchsize],alpha) if iter/batchsize % 50 == 0: loss = model.loss(X[iter:iter+batchsize], Z[iter:iter+batchsize]) if loss < best_loss: best_loss = loss best_W = model._W print 'best loss: {} -- current loss: {}'.format(best_loss, loss) iter += batchsize model._W = best_W
def train(epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): if use_cuda: data, target = data.cuda(), target.cuda() optimizer.zero_grad() output = model(data) criterion = torch.nn.CrossEntropyLoss(reduction='mean') loss = criterion(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.data.item()))
def run_expreminent(look_forward, hidden_size, batch_size, epochs, dropout, dataset): x, y = get_sentiments_prices(dataset['twitter_sentiments'], dataset["reddit_sentiments"], dataset["coin_price"], look_forward) for i in range(x.shape[1]): x[:, i] = normalize_array(x[:, i]) # split into train and test sets train_x, test_x = split(x) train_y, test_y = split(y) train_x = np.reshape(train_x, (train_x.shape[0], look_forward, train_x.shape[1])) test_x = np.reshape(test_x, (test_x.shape[0], look_forward, test_x.shape[1])) model = create_model(hidden_size=hidden_size, look_forward=look_forward, dropout=dropout) model = train(model, train_x, train_y, batch_size=batch_size, epochs=epochs) y_pred = test(model, test_x) score = evaluate(test_y, y_pred) print('Test Score: %.2f RMSE' % score) return score
def train(model, iterator, optimizer, criterion, clip): model.train() epoch_loss = 0 len_iterator = 0 for i, batch in enumerate(iterator): src = batch.src trg = batch.trg optimizer.zero_grad() output = model(src, trg) loss = criterion(output, trg) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), clip) optimizer.step() epoch_loss += loss.item() len_iterator += 1 return epoch_loss / len_iterator
def train(model): model = model.train() optimizer.zero_grad() losses = [] accurate_preds = 0 all_targets = [] all_predictions = [] for d in train_loader: inputs = d['input_ids'].to(device) masks = d['attention_mask'].to(device) all_targets.extend(list(d['targets'].squeeze().numpy())) targets = d['targets'].to(device) outputs = model( input_ids=inputs, attention_mask=masks ) _, preds = torch.max(outputs, dim=1) loss = criterion(outputs, targets) all_predictions.extend(list(preds.cpu().squeeze().numpy())) accurate_preds += torch.sum(preds == targets) losses.append(loss.item()) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) optimizer.step() scheduler.step() return accurate_preds / train_len, np.mean(losses), all_targets, all_predictions
def _fit(self, model, X, Z, batchsize=10, valeval=50, alpha=0.01): iter = 0 best_loss = 1000000 best_W = None while iter + batchsize < X.shape[0]: model.train(X[iter:iter + batchsize], Z[iter:iter + batchsize], alpha) if iter / batchsize % 50 == 0: loss = model.loss(X[iter:iter + batchsize], Z[iter:iter + batchsize]) if loss < best_loss: best_loss = loss best_W = model._W print 'best loss: {} -- current loss: {}'.format( best_loss, loss) iter += batchsize model._W = best_W
def train_model(dataloader, model, optimizer, device, num_epochs, dataset_size): model.to(device) for epoch in range(num_epochs): print('-' * 15) print('Epoch {}/{}'.format(epoch + 1, num_epochs)) for phase in ['train', 'val']: #train and validate every epoch if phase == 'train': model.train() else: model.eval() running_loss = 0.0 for i in tqdm(range(len(dataloader[phase].dataset[0]))): inputs = dataloader[phase].dataset[0][i] #print(inputs.shape) labels = dataloader[phase].dataset[1][i] #print(labels.shape) inputs = inputs.unsqueeze(0) labels = labels.unsqueeze(0) inputs = inputs.to(device) labels = labels.to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): outputs = model(inputs) loss = criterion(outputs, labels) if phase == 'train': loss.backward() optimizer.step() running_loss += loss.item() * inputs.size(0) epoch_loss = running_loss / dataset_size[phase] print('{} Loss: {:.4f} '.format(phase, epoch_loss)) # save the model #saved_model = copy.deepcopy(model.state_dict()) with open(osp.join(Config['path'], "my_model.pth"), "wb") as output_file: torch.save(model.state_dict(), output_file)
def train(epoch): print('#' * 15) print('Epoch {}, Latent Size {}'.format(epoch, model.latent_size)) print('#' * 15) model.train() for index, (x, _) in enumerate(loader): x = x.mean(dim=1, keepdim=True).to(device) optimizer.zero_grad() x_generated, mu, logvar = model(x) loss = get_loss(x_generated, x, mu, logvar) loss.backward() optimizer.step() if index % 100 == 0: print('Loss at iteration {0}: {1:.4f}'.format(index, loss.item())) if epoch == 4: filename = 'epoch{}_ls{}.pkl'.format(epoch, model.latent_size) torch.save(model.state_dict(), os.path.join(weights_dir, filename)) if epoch < 4: scheduler.step()
def train(): model.train() # Turn on the train mode total_loss = 0.0 start_time = time.time() batches = np.random.permutation( range(0, train_data.size(0) - seq_len, seq_len)) for batch_counter, i in enumerate(batches): data, targets = get_batch(train_data, i) optimizer.zero_grad() src_mask = model.generate_square_subsequent_mask( data.size(0)).to(device) output = model( data, src_mask=src_mask, ) loss = criterion(output, targets) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5) optimizer.step() total_loss += loss.item() log_interval = 100 if batch_counter % log_interval == 0 and batch_counter > 0: cur_loss = total_loss / log_interval elapsed = time.time() - start_time print("| epoch {:3d} | {:5d}/{:5d} batches | {:5.6f}" "| ms/batch {:5.4f} | " "loss {:5.4f}".format( epoch, batch_counter, len(train_data) // seq_len, scheduler.get_last_lr()[0], elapsed * 1000 / log_interval, cur_loss, )) total_loss = 0 start_time = time.time()
def train(epoch): model.train() utils.adjust_learning_rate(optimizer, epoch, args.step_size, args.lr, args.gamma) print('epoch =', epoch, 'lr = ', optimizer.param_groups[0]['lr']) for iteration, (lr_tensor, hr_tensor) in enumerate(training_data_loader, 1): if args.cuda: lr_tensor = lr_tensor.to(device) # ranges from [0, 1] hr_tensor = hr_tensor.to(device) # ranges from [0, 1] optimizer.zero_grad() sr_tensor = model(lr_tensor) loss_l1 = l1_criterion(sr_tensor, hr_tensor) loss_sr = loss_l1 loss_sr.backward() optimizer.step() # if iteration % 1000 == 0: print("===> Epoch[{}]({}/{}): Loss_l1: {:.5f}".format( epoch, iteration, len(training_data_loader), loss_l1.item()))
def train(epoch): global niter model.train() for batch_idx, (data, target) in enumerate(train_loader): niter += 1 if use_cuda: data, target = data.cuda(), target.cuda() optimizer.zero_grad() output = model(data) criterion = torch.nn.CrossEntropyLoss(reduction='elementwise_mean') loss = criterion(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.data.item())) writer.add_scalar('data/train_loss', loss.data.item(), niter) writer.add_scalar('data/learning_rate', optimizer.state_dict()['param_groups'][0]['lr'], niter)
def train(train_loader, model, optimizer, epoch): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() model.train() end = time.time() for i, (im, gt) in (enumerate(train_loader)): adjust_learning_rate(optimizer, epoch) # measure data loading time data_time.update(time.time() - end) optimizer.zero_grad() loss, scores, boxes = model((im, gt)) loss.backward() optimizer.step() losses.update(loss.item(), im.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' #'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' #'Prec@5 {top5.val:.3f} ({top5.avg:.3f})' .format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, #top1=top1, top5=top5 )) break;
def train_model(dataloader, model, criterion, optimizer, device, num_epochs, dataset_size): model.to(device) since = time.time() best_model_wts = copy.deepcopy(model.state_dict()) best_acc = 0.0 acc_list = [] loss_list = [] test_acc_list= [] test_loss_list = [] for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) for phase in ['train', 'test']: if phase=='train': model.train() else: model.eval() running_loss = 0.0 running_corrects = 0 for input1, input2, labels in tqdm(dataloaders[phase], position=0, leave=True): input1 = input1.to(device) input2 = input2.to(device) labels = labels.to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase=='train'): outputs = model(input1, input2) outputs = torch.reshape(outputs, (outputs.shape[0],)) outputs = outputs.type(torch.DoubleTensor) labels = labels.type(torch.DoubleTensor) pred = [] for i in outputs: if i>0.5: pred.append(0) else: pred.append(1) pred = torch.FloatTensor(pred) loss = criterion(outputs,labels) if phase=='train': loss.backward() optimizer.step() running_loss += loss.item() * input1.size(0) running_corrects += torch.sum(pred==labels.data) epoch_loss = running_loss / dataset_size[phase] epoch_acc = running_corrects.double() / dataset_size[phase] if phase=='train': acc_list.append(epoch_acc) loss_list.append(epoch_loss) elif phase=='test': test_acc_list.append(epoch_acc) test_loss_list.append(epoch_loss) print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc)) if phase=='test' and epoch_acc > best_acc: best_acc = epoch_acc best_model_wts = copy.deepcopy(model.state_dict()) torch.save(best_model_wts, osp.join(Config['root_path'], Config['checkpoint_path'], 'model.pth')) print('Model saved at: {}'.format(osp.join(Config['root_path'], Config['checkpoint_path'], 'model.pth'))) time_elapsed = time.time() - since print('Time taken to complete training: {:0f}m {:0f}s'.format(time_elapsed // 60, time_elapsed % 60)) print('Best acc: {:.4f}'.format(best_acc)) np.savetxt('acc_list.txt',acc_list) np.savetxt('test_acc_list.txt',test_acc_list) np.savetxt('loss_list.txt',loss_list) np.savetxt('test_loss_list.txt',test_loss_list)
def train_hypergcn(args): if args.dataset_name in ['citeseer', 'cora']: if args.do_svd: data_path = 'data/citeseer.pt' if args.dataset_name == 'citeseer' else 'data/cora_author_10cls300.pt' else: data_path = 'data/citeseer6cls3703.pt' if args.dataset_name == 'citeseer' else 'data/cora_author_10cls1000.pt' args = hypergraph.gen_data_cora(args, data_path=data_path, flip_edge_node=False) elif args.dataset_name in ['dblp', 'pubmed']: data_path = 'data/pubmed_data.pt' if args.dataset_name == 'pubmed' else 'data/dblp_data.pt' args = hypergraph.gen_data_dblp(args, data_path=data_path) else: raise Exception('dataset {} not supported'.format(args.dataset_name)) #data_path = 'data/citeseer.pt' if args.dataset_name == 'citeseer' else 'data/cora_author.pt' #args = hypergraph.gen_data_cora(args, data_path=data_path) #dict_keys(['hypergraph', 'features', 'labels', 'n']) if args.predict_edge: hyp_struct = create_hypergcn_struct(args) n_edge = len(args.edge_X) n_node = len(args.all_labels) labels = torch.cat([args.all_labels, args.edge_classes], 0) X = torch.cat([args.v, args.edge_X], 0) #dict_keys(['hypergraph', 'features', 'labels', 'n']) #pdb.set_trace() datadict = { 'hypergraph': hyp_struct, 'features': args.v, 'labels': labels, 'n': len(args.v) + n_edge } n_labels = max( 1, math.ceil(n_edge * utils.get_label_percent(args.dataset_name))) train_idx = torch.from_numpy( np.random.choice(n_edge, size=(n_labels, ), replace=False)).to(torch.int64) all_idx = torch.LongTensor(list(range(n_edge))) all_idx[train_idx] = -1 test_idx = all_idx[all_idx > -1] train_idx += n_node test_idx += n_node #len(args.all_labels) hg_args = hgcn_config.parse() else: hyp_struct = create_hypergcn_struct(args) #dict_keys(['hypergraph', 'features', 'labels', 'n']) #pdb.set_trace() datadict = { 'hypergraph': hyp_struct, 'features': args.v, 'labels': args.all_labels, 'n': len(args.v) } train_idx = args.label_idx all_idx = torch.LongTensor(list(range(len(args.v)))) all_idx[train_idx] = -1 test_idx = all_idx[all_idx > -1] hg_args = hgcn_config.parse() ''' HyperGCN = model.initialise(dataset, args) # train and test HyperGCN HyperGCN = model.train(HyperGCN, dataset, train, args) ''' hg_args.n_cls = int(args.all_labels.max() + 1) hg_args.fast = args.fast save_data = False #True if save_data: full_datadict = dict(datadict) full_datadict.update({ 'train_idx': train_idx, 'test_idx': test_idx, 'n_cls': hg_args.n_cls }) torch.save(full_datadict, '../hypergcn/data/{}_torch.pt'.format(args.dataset_name)) pdb.set_trace() hg = hgcn_model.initialise(datadict, hg_args) time0 = time.time() hg = hgcn_model.train(hg, datadict, train_idx, hg_args) dur = time.time() - time0 hg_acc = hgcn_model.test(hg, datadict, test_idx, hg_args) print('hg acc ', hg_acc) #pdb.set_trace() return hg_acc, dur
song_folder = "songs_splited" # train and test model = model(X_train, Y_train, S, Tx, Ty, lr=0.005, n_a=128, n_s=64, jump_step=jump_step, epoch=1000, sec=sec, optimizer="Adam") if sys.argv[1] == "-train": model.train(songs) elif sys.argv[1] == "-test": count = 0 for s in songs_test: print("song: ", s) if sys.argv[2] == "-sf": song_folder = sys.argv[3] X_predict, duration = song_preprocessing.graph_spectrogram( "../" + song_folder + "/" + s) if sys.argv[4] == "-f": folder = sys.argv[5] p_s = model.predict(np.transpose(X_predict), songs, folder)
from model import model from dataset.mnist import load_mnist if __name__ == '__main__': model = model() trainset, testset = load_mnist(normalize=True, one_hot_label=True) model.train(trainset, testset)
## add more data: velocity , omega, past position if __name__ == '__main__': train_inputs, train_outputs = read_input() m = m() m.load_state_dict(torch.load('model_1.pt')) optimizer = optim.Adam(m.parameters(), lr=0.005) minibatch_size = 3 num_minibatches = len(train_inputs) // minibatch_size for epoch in (range(30)): # Training print("Training") # Put the model in training mode m.train() start_train = time.time() for group in tqdm(range(num_minibatches)): total_loss = None optimizer.zero_grad() for i in range(group * minibatch_size, (group + 1) * minibatch_size): input_seq = train_inputs[i] gold_seq = torch.tensor(train_outputs[i]) prediction = m(input_seq) loss = m.compute_Loss(prediction, gold_seq) # On the first gradient update if total_loss is None: total_loss = loss else:
def train(): """Train datasets for a number of steps.""" with tf.Graph().as_default(): global_step = tf.Variable(0, trainable=False) # Get images and labels for model. images, labels = model.distorted_inputs() # Build a Graph that computes the logits predictions from the # inference model. logits = model.inference(images) # Calculate loss. loss = model.loss(logits, labels) # Build a Graph that trains the model with one batch of examples and # updates the model parameters. train_op = model.train(loss, global_step) # Create a saver. saver = tf.train.Saver(tf.global_variables()) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() # Build an initialization operation to run below. init = tf.global_variables_initializer() # Start running operations on the Graph. sess = tf.Session(config=tf.ConfigProto( log_device_placement=FLAGS.log_device_placement )) # log_device_placement=True,该参数表示程序会将运行每一个操作的设备输出到屏幕 sess.run(init) # Start the queue runners. tf.train.start_queue_runners(sess=sess) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph_def=sess.graph_def) for step in range(FLAGS.max_steps): start_time = time.time() _, loss_value = sess.run([train_op, loss]) duration = time.time() - start_time assert not np.isnan(loss_value), 'Model diverged with loss = NaN' if step % 10 == 0: num_examples_per_step = FLAGS.batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) format_str = ( '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)' ) print(format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch)) if step % 100 == 0: summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, step) # Save the model checkpoint periodically. if step % 1000 == 0 or (step + 1) == FLAGS.max_steps: checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=step)
flags = tf.app.flags flags.DEFINE_integer("epoch", 25, "Epoch to train [25]") flags.DEFINE_float("learning_rate", 0.0002, "Learning rate of for adam [0.0002]") flags.DEFINE_float("beta1", 0.5, "Momentum term of adam [0.5]") flags.DEFINE_integer("load_size", 250, "The size of images to be loaded [250]") flags.DEFINE_integer("fine_size", 256, "The fine size of images [256]") flags.DEFINE_integer("batch_size", 64, "The size of batch images [64]") flags.DEFINE_integer("c_dim", 3, "The channal size of the images [3]") flags.DEFINE_integer("num_shots", 3, "The number of exposure shots [3]") flags.DEFINE_string("dataset", "dataset/tf_records", "Dataset directory.") flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoint]") flags.DEFINE_string("sample_dir", "samples", "Directory name to save the image samples [samples]") flags.DEFINE_string("log_dir", "logs", "Directory name to save the logs [logs]") flags.DEFINE_integer("save_freq", 0, "Save frequency [0]") FLAGS = flags.FLAGS if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) config = tf.ConfigProto() #config.gpu_options.per_process_gpu_memory_fraction = 0.69 with tf.Session(config=config) as sess: model = model(sess, config=FLAGS, train=True) model.train(FLAGS)
def train_model(model, device, train_data_loader, valid_data_loader, criterion, optimizer, scheduler, num_epochs=5): """ training Parameters -------------- model : DogClassificationModel Network model to be trained. device : device cuda or cpu train_data_loader : dataloader dataloader for training valid_data_loader : dataloader dataloader for validation criterion : Loss function. optimizer : Optimizer. scheduler : Learning rate scheduler. num_epochs : int The number of epochs. Returns -------------- model : DogClassificationModel Trained model. """ since = time.time() model = model.to(device) best_model_wts = copy.deepcopy(model.state_dict()) best_acc = 0.0 for epoch in range(num_epochs): bar = tqdm(total=len(train_data_loader)) bar.set_description("Epoch: {}/{}".format(epoch + 1, num_epochs)) """ Training Phase """ model.train() running_loss = 0.0 running_corrects = 0 for j, (inputs, labels) in enumerate(train_data_loader): optimizer.zero_grad() tmp_loss_item = 0.0 # training with torch.set_grad_enabled(True): outputs = model(inputs.to(device)) torch.cuda.empty_cache() _, preds = torch.max(outputs, 1) loss = criterion(outputs, labels.to(device)) # backward + optimize only if in training phase loss.backward() optimizer.step() tmp_loss_item = loss.item() # statistics running_loss += tmp_loss_item * inputs.size(0) running_corrects += torch.sum(preds.to('cpu') == labels.data) # progress bar bar.update(1) tmp_loss = float(running_loss / (j + 1)) / 32 # 32: mini-batch size tmp_acc = float(running_corrects // (j + 1)) / 32 bar.set_postfix(OrderedDict(loss=tmp_loss, acc=tmp_acc)) # update learning rate scheduler scheduler.step() dataset_size = len(train_data_loader.dataset) epoch_loss = running_loss / dataset_size epoch_acc = running_corrects.double() / dataset_size """ Validation Phase """ model.eval() # Set model to validation mode val_running_loss = 0.0 val_running_corrects = 0 # Iterate over data. for inputs, labels in valid_data_loader: val_inputs = inputs.to(device) val_labels = labels.to(device) # zero the parameter gradients optimizer.zero_grad() # forward # track history if only in train with torch.no_grad(): val_outputs = model(val_inputs) _, preds = torch.max(val_outputs, 1) loss = criterion(val_outputs, val_labels) # statistics val_running_loss += loss.item() * val_inputs.size(0) val_running_corrects += torch.sum(preds == val_labels.data) dataset_size = len(valid_data_loader.dataset) val_epoch_loss = val_running_loss / dataset_size val_epoch_acc = val_running_corrects.double() / dataset_size print('VALIDATION Loss: {:.4f} Acc: {:.4f}'.format( val_epoch_loss, val_epoch_acc)) print("Elapsed time: {} [sec]".format(time.time() - since)) # deep copy the model if val_epoch_acc > best_acc: best_acc = val_epoch_acc best_model_wts = copy.deepcopy(model.state_dict()) time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val Acc: {:4f}'.format(best_acc)) # load best model weights model.load_state_dict(best_model_wts) return model
init_seeds(0) model = SVHN_Model1() criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), 0.001) best_loss = 1000.0 use_cuda = True if use_cuda: model = model.cuda() for epoch in range(17): start = time.time() print('start', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start))) train_loss = train(train_loader, model, criterion, optimizer, epoch) val_loss = validate(val_loader, model, criterion) val_label = [''.join(map(str, x)) for x in val_loader.dataset.img_label] val_predict_label = predict(val_loader, model, 1) val_predict_label = np.vstack([ val_predict_label[:, :11].argmax(1), val_predict_label[:, 11:22].argmax(1), val_predict_label[:, 22:33].argmax(1), val_predict_label[:, 33:44].argmax(1), val_predict_label[:, 44:55].argmax(1), ]).T val_label_pred = [] for x in val_predict_label: val_label_pred.append(''.join(map(str, x[x != 10]))) val_char_acc = np.mean(np.array(val_label_pred) == np.array(val_label))
eval_dataloader = DataLoader(eval_dataset, batch_size=BATCH_SIZE, shuffle=True) train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) #使用交叉熵为 loss,使用 SGD 优化方法 criterion = NN.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) #载入上次训练 model.load_state_dict(torch.load("./chkpoint_res.bin")) #开始训练 for epoch in range(0, 100): model.train() with tqdm(train_dataloader, unit="batch") as tepoch: #进度条 correct = 0 batch = 0 for data, target in tepoch: batch += 1 tepoch.set_description(f"Epoch {epoch}") data, target = data.cuda(), target.cuda() #数据载入 GPU optimizer.zero_grad() #梯度归零 output = model(data) #前向计算 loss = criterion(output, target) #计算 loss loss.backward() #反向传播 optimizer.step() #优化器梯度下降 predictions = output.argmax(dim=1, keepdim=True).squeeze() #预测
train_filename = data_folder + 'dataset_' + keyword + '_' test_filename = data_folder + 'dataset_' + keyword + '_' dropout = [1, 1, 1, 1] wd = 0.0 lr = 0.001 epochs = 10 num_of_threads = 16 min_after_dequeue = 10000 capacity = min_after_dequeue + (num_of_threads + 1) * batch_size # Train train(data_folder, train_filename, test_filename, train_data_count, file_count, weights, dropout, wd, img_size, max_char, class_count, batch_size=batch_size, learning_rate=lr, epochs=epochs, restore=False, var_lr=[None, None])
# In[ ]: from data import data dataset, train, test = data.load(args) print("length of train is", len(train)) # # initialise HyperGCN # In[ ]: from model import model HyperGCN = model.initialise(dataset, args) # # train and test HyperGCN # In[ ]: HyperGCN = model.train(HyperGCN, dataset, train, args) acc = model.test(HyperGCN, dataset, test, args) print(float(acc)) # # store result # In[ ]: # # 'r': [run all cells](https://stackoverflow.com/questions/33143753/jupyter-ipython-notebooks-shortcut-for-run-all) # In[ ]: