def train_model(model, dataset, class_numbers, epochs=50, lr=1e-2, image_name='sample', beta=1., number=500): optimizer = optim.SGD(model.parameters(), lr=lr) test_losses = [] train_x = dataset test_x = dataset for e in range(epochs): model.train() optimizer.zero_grad() x_sample, z_mu, z_var = model(train_x) train_loss, _, _ = utils.loss_function(x_sample, train_x, z_mu, z_var, beta=beta) train_loss.backward() optimizer.step() model.eval() with torch.no_grad(): x_sample, z_mu, z_var = model(test_x) test_loss, BCE, KLD = utils.loss_function(x_sample, test_x, z_mu, z_var, beta=beta) test_losses.append(test_loss.item()) if number in class_numbers: utils.plot_images_2(model, test_x, BCE, z_dim=128, name=image_name) return train_loss, test_loss, BCE, KLD
def main(): lr = 0.00003 n_epochs = 100 n_batches = 200 X = tf.placeholder(tf.float32, [None, 256, 256, 3]) y_ = tf.placeholder(tf.float32, [None, 32, 32, 5]) net_train = get_model(X, is_train=True, reuse=False) net_val = get_model(X, is_train=False, reuse=True) y_train = net_train.outputs y_val = net_val.outputs loss_train, coord_loss, len_loss, is_obj_loss, no_obj_loss = loss_function( y_, y_train) loss_val = loss_function(y_, y_val) op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss_train) sess = tf.InteractiveSession() tl.layers.initialize_global_variables(sess) if os.path.isfile('model.npz'): tl.files.load_and_assign_npz(sess, 'model.npz', net_train) for it in range(n_epochs): tot_loss = tot_loss_1 = tot_loss_2 = tot_loss_3 = tot_loss_4 = 0 for num in range(n_batches): imgs, labels = read_data(num, n_batches) feed_dict = {X: imgs, y_: labels} _, loss, loss_1, loss_2, loss_3, loss_4, pred = sess.run( [ op, loss_train, coord_loss, len_loss, is_obj_loss, no_obj_loss, y_train ], feed_dict=feed_dict) tot_loss += loss tot_loss_1 += loss_1 tot_loss_2 += loss_2 tot_loss_3 += loss_3 tot_loss_4 += loss_4 if it % 10 == 0: print(pred[0, :, :, 0]) show_bbox(imgs[0], pred[0]) show_bbox(imgs[0], labels[0]) if it % 10 == 0: tl.files.save_npz(net_train.all_params, 'model' + str(it), sess) print( 'epoch %d\nloss: %f\ncoord loss: %f\nlen loss: %f\nis object loss: %f\nno object loss: %f' % (it, tot_loss / n_batches, tot_loss_1 / n_batches, tot_loss_2 / n_batches, tot_loss_3 / n_batches, tot_loss_4 / n_batches)) tl.files.save_npz(net_train.all_params, 'model.npz', sess)
def train(train_loader, model, optimizer, epoch, log=None, tb_writer=None): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() model.train() # switch to train mode t_start = time.time() for i, (img_feat, au_feat, labels) in enumerate(train_loader): # measure data load time data_time.update(time.time() - t_start) # print(img_feat.dtype, au_feat.dtype, labels.dtype) img_feat = img_feat.cuda() au_feat = au_feat.cuda() labels = labels.cuda() output = model(img_feat, au_feat) loss = loss_function(output, labels) losses.update(loss.item(), img_feat.size()[0]) # apply different optimizer if args.use_sam: loss.backward() optimizer.first_step(zero_grad=True) loss_function(model(img_feat, au_feat), labels).backward() # reduce_gradients_from_all_accelerators() # optimizer.second_step(zero_grad=True) else: loss.backward() optimizer.step() optimizer.zero_grad() batch_time.update(time.time() - t_start) t_start = time.time() # reset timer if i % args.print_freq == 0 or epoch <= 1: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data Time: {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss: {loss.val:.4f} ({loss.avg:.4f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, lr=optimizer.param_groups[-1]['lr'])) print(output) # print(losses.val, losses.avg, losses.sum, losses.count) if log: log.write(output + '\n') log.flush()
def validate_test(model, epoch, use_loader, use_simple): start_time = time.time() model = model.eval() v_loss = 0. data_loader_use = use_loader _index = 0 for _index, data in enumerate(data_loader_use): spec_input, target = data['mix'], data['binary_mask'] shape = spec_input.size() target = target.view((shape[0], shape[1] * shape[2], -1)) if PARAS.CUDA: spec_input = spec_input.cuda() target = target.cuda() with torch.no_grad(): predicted = model(spec_input) loss_value = loss_function(predicted, target) if not use_simple \ else loss_function_simple(predicted, target) v_loss += loss_value.data.item() v_loss /= (_index + 1) print('End of validation epoch {:3d} | time: {:5.2f}s | LOSS: {:5.4f} |'. format(epoch, (time.time() - start_time), v_loss)) print('-' * 99) return v_loss
def train_step(inp, tar): """ The target (tar) is divided into tar_inp and tar_real tar_inp is passed to the decoder as input. tar_real is the same input shifted by 1: at each position in tar_inp, tar_real contains the next token that should be predicted. """ # sentence = "SOS A lion in the jungle is sleeping EOS" # tar_inp = "SOS A lion in the jungle is sleeping" # tar_real = "A lion in the jungle is sleeping EOS" tar_inp = tar[:, :-1] tar_real = tar[:, 1:] data = create_look_ahead_mask(tf.shape(tar_inp)[1]) enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp) with tf.GradientTape() as tape: predictions, _ = transformer(inp, tar_inp, True, enc_padding_mask, combined_mask, dec_padding_mask) loss = loss_function(tar_real, predictions) gradients = tape.gradient(loss, transformer.trainable_variables) optimizer.apply_gradients(zip(gradients, transformer.trainable_variables)) train_loss(loss) train_accuracy(tar_real, predictions)
def train_step(_input, _target, enc_state): loss = 0 with tf.GradientTape() as tape: enc_output, enc_state = encoder(_input, enc_state) dec_hidden = enc_state dec_input = tf.expand_dims( [target_lang_tokenizer.word_index['<s>']] * batch_size, 1) # First input feeding definition h_t = tf.zeros((batch_size, 1, embedding_dim)) for idx in range(1, _target.shape[1]): # idx means target character index. predictions, dec_hidden, h_t = decoder(dec_input, dec_hidden, enc_output, h_t) #tf.print(tf.argmax(predictions, axis=1)) loss += loss_function(loss_object, _target[:, idx], predictions) dec_input = tf.expand_dims(_target[:, idx], 1) batch_loss = (loss / int(_target.shape[1])) variables = encoder.trainable_variables + decoder.trainable_variables gradients = tape.gradient(loss, variables) optimizer.apply_gradients(zip(gradients, variables)) return batch_loss
def test(epoch): model.eval() with torch.no_grad(): test_loss = 0 for i, (data, _) in enumerate(test_loader): data = Variable(data, volatile=True) final, residual_img, upscaled_image, com_img, orig_im = model( data.cuda()) test_loss += loss_function(final, residual_img, upscaled_image, com_img, orig_im).data.item() if epoch == EPOCHS and i == 0: # save_image(final.data[0],'reconstruction_final',nrow=8) # save_image(com_img.data[0],'com_img',nrow=8) n = min(data.size(0), 6) print("saving the image " + str(n)) comparison = torch.cat([data[:n], final[:n].cpu()]) comparison = comparison.cpu() # print(comparison.data) save_image(com_img[:n].data, 'compressed_' + str(epoch) + '.png', nrow=n) save_image(comparison.data, 'reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
def inner_loop(self, task): # x = task.view(-1, self.input_dim) # train_x = task[self.val_size:, :].to(self.device) # test_x = task[:self.val_size, :].to(self.device) train_x = task.to(self.device) test_x = task.to(self.device) for step in range(self.inner_steps): if step==0: x_sample, z_mu, z_var = self.model(train_x, self.weights) train_loss, BCE, KLD = utils.loss_function(x_sample, train_x, z_mu, z_var, self.beta) grad = torch.autograd.grad(train_loss, self.weights) # gradient clipping for w, g in zip(self.weights, grad): w.grad = g torch.nn.utils.clip_grad_norm_(self.weights, self.clip_value) temp_weights = [w - self.inner_lr * g for w,g in zip(self.weights, grad)] else: x_sample, z_mu, z_var = self.model(train_x, temp_weights) train_loss, BCE, KLD = utils.loss_function(x_sample, train_x, z_mu, z_var, self.beta) grad = torch.autograd.grad(train_loss, temp_weights) # gradient clipping for w, g in zip(temp_weights, grad): w.grad = g torch.nn.utils.clip_grad_norm_(temp_weights, self.clip_value) temp_weights = [w - self.inner_lr * g for w, g in zip(temp_weights, grad)] with open('results/logs/log.txt', 'a+') as f: print("inner iteration: {}, Recon loss: {}, KLD Loss: {}".format(step, BCE, KLD), file=f) x_sample, z_mu, z_var = self.model(test_x, temp_weights) task_loss, BCE, KLD = utils.loss_function(x_sample, test_x, z_mu, z_var, self.beta) self.BCE_losses.append(BCE) self.KLD_losses.append(KLD) with open('results/logs/log.txt','a+') as f: print("Task Recon loss: {}, Task KLD loss: {}".format(BCE, KLD),file=f) return task_loss
def overall_generator_pass(generator, discriminator, img, gt, valid): recon_batch = generator(img) msssim, f1, psnr = loss_function(recon_batch, gt) imgs = recon_batch.data.cpu().numpy()[0, :] imgs = roll_axis(imgs) loss= msssim+f1 G_loss = adversarial_loss(discriminator(recon_batch),valid) g_loss = adversarial_loss(discriminator(recon_batch),valid) + loss return imgs, g_loss, recon_batch, loss, msssim
def train(model, trainset, lr=1e-3, epochs=10, log_interval=10, cuda=True, save_path='./model/trained_vae.pkl'): model = model trainset = trainset device = torch.device("cuda" if cuda else "cpu") #print("Using: {}".format(device)) train_losses = [] train_loss = 0 model.train() # optimizer는 Adam 사용 optimizer = optim.Adam(model.parameters(), lr=lr) for epoch in tnrange(epochs, desc='Training Process'): for batch_idx, (images, _) in enumerate(trainset): images = Variable(images).to(device) if cuda else Variable(images) optimizer.zero_grad() reconstructed, mu, logvar = model(images) loss = loss_function(images, reconstructed, mu, logvar) # loss 계산 loss.backward() # Backpropagation train_losses.append( loss.item() / len(images)) # 배치별로 backprop하여 loss를 loss list에 담는다. train_loss += loss.item() optimizer.step() if batch_idx % log_interval == 0: print("Train Epoch: {} [{}/{} ({:.1f}%)] Loss: {:.4f}".format( epoch + 1, batch_idx * len(images), len(trainset.dataset), batch_idx * 100. / len(trainset), loss.item() / len(images))) print("======= Epoch: {} Average Loss: {:.4f} =======".format( epoch + 1, train_loss / len(trainset.dataset))) train_loss = 0 # save model torch.save(model, save_path) return train_losses
def finetuning(self, task, tune_epochs, number, numbers, name): train_x = task.to(self.device) test_x = task.to(self.device) model_copy = copy.deepcopy(self.model) weights = list(model_copy.parameters()) for epoch in range(tune_epochs): if epoch==0: x_sample, z_mu, z_var = model_copy(train_x, weights) train_loss, BCE, KLD = utils.loss_function(x_sample, train_x, z_mu, z_var, self.beta) grad = torch.autograd.grad(train_loss, weights) # gradient clipping for w, g in zip(weights, grad): w.grad = g torch.nn.utils.clip_grad_norm_(weights, self.clip_value) temp_weights = [w - self.inner_lr * g for w, g in zip(weights, grad)] else: x_sample, z_mu, z_var = model_copy(train_x, temp_weights) train_loss, BCE, KLD = utils.loss_function(x_sample, train_x, z_mu, z_var, self.beta) grad = torch.autograd.grad(train_loss, temp_weights) # gradient clipping for w, g in zip(temp_weights, grad): w.grad = g torch.nn.utils.clip_grad_norm_(temp_weights, self.clip_value) temp_weights = [w - self.inner_lr * g for w, g in zip(temp_weights, grad)] with open("results/logs//tune_log.txt", 'a+') as f: print("inner iteration: {}, Recon_loss = {}, KLD_loss = {}".format(epoch, BCE, KLD), file=f) x_sample, z_mu, z_var = model_copy(test_x, temp_weights) task_loss, BCE, KLD = utils.loss_function(x_sample, test_x, z_mu, z_var, self.beta) if number in numbers: utils.plot_images(model_copy, temp_weights, test_x, BCE, name=name) with open('transfer/tune_log.txt', 'a+') as f: print('Task test loss: {}'.format(task_loss), file=f) return task_loss.item()
def valid_step(x, y_real): y_pred, _, _ = model(x) all_loss, mid_loss = loss_function(y_real, y_pred, PRED_MODE, loss_criterion) all_acc, mid_acc = accuracy_function(y_real, y_pred, PRED_MODE) if MODEL_MODE == 'seq2seq': valid_all_loss(all_loss) valid_mid_loss(mid_loss) if MODEL_MODE == 'seq2seq': valid_all_acc(all_acc) valid_mid_acc(mid_acc) return y_pred
def train_step(inp, tar): tar_inp = tar[:, :-1] tar_real = tar[:, 1:] encoder_padding_mask, decoder_mask, encoder_decoder_padding_mask = create_masks( inp, tar_inp) with tf.GradientTape() as tape: predictions, _ = transformer(inp, tar_inp, True, encoder_padding_mask, decoder_mask, encoder_decoder_padding_mask) loss = loss_function(tar_real, predictions) gradients = tape.gradient(loss, transformer.trainable_variables) optimizer.apply_gradients(zip(gradients, transformer.trainable_variables)) train_loss(loss) train_accuracy(tar_real, predictions)
def train(model, loader, epoch_index, use_simple, optimizer, versatile=True): start_time = time.time() model = model.train() train_loss = 0. batch_num = len(loader) _index = 0 for _index, data in enumerate(loader): spec_input, target = data['mix'], data['binary_mask'] shape = spec_input.size() target = target.view((shape[0], shape[1] * shape[2], -1)) if PARAS.CUDA: spec_input = spec_input.cuda() target = target.cuda() optimizer.zero_grad() predicted = model(spec_input) loss_value = loss_function(predicted, target) if not use_simple \ else loss_function_simple(predicted, target) loss_value.backward() optimizer.step() train_loss += loss_value.data.item() if versatile: if (_index + 1) % PARAS.LOG_STEP == 0: elapsed = time.time() - start_time print( 'Epoch{:3d} | {:3d}/{:3d} batches | {:5.2f}ms/ batch | LOSS: {:5.4f} |' .format( epoch_index, _index + 1, batch_num, elapsed * 1000 / (_index + 1), train_loss / (_index + 1), )) train_loss /= (_index + 1) print('-' * 99) print('End of training epoch {:3d} | time: {:5.2f}s | LOSS: {:5.4f} |'. format(epoch_index, (time.time() - start_time), train_loss)) return train_loss
def train(model, train_loader, optimizer, pitch_criterion, dur_criterion, normal, weights, decay, clip, epoch): model.train() epoch_loss = 0. epoch_pitch_loss = epoch_dur_loss = epoch_kl_loss = 0. num_batch = len(train_loader) for i, batch in enumerate(train_loader): if torch.cuda.is_available(): batch = batch.cuda() optimizer.zero_grad() tfr1 = scheduled_sampling(((epoch + i / num_batch) / N_EPOCH), TFR1[0], TFR1[1]) tfr2 = scheduled_sampling(((epoch + i / num_batch) / N_EPOCH), TFR2[0], TFR2[1]) recon_pitch, recon_dur, dist = model(batch, False, True, tfr1, tfr2) recon_pitch = recon_pitch.view(-1, recon_pitch.size(-1)) recon_dur = recon_dur.view(-1, recon_dur.size(-1)) loss, pitch_loss, dur_loss, kl_loss = \ loss_function(recon_pitch, batch[:, :, 1:, 0].contiguous().view(-1), recon_dur, batch[:, :, 1:, 1:].contiguous().view(-1), dist, pitch_criterion, dur_criterion, normal, weights) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), clip) optimizer.step() epoch_loss += loss.item() epoch_pitch_loss += pitch_loss.item() epoch_dur_loss += dur_loss.item() epoch_kl_loss += kl_loss.item() loss_writer.add_scalar('batch_train_loss %d' % num_batch, loss.item(), epoch * num_batch + i) kl_writer.add_scalar('batch_train_loss %d' % num_batch, kl_loss.item(), epoch * num_batch + i) pitch_writer.add_scalar('batch_train_loss %d' % num_batch, pitch_loss.item(), epoch * num_batch + i) dur_writer.add_scalar('batch_train_loss %d' % num_batch, dur_loss.item(), epoch * num_batch + i) if decay: scheduler.step() return (epoch_loss / num_batch, epoch_pitch_loss / num_batch, epoch_dur_loss / num_batch, epoch_kl_loss / num_batch)
def train_step(inp, tar): tar_inp = tar[:, :-1] tar_real = tar[:, 1:] enc_padding_mask, look_ahead_mask, dec_padding_mask = get_mask( inp, tar_inp) with tf.GradientTape() as tape: prediction, _ = transformer(inp, tar_inp, True, enc_padding_mask, look_ahead_mask, dec_padding_mask) loss = loss_function(tar_real, prediction) gradient = tape.gradient(loss, transformer.trainable_variables) optimizer.apply_gradients(zip(gradient, transformer.trainable_variables)) train_loss(loss) train_accuracy(accuracy_function(tar_real, prediction))
def test(epoch, model, optimizer, test_loader, device, batch_size): model.eval() test_loss = 0 with torch.no_grad(): for i, (data, _) in enumerate(test_loader): data = data.to(device) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).item() if i == 0: n = min(data.size(0), 8) comparison = torch.cat( [data[:n], recon_batch.view(batch_size, 1, 28, 28)[:n]]) save_image(comparison.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
def train(epoch): losses, accuracies, batch_times = np.array([]), np.array([]), np.array([]) step = epoch * samples for batch in train_data: start_time = time.time() step += 1 left_img = batch['left'].to(device) right_img = batch['right'].to(device) target = batch['target'].to(device) _, _, pred = model(left_img, right_img) loss = loss_function(pred, target, class_weights) loss.backward() optimizer.step() scheduler.step() optimizer.zero_grad() acc = pixel_accuracy(pred, target, pixel=2) losses = np.append(losses, loss.item()) accuracies = np.append(accuracies, acc) batch_times = np.append(batch_times, time.time() - start_time) writer.add_scalar("train_loss", loss, global_step=step) writer.add_scalar("train_acc", acc, global_step=step) writer.add_scalar("learning_rate", scheduler.get_lr()[0], global_step=step) if step % 50 == 0: epoch_samples = (batch_size * (step // (epoch + 1))) mean_time = np.mean(batch_times) * 1000 print( "%d/%d samples, train_acc: %f, train_loss: %f, Time per batch: %fms" % (epoch_samples, samples, np.mean(accuracies), np.mean(losses), mean_time)) losses, accuracies, batch_times = np.array([]), np.array( []), np.array([]) if step % 500 == 0: torch.save(model.state_dict(), args.checkpoint) print("Created checkpoint")
def evaluate(epoch): losses, accuracies = np.array([]), np.array([]) for batch in val_data: left_img = batch['left'].to(device) right_img = batch['right'].to(device) target = batch['target'].to(device) _, _, pred = model(left_img, right_img) loss = loss_function(pred, target, class_weights) acc = pixel_accuracy(pred, target, pixel=2) losses = np.append(losses, loss.item()) accuracies = np.append(accuracies, acc) step = (epoch + 1) * samples avg_loss = np.mean(losses) avg_acc = np.mean(accuracies) writer.add_scalar("val_loss", avg_loss, global_step=step) writer.add_scalar("val_acc", avg_acc, global_step=step) print("Evaluation: val_acc: %f, val_loss: %f" % (avg_acc, avg_loss))
def train(epoch, model, optimizer, train_loader, device, log_interval): model.train() train_loss = 0 for batch_idx, (data, _) in enumerate(train_loader): data = data.to(device) optimizer.zero_grad() recon_batch, mu, logvar = model(data) loss = loss_function(recon_batch, data, mu, logvar) loss.backward() train_loss += loss.item() optimizer.step() if batch_idx % log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item() / len(data))) print('====> Epoch: {} Average loss: {:.4f}'.format( epoch, train_loss / len(train_loader.dataset)))
def train(epoch): model.train() train_loss = 0 for batch_idx, (data, _) in enumerate(train_loader): data = Variable(data) optimizer.zero_grad() final, residual_img, upscaled_image, com_img, orig_im = model( data.to(device)) loss = loss_function(final, residual_img, upscaled_image, com_img, orig_im) loss.backward() train_loss += loss.data.item() optimizer.step() if batch_idx % LOG_INTERVAL == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.data.item() / len(data))) print('====> Epoch: {} Average loss: {:.4f}'.format( epoch, train_loss / len(train_loader.dataset)))
def train_step(x, y_real): with tf.GradientTape() as tape: y_pred, attns_forward, attns_backward = model(x, training=True) all_loss, mid_loss = loss_function(y_real, y_pred, PRED_MODE, loss_criterion) all_acc, mid_acc = accuracy_function(y_real, y_pred, PRED_MODE) if MODEL_MODE == 'seq2seq': gradients = tape.gradient(all_loss, model.trainable_variables) elif MODEL_MODE == 'seq2one': gradients = tape.gradient(mid_loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) if MODEL_MODE == 'seq2seq': train_all_loss(all_loss) train_mid_loss(mid_loss) if MODEL_MODE == 'seq2seq': train_all_acc(all_acc) train_mid_acc(mid_acc) return y_pred, attns_forward, attns_backward
def evaluate(model, val_loader, pitch_criterion, dur_criterion, normal, weights, epoch): model.eval() epoch_loss = 0. epoch_pitch_loss = epoch_dur_loss = epoch_kl_loss = 0. num_batch = len(val_loader) with torch.no_grad(): for i, batch in enumerate(val_loader): if torch.cuda.is_available(): batch = batch.cuda() tfr1 = scheduled_sampling(((epoch + i / num_batch) / N_EPOCH), TFR1[0], TFR1[1]) tfr2 = scheduled_sampling(((epoch + i / num_batch) / N_EPOCH), TFR2[0], TFR2[1]) recon_pitch, recon_dur, dist = model(batch, False, True, tfr1, tfr2) recon_pitch = recon_pitch.view(-1, recon_pitch.size(-1)) recon_dur = recon_dur.view(-1, recon_dur.size(-1)) loss, pitch_loss, dur_loss, kl_loss = \ loss_function(recon_pitch, batch[:, :, 1:, 0].contiguous().view(-1), recon_dur, batch[:, :, 1:, 1:].contiguous().view(-1), dist, pitch_criterion, dur_criterion, normal, weights) epoch_loss += loss.item() epoch_pitch_loss += pitch_loss.item() epoch_dur_loss += dur_loss.item() epoch_kl_loss += kl_loss.item() loss_writer.add_scalar('batch_eval_loss %d' % num_batch, loss.item(), epoch * num_batch + i) kl_writer.add_scalar('batch_eval_loss %d' % num_batch, kl_loss.item(), epoch * num_batch + i) pitch_writer.add_scalar('batch_eval_loss %d' % num_batch, pitch_loss.item(), epoch * num_batch + i) dur_writer.add_scalar('batch_eval_loss %d' % num_batch, dur_loss.item(), epoch * num_batch + i) return (epoch_loss / num_batch, epoch_pitch_loss / num_batch, epoch_dur_loss / num_batch, epoch_kl_loss / num_batch)
def train_step(encoder, decoder, optimizer, tokenizer, loss_object, img_tensor, target): loss = 0 hidden = decoder.reset_state(batch_size=target.shape[0]) dec_input = tf.expand_dims([tokenizer.word_index["<start>"]] * target.shape[0], 1) with tf.GradientTape() as tape: features = encoder(img_tensor) for i in range(1, target.shape[1]): predictions, hidden, _ = decoder(dec_input, features, hidden) loss += loss_function(loss_object, target[:, i], predictions) dec_input = tf.expand_dims(target[:, i], 1) total_loss = loss / int(target.shape[1]) trainable_variables = encoder.trainable_variables + decoder.trainable_variables gradients = tape.gradient(loss, trainable_variables) optimizer.apply_gradients(zip(gradients, trainable_variables)) return loss, total_loss
# ceate_feature_map(features) # importance = bst.get_fscore(fmap='xgb.fmap') # importance = sorted(importance.items(), key=operator.itemgetter(1)) # df = pd.DataFrame(importance, columns=['feature', 'fscore']) # df['fscore'] = df['fscore'] / df['fscore'].sum() # df.to_csv("temp/feat_importance.csv", index=False,encoding="utf-8") # df.plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(6, 10)) # plt.rcParams["font.sans-serif"] = ["SimHei"] # plt.title('XGBoost Feature Importance') # plt.xlabel('relative importance') # plt.show() # this is prediction preds = bst.predict(dtest) labels = dtest.get_label() print(tl.loss_function(preds, labels)) # 读取比赛题 # exam_set = tl.load_match_data("data/d_test_A_20180102.csv") exam_set = tl.load_match_data("data/d_test_A_20180102_new.csv") # 数据预处理 exam_set = tl.pre_process(exam_set) del exam_set["乙肝表面抗原"] del exam_set["乙肝表面抗体"] del exam_set["乙肝e抗原"] del exam_set["乙肝e抗体"] del exam_set["乙肝核心抗体"] # 结果预测 exam_set = xgb.DMatrix(exam_set) y_exam = bst.predict(exam_set)
def validate(val_loader, model, accuracy, epoch, log=None, tb_writer=None): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() correlations = AverageMeter() model.eval() t_start = time.time() # load 1 video at a time for now (loader is not batched when batch_size == None) # but since we split one vid into multiple small clips, the input shape is still batch-like with torch.no_grad(): for i, (img_feat, au_feat, labels, frame_count) in enumerate(val_loader): data_time.update(time.time() - t_start) # print(type(img_feat), len(img_feat), img_feat[0].size()) img_feat = torch.stack(img_feat).cuda() au_feat = torch.stack(au_feat).cuda() labels = torch.stack(labels).cuda() output = model(img_feat, au_feat) # [Clip S 15] # rearrange and remove extra padding in the end output = rearrange(output, 'Clip S C -> (Clip S) C') output = torch.cat([output, output[-1:]]) # repeat the last frame to avoid missing if args.train_freq < args.val_freq: output = interpolate_output(output, args.train_freq, args.val_freq) output = output[:frame_count] labels = rearrange(labels, 'Clip S C -> (Clip S) C')[:frame_count] loss = loss_function(output, labels, validate=True) mean_cor, cor = accuracy(output, labels) # mean and per-class correlation # update statistics losses.update(loss.item()) assert not math.isnan(mean_cor.item()), 'at epoch %d' % (epoch) correlations.update(mean_cor.item()) batch_time.update(time.time() - t_start) t_start = time.time() if i % args.print_freq == 0: output = ('Val: [{0}/{1}]\t' 'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss: {loss.val:.4f} ({loss.avg:.4f})\t' 'Corr: {corr.val:.4f} ({corr.avg:.4f}, {corr.count})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, corr=correlations)) print(output) if log is not None: log.write(output + '\n') log.flush() output = ('Validate Results: Corr:{corr.avg:.4f} Loss {loss.avg:.5f}' .format(corr=correlations, loss=losses)) print(output) if log is not None: log.write(output + '\n') log.flush() if tb_writer is not None: tb_writer.add_scalar('loss/validate', losses.avg, epoch) tb_writer.add_scalar('acc/validate_corr', correlations.avg, epoch) return correlations.avg, losses.avg
def validate(val_loader, model, accuracy, epoch, log, tb_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() correlations = AverageMeter() pc_corr = [] model.eval() t_start = time.time() # load 1 video at a time for now (loader is not batched when batch_size == None) # but since we split one vid into multiple small clips, the input shape is still batch-like with torch.no_grad(): for i, (img_feat, au_feat, labels, frame_count) in enumerate(val_loader): data_time.update(time.time() - t_start) # print(type(img_feat), len(img_feat), img_feat[0].size()) img_feat = torch.stack(img_feat).cuda() au_feat = torch.stack(au_feat).cuda() labels = torch.stack(labels).cuda() # normalize # img_feat = F.normalize(img_feat, 2, dim=2) # au_feat = F.normalize(au_feat, 2, dim=2) if args.model == 'EmoBase': output, _, _ = model(img_feat, au_feat) else: if args.repeat_sample: img_feat = rearrange( img_feat, 'Clip R S C -> (Clip R) S C') # Clip R S C output = model_output(model, img_feat, au_feat) # rearrange and remove extra padding in the end if args.repeat_sample: output = rearrange(output, '(Clip R) S C -> (Clip S R) C', R=6) else: output = rearrange(output, 'Clip S C -> (Clip S) C') output = torch.cat([output, output[-1:] ]) # repeat the last frame to avoid missing if not args.repeat_sample and args.train_freq < args.val_freq: output = interpolate_output(output, args.train_freq, args.val_freq) output = output[:frame_count] labels = rearrange(labels, 'Clip S C -> (Clip S) C')[:frame_count] loss, c_loss, t_loss = loss_function(output, labels, args, validate=True) mean_cor, cor = accuracy(output, labels) # mean and per-class correlation if args.cls_mask != None: # mask = [1 if x in args.cls_mask else 0 for x in range(0, 15)] mean_cor = torch.mean(cor[args.cls_mask]) pc_corr.append(cor) # update statistics losses.update(loss.item()) assert not math.isnan(mean_cor.item()), 'at epoch %d' % (epoch) correlations.update(mean_cor.item()) batch_time.update(time.time() - t_start) t_start = time.time() if i % args.print_freq == 0: output = ('Val: [{0}/{1}]\t' 'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss: {loss.val:.4f} ({loss.avg:.4f})\t' 'Corr: {corr.val:.4f} ({corr.avg:.4f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, corr=correlations)) print(output) if log is not None: log.write(output + '\n') log.flush() output = ( 'Validate Results: Corr:{corr.avg:.4f} Loss {loss.avg:.5f}'.format( corr=correlations, loss=losses)) print(output) pc_corr = torch.stack(pc_corr, dim=0) pc_corr = torch.mean(pc_corr, dim=0).cpu().numpy() print('Per-Class Corr:', pc_corr) if log is not None: log.write(output + '\n') log.flush() if tb_writer is not None: tb_writer.add_scalar('loss/validate', losses.avg, epoch) tb_writer.add_scalar('acc/validate_corr', correlations.avg, epoch) return correlations.avg
def train(train_loader, model, optimizer, epoch, log, tb_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() model.train() # switch to train mode t_start = time.time() for i, (img_feat, au_feat, labels) in enumerate(train_loader): # measure data load time data_time.update(time.time() - t_start) # print(img_feat.dtype, au_feat.dtype, labels.dtype) img_feat = img_feat.cuda() au_feat = au_feat.cuda() labels = labels.cuda() # normalize # img_feat = F.normalize(img_feat, 2, dim=2) # au_feat = F.normalize(au_feat, 2, dim=2) if args.model == 'EmoBase': output, mean, std = model(img_feat, au_feat) loss, c_loss, t_loss = loss_function(output, labels, args, mean, std) else: output = model_output(model, img_feat, au_feat) # [B T C] loss, c_loss, t_loss = loss_function(output, labels, args) losses.update(loss.item(), img_feat.size()[0]) # apply different optimizer if args.use_sam: loss.backward() optimizer.first_step(zero_grad=True) if args.model == 'EmoBase': output, mean, std = model(img_feat, au_feat) loss, c_loss, t_loss = loss_function(output, labels, args, mean, std) loss.backward() else: loss_function(model_output(model, img_feat, au_feat), labels, args)[0].backward() # reduce_gradients_from_all_accelerators() # optimizer.second_step(zero_grad=True) else: loss.backward() optimizer.step() optimizer.zero_grad() batch_time.update(time.time() - t_start) t_start = time.time() # reset timer if i % args.print_freq == 0 or epoch <= 1: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss: {loss.val:.4f} ({loss.avg:.4f})\t' 'T_l: {t_loss:.4f} C_l: {c_loss:.4f}'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, t_loss=t_loss.item(), c_loss=c_loss.item(), lr=optimizer.param_groups[-1]['lr'])) print(output) # print(losses.val, losses.avg, losses.sum, losses.count) log.write(output + '\n') log.flush() tb_writer.add_scalar('loss/train', losses.avg, epoch) tb_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
filename='val_40_18_100.bin', start_sample=1280, num_samples=2859136, ) test_data = DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) class_weights = torch.Tensor([1, 4, 10, 4, 1]).to(device) print("%d test samples" % len(test_dataset)) model.eval() losses, accuracies = np.array([]), np.array([]) with torch.no_grad(): for batch in test_data: left_img = batch['left'].to(device) right_img = batch['right'].to(device) target = batch['target'].to(device) _, _, pred = model(left_img, right_img) loss = loss_function(pred, target, class_weights) acc = pixel_accuracy(pred, target, pixel=pixel_dist) losses = np.append(losses, loss.item()) accuracies = np.append(accuracies, acc) avg_loss = np.mean(losses) avg_acc = np.mean(accuracies) print("Accuracy: %f, Loss: %f" % (avg_acc, avg_loss))
# coding=utf-8 # 用knn训练模型 from sklearn.model_selection import train_test_split from sklearn.neighbors import KNeighborsRegressor import utils as tl # 读数据 data = tl.load_good_data("data/d_train_20180102.csv") # 将数据分成特征和标签 X, y = tl.convert_data_to_featrue_label(data) # 随机切分数据 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1) # knn开工 knn = KNeighborsRegressor() knn.fit(X_train, y_train) # 输出结果 predict = knn.predict(X_test) # 查看loss值 print(tl.loss_function(predict, y_test))