def train_steps(self, global_step): # Variables that affect learning rate. decay_steps = int(self.opt_params.NUM_STEPS_PER_DECAY) learning_rate_decay_factor = self.opt_params.LEARNING_RATE_DECAY_FACTOR moving_average_decay = self.opt_params.MOVING_AVERAGE_DECAY self.gen_loss, self.disc_loss = self.get_losses() # Compute gradients. with tf.control_dependencies([self.gen_loss, self.disc_loss]): gen_train_step_op = ops.train( self.gen_loss, global_step, decay_steps, self.opt_params.GEN_INITIAL_LEARNING_RATE, learning_rate_decay_factor, moving_average_decay, target_vars=self.gen_vars, name='gen') disc_train_step_op = ops.train( self.disc_loss, global_step, decay_steps, self.opt_params.DISC_INITIAL_LEARNING_RATE, learning_rate_decay_factor, moving_average_decay, target_vars=self.disc_vars, name='disc') return [gen_train_step_op, disc_train_step_op]
def __init__(self, lr=0.0001, optimizer=tf.train.Optimizer, fine_tuning=True, dropout=False, adaptive_ratio=1.0): ''' ----------Hyperparameters ------------- :param fine_tuning: If True, the parameters of CNN layers will also be fine-tuned. Otherwise, only the parameters of FC layers will be trained. :param dropout: If True, dropout is applied to all fully connected layers except for the last one. Also, dropout_keep_prob should be fed. (default value is 1.0) :param adaptive_ratio: If True, the learning rate of convolutional layer will be learning rate * adaptive_ratio :return: ''' self.desc = "Learning rate : {}, optimizer : {}, fine_tuning : {}, dropout : {}, adaptive ratio : {}"\ .format(lr, optimizer.__name__, fine_tuning, dropout, adaptive_ratio) print(self.desc) self.params = { 'lr': lr, 'optimizer': optimizer, 'fine_tuning': fine_tuning, 'dropout': dropout, 'adaptive_ratio': adaptive_ratio } self.xs = tf.placeholder(tf.float32, [None, 32, 32, 3]) self.ys = tf.placeholder(tf.int32, [None]) self.dropout_keep_prob = tf.placeholder_with_default(1.0, None) pool5 = self.build_convnet(fine_tuning) fc3 = self.build_fcnet(pool5, dropout) self.probs = tf.nn.softmax(fc3, name='softmax') self.loss = ops.loss(logits=self.probs, labels=self.ys, one_hot=False) self.accuracy = ops.accuracy(logits=self.probs, labels=self.ys, one_hot=False) if adaptive_ratio < 1.0: self.train = ops.train(self.loss, optimizer=optimizer, conv_lr=lr * adaptive_ratio, fc_lr=lr) else: self.train = optimizer(learning_rate=lr).minimize(self.loss)
cwd = os.getcwd() os.chdir(os.path.join(path.test, 'images')) # cmd = "ffmpeg -r 30 -f image2 -s 256x256 -i pic_%d-outputs.png -vcodec libx264 -crf 25 -pix_fmt yuv420p ../out.mp4" cmd = 'ffmpeg -r 30 -i pic_%d-outputs.png -c:v libx264 -crf 15 -vf "fps=30,format=yuv420p" ../../out.mp4' os.system(cmd) os.chdir(cwd) elif args.cmd == 'prep': ops.clean_filenames(path.rawA, rename='pic_%s') ops.crop_square_resize(path.rawA, path.A, args.size, args.size) ops.crop_square_resize(path.A, path.B, args.size / 8, args.size / 8, args.size, args.size) ops.combine(path.A, path.B, path.train, args.size) elif args.cmd == 'escalate': escalate() elif args.cmd == 'train': ops.train(path.model, path.train, args.epochs, args.size) elif args.cmd == 'init_remote': # create dirs and git clone repo os.system('ssh %s "mkdir git; cd git; git clone %s; mkdir %s/projects"' % (vm.GPU_INSTANCE, path.GIT_REPO_URL, path.GIT_REPO_NAME)) # install packages os.system( 'ssh %s "sudo apt-get install -y ffmpeg python-imaging python3-pil"' % (vm.GPU_INSTANCE)) elif args.cmd == 'train_remote': """Run on GPU_INSTANCE via ssh and tmux. To keep a process running I use: tmux -d python script.py Manually running tmux first works too. Detaching is done with [ctrl]-[b], [d]. And a running tmux session can be reatached with: tmux attach """
def train(self, global_step): with tf.control_dependencies([self.loss]): return ops.train(self.loss, global_step, learning_rate=self.lr, name='training_step')
def main(): # Settings parser = argparse.ArgumentParser(description='PyTorch Clothing1M') parser.add_argument('--batch_size', type=int, default=256, help='input batch size for training (default: 256)') parser.add_argument('--test_batch_size', type=int, default=256, help='input batch size for testing (default: 256)') parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 120)') parser.add_argument('--gpu_id', type=int, default=0, help='index of gpu to use (default: 0)') parser.add_argument('--lr', type=float, default=0.001, help='init learning rate (default: 0.1)') parser.add_argument('--seed', type=int, default=0, help='random seed (default: 0)') parser.add_argument('--save', action='store_true', default=False, help='For saving softmax_out_avg') parser.add_argument('--SEAL', type=int, default=0, help='Phase of self-evolution') args = parser.parse_args() torch.manual_seed(args.seed) device = torch.device('cuda:'+str(args.gpu_id) if torch.cuda.is_available() else 'cpu') # Datasets root = './data/Clothing1M' num_classes = 14 kwargs = {'num_workers': 32, 'pin_memory': True} if torch.cuda.is_available() else {} transform_train = transforms.Compose([transforms.Resize((256, 256)), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]) transform_test = transforms.Compose([transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]) train_dataset = Clothing1M(root, mode='train', transform=transform_train) val_dataset = Clothing1M(root, mode='val', transform=transform_test) test_dataset = Clothing1M(root, mode='test', transform=transform_test) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, **kwargs) softmax_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) def learning_rate(lr_init, epoch): optim_factor = 0 if(epoch > 5): optim_factor = 1 return lr_init*math.pow(0.1, optim_factor) def load_pretrain(num_classes, device): model_pre = resnet50(num_classes=1000, pretrained=True) # imagenet pretrained, numclasses=1000 if num_classes==1000: return model_pre.to(device) else: model = resnet50(num_classes=num_classes, pretrained=False) params_pre = model_pre.state_dict().copy() params = model.state_dict() for i in params_pre: if not i.startswith('fc'): params[i] = params_pre[i] model.load_state_dict(params) return model.to(device) # results results_root = os.path.join('results', 'clothing') if not os.path.isdir(results_root): os.makedirs(results_root) """ Test model """ if args.SEAL==-1: model = resnet50().to(device) model.load_state_dict(torch.load(os.path.join(results_root, 'seed0_clothing_normal.pt'))) test(args, model, device, test_loader) """ Get softmax_out_avg - normal training on noisy labels """ if args.SEAL==0: # Building model model = load_pretrain(num_classes, device) model = torch.nn.DataParallel(model, device_ids=[0,1,2,3]) # Training best_val_acc = 0 save_path = os.path.join(results_root, 'seed'+str(args.seed)+'_clothing_normal.pt') softmax_out = [] for epoch in range(1, args.epochs + 1): optimizer = optim.SGD(model.parameters(), lr=learning_rate(args.lr, epoch), momentum=0.9, weight_decay=1e-3) train(args, model, device, train_loader, optimizer, epoch) best_val_acc = val_test(args, model, device, val_loader, test_loader, best_val_acc, save_path) softmax_out.append(get_softmax_out(model, softmax_loader, device)) if args.save: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_normal.npy') softmax_out = np.concatenate(softmax_out) np.save(softmax_root, softmax_out) print('new softmax_out saved to', softmax_root, ', shape: ', softmax_out.shape) """ Self Evolution - training on softmax_out_avg """ if args.SEAL>=1: if args.SEAL==1: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_normal.npy') model_path = os.path.join(results_root, 'seed'+str(args.seed)+'_clothing_normal.pt') else: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_SEAL'+str(args.SEAL-1)+'.npy') model_path = os.path.join(results_root, 'seed'+str(args.seed)+'_clothing_SEAL'+str(args.SEAL-1)+'.pt') save_path = os.path.join(results_root, 'seed'+str(args.seed)+'_clothing_SEAL'+str(args.SEAL)+'.pt') # Loading softmax_out_avg of last phase softmax_out_avg = np.load(softmax_root).reshape([-1, len(train_dataset), num_classes]) softmax_out_avg = softmax_out_avg.mean(axis=0) print('softmax_out_avg loaded from', softmax_root, ', shape: ', softmax_out_avg.shape) # Dataset with soft targets train_dataset_soft = Clothing1M_soft(root, targets_soft=torch.Tensor(softmax_out_avg.copy()), mode='train', transform=transform_train) train_loader_soft = torch.utils.data.DataLoader(train_dataset_soft, batch_size=args.batch_size, shuffle=True, **kwargs) # Building model model = load_pretrain(num_classes, device) model = torch.nn.DataParallel(model, device_ids=[0,1,2,3]) model.load_state_dict(torch.load(model_path)) print('Initialize the model using {}.'.format(model_path)) # Training best_val_acc = 0 softmax_out = [] for epoch in range(1, args.epochs + 1): optimizer = optim.SGD(model.parameters(), lr=learning_rate(args.lr, epoch), momentum=0.9, weight_decay=1e-3) train_soft(args, model, device, train_loader_soft, optimizer, epoch) best_val_acc = val_test(args, model, device, val_loader, test_loader, best_val_acc, save_path) softmax_out.append(get_softmax_out(model, softmax_loader, device)) if args.save: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_SEAL'+str(args.SEAL)+'.npy') softmax_out = np.concatenate(softmax_out) np.save(softmax_root, softmax_out) print('new softmax_out saved to', softmax_root, ', shape: ', softmax_out.shape)
def train(args, data, params): train = data['train'] valid = data['valid'] learning_rate = args.learning_rate with tf.Graph().as_default(): input_ph = tf.placeholder(tf.int32, shape=[args.batch_size, params['gram_size']]) targ_ph = tf.placeholder(tf.int32, shape=[args.batch_size]) learning_rate_ph = tf.placeholder(tf.float32, shape=[]) if args.w2v: with h5py.File(args.w2v, 'r') as datafile: embeds = datafile['w2v'][:] scores, normalize_op, vars = ops.model(input_ph, params, embeds) else: scores, normalize_op, vars = ops.model(input_ph, params) loss = ops.loss(scores, targ_ph) train_op, print_op = ops.train(loss, learning_rate_ph, args) #sess = tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=NUM_THREADS,\ # intra_op_parallelism_threads=NUM_THREADS)) sess = tf.Session() init = tf.initialize_all_variables( ) # initialize variables before they can be used saver = tf.train.Saver() sess.run(init) if args.modelfile: saver.restore(sess, args.modelfile) print "Model restored from %s" % args.modelfile valid_loss = 0. for i in xrange(valid.nbatches): valid_feed_dict = get_feed_dict(valid, i, input_ph, targ_ph, learning_rate_ph) batch_loss = sess.run([loss], feed_dict=valid_feed_dict)[0] valid_loss += batch_loss last_valid = valid_loss print 'Initial valid loss: %.3f' % math.exp( valid_loss / valid.nbatches) for epoch in xrange(args.nepochs): print "Training epoch %d with learning rate %.3f" % (epoch + 1, learning_rate) vals = sess.run(vars) start_time = time.time() train_loss = 0. valid_loss = 0. for i in xrange(train.nbatches): train_feed_dict = get_feed_dict(train, i, input_ph, targ_ph, \ learning_rate_ph, learning_rate) #grads = sess.run(print_op, feed_dict=train_feed_dict) _, batch_loss = sess.run([train_op, loss], feed_dict=train_feed_dict) train_loss += batch_loss for i in xrange(valid.nbatches): valid_feed_dict = get_feed_dict(valid, i, input_ph, targ_ph, learning_rate_ph) batch_loss = sess.run([loss], feed_dict=valid_feed_dict)[0] valid_loss += batch_loss if args.normalize: _ = sess.run(normalize_op) duration = time.time() - start_time print "\tloss = %.3f, valid ppl = %.3f, %.3f s" % \ (math.exp(train_loss/train.nbatches), \ math.exp(valid_loss/valid.nbatches), duration) if last_valid < valid_loss: learning_rate /= 2. elif args.outfile: saver.save(sess, args.outfile) if epoch >= args.decay_after: learning_rate /= 1.2 last_valid = valid_loss return sess.run([normalize_op ])[0] # return final normalized embeddings
one_hot = 'no' batch_size = 128 train_loss = [] train_acc = [] valid_acc = [] valid_loss = [] ini_train_loss = 1000000 ini_valid_loss = 1000000 s = timeit.default_timer() for i in tf.range(iteration): rand_index = np.random.choice(len(trainX), size=batch_size, replace=False) rand_x = tf.convert_to_tensor(trainX[rand_index], dtype=tf.float32) rand_y = tf.convert_to_tensor(trainY[rand_index], dtype=tf.int32) # label temp_train_loss = ops.train(model, rand_x, rand_y, optimizer) if (i+1) % 20 == 0: # Record and print results batch_predictions = ops.predict(model, rand_x) valid_index = np.random.choice(len(trainValY), size=batch_size) valid_x = tf.convert_to_tensor(trainValX[valid_index], dtype=tf.float32) valid_y = tf.convert_to_tensor(trainValY[valid_index], dtype=tf.int32) valid_predictions = ops.predict(model, valid_x) batch_predictions = tf.argmax(batch_predictions, axis=1, output_type=tf.int32) temp_train_acc = tf.reduce_mean(tf.cast(tf.equal(batch_predictions, rand_y), tf.float32)) valid_predictions = tf.argmax(valid_predictions, axis=1, output_type=tf.int32) temp_valid_acc = tf.reduce_mean(tf.cast(tf.equal(valid_predictions, valid_y), tf.float32))
def main(): # Settings parser = argparse.ArgumentParser(description='PyTorch MNIST') parser.add_argument('--batch_size', type=int, default=64, help='input batch size for training (default: 64)') parser.add_argument('--test_batch_size', type=int, default=1000, help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=50, help='number of epochs to train (default: 50)') parser.add_argument('--gpu_id', type=int, default=0, help='index of gpu to use (default: 0)') parser.add_argument('--lr', type=float, default=0.01, help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, help='SGD momentum (default: 0.5)') parser.add_argument('--seed', type=int, default=0, help='random seed (default: 0)') parser.add_argument('--noise_pattern', type=str, default='dependent', help='Noise pattern (default: dependent)') parser.add_argument('--noise_rate', type=float, default=0.0, help='Noise rate (default: 0.0)') parser.add_argument('--save', action='store_true', default=False, help='For saving softmax_out_avg') parser.add_argument('--SEAL', type=int, default=0, help='Phase of self-evolution') args = parser.parse_args() torch.manual_seed(args.seed) device = torch.device('cuda:'+str(args.gpu_id) if torch.cuda.is_available() else 'cpu') # Datasets root = './data' kwargs = {'num_workers': 4, 'pin_memory': True} if torch.cuda.is_available() else {} transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) #0.1307, 0.3081 are the mean and std of mnist train_dataset = datasets.MNIST(root, train=True, download=True, transform=transform) train_dataset_noisy = datasets.MNIST(root, train=True, transform=transform) test_dataset = datasets.MNIST(root, train=False, transform=transform) targets_noisy = torch.Tensor(pd.read_csv(os.path.join('./data/MNIST/label_noisy', args.noise_pattern+str(args.noise_rate)+'.csv'))['label_noisy'].values.astype(int)) train_dataset_noisy.targets = targets_noisy train_loader = torch.utils.data.DataLoader(train_dataset_noisy, batch_size=args.batch_size, shuffle=True, **kwargs) softmax_loader = torch.utils.data.DataLoader(train_dataset_noisy, batch_size=args.test_batch_size, shuffle=False, **kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) # results results_root = os.path.join('results', 'mnist_'+args.noise_pattern+str(args.noise_rate)) if not os.path.isdir(results_root): os.makedirs(results_root) """ Get softmax_out_avg - normal training on noisy labels """ if args.SEAL==0: # Building model model = MNIST_CNN().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) # Training softmax_out_avg = np.zeros([len(train_dataset_noisy), 10]) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader) softmax_out_avg += get_softmax_out(model, softmax_loader, device) softmax_out_avg /= args.epochs if args.save: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_avg_'+args.noise_pattern+str(args.noise_rate)+'_normal.npy') np.save(softmax_root, softmax_out_avg) print('new softmax_out_avg saved to', softmax_root, ', shape: ', softmax_out_avg.shape) """ Self Evolution - training on softmax_out_avg """ if args.SEAL>=1: # Loading softmax_out_avg of last phase if args.SEAL==1: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_avg_'+args.noise_pattern+str(args.noise_rate)+'_normal.npy') else: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_avg_'+args.noise_pattern+str(args.noise_rate)+'_SEAL'+str(args.SEAL-1)+'.npy') softmax_out_avg = np.load(softmax_root) print('softmax_out_avg loaded from', softmax_root, ', shape: ', softmax_out_avg.shape) # Dataset with soft targets train_dataset_soft = MNIST_soft(root, targets_soft=torch.Tensor(softmax_out_avg.copy()), train=True, transform=transform) train_dataset_soft.targets = targets_noisy train_loader_soft = torch.utils.data.DataLoader(train_dataset_soft, batch_size=args.batch_size, shuffle=True, **kwargs) # Building model model = MNIST_CNN().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) # Training softmax_out_avg = np.zeros([len(train_dataset_noisy), 10]) for epoch in range(1, args.epochs + 1): train_soft(args, model, device, train_loader_soft, optimizer, epoch) test(args, model, device, test_loader) softmax_out_avg += get_softmax_out(model, softmax_loader, device) softmax_out_avg /= args.epochs if args.save: softmax_root = os.path.join(results_root, 'seed'+str(args.seed)+'_softmax_out_avg_'+args.noise_pattern+str(args.noise_rate)+'_SEAL'+str(args.SEAL)+'.npy') np.save(softmax_root, softmax_out_avg) print('new softmax_out_avg saved to', softmax_root, ', shape: ', softmax_out_avg.shape)
def main(): # Settings parser = argparse.ArgumentParser(description='PyTorch cifar10') parser.add_argument('--batch_size', type=int, default=128, help='input batch size for training (default: 128)') parser.add_argument('--test_batch_size', type=int, default=1000, help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=150, help='number of epochs to train (default: 150)') parser.add_argument('--gpu_id', type=int, default=0, help='index of gpu to use (default: 0)') parser.add_argument('--lr', type=float, default=0.1, help='init learning rate (default: 0.1)') parser.add_argument('--dp', type=float, default=0.0, help='dropout rate (default: 0.0)') parser.add_argument('--seed', type=int, default=0, help='random seed (default: 0)') parser.add_argument('--noise_pattern', type=str, default='dependent', help='Noise pattern (default: dependent)') parser.add_argument('--noise_rate', type=float, default=0.0, help='Noise rate (default: 0.0)') parser.add_argument('--save', action='store_true', default=False, help='For saving softmax_out_avg') parser.add_argument('--SEAL', type=int, default=0, help='Phase of self-evolution') args = parser.parse_args() torch.manual_seed(args.seed) device = torch.device( 'cuda:' + str(args.gpu_id) if torch.cuda.is_available() else 'cpu') # Datasets root = './data/CIFAR10' num_classes = 10 kwargs = { 'num_workers': 4, 'pin_memory': True } if torch.cuda.is_available() else {} transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) train_dataset = datasets.CIFAR10(root, train=True, download=True, transform=transform_train) train_dataset_noisy = datasets.CIFAR10(root, train=True, transform=transform_train) test_dataset = datasets.CIFAR10(root, train=False, transform=transform_test) targets_noisy = list( pd.read_csv( os.path.join('./data/CIFAR10/label_noisy', args.noise_pattern + str(args.noise_rate) + '.csv'))['label_noisy'].values.astype(int)) train_dataset_noisy.targets = targets_noisy train_loader = torch.utils.data.DataLoader(train_dataset_noisy, batch_size=args.batch_size, shuffle=True, **kwargs) softmax_loader = torch.utils.data.DataLoader( train_dataset_noisy, batch_size=args.test_batch_size, shuffle=False, **kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) def learning_rate(lr_init, epoch): optim_factor = 0 if (epoch > 120): optim_factor = 2 elif (epoch > 60): optim_factor = 1 return lr_init * math.pow(0.2, optim_factor) # results results_root = os.path.join( 'results', 'cifar10_' + args.noise_pattern + str(args.noise_rate)) if not os.path.isdir(results_root): os.makedirs(results_root) """ Get softmax_out_avg - normal training on noisy labels """ if args.SEAL == 0: # Building model model = Wide_ResNet(depth=28, widen_factor=10, dropout_rate=args.dp, num_classes=num_classes).to(device) # Training softmax_out_avg = np.zeros([len(train_dataset_noisy), num_classes]) for epoch in range(1, args.epochs + 1): optimizer = optim.SGD(model.parameters(), lr=learning_rate(args.lr, epoch), momentum=0.9, weight_decay=5e-4) train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader) softmax_out_avg += get_softmax_out(model, softmax_loader, device) softmax_out_avg /= args.epochs if args.save: softmax_root = os.path.join( results_root, 'seed' + str(args.seed) + '_softmax_out_avg_' + args.noise_pattern + str(args.noise_rate) + '_normal.npy') np.save(softmax_root, softmax_out_avg) print('new softmax_out_avg saved to', softmax_root, ', shape: ', softmax_out_avg.shape) """ Self Evolution - training on softmax_out_avg """ if args.SEAL >= 1: # Loading softmax_out_avg of last phase if args.SEAL == 1: softmax_root = os.path.join( results_root, 'seed' + str(args.seed) + '_softmax_out_avg_' + args.noise_pattern + str(args.noise_rate) + '_normal.npy') else: softmax_root = os.path.join( results_root, 'seed' + str(args.seed) + '_softmax_out_avg_' + args.noise_pattern + str(args.noise_rate) + '_SEAL' + str(args.SEAL - 1) + '.npy') softmax_out_avg = np.load(softmax_root) print('softmax_out_avg loaded from', softmax_root, ', shape: ', softmax_out_avg.shape) # Dataset with soft targets train_dataset_soft = CIFAR10_soft(root, targets_soft=torch.Tensor( softmax_out_avg.copy()), train=True, transform=transform_train) train_dataset_soft.targets = targets_noisy train_loader_soft = torch.utils.data.DataLoader( train_dataset_soft, batch_size=args.batch_size, shuffle=True, **kwargs) # Building model model = Wide_ResNet(depth=28, widen_factor=10, dropout_rate=args.dp, num_classes=num_classes).to(device) # Training softmax_out_avg = np.zeros([len(train_dataset_noisy), num_classes]) for epoch in range(1, args.epochs + 1): optimizer = optim.SGD(model.parameters(), lr=learning_rate(args.lr, epoch), momentum=0.9, weight_decay=5e-4) train_soft(args, model, device, train_loader_soft, optimizer, epoch) test(args, model, device, test_loader) softmax_out_avg += get_softmax_out(model, softmax_loader, device) softmax_out_avg /= args.epochs if args.save: softmax_root = os.path.join( results_root, 'seed' + str(args.seed) + '_softmax_out_avg_' + args.noise_pattern + str(args.noise_rate) + '_SEAL' + str(args.SEAL) + '.npy') np.save(softmax_root, softmax_out_avg) print('new softmax_out_avg saved to', softmax_root, ', shape: ', softmax_out_avg.shape)
with open(os.path.join(args.save, "loss.json"), "w") as outfile: json.dump([train, val], outfile) for epoch in range(args.startepoch + 1, args.epochs + 1): training_loss = 0 iter_since = time.time() try: for index, dialog in enumerate(training_data): if args.ss: teacher_forcing_ratio = (teacher_lazy_period - epoch + 1) / teacher_lazy_period if teacher_forcing_ratio < 0.5: teacher_forcing_ratio = 0.5 training_loss += train(my_lang, criterion, teacher_forcing_ratio,\ dialog, encoder, context, decoder, \ encoder_optimizer, context_optimizer, decoder_optimizer) if (index) % 500 == 0: print(" @ Iter [", index + 1, "/", len(training_data),"] | avg. loss: ", training_loss / (index + 1), \ " | perplexity: ", math.exp(training_loss / (index + 1))," | usage ", time.time() - iter_since, " seconds | teacher_force: ", \ teacher_forcing_ratio) sample(my_lang, dialog, encoder, context, decoder) iter_since = time.time() if (index + 1) % 2000 == 0: val_since = time.time() validation_score_100 = validate(my_lang, criterion, teacher_forcing_ratio, \ validation_data[:100], encoder, context, decoder, \ encoder_optimizer, context_optimizer, decoder_optimizer) print(" @ Val. [", index + 1, "/", len(training_data),"] | avg. val. loss: ", validation_score_100, \ " | perplexity: ", math.exp(validation_score_100)," | usage ", time.time() - val_since, " seconds") print(" % Best validation score: ",
def main(): # Settings parser = argparse.ArgumentParser(description='PyTorch MNIST') parser.add_argument('--batch_size', type=int, default=64, help='input batch size for training (default: 64)') parser.add_argument('--test_batch_size', type=int, default=1000, help='input batch size for testing (default: 1000)') parser.add_argument( '--epochs', type=int, default=20, help='number of epochs to train (default: 20)' ) # On clean data, 20 is sufficiently large to achiece 100% training accuracy. parser.add_argument('--gpu_id', type=int, default=0, help='index of gpu to use (default: 0)') parser.add_argument('--lr', type=float, default=0.01, help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, help='SGD momentum (default: 0.5)') parser.add_argument('--seed', type=int, default=0, help='random seed (default: 0)') parser.add_argument('--noise_rate', type=float, default=0.0, help='Noise rate (default: 0.0)') parser.add_argument('--load', action='store_true', default=False, help='Load existing averaged softmax') parser.add_argument('--gen', action='store_true', default=False, help='Generate noisy labels') args = parser.parse_args() torch.manual_seed(args.seed) device = torch.device( 'cuda:' + str(args.gpu_id) if torch.cuda.is_available() else 'cpu') # Datasets root = './data' kwargs = { 'num_workers': 4, 'pin_memory': True } if torch.cuda.is_available() else {} transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) #0.1307, 0.3081 are the mean and std of mnist train_dataset = datasets.MNIST(root, train=True, download=True, transform=transform) test_dataset = datasets.MNIST(root, train=False, transform=transform) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) if args.load: softmax_out_avg = np.load('data/MNIST/label_noisy/softmax_out_avg.npy') print('softmax_out_avg loaded, shape: ', softmax_out_avg.shape) else: # Building model model = MNIST_CNN().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) # Training softmax_out_avg = np.zeros([len(train_dataset), 10]) softmax_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader) softmax_out_avg += get_softmax_out(model, softmax_loader, device) softmax_out_avg /= args.epochs np.save('data/MNIST/label_noisy/softmax_out_avg.npy', softmax_out_avg) if args.gen: print('Generating noisy labels according to softmax_out_avg...') label = np.array(train_dataset.targets) label_noisy_cand, label_noisy_prob = [], [] for i in range(len(label)): pred = softmax_out_avg[i, :].copy() pred[label[i]] = -1 label_noisy_cand.append(np.argmax(pred)) label_noisy_prob.append(np.max(pred)) label_noisy = label.copy() index = np.argsort(label_noisy_prob)[-int(args.noise_rate * len(label)):] label_noisy[index] = np.array(label_noisy_cand)[index] save_pth = os.path.join('./data/MNIST/label_noisy', 'dependent' + str(args.noise_rate) + '.csv') pd.DataFrame.from_dict({ 'label': label, 'label_noisy': label_noisy }).to_csv(save_pth, index=False) print('Noisy label data saved to ', save_pth)