def setUp(self): np.random.seed(0) self.train_data = dataset.MNIST( 'data/mnist/train-images-idx3-ubyte.gz', 'data/mnist/train-labels-idx1-ubyte.gz') self.test_data = dataset.MNIST('data/mnist/t10k-images-idx3-ubyte.gz', 'data/mnist/t10k-labels-idx1-ubyte.gz')
def run_mnist( log_dir, save_period, summary_period, im_summary_period, num_iter = int(1e6), batch_size = 64, n_critic = 10, D_lambda = 5, lr = 0.001, z_dim = 10, c_fn_type='l1', ): ds = dataset.MNIST(batch_size) x,_ = ds.train_data_op p_z = arch.Gaussian_P_Z(z_dim) p_z_length = p_z.length Q_arch = partial(arch.fc_arch, input_shape=(784,), output_size=p_z_length, num_layers=3, embed_size=256) G_arch = partial(arch.fc_arch, input_shape=(p_z_length,), output_size=784, # # of generated pixels num_layers=3, embed_size=256) D_arch = partial(arch.fc_arch, input_shape=(p_z_length,), # shape when flattened. output_size=1, num_layers=3, embed_size=64, act_fn='ELU-like') with tf.variable_scope('param_scope') as scope: # To clearly seperate the parameters belong to layers from tf ops. # Make it easier to reuse pass if c_fn_type == 'l2': c_fn = lambda x,y: tf.reduce_sum(tf.abs(x-y),axis=(1,2,3)) #use l1_distance for recon loss else: c_fn = lambda x,y: tf.reduce_sum((x-y)**2,axis=(1,2,3)) #use l2_distance for recon loss model = \ WAE_WGAN(x, p_z, Q_arch, G_arch, D_arch, D_lambda, c_fn, {'lr':lr, 'lambda':10.}, scope) run(**locals())
def main(): mnist = dts.MNIST('./') net = nn.Network([784, 300, 10], cost_func=nn.CrossEntropyCost) times = 80 evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \ net.SGD(mnist.train_data, times, 10, 0.001, evaluation_data=mnist.test_data, monitor_evaluation_accuracy=True, monitor_training_cost=True, monitor_training_accuracy=True) temp = np.tile(100.0, times) evaluation_accuracy = evaluation_accuracy / temp x = np.arange(1, times + 1, 1) fig = plt.figure() ax1 = fig.add_subplot(2, 1, 1) ax1 = plt.plot(x, evaluation_accuracy, 'g-', linewidth=2) plt.xlabel('Epoch') plt.grid() ax2 = fig.add_subplot(2, 1, 2) ax2 = plt.plot(x, training_cost, 'r-', linewidth=2) plt.ylabel('training_lost') plt.xlabel('Epoch') plt.grid() plt.show()
if args.params in ['beta', 'betas', 'b']: params_used = beta_params # mi_params elif args.params in ['mi', 'mis', 'm', 'constraint', 'constraints']: params_used = mi_params elif args.params in ['few_betas', 'small_beta']: params_used = beta_params if args.dmax is not None: params_used["layers.5.layer_kwargs.d_max"] = list(args.dmax) vary_together = False #params = {"activation.encoder": ["softplus", "sigmoid"]} if args.dataset == 'fmnist': d = dataset.fMNIST() elif args.dataset == 'binary_mnist': d = dataset.MNIST(binary= True) elif args.dataset == 'mnist': d = dataset.MNIST() elif args.dataset == 'omniglot': d = dataset.Omniglot() elif args.dataset == 'dsprites': d = dataset.DSprites() if args.per_label is not None: d.shrink_supervised(int(args.per_label)) # name is important! = filesave location if args.time is not None: a = session.Session(name=args.name, config=args.config, dataset = d, parameters= params_used, time = args.time, verbose = args.verbose, per_label = args.per_label, vary_together= vary_together) else:
def main(): """ Main function Here, you should instantiate 1) Dataset objects for training and test datasets 2) DataLoaders for training and testing 3) model 4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9 5) cost function: use torch.nn.CrossEntropyLoss """ # ========== 1. data load ========== transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.1307], [0.3081])]) train_dataset = dataset.MNIST(data_dir = 'data/train.tar', transform=transform) test_dataset = dataset.MNIST(data_dir = 'data/test.tar', transform=transform) train_data = DataLoader(train_dataset, batch_size=64) test_data = DataLoader(test_dataset, batch_size=64) # ========== 2. Lenet 5 model ========== device = 'cuda' if torch.cuda.is_available() else 'cpu' training_epochs = 10 lenet_model = LeNet5().to(device) lenet_optimizer = torch.optim.SGD(lenet_model.parameters(), lr=0.01, momentum=0.9) lenet_cost_function = torch.nn.CrossEntropyLoss().to(device) print('Lenet 5 training start ') lenet_time = time.time() lenet_trn_loss, lenet_trn_acc, lenet_tst_loss, lenet_tst_acc = [],[],[],[] for epoch in range(training_epochs) : lenet_train_loss, lenet_train_acc = train(model=lenet_model, trn_loader=train_data, device=device, criterion=lenet_cost_function, optimizer=lenet_optimizer) lenet_test_loss, lenet_test_acc = test(model=lenet_model, tst_loader=test_data, device=device, criterion=lenet_cost_function) lenet_trn_loss.append(lenet_train_loss); lenet_trn_acc.append(lenet_train_acc) lenet_tst_loss.append(lenet_test_loss); lenet_tst_acc.append(lenet_test_acc) print('epochs {} training loss {} training accuracy {} validation loss {} validation accuracy {}'.format(epoch, lenet_train_loss, lenet_train_acc, lenet_test_loss, lenet_test_acc)) if epoch+1 == 10 : print('lenet execution time : {}'.format(time.time() - lenet_time)) # ========== 3. Regularized Lenet 5 model ========== regularized_lenet_model = regularized_LeNet5().to(device) regularized_lenet_optimizer = torch.optim.SGD(regularized_lenet_model.parameters(), lr=0.01, momentum=0.9, weight_decay=0.001) regularized_lenet_cost_function = torch.nn.CrossEntropyLoss().to(device) print('Regularized Lenet 5 training start ') r_lenet_time = time.time() r_lenet_trn_loss, r_lenet_trn_acc, r_lenet_tst_loss, r_lenet_tst_acc = [],[],[],[] for epoch in range(training_epochs) : regularized_lenet_train_loss, regularized_lenet_train_acc = train(model=regularized_lenet_model, trn_loader=train_data, device=device, criterion=regularized_lenet_cost_function, optimizer=regularized_lenet_optimizer) regularized_lenet_test_loss, regularized_lenet_test_acc = test(model=regularized_lenet_model, tst_loader=test_data, device=device, criterion=regularized_lenet_cost_function) r_lenet_trn_loss.append(regularized_lenet_train_loss); r_lenet_trn_acc.append(regularized_lenet_train_acc) r_lenet_tst_loss.append(regularized_lenet_test_loss); r_lenet_tst_acc.append(regularized_lenet_test_acc) print('epochs {} training loss {} training accuracy {} validation loss {} validation accuracy {}'.format(epoch, regularized_lenet_train_loss, regularized_lenet_train_acc, regularized_lenet_test_loss, regularized_lenet_test_acc)) if epoch+1 == 10 : print('regularized execution time : {}'.format(time.time() - r_lenet_time)) # ========== 4. Custom model Load ========== custom_model = CustomMLP().to(device) custom_optimizer = torch.optim.SGD(custom_model.parameters(), lr=0.01, momentum=0.9) custom_cost_function = torch.nn.CrossEntropyLoss().to(device) print('Custom model training start') custom_time = time.time() custom_trn_loss, custom_trn_acc, custom_tst_loss, custom_tst_acc = [],[],[],[] for epoch in range(training_epochs) : custom_train_loss, custom_train_acc = train(model=custom_model, trn_loader=train_data, device=device, criterion=custom_cost_function, optimizer=custom_optimizer) custom_test_loss, custom_test_acc = test(model=custom_model, tst_loader=test_data, device=device, criterion=custom_cost_function) custom_trn_loss.append(custom_train_loss); custom_trn_acc.append(custom_train_acc) custom_tst_loss.append(custom_test_loss); custom_tst_acc.append(custom_test_acc) print('epochs {} training loss {} training accuracy {} validation loss {} validation accuracy {}'.format(epoch, custom_train_loss, custom_train_acc, custom_test_loss, custom_test_acc)) if epoch+1 == 10 : print('custom model execution time : {}'.format(time.time() - custom_time)) # ========== 5. visualization ========== # make loss and acc list for visualization trn_loss = [lenet_trn_loss, r_lenet_trn_loss, custom_trn_loss] trn_acc = [lenet_trn_acc, r_lenet_trn_acc, custom_trn_acc] tst_loss = [lenet_tst_loss, r_lenet_tst_loss, custom_tst_loss] tst_acc = [lenet_tst_acc, r_lenet_tst_acc, custom_tst_acc] # draw plot draw_plot(trn_loss, trn_acc, tst_loss, tst_acc)
def main(): """ Main function Here, you should instantiate 1) Dataset objects for training and test datasets 2) DataLoaders for training and testing 3) model 4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9 5) cost function: use torch.nn.CrossEntropyLoss """ # write your codes here if torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') batch_size=64 trn_dataset = dataset.MNIST(data_dir='../data/train') trn_loader = torch.utils.data.DataLoader(trn_dataset, batch_size=batch_size, shuffle=True) tst_dataset = dataset.MNIST(data_dir='../data/test') tst_loader = torch.utils.data.DataLoader(tst_dataset, batch_size=batch_size, shuffle=True) LeNet = LeNet5().to(device) Custom = CustomMLP().to(device) # loss criterion = nn.CrossEntropyLoss() optimizer1 = optim.SGD(LeNet.parameters(), lr=0.01, momentum=0.9) optimizer2 = optim.SGD(Custom.parameters(), lr=0.01, momentum=0.9) # hyper-parameters num_epochs = 20 trn_loss_list1 = [] tst_loss_list1 = [] trn_acc1 = [] tst_acc1 = [] trn_loss_list2 = [] tst_loss_list2 = [] trn_acc2 = [] tst_acc2 = [] for epoch in range(num_epochs): print("epoch : ", (epoch+1)) print("LeNet") trn_loss, trn_acc = train(LeNet, trn_loader, device, criterion, optimizer1) trn_loss_list1.append(trn_loss / len(trn_loader)) trn_acc1.append(trn_acc) tst_loss, tst_acc = test(LeNet, tst_loader, device, criterion) tst_loss_list1.append(tst_loss / len(tst_loader)) tst_acc1.append(tst_acc) print("CustomMLP") trn_loss, trn_acc = train(Custom, trn_loader, device, criterion, optimizer2) trn_loss_list2.append(trn_loss / len(trn_loader)) trn_acc2.append(trn_acc) tst_loss, tst_acc = test(Custom, tst_loader, device, criterion) tst_loss_list2.append(tst_loss / len(tst_loader)) tst_acc2.append(tst_acc) plt.figure(figsize=(5,4)) x_range = range(len(trn_loss_list1)) plt.plot(x_range, trn_loss_list1, label="trn") plt.plot(x_range, tst_loss_list1, label="tst") plt.legend() plt.ylim(0, 1) plt.title('LeNet-5 Loss') plt.xlabel("training steps") plt.ylabel("loss") plt.grid() plt.figure(figsize=(5,4)) x_range = range(len(trn_acc1)) plt.plot(x_range, trn_acc1, label="trn") plt.plot(x_range, tst_acc1, label="tst") plt.legend() plt.ylim(0, 100) plt.title('LeNet-5 Accuracy') plt.xlabel("training steps") plt.ylabel("loss") plt.grid() plt.figure(figsize=(5,4)) x_range = range(len(trn_loss_list2)) plt.plot(x_range, trn_loss_list2, label="trn") plt.plot(x_range, tst_loss_list2, label="tst") plt.legend() plt.ylim(0, 1) plt.title('CustomMLP Loss') plt.xlabel("training steps") plt.ylabel("loss") plt.grid() plt.figure(figsize=(5,4)) x_range = range(len(trn_acc2)) plt.plot(x_range, trn_acc2, label="trn") plt.plot(x_range, tst_acc2, label="tst") plt.legend() plt.ylim(0, 100) plt.title('CustomMLP Accuracy') plt.xlabel("training steps") plt.ylabel("acc") plt.grid() # val acc with torch.no_grad(): corr_num = 0 total_num = 0 for j, val in enumerate(tst_loader): val_x, val_label = val if device: val_x = val_x.cuda() val_label =val_label.cuda() val_output = LeNet(val_x) model_label = val_output.argmax(dim=1) corr = val_label[val_label == model_label].size(0) corr_num += corr total_num += val_label.size(0) print("LeNet5 acc: {:.2f}".format(corr_num / total_num * 100)) with torch.no_grad(): corr_num = 0 total_num = 0 for j, val in enumerate(tst_loader): val_x, val_label = val if device: val_x = val_x.cuda() val_label =val_label.cuda() val_output = Custom(val_x) model_label = val_output.argmax(dim=1) corr = val_label[val_label == model_label].size(0) corr_num += corr total_num += val_label.size(0) print("CustomMLP acc: {:.2f}".format(corr_num / total_num * 100))
def main(epochs, batch_size, learn_angle, angle_lr): transform = torchvision.transforms.Compose([ transforms.RandomRotation(30), transforms.Identity() if learn_angle else transforms.Free(), transforms.ToTensor(), transforms.Normalize() ]) set_train = dataset.MNIST(root='./data', train=True, download=True, transform=transform) set_test = dataset.MNIST(root='./data', train=False, download=True, transform=transform) loader_train = torch.utils.data.DataLoader(set_train, batch_size=batch_size, shuffle=True, num_workers=2) loader_test = torch.utils.data.DataLoader(set_test, batch_size=batch_size, shuffle=False, num_workers=2) nn = model.Net(learn_angle) if torch.cuda.is_available(): nn.cuda() optimizer = optim.SGD(nn.parameters(), lr=0.001, momentum=0.9) VIEW_INTERVAL = 100 for epoch in range(epochs): acc_loss = 0.0 running_loss = 0.0 for i, sample in enumerate(loader_train): x, y = pack(sample, learn_angle) optimizer.zero_grad() y_pred = nn(x) loss = model.loss(y_pred, y, angle_lr) loss.backward() optimizer.step() # report loss acc_loss += loss.item() if i % VIEW_INTERVAL == VIEW_INTERVAL - 1: running_loss = acc_loss / VIEW_INTERVAL click.secho( f"\rEpoch {epoch+1}, iteration {i+1}; " f"loss: {(running_loss):.3f}; ", err=True, nl=False) acc_loss = 0.0 # testing count_correct = 0 count_total = 0 for sample in loader_test: x, labels = pack(sample, learn_angle) y_pred = nn(Variable(x)) if learn_angle: labels = labels[0] y_pred = y_pred[0] _, labels_pred = torch.max(y_pred.data, 1) c = (labels_pred == labels).squeeze() count_correct += c.sum().item() count_total += len(c) click.secho( f"\rEpoch {epoch+1}; loss: {(running_loss):.3f}; " f"Test Acc: {100.0 * count_correct / count_total :.2f}%", err=True, nl=False) running_loss = 0 click.secho('', err=True)
def main(): # Training settings parser = argparse.ArgumentParser( description='Reducing Communication Rounds in Federated Learning') parser.add_argument('--batch-size', type=int, default=32, metavar='N', help='input batch size for training (default: 32)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument( '--max_iterations', type=int, default=20, metavar='N', help='number of iterations for global training (default:20)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum (default: 0.9)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') parser.add_argument('--client-num', type=int, default=100, help='Number of clients(locals) to use') parser.add_argument('--cli-ite-num', type=int, default=4, help='Number of epochs to train the local clients for') parser.add_argument('--start_threshold', type=float, default=0.8, help='Starting threshold for Check relevance function') parser.add_argument('--use_wandb', action='store_true', default=False, help='Use wandb for logging') parser.add_argument('--anonymous_mode', action='store_true', default=False, help='Use anonymous mode for visualizing results') parser.add_argument( '--force_client_train', type=int, default=5, help='After these many epochs, clients are selectively trained') parser.add_argument('--topk', type=float, default=0.1, help='topk') args = parser.parse_args() #We use wandb for code logging. First run would require you to set it up on wandb.ai. #To use wandb, set the flab --use_wandb. If not, matlplotlib plots are stored if args.use_wandb: if args.anonymous_mode: wandb.login() wandb.init(project="cloud-federated", anonymous="must") else: wandb.init(project="cloud-federated", entity="cloud") wandb.config.update(args) use_cuda = not args.no_cuda and torch.cuda.is_available() set_random_seeds(args.seed) device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 0, 'pin_memory': True} if use_cuda else {} #All trainloders to be appended to this list train_loaders = [] #We modify the original MNIST dataloader to work for our setting as different clients have different subsets of the data #Refer to dataset.py for changes for i in range(args.client_num): train_loaders.append( torch.utils.data.DataLoader(dataset.MNIST( '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]), client_id=i, num_clients=args.client_num), batch_size=args.batch_size, shuffle=True, **kwargs)) #Test uses the standard dataloader used for MNIST test_loader = torch.utils.data.DataLoader(dataset.MNIST( '../data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]), client_id=-1, num_clients=1), batch_size=args.test_batch_size, shuffle=False, **kwargs) model = Net().to(device) if not os.path.exists('generated_data'): os.mkdir('generated_data') if not os.path.exists('generated_data/thresh' + str(args.start_threshold)): os.mkdir('generated_data/thresh' + str(args.start_threshold)) communication_rounds = [] last_update = None # Last update is initialized to None --> implies that for the first run, all updates are relevant previous_relevances = [ True ] * args.client_num # Last update is initialized to None --> implies that for the first run, all updates are relevant prev_local_updates = None # Last update is initialized to None --> implies that for the first run, all updates are relevant all_relevances = [] test_losses = [] test_accuracies = [] thresholds = [] average_signs = [] when_above_60 = -1 for it in tqdm(range(1, args.max_iterations)): ( c_round_cmfl, c_round_after_sel ), last_update, avg_sign, thresh, rel_it, prev_local_updates = global_train( args, model, device, train_loaders, it, last_update, previous_relevances, prev_local_updates) if it > args.force_client_train: previous_relevances = rel_it print('Not training all! ') c_round = c_round_after_sel else: c_round = c_round_cmfl + args.client_num communication_rounds.append(c_round) test_loss, test_acc = test(args, model, device, test_loader) test_losses.append(test_loss) test_accuracies.append(test_acc) print('Cumulative Communication Rounds : {}'.format( sum(communication_rounds))) thresholds.append(thresh) average_signs.append(avg_sign) all_relevances.append(rel_it) if args.use_wandb: heatmap = draw_heatmap( np.stack(all_relevances, 1).astype(np.float)) wand_dict = { 'Communication Rounds': sum(communication_rounds), 'Test loss': test_loss, 'Test Acc': test_acc, 'Average sign': avg_sign, 'Threshold': thresh, 'relevance_it': wandb.Image(heatmap) } wandb.log(wand_dict) if last_update == None: print('exiting!') break if (args.save_model): torch.save(model.state_dict(), "model_" + str(it) + ".pt") if when_above_60 == -1 and test_acc > 60: when_above_60 = sum(communication_rounds) all_stats = { 'communication_rounds': communication_rounds, 'test_loss': test_losses, 'test_acc': test_accuracies, 'avg_sign': average_signs, 'threshold': thresholds, #'heatmap':np.stack(all_relevances,1).astype(np.float) } print( 'Training done, the model reached above 60 percent accuracy after {} communication rounds' .format(when_above_60)) with open( 'generated_data/thresh' + str(args.start_threshold) + '/' + 'all_stats.pkl', 'wb') as f: pickle.dump(all_stats, f) plot_data(all_stats, 'generated_data/thresh' + str(args.start_threshold) + '/')
def _parse_args(self): # Note : can also pass dataset directly if isinstance(self.dataset, str): if self.dataset == 'mnist': self.dataset = dataset.MNIST(binary=False) elif self.dataset == 'binary_mnist': self.dataset = dataset.MNIST(binary=True) elif self.dataset == 'omniglot': self.dataset = dataset.Omniglot() elif self.dataset == 'celeb_a': pass elif self.dataset == 'dsprites': self.dataset = dataset.DSprites() # selects per_label examples from each class for reduced training data if self.per_label is not None: self.dataset.shrink_supervised(self.per_label) self.dataset_clean = copy(self.dataset) if self.optimizer_params.get("norm", False): opt_norm = self.optimizer_params["norm"] del self.optimizer_params["norm"] else: opt_norm = None self.optimizer = getattr(keras.optimizers, self.optimizer)(**self.optimizer_params) #if opt_norm is not None: # self.optimizer = NormalizedOptimizer(self.optimizer, normalization = opt_norm) self.lr_callback = False if isinstance(self.lr, str): try: mod = importlib.import_module(str('lr_sched')) # LR Callback will be True /// self.lr = function of epochs -> lr self.lr_callback = isinstance(self.lr, str) self.lr = getattr(mod, self.lr) except: try: mod = importlib.import_module( str('custom_functions.lr_sched')) # LR Callback will be True /// self.lr = function of epochs -> lr self.lr_callback = isinstance(self.lr, str) self.lr = getattr(mod, self.lr) except: #self.lr = dflt.get('lr', .001) #print() warnings.warn( "Cannot find LR Schedule function. Proceeding with default, constant learning rate of 0.001." ) #print() # Architecture Args if self.encoder_dims is None: try: self.encoder_dims = self.latent_dims except Exception as e: print(e) raise ValueError if self.decoder_dims is None: self.decoder_dims = list(reversed(self.encoder_dims[:-1])) self.decoder_dims.append(self.dataset.dim) else: pass
import nn import dataset import dataset.transforms if __name__ == '__main__': model = nn.model.SequentialModel(input_shape=[-1, 784]) model.add(nn.layer.Dense(128, activation=nn.activation.Tanh())) model.add(nn.layer.Dense(128, activation=nn.activation.Tanh())) model.add(nn.layer.Dense(10, activation=nn.activation.Softmax())) model.setup(nn.loss.Cross_Entropy_With_Softmax(), nn.metric.CategoricalAccuracy()) # model.save("MNISTDNN.model") # # model = nn.model.Model.load('MNISTDNN.model') # data = dataset.MNIST() trans = dataset.transforms.Shuffle().add(dataset.transforms.ImageCls()) x, y, x_t, y_t = trans(*data.load()) model.compile(nn.gradient_descent.GradientDecay(0.1)) model.fit(x, y, batch_size=64, epoch=10)
def run_mnist( log_dir, save_period, summary_period, im_summary_period, num_iter=int(1e6), batch_size=128, lr=0.0001, lamb=10, z_dim=8, c_fn_type='l2_sum', ): ds = dataset.MNIST(batch_size) x, _ = ds.train_data_op p_z = arch.Gaussian_P_Z(z_dim) p_z_length = p_z.length Q_arch = partial(arch.fc_arch, input_shape=(784, ), output_size=p_z_length, num_layers=3, embed_size=256) G_arch = partial( arch.fc_arch, input_shape=(p_z_length, ), output_size=784, # # of generated pixels num_layers=3, embed_size=256) with tf.variable_scope('param_scope') as scope: # To clearly seperate the parameters belong to layers from tf ops. # Make it easier to reuse pass if c_fn_type == 'l1': c_fn = lambda x, y: tf.reduce_mean(tf.abs(x - y), axis=(1, 2, 3) ) #use l1_distance for recon loss if c_fn_type == 'l1_sum': c_fn = lambda x, y: tf.reduce_sum(tf.abs(x - y), axis=(1, 2, 3) ) #use l1_distance for recon loss elif c_fn_type == 'l2': c_fn = lambda x, y: tf.reduce_mean( (x - y)**2, axis=(1, 2, 3)) #use l2_distance for recon loss elif c_fn_type == 'l2_sum': c_fn = lambda x, y: tf.reduce_sum( (x - y)**2, axis=(1, 2, 3)) #use l2_distance for recon loss else: assert False, 'not supported cost type' model = \ WAE_MMD(x, p_z, Q_arch, G_arch, c_fn, {'lr':lr, 'lambda':lamb}, scope) run(**locals())
def main(net="lenet5", augmentation=False): device = torch.device("cpu") if net == "lenet5" : net = LeNet5() elif net == "mlp" : net = CustomMLP() else : print("Not support Networks") loss = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(net.parameters(), lr=0.01, momentum=0.9) if os.path.isdir("../data/train_mnist") == True : pass else : print("Unzip train mnist") trn_dataset = '../data/train.tar' trn_dst_path = '../data/train_mnist' os.makedirs(trn_dst_path) with tarfile.TarFile(trn_dataset, 'r') as file: file.extractall(trn_dst_path) if os.path.isdir("../data/test_mnist") == True : pass else : print("Unzip test mnist") tst_dataset = '../data/test.tar' tst_dst_path = '../data/test_mnist' os.makedirs(tst_dst_path) with tarfile.TarFile(tst_dataset, 'r') as file: file.extractall(tst_dst_path) train_folder = "../data/train_mnist/train" test_folder = "../data/test_mnist/test" train_set = dataset.MNIST(train_folder) test_set = dataset.MNIST(test_folder) if augmentation == True : aug_train_set = dataset.MNIST(train_folder, aug=True) aug_test_Set = dataset.MNIST(test_folder) trn_loader = DataLoader(train_set, batch_size=128) tst_loader = DataLoader(test_set, batch_size=128) if augmentation == True : aug_train_loader = DataLoader(aug_train_set, batch_size=128) aug_test_loader = DataLoader(aug_test_Set, batch_size=128) train_logger = [] test_logger = [] for epoch in range(5) : if augmentation == False : training = train(net.to(device), trn_loader, device, loss, optimizer, epoch) tests = test(net.to(device), tst_loader, device, loss, epoch) train_logger.append(training) test_logger.append(tests) else : training = train(net.to(device), aug_train_loader, device, loss, optimizer, epoch) tests = test(net.to(device), tst_loader, device, loss, epoch) train_logger.append(training) test_logger.append(tests) return train_logger, test_logger
def main(): """ Main function Here, you should instantiate 1) Dataset objects for training and test datasets 2) DataLoaders for training and testing 3) model 4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9 5) cost function: use torch.nn.CrossEntropyLoss """ n_cpu = multiprocessing.cpu_count() data_dir_dict = {'train': '../data/train', 'test': '../data/test'} for _, data_dir in data_dir_dict.items(): if not os.path.isdir(data_dir): tar = tarfile.open('%s.tar' % data_dir, mode='r') tar.extractall(path='./data/') device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') num_epochs = 50 batch_size = 512 criterion = nn.CrossEntropyLoss() train_dataset = dataset.MNIST(data_dir_dict['train']) train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=n_cpu) test_dataset = dataset.MNIST(data_dir_dict['test']) test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=n_cpu) net = LeNet5().to(device) net_optimizer = SGD(net.parameters(), lr=.01, momentum=.9) net_trn_loss_list, net_trn_acc_list, net_tst_loss_list, net_tst_acc_list= [], [], [], [] print('--------------LeNet5--------------') for epoch in range(num_epochs): net_time = time.time() trn_loss, trn_acc = train(net, train_loader, device, criterion, net_optimizer) tst_loss, tst_acc = test(net, test_loader, device, criterion) print('%s epoch || %.5f time' % ((epoch + 1), time.time() - net_time)) print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\ %(trn_loss, trn_acc, tst_loss, tst_acc)) print('') net_trn_loss_list.append(trn_loss) net_trn_acc_list.append(trn_acc) net_tst_loss_list.append(tst_loss) net_tst_acc_list.append(tst_acc) mlp = CustomMLP().to(device) mlp_optimizer = SGD(mlp.parameters(), lr=.01, momentum=.9) mlp_trn_loss_list, mlp_trn_acc_list, mlp_tst_loss_list, mlp_tst_acc_list= [], [], [], [] print('--------------CustomMLP--------------') for epoch in range(num_epochs): mlp_time = time.time() trn_loss, trn_acc = train(mlp, train_loader, device, criterion, mlp_optimizer) tst_loss, tst_acc = test(mlp, test_loader, device, criterion) print('%s epoch || %.5f time' % ((epoch + 1), time.time() - mlp_time)) print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\ %(trn_loss, trn_acc, tst_loss, tst_acc)) print('') mlp_trn_loss_list.append(trn_loss) mlp_trn_acc_list.append(trn_acc) mlp_tst_loss_list.append(tst_loss) mlp_tst_acc_list.append(tst_acc) del train_dataset del train_loader train_dataset_aug = dataset.MNIST(data_dir_dict['train'], aug_option=True) train_loader_aug = DataLoader(dataset=train_dataset_aug, batch_size=batch_size, shuffle=True, num_workers=n_cpu) net_aug = LeNet5().to(device) net_optimizer_aug = SGD(net_aug.parameters(), lr=.01, momentum=.9) net_aug_trn_loss_list, net_aug_trn_acc_list, net_aug_tst_loss_list, net_aug_tst_acc_list= [], [], [], [] print('--------------LeNet5 with augmentation--------------') for epoch in range(num_epochs): net_time = time.time() trn_loss, trn_acc = train(net_aug, train_loader_aug, device, criterion, net_optimizer_aug) tst_loss, tst_acc = test(net_aug, test_loader, device, criterion) print('%s epoch || %.5f time' % ((epoch + 1), time.time() - net_time)) print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\ %(trn_loss, trn_acc, tst_loss, tst_acc)) print('') net_aug_trn_loss_list.append(trn_loss) net_aug_trn_acc_list.append(trn_acc) net_aug_tst_loss_list.append(tst_loss) net_aug_tst_acc_list.append(tst_acc) mlp_aug = CustomMLP().to(device) mlp_optimizer_aug = SGD(mlp_aug.parameters(), lr=.01, momentum=.9) mlp_aug_trn_loss_list, mlp_aug_trn_acc_list, mlp_aug_tst_loss_list, mlp_aug_tst_acc_list= [], [], [], [] print('--------------CustomMLP with augmentation--------------') for epoch in range(num_epochs): mlp_time = time.time() trn_loss, trn_acc = train(mlp_aug, train_loader_aug, device, criterion, mlp_optimizer_aug) tst_loss, tst_acc = test(mlp_aug, test_loader, device, criterion) print('%s epoch || %.5f time' % ((epoch + 1), time.time() - mlp_time)) print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\ %(trn_loss, trn_acc, tst_loss, tst_acc)) print('') mlp_aug_trn_loss_list.append(trn_loss) mlp_aug_trn_acc_list.append(trn_acc) mlp_aug_tst_loss_list.append(tst_loss) mlp_aug_tst_acc_list.append(tst_acc) net_result, net_aug_result, mlp_result, mlp_aug_result = {}, {}, {}, {} net_result['train_loss'] = net_trn_loss_list net_result['train_acc'] = net_trn_acc_list net_result['test_loss'] = net_tst_loss_list net_result['test_acc'] = net_tst_acc_list net_aug_result['train_loss'] = net_aug_trn_loss_list net_aug_result['train_acc'] = net_aug_trn_acc_list net_aug_result['test_loss'] = net_aug_tst_loss_list net_aug_result['test_acc'] = net_aug_tst_acc_list mlp_result['train_loss'] = mlp_trn_loss_list mlp_result['train_acc'] = mlp_trn_acc_list mlp_result['test_loss'] = mlp_tst_loss_list mlp_result['test_acc'] = mlp_tst_acc_list mlp_aug_result['train_loss'] = mlp_aug_trn_loss_list mlp_aug_result['train_acc'] = mlp_aug_trn_acc_list mlp_aug_result['test_loss'] = mlp_aug_tst_loss_list mlp_aug_result['test_acc'] = mlp_aug_tst_acc_list net_result = pd.DataFrame( net_result, columns=['train_loss', 'train_acc', 'test_loss', 'test_acc']) net_aug_result = pd.DataFrame( net_aug_result, columns=['train_loss', 'train_acc', 'test_loss', 'test_acc']) mlp_result = pd.DataFrame( mlp_result, columns=['train_loss', 'train_acc', 'test_loss', 'test_acc']) mlp_aug_result = pd.DataFrame( mlp_aug_result, columns=['train_loss', 'train_acc', 'test_loss', 'test_acc']) if not os.path.exists('../results/'): os.makedirs('../results/') net_result.to_csv('../results/LeNet5.csv', index=False) net_aug_result.to_csv('../results/LeNet5_aug.csv', index=False) mlp_result.to_csv('../results/MLP.csv', index=False) mlp_aug_result.to_csv('../results/MLP_aug.csv', index=False)
def __init__(self, sess, epoch, batch_size, z_dim, dataset_name, compute_metrics_it, checkpoint_dir, result_dir, log_dir, gpu_id, bot, redo, verbosity): self.sess = sess self.dataset_name = dataset_name.lower() self.checkpoint_dir = checkpoint_dir self.result_dir = result_dir self.log_dir = log_dir self.epoch = epoch self.batch_size = batch_size self.compute_metrics_it = compute_metrics_it self.gpu_id = gpu_id self.bot = bot self.redo = redo self.verbosity = verbosity if self.dataset_name in ['mnist']: # parameters self.input_height = 28 self.input_width = 28 self.z_dim = z_dim # dimension of noise-vector self.c_dim = 1 # test self.sample_num = 64 # number of generated images to be saved # load mnist self.ds = dataset.MNIST(self.batch_size) self.num_batches = self.ds.N_TRAIN_SAMPLES // self.batch_size # architecture hyper parameters self.data_format = 'NHWC' elif self.dataset_name in ['fashion-mnist']: # parameters self.input_height = 28 self.input_width = 28 self.z_dim = z_dim # dimension of noise-vector self.c_dim = 1 # test self.sample_num = 64 # number of generated images to be saved # load mnist self.ds = dataset.FASHION_MNIST(self.batch_size) self.num_batches = self.ds.N_TRAIN_SAMPLES // self.batch_size # architecture hyper parameters self.data_format = 'NHWC' elif self.dataset_name in ['celeba']: # parameters self.input_height = 64 self.input_width = 64 self.z_dim = z_dim # dimension of noise-vector self.c_dim = 3 # test self.sample_num = 64 # number of generated images to be saved # load CelebA self.ds = dataset.CelebA(self.batch_size) self.num_batches = self.ds.N_TRAIN_SAMPLES // self.batch_size else: raise NotImplementedError