def setUp(self):
np.random.seed(0)
self.train_data = dataset.MNIST(
'data/mnist/train-images-idx3-ubyte.gz',
'data/mnist/train-labels-idx1-ubyte.gz')
self.test_data = dataset.MNIST('data/mnist/t10k-images-idx3-ubyte.gz',
'data/mnist/t10k-labels-idx1-ubyte.gz')
def run_mnist(
log_dir,
save_period,
summary_period,
im_summary_period,
num_iter = int(1e6),
batch_size = 64,
n_critic = 10,
D_lambda = 5,
lr = 0.001,
z_dim = 10,
c_fn_type='l1',
):
ds = dataset.MNIST(batch_size)
x,_ = ds.train_data_op
p_z = arch.Gaussian_P_Z(z_dim)
p_z_length = p_z.length
Q_arch = partial(arch.fc_arch,
input_shape=(784,),
output_size=p_z_length,
num_layers=3,
embed_size=256)
G_arch = partial(arch.fc_arch,
input_shape=(p_z_length,),
output_size=784, # # of generated pixels
num_layers=3,
embed_size=256)
D_arch = partial(arch.fc_arch,
input_shape=(p_z_length,), # shape when flattened.
output_size=1,
num_layers=3,
embed_size=64,
act_fn='ELU-like')
with tf.variable_scope('param_scope') as scope:
# To clearly seperate the parameters belong to layers from tf ops.
# Make it easier to reuse
pass
if c_fn_type == 'l2':
c_fn = lambda x,y: tf.reduce_sum(tf.abs(x-y),axis=(1,2,3)) #use l1_distance for recon loss
else:
c_fn = lambda x,y: tf.reduce_sum((x-y)**2,axis=(1,2,3)) #use l2_distance for recon loss
model = \
WAE_WGAN(x,
p_z,
Q_arch,
G_arch,
D_arch,
D_lambda,
c_fn,
{'lr':lr, 'lambda':10.},
scope)
run(**locals())
def main():
mnist = dts.MNIST('./')
net = nn.Network([784, 300, 10], cost_func=nn.CrossEntropyCost)
times = 80
evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \
net.SGD(mnist.train_data, times, 10, 0.001, evaluation_data=mnist.test_data,
monitor_evaluation_accuracy=True, monitor_training_cost=True, monitor_training_accuracy=True)
temp = np.tile(100.0, times)
evaluation_accuracy = evaluation_accuracy / temp
x = np.arange(1, times + 1, 1)
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax1 = plt.plot(x, evaluation_accuracy, 'g-', linewidth=2)
plt.xlabel('Epoch')
plt.grid()
ax2 = fig.add_subplot(2, 1, 2)
ax2 = plt.plot(x, training_cost, 'r-', linewidth=2)
plt.ylabel('training_lost')
plt.xlabel('Epoch')
plt.grid()
plt.show()
if args.params in ['beta', 'betas', 'b']:
params_used = beta_params # mi_params
elif args.params in ['mi', 'mis', 'm', 'constraint', 'constraints']:
params_used = mi_params
elif args.params in ['few_betas', 'small_beta']:
params_used = beta_params
if args.dmax is not None:
params_used["layers.5.layer_kwargs.d_max"] = list(args.dmax)
vary_together = False
#params = {"activation.encoder": ["softplus", "sigmoid"]}
if args.dataset == 'fmnist':
d = dataset.fMNIST()
elif args.dataset == 'binary_mnist':
d = dataset.MNIST(binary= True)
elif args.dataset == 'mnist':
d = dataset.MNIST()
elif args.dataset == 'omniglot':
d = dataset.Omniglot()
elif args.dataset == 'dsprites':
d = dataset.DSprites()
if args.per_label is not None:
d.shrink_supervised(int(args.per_label))
# name is important! = filesave location
if args.time is not None:
a = session.Session(name=args.name, config=args.config, dataset = d, parameters= params_used, time = args.time, verbose = args.verbose, per_label = args.per_label, vary_together= vary_together)
else:
def main():
""" Main function
Here, you should instantiate
1) Dataset objects for training and test datasets
2) DataLoaders for training and testing
3) model
4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9
5) cost function: use torch.nn.CrossEntropyLoss
"""
# ========== 1. data load ==========
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.1307], [0.3081])])
train_dataset = dataset.MNIST(data_dir = 'data/train.tar', transform=transform)
test_dataset = dataset.MNIST(data_dir = 'data/test.tar', transform=transform)
train_data = DataLoader(train_dataset, batch_size=64)
test_data = DataLoader(test_dataset, batch_size=64)
# ========== 2. Lenet 5 model ==========
device = 'cuda' if torch.cuda.is_available() else 'cpu'
training_epochs = 10
lenet_model = LeNet5().to(device)
lenet_optimizer = torch.optim.SGD(lenet_model.parameters(), lr=0.01, momentum=0.9)
lenet_cost_function = torch.nn.CrossEntropyLoss().to(device)
print('Lenet 5 training start ')
lenet_time = time.time()
lenet_trn_loss, lenet_trn_acc, lenet_tst_loss, lenet_tst_acc = [],[],[],[]
for epoch in range(training_epochs) :
lenet_train_loss, lenet_train_acc = train(model=lenet_model, trn_loader=train_data, device=device, criterion=lenet_cost_function,
optimizer=lenet_optimizer)
lenet_test_loss, lenet_test_acc = test(model=lenet_model, tst_loader=test_data, device=device, criterion=lenet_cost_function)
lenet_trn_loss.append(lenet_train_loss); lenet_trn_acc.append(lenet_train_acc)
lenet_tst_loss.append(lenet_test_loss); lenet_tst_acc.append(lenet_test_acc)
print('epochs {} training loss {} training accuracy {} validation loss {} validation accuracy {}'.format(epoch, lenet_train_loss, lenet_train_acc,
lenet_test_loss, lenet_test_acc))
if epoch+1 == 10 :
print('lenet execution time : {}'.format(time.time() - lenet_time))
# ========== 3. Regularized Lenet 5 model ==========
regularized_lenet_model = regularized_LeNet5().to(device)
regularized_lenet_optimizer = torch.optim.SGD(regularized_lenet_model.parameters(), lr=0.01, momentum=0.9, weight_decay=0.001)
regularized_lenet_cost_function = torch.nn.CrossEntropyLoss().to(device)
print('Regularized Lenet 5 training start ')
r_lenet_time = time.time()
r_lenet_trn_loss, r_lenet_trn_acc, r_lenet_tst_loss, r_lenet_tst_acc = [],[],[],[]
for epoch in range(training_epochs) :
regularized_lenet_train_loss, regularized_lenet_train_acc = train(model=regularized_lenet_model, trn_loader=train_data, device=device,
criterion=regularized_lenet_cost_function, optimizer=regularized_lenet_optimizer)
regularized_lenet_test_loss, regularized_lenet_test_acc = test(model=regularized_lenet_model, tst_loader=test_data, device=device,
criterion=regularized_lenet_cost_function)
r_lenet_trn_loss.append(regularized_lenet_train_loss); r_lenet_trn_acc.append(regularized_lenet_train_acc)
r_lenet_tst_loss.append(regularized_lenet_test_loss); r_lenet_tst_acc.append(regularized_lenet_test_acc)
print('epochs {} training loss {} training accuracy {} validation loss {} validation accuracy {}'.format(epoch, regularized_lenet_train_loss, regularized_lenet_train_acc,
regularized_lenet_test_loss, regularized_lenet_test_acc))
if epoch+1 == 10 :
print('regularized execution time : {}'.format(time.time() - r_lenet_time))
# ========== 4. Custom model Load ==========
custom_model = CustomMLP().to(device)
custom_optimizer = torch.optim.SGD(custom_model.parameters(), lr=0.01, momentum=0.9)
custom_cost_function = torch.nn.CrossEntropyLoss().to(device)
print('Custom model training start')
custom_time = time.time()
custom_trn_loss, custom_trn_acc, custom_tst_loss, custom_tst_acc = [],[],[],[]
for epoch in range(training_epochs) :
custom_train_loss, custom_train_acc = train(model=custom_model, trn_loader=train_data, device=device, criterion=custom_cost_function, optimizer=custom_optimizer)
custom_test_loss, custom_test_acc = test(model=custom_model, tst_loader=test_data, device=device, criterion=custom_cost_function)
custom_trn_loss.append(custom_train_loss); custom_trn_acc.append(custom_train_acc)
custom_tst_loss.append(custom_test_loss); custom_tst_acc.append(custom_test_acc)
print('epochs {} training loss {} training accuracy {} validation loss {} validation accuracy {}'.format(epoch, custom_train_loss, custom_train_acc,
custom_test_loss, custom_test_acc))
if epoch+1 == 10 :
print('custom model execution time : {}'.format(time.time() - custom_time))
# ========== 5. visualization ==========
# make loss and acc list for visualization
trn_loss = [lenet_trn_loss, r_lenet_trn_loss, custom_trn_loss]
trn_acc = [lenet_trn_acc, r_lenet_trn_acc, custom_trn_acc]
tst_loss = [lenet_tst_loss, r_lenet_tst_loss, custom_tst_loss]
tst_acc = [lenet_tst_acc, r_lenet_tst_acc, custom_tst_acc]
# draw plot
draw_plot(trn_loss, trn_acc, tst_loss, tst_acc)
def main():
""" Main function
Here, you should instantiate
1) Dataset objects for training and test datasets
2) DataLoaders for training and testing
3) model
4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9
5) cost function: use torch.nn.CrossEntropyLoss
"""
# write your codes here
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
batch_size=64
trn_dataset = dataset.MNIST(data_dir='../data/train')
trn_loader = torch.utils.data.DataLoader(trn_dataset,
batch_size=batch_size,
shuffle=True)
tst_dataset = dataset.MNIST(data_dir='../data/test')
tst_loader = torch.utils.data.DataLoader(tst_dataset,
batch_size=batch_size,
shuffle=True)
LeNet = LeNet5().to(device)
Custom = CustomMLP().to(device)
# loss
criterion = nn.CrossEntropyLoss()
optimizer1 = optim.SGD(LeNet.parameters(), lr=0.01, momentum=0.9)
optimizer2 = optim.SGD(Custom.parameters(), lr=0.01, momentum=0.9)
# hyper-parameters
num_epochs = 20
trn_loss_list1 = []
tst_loss_list1 = []
trn_acc1 = []
tst_acc1 = []
trn_loss_list2 = []
tst_loss_list2 = []
trn_acc2 = []
tst_acc2 = []
for epoch in range(num_epochs):
print("epoch : ", (epoch+1))
print("LeNet")
trn_loss, trn_acc = train(LeNet, trn_loader, device, criterion, optimizer1)
trn_loss_list1.append(trn_loss / len(trn_loader))
trn_acc1.append(trn_acc)
tst_loss, tst_acc = test(LeNet, tst_loader, device, criterion)
tst_loss_list1.append(tst_loss / len(tst_loader))
tst_acc1.append(tst_acc)
print("CustomMLP")
trn_loss, trn_acc = train(Custom, trn_loader, device, criterion, optimizer2)
trn_loss_list2.append(trn_loss / len(trn_loader))
trn_acc2.append(trn_acc)
tst_loss, tst_acc = test(Custom, tst_loader, device, criterion)
tst_loss_list2.append(tst_loss / len(tst_loader))
tst_acc2.append(tst_acc)
plt.figure(figsize=(5,4))
x_range = range(len(trn_loss_list1))
plt.plot(x_range, trn_loss_list1, label="trn")
plt.plot(x_range, tst_loss_list1, label="tst")
plt.legend()
plt.ylim(0, 1)
plt.title('LeNet-5 Loss')
plt.xlabel("training steps")
plt.ylabel("loss")
plt.grid()
plt.figure(figsize=(5,4))
x_range = range(len(trn_acc1))
plt.plot(x_range, trn_acc1, label="trn")
plt.plot(x_range, tst_acc1, label="tst")
plt.legend()
plt.ylim(0, 100)
plt.title('LeNet-5 Accuracy')
plt.xlabel("training steps")
plt.ylabel("loss")
plt.grid()
plt.figure(figsize=(5,4))
x_range = range(len(trn_loss_list2))
plt.plot(x_range, trn_loss_list2, label="trn")
plt.plot(x_range, tst_loss_list2, label="tst")
plt.legend()
plt.ylim(0, 1)
plt.title('CustomMLP Loss')
plt.xlabel("training steps")
plt.ylabel("loss")
plt.grid()
plt.figure(figsize=(5,4))
x_range = range(len(trn_acc2))
plt.plot(x_range, trn_acc2, label="trn")
plt.plot(x_range, tst_acc2, label="tst")
plt.legend()
plt.ylim(0, 100)
plt.title('CustomMLP Accuracy')
plt.xlabel("training steps")
plt.ylabel("acc")
plt.grid()
# val acc
with torch.no_grad():
corr_num = 0
total_num = 0
for j, val in enumerate(tst_loader):
val_x, val_label = val
if device:
val_x = val_x.cuda()
val_label =val_label.cuda()
val_output = LeNet(val_x)
model_label = val_output.argmax(dim=1)
corr = val_label[val_label == model_label].size(0)
corr_num += corr
total_num += val_label.size(0)
print("LeNet5 acc: {:.2f}".format(corr_num / total_num * 100))
with torch.no_grad():
corr_num = 0
total_num = 0
for j, val in enumerate(tst_loader):
val_x, val_label = val
if device:
val_x = val_x.cuda()
val_label =val_label.cuda()
val_output = Custom(val_x)
model_label = val_output.argmax(dim=1)
corr = val_label[val_label == model_label].size(0)
corr_num += corr
total_num += val_label.size(0)
print("CustomMLP acc: {:.2f}".format(corr_num / total_num * 100))
def main(epochs, batch_size, learn_angle, angle_lr):
transform = torchvision.transforms.Compose([
transforms.RandomRotation(30),
transforms.Identity() if learn_angle else transforms.Free(),
transforms.ToTensor(),
transforms.Normalize()
])
set_train = dataset.MNIST(root='./data',
train=True,
download=True,
transform=transform)
set_test = dataset.MNIST(root='./data',
train=False,
download=True,
transform=transform)
loader_train = torch.utils.data.DataLoader(set_train,
batch_size=batch_size,
shuffle=True,
num_workers=2)
loader_test = torch.utils.data.DataLoader(set_test,
batch_size=batch_size,
shuffle=False,
num_workers=2)
nn = model.Net(learn_angle)
if torch.cuda.is_available():
nn.cuda()
optimizer = optim.SGD(nn.parameters(), lr=0.001, momentum=0.9)
VIEW_INTERVAL = 100
for epoch in range(epochs):
acc_loss = 0.0
running_loss = 0.0
for i, sample in enumerate(loader_train):
x, y = pack(sample, learn_angle)
optimizer.zero_grad()
y_pred = nn(x)
loss = model.loss(y_pred, y, angle_lr)
loss.backward()
optimizer.step()
# report loss
acc_loss += loss.item()
if i % VIEW_INTERVAL == VIEW_INTERVAL - 1:
running_loss = acc_loss / VIEW_INTERVAL
click.secho(
f"\rEpoch {epoch+1}, iteration {i+1}; "
f"loss: {(running_loss):.3f}; ",
err=True,
nl=False)
acc_loss = 0.0
# testing
count_correct = 0
count_total = 0
for sample in loader_test:
x, labels = pack(sample, learn_angle)
y_pred = nn(Variable(x))
if learn_angle:
labels = labels[0]
y_pred = y_pred[0]
_, labels_pred = torch.max(y_pred.data, 1)
c = (labels_pred == labels).squeeze()
count_correct += c.sum().item()
count_total += len(c)
click.secho(
f"\rEpoch {epoch+1}; loss: {(running_loss):.3f}; "
f"Test Acc: {100.0 * count_correct / count_total :.2f}%",
err=True,
nl=False)
running_loss = 0
click.secho('', err=True)
def main():
# Training settings
parser = argparse.ArgumentParser(
description='Reducing Communication Rounds in Federated Learning')
parser.add_argument('--batch-size',
type=int,
default=32,
metavar='N',
help='input batch size for training (default: 32)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument(
'--max_iterations',
type=int,
default=20,
metavar='N',
help='number of iterations for global training (default:20)')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.9,
metavar='M',
help='SGD momentum (default: 0.9)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
parser.add_argument('--client-num',
type=int,
default=100,
help='Number of clients(locals) to use')
parser.add_argument('--cli-ite-num',
type=int,
default=4,
help='Number of epochs to train the local clients for')
parser.add_argument('--start_threshold',
type=float,
default=0.8,
help='Starting threshold for Check relevance function')
parser.add_argument('--use_wandb',
action='store_true',
default=False,
help='Use wandb for logging')
parser.add_argument('--anonymous_mode',
action='store_true',
default=False,
help='Use anonymous mode for visualizing results')
parser.add_argument(
'--force_client_train',
type=int,
default=5,
help='After these many epochs, clients are selectively trained')
parser.add_argument('--topk', type=float, default=0.1, help='topk')
args = parser.parse_args()
#We use wandb for code logging. First run would require you to set it up on wandb.ai.
#To use wandb, set the flab --use_wandb. If not, matlplotlib plots are stored
if args.use_wandb:
if args.anonymous_mode:
wandb.login()
wandb.init(project="cloud-federated", anonymous="must")
else:
wandb.init(project="cloud-federated", entity="cloud")
wandb.config.update(args)
use_cuda = not args.no_cuda and torch.cuda.is_available()
set_random_seeds(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 0, 'pin_memory': True} if use_cuda else {}
#All trainloders to be appended to this list
train_loaders = []
#We modify the original MNIST dataloader to work for our setting as different clients have different subsets of the data
#Refer to dataset.py for changes
for i in range(args.client_num):
train_loaders.append(
torch.utils.data.DataLoader(dataset.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]),
client_id=i,
num_clients=args.client_num),
batch_size=args.batch_size,
shuffle=True,
**kwargs))
#Test uses the standard dataloader used for MNIST
test_loader = torch.utils.data.DataLoader(dataset.MNIST(
'../data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]),
client_id=-1,
num_clients=1),
batch_size=args.test_batch_size,
shuffle=False,
**kwargs)
model = Net().to(device)
if not os.path.exists('generated_data'):
os.mkdir('generated_data')
if not os.path.exists('generated_data/thresh' + str(args.start_threshold)):
os.mkdir('generated_data/thresh' + str(args.start_threshold))
communication_rounds = []
last_update = None # Last update is initialized to None --> implies that for the first run, all updates are relevant
previous_relevances = [
True
] * args.client_num # Last update is initialized to None --> implies that for the first run, all updates are relevant
prev_local_updates = None # Last update is initialized to None --> implies that for the first run, all updates are relevant
all_relevances = []
test_losses = []
test_accuracies = []
thresholds = []
average_signs = []
when_above_60 = -1
for it in tqdm(range(1, args.max_iterations)):
(
c_round_cmfl, c_round_after_sel
), last_update, avg_sign, thresh, rel_it, prev_local_updates = global_train(
args, model, device, train_loaders, it, last_update,
previous_relevances, prev_local_updates)
if it > args.force_client_train:
previous_relevances = rel_it
print('Not training all! ')
c_round = c_round_after_sel
else:
c_round = c_round_cmfl + args.client_num
communication_rounds.append(c_round)
test_loss, test_acc = test(args, model, device, test_loader)
test_losses.append(test_loss)
test_accuracies.append(test_acc)
print('Cumulative Communication Rounds : {}'.format(
sum(communication_rounds)))
thresholds.append(thresh)
average_signs.append(avg_sign)
all_relevances.append(rel_it)
if args.use_wandb:
heatmap = draw_heatmap(
np.stack(all_relevances, 1).astype(np.float))
wand_dict = {
'Communication Rounds': sum(communication_rounds),
'Test loss': test_loss,
'Test Acc': test_acc,
'Average sign': avg_sign,
'Threshold': thresh,
'relevance_it': wandb.Image(heatmap)
}
wandb.log(wand_dict)
if last_update == None:
print('exiting!')
break
if (args.save_model):
torch.save(model.state_dict(), "model_" + str(it) + ".pt")
if when_above_60 == -1 and test_acc > 60:
when_above_60 = sum(communication_rounds)
all_stats = {
'communication_rounds': communication_rounds,
'test_loss': test_losses,
'test_acc': test_accuracies,
'avg_sign': average_signs,
'threshold': thresholds,
#'heatmap':np.stack(all_relevances,1).astype(np.float)
}
print(
'Training done, the model reached above 60 percent accuracy after {} communication rounds'
.format(when_above_60))
with open(
'generated_data/thresh' + str(args.start_threshold) + '/' +
'all_stats.pkl', 'wb') as f:
pickle.dump(all_stats, f)
plot_data(all_stats,
'generated_data/thresh' + str(args.start_threshold) + '/')
def _parse_args(self):
# Note : can also pass dataset directly
if isinstance(self.dataset, str):
if self.dataset == 'mnist':
self.dataset = dataset.MNIST(binary=False)
elif self.dataset == 'binary_mnist':
self.dataset = dataset.MNIST(binary=True)
elif self.dataset == 'omniglot':
self.dataset = dataset.Omniglot()
elif self.dataset == 'celeb_a':
pass
elif self.dataset == 'dsprites':
self.dataset = dataset.DSprites()
# selects per_label examples from each class for reduced training data
if self.per_label is not None:
self.dataset.shrink_supervised(self.per_label)
self.dataset_clean = copy(self.dataset)
if self.optimizer_params.get("norm", False):
opt_norm = self.optimizer_params["norm"]
del self.optimizer_params["norm"]
else:
opt_norm = None
self.optimizer = getattr(keras.optimizers,
self.optimizer)(**self.optimizer_params)
#if opt_norm is not None:
# self.optimizer = NormalizedOptimizer(self.optimizer, normalization = opt_norm)
self.lr_callback = False
if isinstance(self.lr, str):
try:
mod = importlib.import_module(str('lr_sched'))
# LR Callback will be True /// self.lr = function of epochs -> lr
self.lr_callback = isinstance(self.lr, str)
self.lr = getattr(mod, self.lr)
except:
try:
mod = importlib.import_module(
str('custom_functions.lr_sched'))
# LR Callback will be True /// self.lr = function of epochs -> lr
self.lr_callback = isinstance(self.lr, str)
self.lr = getattr(mod, self.lr)
except:
#self.lr = dflt.get('lr', .001)
#print()
warnings.warn(
"Cannot find LR Schedule function. Proceeding with default, constant learning rate of 0.001."
)
#print()
# Architecture Args
if self.encoder_dims is None:
try:
self.encoder_dims = self.latent_dims
except Exception as e:
print(e)
raise ValueError
if self.decoder_dims is None:
self.decoder_dims = list(reversed(self.encoder_dims[:-1]))
self.decoder_dims.append(self.dataset.dim)
else:
pass
import nn
import dataset
import dataset.transforms
if __name__ == '__main__':
model = nn.model.SequentialModel(input_shape=[-1, 784])
model.add(nn.layer.Dense(128, activation=nn.activation.Tanh()))
model.add(nn.layer.Dense(128, activation=nn.activation.Tanh()))
model.add(nn.layer.Dense(10, activation=nn.activation.Softmax()))
model.setup(nn.loss.Cross_Entropy_With_Softmax(),
nn.metric.CategoricalAccuracy())
# model.save("MNISTDNN.model")
# # model = nn.model.Model.load('MNISTDNN.model')
#
data = dataset.MNIST()
trans = dataset.transforms.Shuffle().add(dataset.transforms.ImageCls())
x, y, x_t, y_t = trans(*data.load())
model.compile(nn.gradient_descent.GradientDecay(0.1))
model.fit(x, y, batch_size=64, epoch=10)
def run_mnist(
log_dir,
save_period,
summary_period,
im_summary_period,
num_iter=int(1e6),
batch_size=128,
lr=0.0001,
lamb=10,
z_dim=8,
c_fn_type='l2_sum',
):
ds = dataset.MNIST(batch_size)
x, _ = ds.train_data_op
p_z = arch.Gaussian_P_Z(z_dim)
p_z_length = p_z.length
Q_arch = partial(arch.fc_arch,
input_shape=(784, ),
output_size=p_z_length,
num_layers=3,
embed_size=256)
G_arch = partial(
arch.fc_arch,
input_shape=(p_z_length, ),
output_size=784, # # of generated pixels
num_layers=3,
embed_size=256)
with tf.variable_scope('param_scope') as scope:
# To clearly seperate the parameters belong to layers from tf ops.
# Make it easier to reuse
pass
if c_fn_type == 'l1':
c_fn = lambda x, y: tf.reduce_mean(tf.abs(x - y), axis=(1, 2, 3)
) #use l1_distance for recon loss
if c_fn_type == 'l1_sum':
c_fn = lambda x, y: tf.reduce_sum(tf.abs(x - y), axis=(1, 2, 3)
) #use l1_distance for recon loss
elif c_fn_type == 'l2':
c_fn = lambda x, y: tf.reduce_mean(
(x - y)**2, axis=(1, 2, 3)) #use l2_distance for recon loss
elif c_fn_type == 'l2_sum':
c_fn = lambda x, y: tf.reduce_sum(
(x - y)**2, axis=(1, 2, 3)) #use l2_distance for recon loss
else:
assert False, 'not supported cost type'
model = \
WAE_MMD(x,
p_z,
Q_arch,
G_arch,
c_fn,
{'lr':lr, 'lambda':lamb},
scope)
run(**locals())
def main(net="lenet5", augmentation=False):
device = torch.device("cpu")
if net == "lenet5" :
net = LeNet5()
elif net == "mlp" :
net = CustomMLP()
else :
print("Not support Networks")
loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(), lr=0.01, momentum=0.9)
if os.path.isdir("../data/train_mnist") == True :
pass
else :
print("Unzip train mnist")
trn_dataset = '../data/train.tar'
trn_dst_path = '../data/train_mnist'
os.makedirs(trn_dst_path)
with tarfile.TarFile(trn_dataset, 'r') as file:
file.extractall(trn_dst_path)
if os.path.isdir("../data/test_mnist") == True :
pass
else :
print("Unzip test mnist")
tst_dataset = '../data/test.tar'
tst_dst_path = '../data/test_mnist'
os.makedirs(tst_dst_path)
with tarfile.TarFile(tst_dataset, 'r') as file:
file.extractall(tst_dst_path)
train_folder = "../data/train_mnist/train"
test_folder = "../data/test_mnist/test"
train_set = dataset.MNIST(train_folder)
test_set = dataset.MNIST(test_folder)
if augmentation == True :
aug_train_set = dataset.MNIST(train_folder, aug=True)
aug_test_Set = dataset.MNIST(test_folder)
trn_loader = DataLoader(train_set, batch_size=128)
tst_loader = DataLoader(test_set, batch_size=128)
if augmentation == True :
aug_train_loader = DataLoader(aug_train_set, batch_size=128)
aug_test_loader = DataLoader(aug_test_Set, batch_size=128)
train_logger = []
test_logger = []
for epoch in range(5) :
if augmentation == False :
training = train(net.to(device), trn_loader, device, loss, optimizer, epoch)
tests = test(net.to(device), tst_loader, device, loss, epoch)
train_logger.append(training)
test_logger.append(tests)
else :
training = train(net.to(device), aug_train_loader, device, loss, optimizer, epoch)
tests = test(net.to(device), tst_loader, device, loss, epoch)
train_logger.append(training)
test_logger.append(tests)
return train_logger, test_logger
def main():
""" Main function
Here, you should instantiate
1) Dataset objects for training and test datasets
2) DataLoaders for training and testing
3) model
4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9
5) cost function: use torch.nn.CrossEntropyLoss
"""
n_cpu = multiprocessing.cpu_count()
data_dir_dict = {'train': '../data/train', 'test': '../data/test'}
for _, data_dir in data_dir_dict.items():
if not os.path.isdir(data_dir):
tar = tarfile.open('%s.tar' % data_dir, mode='r')
tar.extractall(path='./data/')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
num_epochs = 50
batch_size = 512
criterion = nn.CrossEntropyLoss()
train_dataset = dataset.MNIST(data_dir_dict['train'])
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=n_cpu)
test_dataset = dataset.MNIST(data_dir_dict['test'])
test_loader = DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=n_cpu)
net = LeNet5().to(device)
net_optimizer = SGD(net.parameters(), lr=.01, momentum=.9)
net_trn_loss_list, net_trn_acc_list, net_tst_loss_list, net_tst_acc_list= [], [], [], []
print('--------------LeNet5--------------')
for epoch in range(num_epochs):
net_time = time.time()
trn_loss, trn_acc = train(net, train_loader, device, criterion,
net_optimizer)
tst_loss, tst_acc = test(net, test_loader, device, criterion)
print('%s epoch || %.5f time' % ((epoch + 1), time.time() - net_time))
print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\
%(trn_loss, trn_acc, tst_loss, tst_acc))
print('')
net_trn_loss_list.append(trn_loss)
net_trn_acc_list.append(trn_acc)
net_tst_loss_list.append(tst_loss)
net_tst_acc_list.append(tst_acc)
mlp = CustomMLP().to(device)
mlp_optimizer = SGD(mlp.parameters(), lr=.01, momentum=.9)
mlp_trn_loss_list, mlp_trn_acc_list, mlp_tst_loss_list, mlp_tst_acc_list= [], [], [], []
print('--------------CustomMLP--------------')
for epoch in range(num_epochs):
mlp_time = time.time()
trn_loss, trn_acc = train(mlp, train_loader, device, criterion,
mlp_optimizer)
tst_loss, tst_acc = test(mlp, test_loader, device, criterion)
print('%s epoch || %.5f time' % ((epoch + 1), time.time() - mlp_time))
print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\
%(trn_loss, trn_acc, tst_loss, tst_acc))
print('')
mlp_trn_loss_list.append(trn_loss)
mlp_trn_acc_list.append(trn_acc)
mlp_tst_loss_list.append(tst_loss)
mlp_tst_acc_list.append(tst_acc)
del train_dataset
del train_loader
train_dataset_aug = dataset.MNIST(data_dir_dict['train'], aug_option=True)
train_loader_aug = DataLoader(dataset=train_dataset_aug,
batch_size=batch_size,
shuffle=True,
num_workers=n_cpu)
net_aug = LeNet5().to(device)
net_optimizer_aug = SGD(net_aug.parameters(), lr=.01, momentum=.9)
net_aug_trn_loss_list, net_aug_trn_acc_list, net_aug_tst_loss_list, net_aug_tst_acc_list= [], [], [], []
print('--------------LeNet5 with augmentation--------------')
for epoch in range(num_epochs):
net_time = time.time()
trn_loss, trn_acc = train(net_aug, train_loader_aug, device, criterion,
net_optimizer_aug)
tst_loss, tst_acc = test(net_aug, test_loader, device, criterion)
print('%s epoch || %.5f time' % ((epoch + 1), time.time() - net_time))
print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\
%(trn_loss, trn_acc, tst_loss, tst_acc))
print('')
net_aug_trn_loss_list.append(trn_loss)
net_aug_trn_acc_list.append(trn_acc)
net_aug_tst_loss_list.append(tst_loss)
net_aug_tst_acc_list.append(tst_acc)
mlp_aug = CustomMLP().to(device)
mlp_optimizer_aug = SGD(mlp_aug.parameters(), lr=.01, momentum=.9)
mlp_aug_trn_loss_list, mlp_aug_trn_acc_list, mlp_aug_tst_loss_list, mlp_aug_tst_acc_list= [], [], [], []
print('--------------CustomMLP with augmentation--------------')
for epoch in range(num_epochs):
mlp_time = time.time()
trn_loss, trn_acc = train(mlp_aug, train_loader_aug, device, criterion,
mlp_optimizer_aug)
tst_loss, tst_acc = test(mlp_aug, test_loader, device, criterion)
print('%s epoch || %.5f time' % ((epoch + 1), time.time() - mlp_time))
print('avg train loss: %.5f | avg train acc: %.5f ||| avg test loss: %.5f | avg test acc: %.5f'\
%(trn_loss, trn_acc, tst_loss, tst_acc))
print('')
mlp_aug_trn_loss_list.append(trn_loss)
mlp_aug_trn_acc_list.append(trn_acc)
mlp_aug_tst_loss_list.append(tst_loss)
mlp_aug_tst_acc_list.append(tst_acc)
net_result, net_aug_result, mlp_result, mlp_aug_result = {}, {}, {}, {}
net_result['train_loss'] = net_trn_loss_list
net_result['train_acc'] = net_trn_acc_list
net_result['test_loss'] = net_tst_loss_list
net_result['test_acc'] = net_tst_acc_list
net_aug_result['train_loss'] = net_aug_trn_loss_list
net_aug_result['train_acc'] = net_aug_trn_acc_list
net_aug_result['test_loss'] = net_aug_tst_loss_list
net_aug_result['test_acc'] = net_aug_tst_acc_list
mlp_result['train_loss'] = mlp_trn_loss_list
mlp_result['train_acc'] = mlp_trn_acc_list
mlp_result['test_loss'] = mlp_tst_loss_list
mlp_result['test_acc'] = mlp_tst_acc_list
mlp_aug_result['train_loss'] = mlp_aug_trn_loss_list
mlp_aug_result['train_acc'] = mlp_aug_trn_acc_list
mlp_aug_result['test_loss'] = mlp_aug_tst_loss_list
mlp_aug_result['test_acc'] = mlp_aug_tst_acc_list
net_result = pd.DataFrame(
net_result,
columns=['train_loss', 'train_acc', 'test_loss', 'test_acc'])
net_aug_result = pd.DataFrame(
net_aug_result,
columns=['train_loss', 'train_acc', 'test_loss', 'test_acc'])
mlp_result = pd.DataFrame(
mlp_result,
columns=['train_loss', 'train_acc', 'test_loss', 'test_acc'])
mlp_aug_result = pd.DataFrame(
mlp_aug_result,
columns=['train_loss', 'train_acc', 'test_loss', 'test_acc'])
if not os.path.exists('../results/'):
os.makedirs('../results/')
net_result.to_csv('../results/LeNet5.csv', index=False)
net_aug_result.to_csv('../results/LeNet5_aug.csv', index=False)
mlp_result.to_csv('../results/MLP.csv', index=False)
mlp_aug_result.to_csv('../results/MLP_aug.csv', index=False)
def __init__(self, sess, epoch, batch_size, z_dim, dataset_name,
compute_metrics_it, checkpoint_dir, result_dir, log_dir,
gpu_id, bot, redo, verbosity):
self.sess = sess
self.dataset_name = dataset_name.lower()
self.checkpoint_dir = checkpoint_dir
self.result_dir = result_dir
self.log_dir = log_dir
self.epoch = epoch
self.batch_size = batch_size
self.compute_metrics_it = compute_metrics_it
self.gpu_id = gpu_id
self.bot = bot
self.redo = redo
self.verbosity = verbosity
if self.dataset_name in ['mnist']:
# parameters
self.input_height = 28
self.input_width = 28
self.z_dim = z_dim # dimension of noise-vector
self.c_dim = 1
# test
self.sample_num = 64 # number of generated images to be saved
# load mnist
self.ds = dataset.MNIST(self.batch_size)
self.num_batches = self.ds.N_TRAIN_SAMPLES // self.batch_size
# architecture hyper parameters
self.data_format = 'NHWC'
elif self.dataset_name in ['fashion-mnist']:
# parameters
self.input_height = 28
self.input_width = 28
self.z_dim = z_dim # dimension of noise-vector
self.c_dim = 1
# test
self.sample_num = 64 # number of generated images to be saved
# load mnist
self.ds = dataset.FASHION_MNIST(self.batch_size)
self.num_batches = self.ds.N_TRAIN_SAMPLES // self.batch_size
# architecture hyper parameters
self.data_format = 'NHWC'
elif self.dataset_name in ['celeba']:
# parameters
self.input_height = 64
self.input_width = 64
self.z_dim = z_dim # dimension of noise-vector
self.c_dim = 3
# test
self.sample_num = 64 # number of generated images to be saved
# load CelebA
self.ds = dataset.CelebA(self.batch_size)
self.num_batches = self.ds.N_TRAIN_SAMPLES // self.batch_size
else:
raise NotImplementedError
